diff options
Diffstat (limited to 'include/asm-generic')
140 files changed, 5487 insertions, 4896 deletions
diff --git a/include/asm-generic/4level-fixup.h b/include/asm-generic/4level-fixup.h deleted file mode 100644 index e3667c9a33a5..000000000000 --- a/include/asm-generic/4level-fixup.h +++ /dev/null @@ -1,40 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _4LEVEL_FIXUP_H -#define _4LEVEL_FIXUP_H - -#define __ARCH_HAS_4LEVEL_HACK -#define __PAGETABLE_PUD_FOLDED 1 - -#define PUD_SHIFT PGDIR_SHIFT -#define PUD_SIZE PGDIR_SIZE -#define PUD_MASK PGDIR_MASK -#define PTRS_PER_PUD 1 - -#define pud_t pgd_t - -#define pmd_alloc(mm, pud, address) \ - ((unlikely(pgd_none(*(pud))) && __pmd_alloc(mm, pud, address))? \ - NULL: pmd_offset(pud, address)) - -#define pud_offset(pgd, start) (pgd) -#define pud_none(pud) 0 -#define pud_bad(pud) 0 -#define pud_present(pud) 1 -#define pud_ERROR(pud) do { } while (0) -#define pud_clear(pud) pgd_clear(pud) -#define pud_val(pud) pgd_val(pud) -#define pud_populate(mm, pud, pmd) pgd_populate(mm, pud, pmd) -#define pud_page(pud) pgd_page(pud) -#define pud_page_vaddr(pud) pgd_page_vaddr(pud) - -#undef pud_free_tlb -#define pud_free_tlb(tlb, x, addr) do { } while (0) -#define pud_free(mm, x) do { } while (0) -#define __pud_free_tlb(tlb, x, addr) do { } while (0) - -#undef pud_addr_end -#define pud_addr_end(addr, end) (end) - -#include <asm-generic/5level-fixup.h> - -#endif diff --git a/include/asm-generic/5level-fixup.h b/include/asm-generic/5level-fixup.h deleted file mode 100644 index bb6cb347018c..000000000000 --- a/include/asm-generic/5level-fixup.h +++ /dev/null @@ -1,44 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _5LEVEL_FIXUP_H -#define _5LEVEL_FIXUP_H - -#define __ARCH_HAS_5LEVEL_HACK -#define __PAGETABLE_P4D_FOLDED 1 - -#define P4D_SHIFT PGDIR_SHIFT -#define P4D_SIZE PGDIR_SIZE -#define P4D_MASK PGDIR_MASK -#define MAX_PTRS_PER_P4D 1 -#define PTRS_PER_P4D 1 - -#define p4d_t pgd_t - -#define pud_alloc(mm, p4d, address) \ - ((unlikely(pgd_none(*(p4d))) && __pud_alloc(mm, p4d, address)) ? \ - NULL : pud_offset(p4d, address)) - -#define p4d_alloc(mm, pgd, address) (pgd) -#define p4d_offset(pgd, start) (pgd) -#define p4d_none(p4d) 0 -#define p4d_bad(p4d) 0 -#define p4d_present(p4d) 1 -#define p4d_ERROR(p4d) do { } while (0) -#define p4d_clear(p4d) pgd_clear(p4d) -#define p4d_val(p4d) pgd_val(p4d) -#define p4d_populate(mm, p4d, pud) pgd_populate(mm, p4d, pud) -#define p4d_populate_safe(mm, p4d, pud) pgd_populate(mm, p4d, pud) -#define p4d_page(p4d) pgd_page(p4d) -#define p4d_page_vaddr(p4d) pgd_page_vaddr(p4d) - -#define __p4d(x) __pgd(x) -#define set_p4d(p4dp, p4d) set_pgd(p4dp, p4d) - -#undef p4d_free_tlb -#define p4d_free_tlb(tlb, x, addr) do { } while (0) -#define p4d_free(mm, x) do { } while (0) -#define __p4d_free_tlb(tlb, x, addr) do { } while (0) - -#undef p4d_addr_end -#define p4d_addr_end(addr, end) (end) - -#endif diff --git a/include/asm-generic/Kbuild b/include/asm-generic/Kbuild new file mode 100644 index 000000000000..295c94a3ccc1 --- /dev/null +++ b/include/asm-generic/Kbuild @@ -0,0 +1,67 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# asm headers that all architectures except um should have +# (This file is not included when SRCARCH=um since UML borrows several +# asm headers from the host architecture.) + +mandatory-y += atomic.h +mandatory-y += archrandom.h +mandatory-y += barrier.h +mandatory-y += bitops.h +mandatory-y += bug.h +mandatory-y += cacheflush.h +mandatory-y += cfi.h +mandatory-y += checksum.h +mandatory-y += compat.h +mandatory-y += current.h +mandatory-y += delay.h +mandatory-y += device.h +mandatory-y += div64.h +mandatory-y += dma-mapping.h +mandatory-y += dma.h +mandatory-y += emergency-restart.h +mandatory-y += exec.h +mandatory-y += ftrace.h +mandatory-y += futex.h +mandatory-y += hardirq.h +mandatory-y += hw_irq.h +mandatory-y += io.h +mandatory-y += irq.h +mandatory-y += irq_regs.h +mandatory-y += irq_work.h +mandatory-y += kdebug.h +mandatory-y += kmap_size.h +mandatory-y += kprobes.h +mandatory-y += linkage.h +mandatory-y += local.h +mandatory-y += local64.h +mandatory-y += mmiowb.h +mandatory-y += mmu.h +mandatory-y += mmu_context.h +mandatory-y += module.h +mandatory-y += module.lds.h +mandatory-y += msi.h +mandatory-y += pci.h +mandatory-y += percpu.h +mandatory-y += pgalloc.h +mandatory-y += preempt.h +mandatory-y += rqspinlock.h +mandatory-y += runtime-const.h +mandatory-y += rwonce.h +mandatory-y += sections.h +mandatory-y += serial.h +mandatory-y += shmparam.h +mandatory-y += simd.h +mandatory-y += softirq_stack.h +mandatory-y += switch_to.h +mandatory-y += timex.h +mandatory-y += tlbflush.h +mandatory-y += topology.h +mandatory-y += trace_clock.h +mandatory-y += uaccess.h +mandatory-y += unwind_user.h +mandatory-y += vermagic.h +mandatory-y += vga.h +mandatory-y += video.h +mandatory-y += word-at-a-time.h +mandatory-y += xor.h diff --git a/include/asm-generic/access_ok.h b/include/asm-generic/access_ok.h new file mode 100644 index 000000000000..2866ae61b1cd --- /dev/null +++ b/include/asm-generic/access_ok.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_GENERIC_ACCESS_OK_H__ +#define __ASM_GENERIC_ACCESS_OK_H__ + +/* + * Checking whether a pointer is valid for user space access. + * These definitions work on most architectures, but overrides can + * be used where necessary. + */ + +/* + * architectures with compat tasks have a variable TASK_SIZE and should + * override this to a constant. + */ +#ifndef TASK_SIZE_MAX +#define TASK_SIZE_MAX TASK_SIZE +#endif + +#ifndef __access_ok +/* + * 'size' is a compile-time constant for most callers, so optimize for + * this case to turn the check into a single comparison against a constant + * limit and catch all possible overflows. + * On architectures with separate user address space (m68k, s390, parisc, + * sparc64) or those without an MMU, this should always return true. + * + * This version was originally contributed by Jonas Bonn for the + * OpenRISC architecture, and was found to be the most efficient + * for constant 'size' and 'limit' values. + */ +static inline int __access_ok(const void __user *ptr, unsigned long size) +{ + unsigned long limit = TASK_SIZE_MAX; + unsigned long addr = (unsigned long)ptr; + + if (IS_ENABLED(CONFIG_ALTERNATE_USER_ADDRESS_SPACE) || + !IS_ENABLED(CONFIG_MMU)) + return true; + + return (size <= limit) && (addr <= (limit - size)); +} +#endif + +#ifndef access_ok +#define access_ok(addr, size) likely(__access_ok(addr, size)) +#endif + +#endif diff --git a/include/asm-generic/agp.h b/include/asm-generic/agp.h new file mode 100644 index 000000000000..10db92ede168 --- /dev/null +++ b/include/asm-generic/agp.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_GENERIC_AGP_H +#define _ASM_GENERIC_AGP_H + +#include <asm/io.h> + +#define map_page_into_agp(page) do {} while (0) +#define unmap_page_from_agp(page) do {} while (0) +#define flush_agp_cache() mb() + +#endif /* _ASM_GENERIC_AGP_H */ diff --git a/include/asm-generic/archrandom.h b/include/asm-generic/archrandom.h new file mode 100644 index 000000000000..3cd7f980cfdc --- /dev/null +++ b/include/asm-generic/archrandom.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_GENERIC_ARCHRANDOM_H__ +#define __ASM_GENERIC_ARCHRANDOM_H__ + +static inline size_t __must_check arch_get_random_longs(unsigned long *v, size_t max_longs) +{ + return 0; +} + +static inline size_t __must_check arch_get_random_seed_longs(unsigned long *v, size_t max_longs) +{ + return 0; +} + +#endif diff --git a/include/asm-generic/atomic-instrumented.h b/include/asm-generic/atomic-instrumented.h deleted file mode 100644 index 0d4b1d3dbc1e..000000000000 --- a/include/asm-generic/atomic-instrumented.h +++ /dev/null @@ -1,467 +0,0 @@ -/* - * This file provides wrappers with KASAN instrumentation for atomic operations. - * To use this functionality an arch's atomic.h file needs to define all - * atomic operations with arch_ prefix (e.g. arch_atomic_read()) and include - * this file at the end. This file provides atomic_read() that forwards to - * arch_atomic_read() for actual atomic operation. - * Note: if an arch atomic operation is implemented by means of other atomic - * operations (e.g. atomic_read()/atomic_cmpxchg() loop), then it needs to use - * arch_ variants (i.e. arch_atomic_read()/arch_atomic_cmpxchg()) to avoid - * double instrumentation. - */ - -#ifndef _LINUX_ATOMIC_INSTRUMENTED_H -#define _LINUX_ATOMIC_INSTRUMENTED_H - -#include <linux/build_bug.h> -#include <linux/kasan-checks.h> - -static __always_inline int atomic_read(const atomic_t *v) -{ - kasan_check_read(v, sizeof(*v)); - return arch_atomic_read(v); -} - -static __always_inline s64 atomic64_read(const atomic64_t *v) -{ - kasan_check_read(v, sizeof(*v)); - return arch_atomic64_read(v); -} - -static __always_inline void atomic_set(atomic_t *v, int i) -{ - kasan_check_write(v, sizeof(*v)); - arch_atomic_set(v, i); -} - -static __always_inline void atomic64_set(atomic64_t *v, s64 i) -{ - kasan_check_write(v, sizeof(*v)); - arch_atomic64_set(v, i); -} - -static __always_inline int atomic_xchg(atomic_t *v, int i) -{ - kasan_check_write(v, sizeof(*v)); - return arch_atomic_xchg(v, i); -} - -static __always_inline s64 atomic64_xchg(atomic64_t *v, s64 i) -{ - kasan_check_write(v, sizeof(*v)); - return arch_atomic64_xchg(v, i); -} - -static __always_inline int atomic_cmpxchg(atomic_t *v, int old, int new) -{ - kasan_check_write(v, sizeof(*v)); - return arch_atomic_cmpxchg(v, old, new); -} - -static __always_inline s64 atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) -{ - kasan_check_write(v, sizeof(*v)); - return arch_atomic64_cmpxchg(v, old, new); -} - -#ifdef arch_atomic_try_cmpxchg -#define atomic_try_cmpxchg atomic_try_cmpxchg -static __always_inline bool atomic_try_cmpxchg(atomic_t *v, int *old, int new) -{ - kasan_check_write(v, sizeof(*v)); - kasan_check_read(old, sizeof(*old)); - return arch_atomic_try_cmpxchg(v, old, new); -} -#endif - -#ifdef arch_atomic64_try_cmpxchg -#define atomic64_try_cmpxchg atomic64_try_cmpxchg -static __always_inline bool atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) -{ - kasan_check_write(v, sizeof(*v)); - kasan_check_read(old, sizeof(*old)); - return arch_atomic64_try_cmpxchg(v, old, new); -} -#endif - -#ifdef arch_atomic_fetch_add_unless -#define atomic_fetch_add_unless atomic_fetch_add_unless -static __always_inline int atomic_fetch_add_unless(atomic_t *v, int a, int u) -{ - kasan_check_write(v, sizeof(*v)); - return arch_atomic_fetch_add_unless(v, a, u); -} -#endif - -#ifdef arch_atomic64_fetch_add_unless -#define atomic64_fetch_add_unless atomic64_fetch_add_unless -static __always_inline s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) -{ - kasan_check_write(v, sizeof(*v)); - return arch_atomic64_fetch_add_unless(v, a, u); -} -#endif - -#ifdef arch_atomic_inc -#define atomic_inc atomic_inc -static __always_inline void atomic_inc(atomic_t *v) -{ - kasan_check_write(v, sizeof(*v)); - arch_atomic_inc(v); -} -#endif - -#ifdef arch_atomic64_inc -#define atomic64_inc atomic64_inc -static __always_inline void atomic64_inc(atomic64_t *v) -{ - kasan_check_write(v, sizeof(*v)); - arch_atomic64_inc(v); -} -#endif - -#ifdef arch_atomic_dec -#define atomic_dec atomic_dec -static __always_inline void atomic_dec(atomic_t *v) -{ - kasan_check_write(v, sizeof(*v)); - arch_atomic_dec(v); -} -#endif - -#ifdef atch_atomic64_dec -#define atomic64_dec -static __always_inline void atomic64_dec(atomic64_t *v) -{ - kasan_check_write(v, sizeof(*v)); - arch_atomic64_dec(v); -} -#endif - -static __always_inline void atomic_add(int i, atomic_t *v) -{ - kasan_check_write(v, sizeof(*v)); - arch_atomic_add(i, v); -} - -static __always_inline void atomic64_add(s64 i, atomic64_t *v) -{ - kasan_check_write(v, sizeof(*v)); - arch_atomic64_add(i, v); -} - -static __always_inline void atomic_sub(int i, atomic_t *v) -{ - kasan_check_write(v, sizeof(*v)); - arch_atomic_sub(i, v); -} - -static __always_inline void atomic64_sub(s64 i, atomic64_t *v) -{ - kasan_check_write(v, sizeof(*v)); - arch_atomic64_sub(i, v); -} - -static __always_inline void atomic_and(int i, atomic_t *v) -{ - kasan_check_write(v, sizeof(*v)); - arch_atomic_and(i, v); -} - -static __always_inline void atomic64_and(s64 i, atomic64_t *v) -{ - kasan_check_write(v, sizeof(*v)); - arch_atomic64_and(i, v); -} - -static __always_inline void atomic_or(int i, atomic_t *v) -{ - kasan_check_write(v, sizeof(*v)); - arch_atomic_or(i, v); -} - -static __always_inline void atomic64_or(s64 i, atomic64_t *v) -{ - kasan_check_write(v, sizeof(*v)); - arch_atomic64_or(i, v); -} - -static __always_inline void atomic_xor(int i, atomic_t *v) -{ - kasan_check_write(v, sizeof(*v)); - arch_atomic_xor(i, v); -} - -static __always_inline void atomic64_xor(s64 i, atomic64_t *v) -{ - kasan_check_write(v, sizeof(*v)); - arch_atomic64_xor(i, v); -} - -#ifdef arch_atomic_inc_return -#define atomic_inc_return atomic_inc_return -static __always_inline int atomic_inc_return(atomic_t *v) -{ - kasan_check_write(v, sizeof(*v)); - return arch_atomic_inc_return(v); -} -#endif - -#ifdef arch_atomic64_in_return -#define atomic64_inc_return atomic64_inc_return -static __always_inline s64 atomic64_inc_return(atomic64_t *v) -{ - kasan_check_write(v, sizeof(*v)); - return arch_atomic64_inc_return(v); -} -#endif - -#ifdef arch_atomic_dec_return -#define atomic_dec_return atomic_dec_return -static __always_inline int atomic_dec_return(atomic_t *v) -{ - kasan_check_write(v, sizeof(*v)); - return arch_atomic_dec_return(v); -} -#endif - -#ifdef arch_atomic64_dec_return -#define atomic64_dec_return atomic64_dec_return -static __always_inline s64 atomic64_dec_return(atomic64_t *v) -{ - kasan_check_write(v, sizeof(*v)); - return arch_atomic64_dec_return(v); -} -#endif - -#ifdef arch_atomic64_inc_not_zero -#define atomic64_inc_not_zero atomic64_inc_not_zero -static __always_inline bool atomic64_inc_not_zero(atomic64_t *v) -{ - kasan_check_write(v, sizeof(*v)); - return arch_atomic64_inc_not_zero(v); -} -#endif - -#ifdef arch_atomic64_dec_if_positive -#define atomic64_dec_if_positive atomic64_dec_if_positive -static __always_inline s64 atomic64_dec_if_positive(atomic64_t *v) -{ - kasan_check_write(v, sizeof(*v)); - return arch_atomic64_dec_if_positive(v); -} -#endif - -#ifdef arch_atomic_dec_and_test -#define atomic_dec_and_test atomic_dec_and_test -static __always_inline bool atomic_dec_and_test(atomic_t *v) -{ - kasan_check_write(v, sizeof(*v)); - return arch_atomic_dec_and_test(v); -} -#endif - -#ifdef arch_atomic64_dec_and_test -#define atomic64_dec_and_test atomic64_dec_and_test -static __always_inline bool atomic64_dec_and_test(atomic64_t *v) -{ - kasan_check_write(v, sizeof(*v)); - return arch_atomic64_dec_and_test(v); -} -#endif - -#ifdef arch_atomic_inc_and_test -#define atomic_inc_and_test atomic_inc_and_test -static __always_inline bool atomic_inc_and_test(atomic_t *v) -{ - kasan_check_write(v, sizeof(*v)); - return arch_atomic_inc_and_test(v); -} -#endif - -#ifdef arch_atomic64_inc_and_test -#define atomic64_inc_and_test atomic64_inc_and_test -static __always_inline bool atomic64_inc_and_test(atomic64_t *v) -{ - kasan_check_write(v, sizeof(*v)); - return arch_atomic64_inc_and_test(v); -} -#endif - -static __always_inline int atomic_add_return(int i, atomic_t *v) -{ - kasan_check_write(v, sizeof(*v)); - return arch_atomic_add_return(i, v); -} - -static __always_inline s64 atomic64_add_return(s64 i, atomic64_t *v) -{ - kasan_check_write(v, sizeof(*v)); - return arch_atomic64_add_return(i, v); -} - -static __always_inline int atomic_sub_return(int i, atomic_t *v) -{ - kasan_check_write(v, sizeof(*v)); - return arch_atomic_sub_return(i, v); -} - -static __always_inline s64 atomic64_sub_return(s64 i, atomic64_t *v) -{ - kasan_check_write(v, sizeof(*v)); - return arch_atomic64_sub_return(i, v); -} - -static __always_inline int atomic_fetch_add(int i, atomic_t *v) -{ - kasan_check_write(v, sizeof(*v)); - return arch_atomic_fetch_add(i, v); -} - -static __always_inline s64 atomic64_fetch_add(s64 i, atomic64_t *v) -{ - kasan_check_write(v, sizeof(*v)); - return arch_atomic64_fetch_add(i, v); -} - -static __always_inline int atomic_fetch_sub(int i, atomic_t *v) -{ - kasan_check_write(v, sizeof(*v)); - return arch_atomic_fetch_sub(i, v); -} - -static __always_inline s64 atomic64_fetch_sub(s64 i, atomic64_t *v) -{ - kasan_check_write(v, sizeof(*v)); - return arch_atomic64_fetch_sub(i, v); -} - -static __always_inline int atomic_fetch_and(int i, atomic_t *v) -{ - kasan_check_write(v, sizeof(*v)); - return arch_atomic_fetch_and(i, v); -} - -static __always_inline s64 atomic64_fetch_and(s64 i, atomic64_t *v) -{ - kasan_check_write(v, sizeof(*v)); - return arch_atomic64_fetch_and(i, v); -} - -static __always_inline int atomic_fetch_or(int i, atomic_t *v) -{ - kasan_check_write(v, sizeof(*v)); - return arch_atomic_fetch_or(i, v); -} - -static __always_inline s64 atomic64_fetch_or(s64 i, atomic64_t *v) -{ - kasan_check_write(v, sizeof(*v)); - return arch_atomic64_fetch_or(i, v); -} - -static __always_inline int atomic_fetch_xor(int i, atomic_t *v) -{ - kasan_check_write(v, sizeof(*v)); - return arch_atomic_fetch_xor(i, v); -} - -static __always_inline s64 atomic64_fetch_xor(s64 i, atomic64_t *v) -{ - kasan_check_write(v, sizeof(*v)); - return arch_atomic64_fetch_xor(i, v); -} - -#ifdef arch_atomic_sub_and_test -#define atomic_sub_and_test atomic_sub_and_test -static __always_inline bool atomic_sub_and_test(int i, atomic_t *v) -{ - kasan_check_write(v, sizeof(*v)); - return arch_atomic_sub_and_test(i, v); -} -#endif - -#ifdef arch_atomic64_sub_and_test -#define atomic64_sub_and_test atomic64_sub_and_test -static __always_inline bool atomic64_sub_and_test(s64 i, atomic64_t *v) -{ - kasan_check_write(v, sizeof(*v)); - return arch_atomic64_sub_and_test(i, v); -} -#endif - -#ifdef arch_atomic_add_negative -#define atomic_add_negative atomic_add_negative -static __always_inline bool atomic_add_negative(int i, atomic_t *v) -{ - kasan_check_write(v, sizeof(*v)); - return arch_atomic_add_negative(i, v); -} -#endif - -#ifdef arch_atomic64_add_negative -#define atomic64_add_negative atomic64_add_negative -static __always_inline bool atomic64_add_negative(s64 i, atomic64_t *v) -{ - kasan_check_write(v, sizeof(*v)); - return arch_atomic64_add_negative(i, v); -} -#endif - -#define xchg(ptr, new) \ -({ \ - typeof(ptr) __ai_ptr = (ptr); \ - kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_xchg(__ai_ptr, (new)); \ -}) - -#define cmpxchg(ptr, old, new) \ -({ \ - typeof(ptr) __ai_ptr = (ptr); \ - kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_cmpxchg(__ai_ptr, (old), (new)); \ -}) - -#define sync_cmpxchg(ptr, old, new) \ -({ \ - typeof(ptr) __ai_ptr = (ptr); \ - kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_sync_cmpxchg(__ai_ptr, (old), (new)); \ -}) - -#define cmpxchg_local(ptr, old, new) \ -({ \ - typeof(ptr) __ai_ptr = (ptr); \ - kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_cmpxchg_local(__ai_ptr, (old), (new)); \ -}) - -#define cmpxchg64(ptr, old, new) \ -({ \ - typeof(ptr) __ai_ptr = (ptr); \ - kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_cmpxchg64(__ai_ptr, (old), (new)); \ -}) - -#define cmpxchg64_local(ptr, old, new) \ -({ \ - typeof(ptr) __ai_ptr = (ptr); \ - kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_cmpxchg64_local(__ai_ptr, (old), (new)); \ -}) - -#define cmpxchg_double(p1, p2, o1, o2, n1, n2) \ -({ \ - typeof(p1) __ai_p1 = (p1); \ - kasan_check_write(__ai_p1, 2 * sizeof(*__ai_p1)); \ - arch_cmpxchg_double(__ai_p1, (p2), (o1), (o2), (n1), (n2)); \ -}) - -#define cmpxchg_double_local(p1, p2, o1, o2, n1, n2) \ -({ \ - typeof(p1) __ai_p1 = (p1); \ - kasan_check_write(__ai_p1, 2 * sizeof(*__ai_p1)); \ - arch_cmpxchg_double_local(__ai_p1, (p2), (o1), (o2), (n1), (n2)); \ -}) - -#endif /* _LINUX_ATOMIC_INSTRUMENTED_H */ diff --git a/include/asm-generic/atomic-long.h b/include/asm-generic/atomic-long.h deleted file mode 100644 index 87d14476edc2..000000000000 --- a/include/asm-generic/atomic-long.h +++ /dev/null @@ -1,269 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _ASM_GENERIC_ATOMIC_LONG_H -#define _ASM_GENERIC_ATOMIC_LONG_H -/* - * Copyright (C) 2005 Silicon Graphics, Inc. - * Christoph Lameter - * - * Allows to provide arch independent atomic definitions without the need to - * edit all arch specific atomic.h files. - */ - -#include <asm/types.h> - -/* - * Suppport for atomic_long_t - * - * Casts for parameters are avoided for existing atomic functions in order to - * avoid issues with cast-as-lval under gcc 4.x and other limitations that the - * macros of a platform may have. - */ - -#if BITS_PER_LONG == 64 - -typedef atomic64_t atomic_long_t; - -#define ATOMIC_LONG_INIT(i) ATOMIC64_INIT(i) -#define ATOMIC_LONG_PFX(x) atomic64 ## x -#define ATOMIC_LONG_TYPE s64 - -#else - -typedef atomic_t atomic_long_t; - -#define ATOMIC_LONG_INIT(i) ATOMIC_INIT(i) -#define ATOMIC_LONG_PFX(x) atomic ## x -#define ATOMIC_LONG_TYPE int - -#endif - -#define ATOMIC_LONG_READ_OP(mo) \ -static inline long atomic_long_read##mo(const atomic_long_t *l) \ -{ \ - ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l; \ - \ - return (long)ATOMIC_LONG_PFX(_read##mo)(v); \ -} -ATOMIC_LONG_READ_OP() -ATOMIC_LONG_READ_OP(_acquire) - -#undef ATOMIC_LONG_READ_OP - -#define ATOMIC_LONG_SET_OP(mo) \ -static inline void atomic_long_set##mo(atomic_long_t *l, long i) \ -{ \ - ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l; \ - \ - ATOMIC_LONG_PFX(_set##mo)(v, i); \ -} -ATOMIC_LONG_SET_OP() -ATOMIC_LONG_SET_OP(_release) - -#undef ATOMIC_LONG_SET_OP - -#define ATOMIC_LONG_ADD_SUB_OP(op, mo) \ -static inline long \ -atomic_long_##op##_return##mo(long i, atomic_long_t *l) \ -{ \ - ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l; \ - \ - return (long)ATOMIC_LONG_PFX(_##op##_return##mo)(i, v); \ -} -ATOMIC_LONG_ADD_SUB_OP(add,) -ATOMIC_LONG_ADD_SUB_OP(add, _relaxed) -ATOMIC_LONG_ADD_SUB_OP(add, _acquire) -ATOMIC_LONG_ADD_SUB_OP(add, _release) -ATOMIC_LONG_ADD_SUB_OP(sub,) -ATOMIC_LONG_ADD_SUB_OP(sub, _relaxed) -ATOMIC_LONG_ADD_SUB_OP(sub, _acquire) -ATOMIC_LONG_ADD_SUB_OP(sub, _release) - -#undef ATOMIC_LONG_ADD_SUB_OP - -#define atomic_long_cmpxchg_relaxed(l, old, new) \ - (ATOMIC_LONG_PFX(_cmpxchg_relaxed)((ATOMIC_LONG_PFX(_t) *)(l), \ - (old), (new))) -#define atomic_long_cmpxchg_acquire(l, old, new) \ - (ATOMIC_LONG_PFX(_cmpxchg_acquire)((ATOMIC_LONG_PFX(_t) *)(l), \ - (old), (new))) -#define atomic_long_cmpxchg_release(l, old, new) \ - (ATOMIC_LONG_PFX(_cmpxchg_release)((ATOMIC_LONG_PFX(_t) *)(l), \ - (old), (new))) -#define atomic_long_cmpxchg(l, old, new) \ - (ATOMIC_LONG_PFX(_cmpxchg)((ATOMIC_LONG_PFX(_t) *)(l), (old), (new))) - - -#define atomic_long_try_cmpxchg_relaxed(l, old, new) \ - (ATOMIC_LONG_PFX(_try_cmpxchg_relaxed)((ATOMIC_LONG_PFX(_t) *)(l), \ - (ATOMIC_LONG_TYPE *)(old), (ATOMIC_LONG_TYPE)(new))) -#define atomic_long_try_cmpxchg_acquire(l, old, new) \ - (ATOMIC_LONG_PFX(_try_cmpxchg_acquire)((ATOMIC_LONG_PFX(_t) *)(l), \ - (ATOMIC_LONG_TYPE *)(old), (ATOMIC_LONG_TYPE)(new))) -#define atomic_long_try_cmpxchg_release(l, old, new) \ - (ATOMIC_LONG_PFX(_try_cmpxchg_release)((ATOMIC_LONG_PFX(_t) *)(l), \ - (ATOMIC_LONG_TYPE *)(old), (ATOMIC_LONG_TYPE)(new))) -#define atomic_long_try_cmpxchg(l, old, new) \ - (ATOMIC_LONG_PFX(_try_cmpxchg)((ATOMIC_LONG_PFX(_t) *)(l), \ - (ATOMIC_LONG_TYPE *)(old), (ATOMIC_LONG_TYPE)(new))) - - -#define atomic_long_xchg_relaxed(v, new) \ - (ATOMIC_LONG_PFX(_xchg_relaxed)((ATOMIC_LONG_PFX(_t) *)(v), (new))) -#define atomic_long_xchg_acquire(v, new) \ - (ATOMIC_LONG_PFX(_xchg_acquire)((ATOMIC_LONG_PFX(_t) *)(v), (new))) -#define atomic_long_xchg_release(v, new) \ - (ATOMIC_LONG_PFX(_xchg_release)((ATOMIC_LONG_PFX(_t) *)(v), (new))) -#define atomic_long_xchg(v, new) \ - (ATOMIC_LONG_PFX(_xchg)((ATOMIC_LONG_PFX(_t) *)(v), (new))) - -static __always_inline void atomic_long_inc(atomic_long_t *l) -{ - ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l; - - ATOMIC_LONG_PFX(_inc)(v); -} - -static __always_inline void atomic_long_dec(atomic_long_t *l) -{ - ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l; - - ATOMIC_LONG_PFX(_dec)(v); -} - -#define ATOMIC_LONG_FETCH_OP(op, mo) \ -static inline long \ -atomic_long_fetch_##op##mo(long i, atomic_long_t *l) \ -{ \ - ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l; \ - \ - return (long)ATOMIC_LONG_PFX(_fetch_##op##mo)(i, v); \ -} - -ATOMIC_LONG_FETCH_OP(add, ) -ATOMIC_LONG_FETCH_OP(add, _relaxed) -ATOMIC_LONG_FETCH_OP(add, _acquire) -ATOMIC_LONG_FETCH_OP(add, _release) -ATOMIC_LONG_FETCH_OP(sub, ) -ATOMIC_LONG_FETCH_OP(sub, _relaxed) -ATOMIC_LONG_FETCH_OP(sub, _acquire) -ATOMIC_LONG_FETCH_OP(sub, _release) -ATOMIC_LONG_FETCH_OP(and, ) -ATOMIC_LONG_FETCH_OP(and, _relaxed) -ATOMIC_LONG_FETCH_OP(and, _acquire) -ATOMIC_LONG_FETCH_OP(and, _release) -ATOMIC_LONG_FETCH_OP(andnot, ) -ATOMIC_LONG_FETCH_OP(andnot, _relaxed) -ATOMIC_LONG_FETCH_OP(andnot, _acquire) -ATOMIC_LONG_FETCH_OP(andnot, _release) -ATOMIC_LONG_FETCH_OP(or, ) -ATOMIC_LONG_FETCH_OP(or, _relaxed) -ATOMIC_LONG_FETCH_OP(or, _acquire) -ATOMIC_LONG_FETCH_OP(or, _release) -ATOMIC_LONG_FETCH_OP(xor, ) -ATOMIC_LONG_FETCH_OP(xor, _relaxed) -ATOMIC_LONG_FETCH_OP(xor, _acquire) -ATOMIC_LONG_FETCH_OP(xor, _release) - -#undef ATOMIC_LONG_FETCH_OP - -#define ATOMIC_LONG_FETCH_INC_DEC_OP(op, mo) \ -static inline long \ -atomic_long_fetch_##op##mo(atomic_long_t *l) \ -{ \ - ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l; \ - \ - return (long)ATOMIC_LONG_PFX(_fetch_##op##mo)(v); \ -} - -ATOMIC_LONG_FETCH_INC_DEC_OP(inc,) -ATOMIC_LONG_FETCH_INC_DEC_OP(inc, _relaxed) -ATOMIC_LONG_FETCH_INC_DEC_OP(inc, _acquire) -ATOMIC_LONG_FETCH_INC_DEC_OP(inc, _release) -ATOMIC_LONG_FETCH_INC_DEC_OP(dec,) -ATOMIC_LONG_FETCH_INC_DEC_OP(dec, _relaxed) -ATOMIC_LONG_FETCH_INC_DEC_OP(dec, _acquire) -ATOMIC_LONG_FETCH_INC_DEC_OP(dec, _release) - -#undef ATOMIC_LONG_FETCH_INC_DEC_OP - -#define ATOMIC_LONG_OP(op) \ -static __always_inline void \ -atomic_long_##op(long i, atomic_long_t *l) \ -{ \ - ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l; \ - \ - ATOMIC_LONG_PFX(_##op)(i, v); \ -} - -ATOMIC_LONG_OP(add) -ATOMIC_LONG_OP(sub) -ATOMIC_LONG_OP(and) -ATOMIC_LONG_OP(andnot) -ATOMIC_LONG_OP(or) -ATOMIC_LONG_OP(xor) - -#undef ATOMIC_LONG_OP - -static inline int atomic_long_sub_and_test(long i, atomic_long_t *l) -{ - ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l; - - return ATOMIC_LONG_PFX(_sub_and_test)(i, v); -} - -static inline int atomic_long_dec_and_test(atomic_long_t *l) -{ - ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l; - - return ATOMIC_LONG_PFX(_dec_and_test)(v); -} - -static inline int atomic_long_inc_and_test(atomic_long_t *l) -{ - ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l; - - return ATOMIC_LONG_PFX(_inc_and_test)(v); -} - -static inline int atomic_long_add_negative(long i, atomic_long_t *l) -{ - ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l; - - return ATOMIC_LONG_PFX(_add_negative)(i, v); -} - -#define ATOMIC_LONG_INC_DEC_OP(op, mo) \ -static inline long \ -atomic_long_##op##_return##mo(atomic_long_t *l) \ -{ \ - ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l; \ - \ - return (long)ATOMIC_LONG_PFX(_##op##_return##mo)(v); \ -} -ATOMIC_LONG_INC_DEC_OP(inc,) -ATOMIC_LONG_INC_DEC_OP(inc, _relaxed) -ATOMIC_LONG_INC_DEC_OP(inc, _acquire) -ATOMIC_LONG_INC_DEC_OP(inc, _release) -ATOMIC_LONG_INC_DEC_OP(dec,) -ATOMIC_LONG_INC_DEC_OP(dec, _relaxed) -ATOMIC_LONG_INC_DEC_OP(dec, _acquire) -ATOMIC_LONG_INC_DEC_OP(dec, _release) - -#undef ATOMIC_LONG_INC_DEC_OP - -static inline long atomic_long_add_unless(atomic_long_t *l, long a, long u) -{ - ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l; - - return (long)ATOMIC_LONG_PFX(_add_unless)(v, a, u); -} - -#define atomic_long_inc_not_zero(l) \ - ATOMIC_LONG_PFX(_inc_not_zero)((ATOMIC_LONG_PFX(_t) *)(l)) - -#define atomic_long_cond_read_relaxed(v, c) \ - ATOMIC_LONG_PFX(_cond_read_relaxed)((ATOMIC_LONG_PFX(_t) *)(v), (c)) -#define atomic_long_cond_read_acquire(v, c) \ - ATOMIC_LONG_PFX(_cond_read_acquire)((ATOMIC_LONG_PFX(_t) *)(v), (c)) - -#endif /* _ASM_GENERIC_ATOMIC_LONG_H */ diff --git a/include/asm-generic/atomic.h b/include/asm-generic/atomic.h index 13324aa828eb..22142c71d35a 100644 --- a/include/asm-generic/atomic.h +++ b/include/asm-generic/atomic.h @@ -1,14 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* - * Generic C implementation of atomic counter operations. Usable on - * UP systems only. Do not include in machine independent code. + * Generic C implementation of atomic counter operations. Do not include in + * machine independent code. * * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved. * Written by David Howells (dhowells@redhat.com) - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public Licence - * as published by the Free Software Foundation; either version - * 2 of the Licence, or (at your option) any later version. */ #ifndef __ASM_GENERIC_ATOMIC_H #define __ASM_GENERIC_ATOMIC_H @@ -16,56 +12,39 @@ #include <asm/cmpxchg.h> #include <asm/barrier.h> -/* - * atomic_$op() - $op integer to atomic variable - * @i: integer value to $op - * @v: pointer to the atomic variable - * - * Atomically $ops @i to @v. Does not strictly guarantee a memory-barrier, use - * smp_mb__{before,after}_atomic(). - */ - -/* - * atomic_$op_return() - $op interer to atomic variable and returns the result - * @i: integer value to $op - * @v: pointer to the atomic variable - * - * Atomically $ops @i to @v. Does imply a full memory barrier. - */ - #ifdef CONFIG_SMP /* we can build all atomic primitives from cmpxchg */ #define ATOMIC_OP(op, c_op) \ -static inline void atomic_##op(int i, atomic_t *v) \ +static inline void generic_atomic_##op(int i, atomic_t *v) \ { \ int c, old; \ \ c = v->counter; \ - while ((old = cmpxchg(&v->counter, c, c c_op i)) != c) \ + while ((old = arch_cmpxchg(&v->counter, c, c c_op i)) != c) \ c = old; \ } #define ATOMIC_OP_RETURN(op, c_op) \ -static inline int atomic_##op##_return(int i, atomic_t *v) \ +static inline int generic_atomic_##op##_return(int i, atomic_t *v) \ { \ int c, old; \ \ c = v->counter; \ - while ((old = cmpxchg(&v->counter, c, c c_op i)) != c) \ + while ((old = arch_cmpxchg(&v->counter, c, c c_op i)) != c) \ c = old; \ \ return c c_op i; \ } #define ATOMIC_FETCH_OP(op, c_op) \ -static inline int atomic_fetch_##op(int i, atomic_t *v) \ +static inline int generic_atomic_fetch_##op(int i, atomic_t *v) \ { \ int c, old; \ \ c = v->counter; \ - while ((old = cmpxchg(&v->counter, c, c c_op i)) != c) \ + while ((old = arch_cmpxchg(&v->counter, c, c c_op i)) != c) \ c = old; \ \ return c; \ @@ -76,7 +55,7 @@ static inline int atomic_fetch_##op(int i, atomic_t *v) \ #include <linux/irqflags.h> #define ATOMIC_OP(op, c_op) \ -static inline void atomic_##op(int i, atomic_t *v) \ +static inline void generic_atomic_##op(int i, atomic_t *v) \ { \ unsigned long flags; \ \ @@ -86,7 +65,7 @@ static inline void atomic_##op(int i, atomic_t *v) \ } #define ATOMIC_OP_RETURN(op, c_op) \ -static inline int atomic_##op##_return(int i, atomic_t *v) \ +static inline int generic_atomic_##op##_return(int i, atomic_t *v) \ { \ unsigned long flags; \ int ret; \ @@ -99,7 +78,7 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ } #define ATOMIC_FETCH_OP(op, c_op) \ -static inline int atomic_fetch_##op(int i, atomic_t *v) \ +static inline int generic_atomic_fetch_##op(int i, atomic_t *v) \ { \ unsigned long flags; \ int ret; \ @@ -114,89 +93,41 @@ static inline int atomic_fetch_##op(int i, atomic_t *v) \ #endif /* CONFIG_SMP */ -#ifndef atomic_add_return ATOMIC_OP_RETURN(add, +) -#endif - -#ifndef atomic_sub_return ATOMIC_OP_RETURN(sub, -) -#endif -#ifndef atomic_fetch_add ATOMIC_FETCH_OP(add, +) -#endif - -#ifndef atomic_fetch_sub ATOMIC_FETCH_OP(sub, -) -#endif - -#ifndef atomic_fetch_and ATOMIC_FETCH_OP(and, &) -#endif - -#ifndef atomic_fetch_or ATOMIC_FETCH_OP(or, |) -#endif - -#ifndef atomic_fetch_xor ATOMIC_FETCH_OP(xor, ^) -#endif -#ifndef atomic_and +ATOMIC_OP(add, +) +ATOMIC_OP(sub, -) ATOMIC_OP(and, &) -#endif - -#ifndef atomic_or ATOMIC_OP(or, |) -#endif - -#ifndef atomic_xor ATOMIC_OP(xor, ^) -#endif #undef ATOMIC_FETCH_OP #undef ATOMIC_OP_RETURN #undef ATOMIC_OP -/* - * Atomic operations that C can't guarantee us. Useful for - * resource counting etc.. - */ - -#define ATOMIC_INIT(i) { (i) } - -/** - * atomic_read - read atomic variable - * @v: pointer of type atomic_t - * - * Atomically reads the value of @v. - */ -#ifndef atomic_read -#define atomic_read(v) READ_ONCE((v)->counter) -#endif - -/** - * atomic_set - set atomic variable - * @v: pointer of type atomic_t - * @i: required value - * - * Atomically sets the value of @v to @i. - */ -#define atomic_set(v, i) WRITE_ONCE(((v)->counter), (i)) - -#include <linux/irqflags.h> +#define arch_atomic_add_return generic_atomic_add_return +#define arch_atomic_sub_return generic_atomic_sub_return -static inline void atomic_add(int i, atomic_t *v) -{ - atomic_add_return(i, v); -} +#define arch_atomic_fetch_add generic_atomic_fetch_add +#define arch_atomic_fetch_sub generic_atomic_fetch_sub +#define arch_atomic_fetch_and generic_atomic_fetch_and +#define arch_atomic_fetch_or generic_atomic_fetch_or +#define arch_atomic_fetch_xor generic_atomic_fetch_xor -static inline void atomic_sub(int i, atomic_t *v) -{ - atomic_sub_return(i, v); -} +#define arch_atomic_add generic_atomic_add +#define arch_atomic_sub generic_atomic_sub +#define arch_atomic_and generic_atomic_and +#define arch_atomic_or generic_atomic_or +#define arch_atomic_xor generic_atomic_xor -#define atomic_xchg(ptr, v) (xchg(&(ptr)->counter, (v))) -#define atomic_cmpxchg(v, old, new) (cmpxchg(&((v)->counter), (old), (new))) +#define arch_atomic_read(v) READ_ONCE((v)->counter) +#define arch_atomic_set(v, i) WRITE_ONCE(((v)->counter), (i)) #endif /* __ASM_GENERIC_ATOMIC_H */ diff --git a/include/asm-generic/atomic64.h b/include/asm-generic/atomic64.h index 97b28b7f1f29..100d24b02e52 100644 --- a/include/asm-generic/atomic64.h +++ b/include/asm-generic/atomic64.h @@ -1,37 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * Generic implementation of 64-bit atomics using spinlocks, * useful on processors that don't have 64-bit atomic instructions. * * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com> - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. */ #ifndef _ASM_GENERIC_ATOMIC64_H #define _ASM_GENERIC_ATOMIC64_H #include <linux/types.h> typedef struct { - long long counter; + s64 counter; } atomic64_t; #define ATOMIC64_INIT(i) { (i) } -extern long long atomic64_read(const atomic64_t *v); -extern void atomic64_set(atomic64_t *v, long long i); - -#define atomic64_set_release(v, i) atomic64_set((v), (i)) +extern s64 generic_atomic64_read(const atomic64_t *v); +extern void generic_atomic64_set(atomic64_t *v, s64 i); #define ATOMIC64_OP(op) \ -extern void atomic64_##op(long long a, atomic64_t *v); +extern void generic_atomic64_##op(s64 a, atomic64_t *v); #define ATOMIC64_OP_RETURN(op) \ -extern long long atomic64_##op##_return(long long a, atomic64_t *v); +extern s64 generic_atomic64_##op##_return(s64 a, atomic64_t *v); #define ATOMIC64_FETCH_OP(op) \ -extern long long atomic64_fetch_##op(long long a, atomic64_t *v); +extern s64 generic_atomic64_fetch_##op(s64 a, atomic64_t *v); #define ATOMIC64_OPS(op) ATOMIC64_OP(op) ATOMIC64_OP_RETURN(op) ATOMIC64_FETCH_OP(op) @@ -50,11 +44,32 @@ ATOMIC64_OPS(xor) #undef ATOMIC64_OP_RETURN #undef ATOMIC64_OP -extern long long atomic64_dec_if_positive(atomic64_t *v); -#define atomic64_dec_if_positive atomic64_dec_if_positive -extern long long atomic64_cmpxchg(atomic64_t *v, long long o, long long n); -extern long long atomic64_xchg(atomic64_t *v, long long new); -extern long long atomic64_fetch_add_unless(atomic64_t *v, long long a, long long u); -#define atomic64_fetch_add_unless atomic64_fetch_add_unless +extern s64 generic_atomic64_dec_if_positive(atomic64_t *v); +extern s64 generic_atomic64_cmpxchg(atomic64_t *v, s64 o, s64 n); +extern s64 generic_atomic64_xchg(atomic64_t *v, s64 new); +extern s64 generic_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u); + +#define arch_atomic64_read generic_atomic64_read +#define arch_atomic64_set generic_atomic64_set +#define arch_atomic64_set_release generic_atomic64_set + +#define arch_atomic64_add generic_atomic64_add +#define arch_atomic64_add_return generic_atomic64_add_return +#define arch_atomic64_fetch_add generic_atomic64_fetch_add +#define arch_atomic64_sub generic_atomic64_sub +#define arch_atomic64_sub_return generic_atomic64_sub_return +#define arch_atomic64_fetch_sub generic_atomic64_fetch_sub + +#define arch_atomic64_and generic_atomic64_and +#define arch_atomic64_fetch_and generic_atomic64_fetch_and +#define arch_atomic64_or generic_atomic64_or +#define arch_atomic64_fetch_or generic_atomic64_fetch_or +#define arch_atomic64_xor generic_atomic64_xor +#define arch_atomic64_fetch_xor generic_atomic64_fetch_xor + +#define arch_atomic64_dec_if_positive generic_atomic64_dec_if_positive +#define arch_atomic64_cmpxchg generic_atomic64_cmpxchg +#define arch_atomic64_xchg generic_atomic64_xchg +#define arch_atomic64_fetch_add_unless generic_atomic64_fetch_add_unless #endif /* _ASM_GENERIC_ATOMIC64_H */ diff --git a/include/asm-generic/audit_change_attr.h b/include/asm-generic/audit_change_attr.h index 331670807cf0..cc840537885f 100644 --- a/include/asm-generic/audit_change_attr.h +++ b/include/asm-generic/audit_change_attr.h @@ -11,9 +11,15 @@ __NR_lchown, __NR_fchown, #endif __NR_setxattr, +#ifdef __NR_setxattrat +__NR_setxattrat, +#endif __NR_lsetxattr, __NR_fsetxattr, __NR_removexattr, +#ifdef __NR_removexattrat +__NR_removexattrat, +#endif __NR_lremovexattr, __NR_fremovexattr, #ifdef __NR_fchownat diff --git a/include/asm-generic/barrier.h b/include/asm-generic/barrier.h index 2cafdbb9ae4c..d4f581c1e21d 100644 --- a/include/asm-generic/barrier.h +++ b/include/asm-generic/barrier.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * Generic barrier definitions. * @@ -6,11 +7,6 @@ * * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved. * Written by David Howells (dhowells@redhat.com) - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public Licence - * as published by the Free Software Foundation; either version - * 2 of the Licence, or (at your option) any later version. */ #ifndef __ASM_GENERIC_BARRIER_H #define __ASM_GENERIC_BARRIER_H @@ -18,12 +14,43 @@ #ifndef __ASSEMBLY__ #include <linux/compiler.h> +#include <linux/kcsan-checks.h> +#include <asm/rwonce.h> #ifndef nop #define nop() asm volatile ("nop") #endif /* + * Architectures that want generic instrumentation can define __ prefixed + * variants of all barriers. + */ + +#ifdef __mb +#define mb() do { kcsan_mb(); __mb(); } while (0) +#endif + +#ifdef __rmb +#define rmb() do { kcsan_rmb(); __rmb(); } while (0) +#endif + +#ifdef __wmb +#define wmb() do { kcsan_wmb(); __wmb(); } while (0) +#endif + +#ifdef __dma_mb +#define dma_mb() do { kcsan_mb(); __dma_mb(); } while (0) +#endif + +#ifdef __dma_rmb +#define dma_rmb() do { kcsan_rmb(); __dma_rmb(); } while (0) +#endif + +#ifdef __dma_wmb +#define dma_wmb() do { kcsan_wmb(); __dma_wmb(); } while (0) +#endif + +/* * Force strict CPU ordering. And yes, this is required on UP too when we're * talking to devices. * @@ -42,6 +69,10 @@ #define wmb() mb() #endif +#ifndef dma_mb +#define dma_mb() mb() +#endif + #ifndef dma_rmb #define dma_rmb() rmb() #endif @@ -50,10 +81,6 @@ #define dma_wmb() wmb() #endif -#ifndef read_barrier_depends -#define read_barrier_depends() do { } while (0) -#endif - #ifndef __smp_mb #define __smp_mb() mb() #endif @@ -66,26 +93,18 @@ #define __smp_wmb() wmb() #endif -#ifndef __smp_read_barrier_depends -#define __smp_read_barrier_depends() read_barrier_depends() -#endif - #ifdef CONFIG_SMP #ifndef smp_mb -#define smp_mb() __smp_mb() +#define smp_mb() do { kcsan_mb(); __smp_mb(); } while (0) #endif #ifndef smp_rmb -#define smp_rmb() __smp_rmb() +#define smp_rmb() do { kcsan_rmb(); __smp_rmb(); } while (0) #endif #ifndef smp_wmb -#define smp_wmb() __smp_wmb() -#endif - -#ifndef smp_read_barrier_depends -#define smp_read_barrier_depends() __smp_read_barrier_depends() +#define smp_wmb() do { kcsan_wmb(); __smp_wmb(); } while (0) #endif #else /* !CONFIG_SMP */ @@ -102,10 +121,6 @@ #define smp_wmb() barrier() #endif -#ifndef smp_read_barrier_depends -#define smp_read_barrier_depends() do { } while (0) -#endif - #endif /* CONFIG_SMP */ #ifndef __smp_store_mb @@ -132,29 +147,29 @@ do { \ #ifndef __smp_load_acquire #define __smp_load_acquire(p) \ ({ \ - typeof(*p) ___p1 = READ_ONCE(*p); \ + __unqual_scalar_typeof(*p) ___p1 = READ_ONCE(*p); \ compiletime_assert_atomic_type(*p); \ __smp_mb(); \ - ___p1; \ + (typeof(*p))___p1; \ }) #endif #ifdef CONFIG_SMP #ifndef smp_store_mb -#define smp_store_mb(var, value) __smp_store_mb(var, value) +#define smp_store_mb(var, value) do { kcsan_mb(); __smp_store_mb(var, value); } while (0) #endif #ifndef smp_mb__before_atomic -#define smp_mb__before_atomic() __smp_mb__before_atomic() +#define smp_mb__before_atomic() do { kcsan_mb(); __smp_mb__before_atomic(); } while (0) #endif #ifndef smp_mb__after_atomic -#define smp_mb__after_atomic() __smp_mb__after_atomic() +#define smp_mb__after_atomic() do { kcsan_mb(); __smp_mb__after_atomic(); } while (0) #endif #ifndef smp_store_release -#define smp_store_release(p, v) __smp_store_release(p, v) +#define smp_store_release(p, v) do { kcsan_release(); __smp_store_release(p, v); } while (0) #endif #ifndef smp_load_acquire @@ -178,7 +193,6 @@ do { \ #ifndef smp_store_release #define smp_store_release(p, v) \ do { \ - compiletime_assert_atomic_type(*p); \ barrier(); \ WRITE_ONCE(*p, v); \ } while (0) @@ -187,24 +201,22 @@ do { \ #ifndef smp_load_acquire #define smp_load_acquire(p) \ ({ \ - typeof(*p) ___p1 = READ_ONCE(*p); \ - compiletime_assert_atomic_type(*p); \ + __unqual_scalar_typeof(*p) ___p1 = READ_ONCE(*p); \ barrier(); \ - ___p1; \ + (typeof(*p))___p1; \ }) #endif #endif /* CONFIG_SMP */ /* Barriers for virtual machine guests when talking to an SMP host */ -#define virt_mb() __smp_mb() -#define virt_rmb() __smp_rmb() -#define virt_wmb() __smp_wmb() -#define virt_read_barrier_depends() __smp_read_barrier_depends() -#define virt_store_mb(var, value) __smp_store_mb(var, value) -#define virt_mb__before_atomic() __smp_mb__before_atomic() -#define virt_mb__after_atomic() __smp_mb__after_atomic() -#define virt_store_release(p, v) __smp_store_release(p, v) +#define virt_mb() do { kcsan_mb(); __smp_mb(); } while (0) +#define virt_rmb() do { kcsan_rmb(); __smp_rmb(); } while (0) +#define virt_wmb() do { kcsan_wmb(); __smp_wmb(); } while (0) +#define virt_store_mb(var, value) do { kcsan_mb(); __smp_store_mb(var, value); } while (0) +#define virt_mb__before_atomic() do { kcsan_mb(); __smp_mb__before_atomic(); } while (0) +#define virt_mb__after_atomic() do { kcsan_mb(); __smp_mb__after_atomic(); } while (0) +#define virt_store_release(p, v) do { kcsan_release(); __smp_store_release(p, v); } while (0) #define virt_load_acquire(p) __smp_load_acquire(p) /** @@ -233,14 +245,14 @@ do { \ #ifndef smp_cond_load_relaxed #define smp_cond_load_relaxed(ptr, cond_expr) ({ \ typeof(ptr) __PTR = (ptr); \ - typeof(*ptr) VAL; \ + __unqual_scalar_typeof(*ptr) VAL; \ for (;;) { \ VAL = READ_ONCE(*__PTR); \ if (cond_expr) \ break; \ cpu_relax(); \ } \ - VAL; \ + (typeof(*ptr))VAL; \ }) #endif @@ -254,12 +266,41 @@ do { \ */ #ifndef smp_cond_load_acquire #define smp_cond_load_acquire(ptr, cond_expr) ({ \ - typeof(*ptr) _val; \ + __unqual_scalar_typeof(*ptr) _val; \ _val = smp_cond_load_relaxed(ptr, cond_expr); \ smp_acquire__after_ctrl_dep(); \ - _val; \ + (typeof(*ptr))_val; \ }) #endif +/* + * pmem_wmb() ensures that all stores for which the modification + * are written to persistent storage by preceding instructions have + * updated persistent storage before any data access or data transfer + * caused by subsequent instructions is initiated. + */ +#ifndef pmem_wmb +#define pmem_wmb() wmb() +#endif + +/* + * ioremap_wc() maps I/O memory as memory with write-combining attributes. For + * this kind of memory accesses, the CPU may wait for prior accesses to be + * merged with subsequent ones. In some situation, such wait is bad for the + * performance. io_stop_wc() can be used to prevent the merging of + * write-combining memory accesses before this macro with those after it. + */ +#ifndef io_stop_wc +#define io_stop_wc() do { } while (0) +#endif + +/* + * Architectures that guarantee an implicit smp_mb() in switch_mm() + * can override smp_mb__after_switch_mm. + */ +#ifndef smp_mb__after_switch_mm +# define smp_mb__after_switch_mm() smp_mb() +#endif + #endif /* !__ASSEMBLY__ */ #endif /* __ASM_GENERIC_BARRIER_H */ diff --git a/include/asm-generic/bitops.h b/include/asm-generic/bitops.h index bfc96bf6606e..a47b8a71d6fe 100644 --- a/include/asm-generic/bitops.h +++ b/include/asm-generic/bitops.h @@ -4,8 +4,9 @@ /* * For the benefit of those who are trying to port Linux to another - * architecture, here are some C-language equivalents. You should - * recode these in the native assembly language, if at all possible. + * architecture, here are some C-language equivalents. They should + * generate reasonable code, so take a look at what your compiler spits + * out before rolling your own buggy implementation in assembly language. * * C language equivalents written by Theodore Ts'o, 9/26/92 */ @@ -19,7 +20,6 @@ #include <asm-generic/bitops/fls.h> #include <asm-generic/bitops/__fls.h> #include <asm-generic/bitops/fls64.h> -#include <asm-generic/bitops/find.h> #ifndef _LINUX_BITOPS_H #error only <linux/bitops.h> can be included directly diff --git a/include/asm-generic/bitops/__ffs.h b/include/asm-generic/bitops/__ffs.h index 39e56e1c7203..3a899c626fdc 100644 --- a/include/asm-generic/bitops/__ffs.h +++ b/include/asm-generic/bitops/__ffs.h @@ -5,14 +5,14 @@ #include <asm/types.h> /** - * __ffs - find first bit in word. + * generic___ffs - find first bit in word. * @word: The word to search * * Undefined if no bit exists, so code should check against 0 first. */ -static __always_inline unsigned long __ffs(unsigned long word) +static __always_inline __attribute_const__ unsigned int generic___ffs(unsigned long word) { - int num = 0; + unsigned int num = 0; #if BITS_PER_LONG == 64 if ((word & 0xffffffff) == 0) { @@ -41,4 +41,8 @@ static __always_inline unsigned long __ffs(unsigned long word) return num; } +#ifndef __HAVE_ARCH___FFS +#define __ffs(word) generic___ffs(word) +#endif + #endif /* _ASM_GENERIC_BITOPS___FFS_H_ */ diff --git a/include/asm-generic/bitops/__fls.h b/include/asm-generic/bitops/__fls.h index 03f721a8a2b1..35f33780ca6c 100644 --- a/include/asm-generic/bitops/__fls.h +++ b/include/asm-generic/bitops/__fls.h @@ -5,14 +5,14 @@ #include <asm/types.h> /** - * __fls - find last (most-significant) set bit in a long word + * generic___fls - find last (most-significant) set bit in a long word * @word: the word to search * * Undefined if no set bit exists, so code should check against 0 first. */ -static __always_inline unsigned long __fls(unsigned long word) +static __always_inline __attribute_const__ unsigned int generic___fls(unsigned long word) { - int num = BITS_PER_LONG - 1; + unsigned int num = BITS_PER_LONG - 1; #if BITS_PER_LONG == 64 if (!(word & (~0ul << 32))) { @@ -41,4 +41,8 @@ static __always_inline unsigned long __fls(unsigned long word) return num; } +#ifndef __HAVE_ARCH___FLS +#define __fls(word) generic___fls(word) +#endif + #endif /* _ASM_GENERIC_BITOPS___FLS_H_ */ diff --git a/include/asm-generic/bitops/atomic.h b/include/asm-generic/bitops/atomic.h index dd90c9792909..e076e079f6b2 100644 --- a/include/asm-generic/bitops/atomic.h +++ b/include/asm-generic/bitops/atomic.h @@ -11,58 +11,60 @@ * See Documentation/atomic_bitops.txt for details. */ -static inline void set_bit(unsigned int nr, volatile unsigned long *p) +static __always_inline void +arch_set_bit(unsigned int nr, volatile unsigned long *p) { p += BIT_WORD(nr); - atomic_long_or(BIT_MASK(nr), (atomic_long_t *)p); + raw_atomic_long_or(BIT_MASK(nr), (atomic_long_t *)p); } -static inline void clear_bit(unsigned int nr, volatile unsigned long *p) +static __always_inline void +arch_clear_bit(unsigned int nr, volatile unsigned long *p) { p += BIT_WORD(nr); - atomic_long_andnot(BIT_MASK(nr), (atomic_long_t *)p); + raw_atomic_long_andnot(BIT_MASK(nr), (atomic_long_t *)p); } -static inline void change_bit(unsigned int nr, volatile unsigned long *p) +static __always_inline void +arch_change_bit(unsigned int nr, volatile unsigned long *p) { p += BIT_WORD(nr); - atomic_long_xor(BIT_MASK(nr), (atomic_long_t *)p); + raw_atomic_long_xor(BIT_MASK(nr), (atomic_long_t *)p); } -static inline int test_and_set_bit(unsigned int nr, volatile unsigned long *p) +static __always_inline int +arch_test_and_set_bit(unsigned int nr, volatile unsigned long *p) { long old; unsigned long mask = BIT_MASK(nr); p += BIT_WORD(nr); - if (READ_ONCE(*p) & mask) - return 1; - - old = atomic_long_fetch_or(mask, (atomic_long_t *)p); + old = raw_atomic_long_fetch_or(mask, (atomic_long_t *)p); return !!(old & mask); } -static inline int test_and_clear_bit(unsigned int nr, volatile unsigned long *p) +static __always_inline int +arch_test_and_clear_bit(unsigned int nr, volatile unsigned long *p) { long old; unsigned long mask = BIT_MASK(nr); p += BIT_WORD(nr); - if (!(READ_ONCE(*p) & mask)) - return 0; - - old = atomic_long_fetch_andnot(mask, (atomic_long_t *)p); + old = raw_atomic_long_fetch_andnot(mask, (atomic_long_t *)p); return !!(old & mask); } -static inline int test_and_change_bit(unsigned int nr, volatile unsigned long *p) +static __always_inline int +arch_test_and_change_bit(unsigned int nr, volatile unsigned long *p) { long old; unsigned long mask = BIT_MASK(nr); p += BIT_WORD(nr); - old = atomic_long_fetch_xor(mask, (atomic_long_t *)p); + old = raw_atomic_long_fetch_xor(mask, (atomic_long_t *)p); return !!(old & mask); } +#include <asm-generic/bitops/instrumented-atomic.h> + #endif /* _ASM_GENERIC_BITOPS_ATOMIC_H */ diff --git a/include/asm-generic/bitops/builtin-__ffs.h b/include/asm-generic/bitops/builtin-__ffs.h index 87024da44d10..d3c3f567045d 100644 --- a/include/asm-generic/bitops/builtin-__ffs.h +++ b/include/asm-generic/bitops/builtin-__ffs.h @@ -8,7 +8,7 @@ * * Undefined if no bit exists, so code should check against 0 first. */ -static __always_inline unsigned long __ffs(unsigned long word) +static __always_inline __attribute_const__ unsigned int __ffs(unsigned long word) { return __builtin_ctzl(word); } diff --git a/include/asm-generic/bitops/builtin-__fls.h b/include/asm-generic/bitops/builtin-__fls.h index 43a5aa9afbdb..7770c4f1bfcd 100644 --- a/include/asm-generic/bitops/builtin-__fls.h +++ b/include/asm-generic/bitops/builtin-__fls.h @@ -8,7 +8,7 @@ * * Undefined if no set bit exists, so code should check against 0 first. */ -static __always_inline unsigned long __fls(unsigned long word) +static __always_inline __attribute_const__ unsigned int __fls(unsigned long word) { return (sizeof(word) * 8) - 1 - __builtin_clzl(word); } diff --git a/include/asm-generic/bitops/builtin-ffs.h b/include/asm-generic/bitops/builtin-ffs.h index 458c85ebcd15..7b129329046b 100644 --- a/include/asm-generic/bitops/builtin-ffs.h +++ b/include/asm-generic/bitops/builtin-ffs.h @@ -8,11 +8,8 @@ * * This is defined the same way as * the libc and compiler builtin ffs routines, therefore - * differs in spirit from the above ffz (man ffs). + * differs in spirit from ffz (man ffs). */ -static __always_inline int ffs(int x) -{ - return __builtin_ffs(x); -} +#define ffs(x) __builtin_ffs(x) #endif diff --git a/include/asm-generic/bitops/builtin-fls.h b/include/asm-generic/bitops/builtin-fls.h index c8455cc28841..be707da8c7cd 100644 --- a/include/asm-generic/bitops/builtin-fls.h +++ b/include/asm-generic/bitops/builtin-fls.h @@ -9,7 +9,7 @@ * This is defined the same way as ffs. * Note fls(0) = 0, fls(1) = 1, fls(0x80000000) = 32. */ -static __always_inline int fls(unsigned int x) +static __always_inline __attribute_const__ int fls(unsigned int x) { return x ? sizeof(x) * 8 - __builtin_clz(x) : 0; } diff --git a/include/asm-generic/bitops/ffs.h b/include/asm-generic/bitops/ffs.h index e81868b2c0f0..5ff2b7fbda6d 100644 --- a/include/asm-generic/bitops/ffs.h +++ b/include/asm-generic/bitops/ffs.h @@ -3,14 +3,14 @@ #define _ASM_GENERIC_BITOPS_FFS_H_ /** - * ffs - find first bit set + * generic_ffs - find first bit set * @x: the word to search * * This is defined the same way as * the libc and compiler builtin ffs routines, therefore - * differs in spirit from the above ffz (man ffs). + * differs in spirit from ffz (man ffs). */ -static inline int ffs(int x) +static inline __attribute_const__ int generic_ffs(int x) { int r = 1; @@ -39,4 +39,8 @@ static inline int ffs(int x) return r; } +#ifndef __HAVE_ARCH_FFS +#define ffs(x) generic_ffs(x) +#endif + #endif /* _ASM_GENERIC_BITOPS_FFS_H_ */ diff --git a/include/asm-generic/bitops/find.h b/include/asm-generic/bitops/find.h deleted file mode 100644 index 8a1ee10014de..000000000000 --- a/include/asm-generic/bitops/find.h +++ /dev/null @@ -1,83 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _ASM_GENERIC_BITOPS_FIND_H_ -#define _ASM_GENERIC_BITOPS_FIND_H_ - -#ifndef find_next_bit -/** - * find_next_bit - find the next set bit in a memory region - * @addr: The address to base the search on - * @offset: The bitnumber to start searching at - * @size: The bitmap size in bits - * - * Returns the bit number for the next set bit - * If no bits are set, returns @size. - */ -extern unsigned long find_next_bit(const unsigned long *addr, unsigned long - size, unsigned long offset); -#endif - -#ifndef find_next_and_bit -/** - * find_next_and_bit - find the next set bit in both memory regions - * @addr1: The first address to base the search on - * @addr2: The second address to base the search on - * @offset: The bitnumber to start searching at - * @size: The bitmap size in bits - * - * Returns the bit number for the next set bit - * If no bits are set, returns @size. - */ -extern unsigned long find_next_and_bit(const unsigned long *addr1, - const unsigned long *addr2, unsigned long size, - unsigned long offset); -#endif - -#ifndef find_next_zero_bit -/** - * find_next_zero_bit - find the next cleared bit in a memory region - * @addr: The address to base the search on - * @offset: The bitnumber to start searching at - * @size: The bitmap size in bits - * - * Returns the bit number of the next zero bit - * If no bits are zero, returns @size. - */ -extern unsigned long find_next_zero_bit(const unsigned long *addr, unsigned - long size, unsigned long offset); -#endif - -#ifdef CONFIG_GENERIC_FIND_FIRST_BIT - -/** - * find_first_bit - find the first set bit in a memory region - * @addr: The address to start the search at - * @size: The maximum number of bits to search - * - * Returns the bit number of the first set bit. - * If no bits are set, returns @size. - */ -extern unsigned long find_first_bit(const unsigned long *addr, - unsigned long size); - -/** - * find_first_zero_bit - find the first cleared bit in a memory region - * @addr: The address to start the search at - * @size: The maximum number of bits to search - * - * Returns the bit number of the first cleared bit. - * If no bits are zero, returns @size. - */ -extern unsigned long find_first_zero_bit(const unsigned long *addr, - unsigned long size); -#else /* CONFIG_GENERIC_FIND_FIRST_BIT */ - -#ifndef find_first_bit -#define find_first_bit(addr, size) find_next_bit((addr), (size), 0) -#endif -#ifndef find_first_zero_bit -#define find_first_zero_bit(addr, size) find_next_zero_bit((addr), (size), 0) -#endif - -#endif /* CONFIG_GENERIC_FIND_FIRST_BIT */ - -#endif /*_ASM_GENERIC_BITOPS_FIND_H_ */ diff --git a/include/asm-generic/bitops/fls.h b/include/asm-generic/bitops/fls.h index b168bb10e1be..8eed3437edb9 100644 --- a/include/asm-generic/bitops/fls.h +++ b/include/asm-generic/bitops/fls.h @@ -3,14 +3,14 @@ #define _ASM_GENERIC_BITOPS_FLS_H_ /** - * fls - find last (most-significant) bit set + * generic_fls - find last (most-significant) bit set * @x: the word to search * * This is defined the same way as ffs. * Note fls(0) = 0, fls(1) = 1, fls(0x80000000) = 32. */ -static __always_inline int fls(unsigned int x) +static __always_inline __attribute_const__ int generic_fls(unsigned int x) { int r = 32; @@ -39,4 +39,8 @@ static __always_inline int fls(unsigned int x) return r; } +#ifndef __HAVE_ARCH_FLS +#define fls(x) generic_fls(x) +#endif + #endif /* _ASM_GENERIC_BITOPS_FLS_H_ */ diff --git a/include/asm-generic/bitops/fls64.h b/include/asm-generic/bitops/fls64.h index 866f2b2304ff..b5f58dd261a3 100644 --- a/include/asm-generic/bitops/fls64.h +++ b/include/asm-generic/bitops/fls64.h @@ -16,7 +16,7 @@ * at position 64. */ #if BITS_PER_LONG == 32 -static __always_inline int fls64(__u64 x) +static __always_inline __attribute_const__ int fls64(__u64 x) { __u32 h = x >> 32; if (h) @@ -24,7 +24,7 @@ static __always_inline int fls64(__u64 x) return fls(x); } #elif BITS_PER_LONG == 64 -static __always_inline int fls64(__u64 x) +static __always_inline __attribute_const__ int fls64(__u64 x) { if (x == 0) return 0; diff --git a/include/asm-generic/bitops/generic-non-atomic.h b/include/asm-generic/bitops/generic-non-atomic.h new file mode 100644 index 000000000000..564a8c675d85 --- /dev/null +++ b/include/asm-generic/bitops/generic-non-atomic.h @@ -0,0 +1,175 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef __ASM_GENERIC_BITOPS_GENERIC_NON_ATOMIC_H +#define __ASM_GENERIC_BITOPS_GENERIC_NON_ATOMIC_H + +#include <linux/bits.h> +#include <asm/barrier.h> + +#ifndef _LINUX_BITOPS_H +#error only <linux/bitops.h> can be included directly +#endif + +/* + * Generic definitions for bit operations, should not be used in regular code + * directly. + */ + +/** + * generic___set_bit - Set a bit in memory + * @nr: the bit to set + * @addr: the address to start counting from + * + * Unlike set_bit(), this function is non-atomic and may be reordered. + * If it's called on the same region of memory simultaneously, the effect + * may be that only one operation succeeds. + */ +static __always_inline void +generic___set_bit(unsigned long nr, volatile unsigned long *addr) +{ + unsigned long mask = BIT_MASK(nr); + unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); + + *p |= mask; +} + +static __always_inline void +generic___clear_bit(unsigned long nr, volatile unsigned long *addr) +{ + unsigned long mask = BIT_MASK(nr); + unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); + + *p &= ~mask; +} + +/** + * generic___change_bit - Toggle a bit in memory + * @nr: the bit to change + * @addr: the address to start counting from + * + * Unlike change_bit(), this function is non-atomic and may be reordered. + * If it's called on the same region of memory simultaneously, the effect + * may be that only one operation succeeds. + */ +static __always_inline void +generic___change_bit(unsigned long nr, volatile unsigned long *addr) +{ + unsigned long mask = BIT_MASK(nr); + unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); + + *p ^= mask; +} + +/** + * generic___test_and_set_bit - Set a bit and return its old value + * @nr: Bit to set + * @addr: Address to count from + * + * This operation is non-atomic and can be reordered. + * If two examples of this operation race, one can appear to succeed + * but actually fail. You must protect multiple accesses with a lock. + */ +static __always_inline bool +generic___test_and_set_bit(unsigned long nr, volatile unsigned long *addr) +{ + unsigned long mask = BIT_MASK(nr); + unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); + unsigned long old = *p; + + *p = old | mask; + return (old & mask) != 0; +} + +/** + * generic___test_and_clear_bit - Clear a bit and return its old value + * @nr: Bit to clear + * @addr: Address to count from + * + * This operation is non-atomic and can be reordered. + * If two examples of this operation race, one can appear to succeed + * but actually fail. You must protect multiple accesses with a lock. + */ +static __always_inline bool +generic___test_and_clear_bit(unsigned long nr, volatile unsigned long *addr) +{ + unsigned long mask = BIT_MASK(nr); + unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); + unsigned long old = *p; + + *p = old & ~mask; + return (old & mask) != 0; +} + +/* WARNING: non atomic and it can be reordered! */ +static __always_inline bool +generic___test_and_change_bit(unsigned long nr, volatile unsigned long *addr) +{ + unsigned long mask = BIT_MASK(nr); + unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); + unsigned long old = *p; + + *p = old ^ mask; + return (old & mask) != 0; +} + +/** + * generic_test_bit - Determine whether a bit is set + * @nr: bit number to test + * @addr: Address to start counting from + */ +static __always_inline bool +generic_test_bit(unsigned long nr, const volatile unsigned long *addr) +{ + /* + * Unlike the bitops with the '__' prefix above, this one *is* atomic, + * so `volatile` must always stay here with no cast-aways. See + * `Documentation/atomic_bitops.txt` for the details. + */ + return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); +} + +/** + * generic_test_bit_acquire - Determine, with acquire semantics, whether a bit is set + * @nr: bit number to test + * @addr: Address to start counting from + */ +static __always_inline bool +generic_test_bit_acquire(unsigned long nr, const volatile unsigned long *addr) +{ + unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); + return 1UL & (smp_load_acquire(p) >> (nr & (BITS_PER_LONG-1))); +} + +/* + * const_*() definitions provide good compile-time optimizations when + * the passed arguments can be resolved at compile time. + */ +#define const___set_bit generic___set_bit +#define const___clear_bit generic___clear_bit +#define const___change_bit generic___change_bit +#define const___test_and_set_bit generic___test_and_set_bit +#define const___test_and_clear_bit generic___test_and_clear_bit +#define const___test_and_change_bit generic___test_and_change_bit +#define const_test_bit_acquire generic_test_bit_acquire + +/** + * const_test_bit - Determine whether a bit is set + * @nr: bit number to test + * @addr: Address to start counting from + * + * A version of generic_test_bit() which discards the `volatile` qualifier to + * allow a compiler to optimize code harder. Non-atomic and to be called only + * for testing compile-time constants, e.g. by the corresponding macros, not + * directly from "regular" code. + */ +static __always_inline bool +const_test_bit(unsigned long nr, const volatile unsigned long *addr) +{ + const unsigned long *p = (const unsigned long *)addr + BIT_WORD(nr); + unsigned long mask = BIT_MASK(nr); + unsigned long val = *p; + + return !!(val & mask); +} + +#endif /* __ASM_GENERIC_BITOPS_GENERIC_NON_ATOMIC_H */ diff --git a/include/asm-generic/bitops/instrumented-atomic.h b/include/asm-generic/bitops/instrumented-atomic.h new file mode 100644 index 000000000000..4225a8ca9c1a --- /dev/null +++ b/include/asm-generic/bitops/instrumented-atomic.h @@ -0,0 +1,103 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* + * This file provides wrappers with sanitizer instrumentation for atomic bit + * operations. + * + * To use this functionality, an arch's bitops.h file needs to define each of + * the below bit operations with an arch_ prefix (e.g. arch_set_bit(), + * arch___set_bit(), etc.). + */ +#ifndef _ASM_GENERIC_BITOPS_INSTRUMENTED_ATOMIC_H +#define _ASM_GENERIC_BITOPS_INSTRUMENTED_ATOMIC_H + +#include <linux/instrumented.h> + +/** + * set_bit - Atomically set a bit in memory + * @nr: the bit to set + * @addr: the address to start counting from + * + * This is a relaxed atomic operation (no implied memory barriers). + * + * Note that @nr may be almost arbitrarily large; this function is not + * restricted to acting on a single-word quantity. + */ +static __always_inline void set_bit(long nr, volatile unsigned long *addr) +{ + instrument_atomic_write(addr + BIT_WORD(nr), sizeof(long)); + arch_set_bit(nr, addr); +} + +/** + * clear_bit - Clears a bit in memory + * @nr: Bit to clear + * @addr: Address to start counting from + * + * This is a relaxed atomic operation (no implied memory barriers). + */ +static __always_inline void clear_bit(long nr, volatile unsigned long *addr) +{ + instrument_atomic_write(addr + BIT_WORD(nr), sizeof(long)); + arch_clear_bit(nr, addr); +} + +/** + * change_bit - Toggle a bit in memory + * @nr: Bit to change + * @addr: Address to start counting from + * + * This is a relaxed atomic operation (no implied memory barriers). + * + * Note that @nr may be almost arbitrarily large; this function is not + * restricted to acting on a single-word quantity. + */ +static __always_inline void change_bit(long nr, volatile unsigned long *addr) +{ + instrument_atomic_write(addr + BIT_WORD(nr), sizeof(long)); + arch_change_bit(nr, addr); +} + +/** + * test_and_set_bit - Set a bit and return its old value + * @nr: Bit to set + * @addr: Address to count from + * + * This is an atomic fully-ordered operation (implied full memory barrier). + */ +static __always_inline bool test_and_set_bit(long nr, volatile unsigned long *addr) +{ + kcsan_mb(); + instrument_atomic_read_write(addr + BIT_WORD(nr), sizeof(long)); + return arch_test_and_set_bit(nr, addr); +} + +/** + * test_and_clear_bit - Clear a bit and return its old value + * @nr: Bit to clear + * @addr: Address to count from + * + * This is an atomic fully-ordered operation (implied full memory barrier). + */ +static __always_inline bool test_and_clear_bit(long nr, volatile unsigned long *addr) +{ + kcsan_mb(); + instrument_atomic_read_write(addr + BIT_WORD(nr), sizeof(long)); + return arch_test_and_clear_bit(nr, addr); +} + +/** + * test_and_change_bit - Change a bit and return its old value + * @nr: Bit to change + * @addr: Address to count from + * + * This is an atomic fully-ordered operation (implied full memory barrier). + */ +static __always_inline bool test_and_change_bit(long nr, volatile unsigned long *addr) +{ + kcsan_mb(); + instrument_atomic_read_write(addr + BIT_WORD(nr), sizeof(long)); + return arch_test_and_change_bit(nr, addr); +} + +#endif /* _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H */ diff --git a/include/asm-generic/bitops/instrumented-lock.h b/include/asm-generic/bitops/instrumented-lock.h new file mode 100644 index 000000000000..542d3727ee4e --- /dev/null +++ b/include/asm-generic/bitops/instrumented-lock.h @@ -0,0 +1,82 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* + * This file provides wrappers with sanitizer instrumentation for bit + * locking operations. + * + * To use this functionality, an arch's bitops.h file needs to define each of + * the below bit operations with an arch_ prefix (e.g. arch_set_bit(), + * arch___set_bit(), etc.). + */ +#ifndef _ASM_GENERIC_BITOPS_INSTRUMENTED_LOCK_H +#define _ASM_GENERIC_BITOPS_INSTRUMENTED_LOCK_H + +#include <linux/instrumented.h> + +/** + * clear_bit_unlock - Clear a bit in memory, for unlock + * @nr: the bit to set + * @addr: the address to start counting from + * + * This operation is atomic and provides release barrier semantics. + */ +static inline void clear_bit_unlock(long nr, volatile unsigned long *addr) +{ + kcsan_release(); + instrument_atomic_write(addr + BIT_WORD(nr), sizeof(long)); + arch_clear_bit_unlock(nr, addr); +} + +/** + * __clear_bit_unlock - Clears a bit in memory + * @nr: Bit to clear + * @addr: Address to start counting from + * + * This is a non-atomic operation but implies a release barrier before the + * memory operation. It can be used for an unlock if no other CPUs can + * concurrently modify other bits in the word. + */ +static inline void __clear_bit_unlock(long nr, volatile unsigned long *addr) +{ + kcsan_release(); + instrument_write(addr + BIT_WORD(nr), sizeof(long)); + arch___clear_bit_unlock(nr, addr); +} + +/** + * test_and_set_bit_lock - Set a bit and return its old value, for lock + * @nr: Bit to set + * @addr: Address to count from + * + * This operation is atomic and provides acquire barrier semantics if + * the returned value is 0. + * It can be used to implement bit locks. + */ +static inline bool test_and_set_bit_lock(long nr, volatile unsigned long *addr) +{ + instrument_atomic_read_write(addr + BIT_WORD(nr), sizeof(long)); + return arch_test_and_set_bit_lock(nr, addr); +} + +/** + * xor_unlock_is_negative_byte - XOR a single byte in memory and test if + * it is negative, for unlock. + * @mask: Change the bits which are set in this mask. + * @addr: The address of the word containing the byte to change. + * + * Changes some of bits 0-6 in the word pointed to by @addr. + * This operation is atomic and provides release barrier semantics. + * Used to optimise some folio operations which are commonly paired + * with an unlock or end of writeback. Bit 7 is used as PG_waiters to + * indicate whether anybody is waiting for the unlock. + * + * Return: Whether the top bit of the byte is set. + */ +static inline bool xor_unlock_is_negative_byte(unsigned long mask, + volatile unsigned long *addr) +{ + kcsan_release(); + instrument_atomic_write(addr, sizeof(long)); + return arch_xor_unlock_is_negative_byte(mask, addr); +} +#endif /* _ASM_GENERIC_BITOPS_INSTRUMENTED_LOCK_H */ diff --git a/include/asm-generic/bitops/instrumented-non-atomic.h b/include/asm-generic/bitops/instrumented-non-atomic.h new file mode 100644 index 000000000000..2b238b161a62 --- /dev/null +++ b/include/asm-generic/bitops/instrumented-non-atomic.h @@ -0,0 +1,157 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* + * This file provides wrappers with sanitizer instrumentation for non-atomic + * bit operations. + * + * To use this functionality, an arch's bitops.h file needs to define each of + * the below bit operations with an arch_ prefix (e.g. arch_set_bit(), + * arch___set_bit(), etc.). + */ +#ifndef _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H +#define _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H + +#include <linux/instrumented.h> + +/** + * ___set_bit - Set a bit in memory + * @nr: the bit to set + * @addr: the address to start counting from + * + * Unlike set_bit(), this function is non-atomic. If it is called on the same + * region of memory concurrently, the effect may be that only one operation + * succeeds. + */ +static __always_inline void +___set_bit(unsigned long nr, volatile unsigned long *addr) +{ + instrument_write(addr + BIT_WORD(nr), sizeof(long)); + arch___set_bit(nr, addr); +} + +/** + * ___clear_bit - Clears a bit in memory + * @nr: the bit to clear + * @addr: the address to start counting from + * + * Unlike clear_bit(), this function is non-atomic. If it is called on the same + * region of memory concurrently, the effect may be that only one operation + * succeeds. + */ +static __always_inline void +___clear_bit(unsigned long nr, volatile unsigned long *addr) +{ + instrument_write(addr + BIT_WORD(nr), sizeof(long)); + arch___clear_bit(nr, addr); +} + +/** + * ___change_bit - Toggle a bit in memory + * @nr: the bit to change + * @addr: the address to start counting from + * + * Unlike change_bit(), this function is non-atomic. If it is called on the same + * region of memory concurrently, the effect may be that only one operation + * succeeds. + */ +static __always_inline void +___change_bit(unsigned long nr, volatile unsigned long *addr) +{ + instrument_write(addr + BIT_WORD(nr), sizeof(long)); + arch___change_bit(nr, addr); +} + +static __always_inline void __instrument_read_write_bitop(long nr, volatile unsigned long *addr) +{ + if (IS_ENABLED(CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC)) { + /* + * We treat non-atomic read-write bitops a little more special. + * Given the operations here only modify a single bit, assuming + * non-atomicity of the writer is sufficient may be reasonable + * for certain usage (and follows the permissible nature of the + * assume-plain-writes-atomic rule): + * 1. report read-modify-write races -> check read; + * 2. do not report races with marked readers, but do report + * races with unmarked readers -> check "atomic" write. + */ + kcsan_check_read(addr + BIT_WORD(nr), sizeof(long)); + /* + * Use generic write instrumentation, in case other sanitizers + * or tools are enabled alongside KCSAN. + */ + instrument_write(addr + BIT_WORD(nr), sizeof(long)); + } else { + instrument_read_write(addr + BIT_WORD(nr), sizeof(long)); + } +} + +/** + * ___test_and_set_bit - Set a bit and return its old value + * @nr: Bit to set + * @addr: Address to count from + * + * This operation is non-atomic. If two instances of this operation race, one + * can appear to succeed but actually fail. + */ +static __always_inline bool +___test_and_set_bit(unsigned long nr, volatile unsigned long *addr) +{ + __instrument_read_write_bitop(nr, addr); + return arch___test_and_set_bit(nr, addr); +} + +/** + * ___test_and_clear_bit - Clear a bit and return its old value + * @nr: Bit to clear + * @addr: Address to count from + * + * This operation is non-atomic. If two instances of this operation race, one + * can appear to succeed but actually fail. + */ +static __always_inline bool +___test_and_clear_bit(unsigned long nr, volatile unsigned long *addr) +{ + __instrument_read_write_bitop(nr, addr); + return arch___test_and_clear_bit(nr, addr); +} + +/** + * ___test_and_change_bit - Change a bit and return its old value + * @nr: Bit to change + * @addr: Address to count from + * + * This operation is non-atomic. If two instances of this operation race, one + * can appear to succeed but actually fail. + */ +static __always_inline bool +___test_and_change_bit(unsigned long nr, volatile unsigned long *addr) +{ + __instrument_read_write_bitop(nr, addr); + return arch___test_and_change_bit(nr, addr); +} + +/** + * _test_bit - Determine whether a bit is set + * @nr: bit number to test + * @addr: Address to start counting from + */ +static __always_inline bool +_test_bit(unsigned long nr, const volatile unsigned long *addr) +{ + instrument_atomic_read(addr + BIT_WORD(nr), sizeof(long)); + return arch_test_bit(nr, addr); +} + +/** + * _test_bit_acquire - Determine, with acquire semantics, whether a bit is set + * @nr: bit number to test + * @addr: Address to start counting from + */ +static __always_inline bool +_test_bit_acquire(unsigned long nr, const volatile unsigned long *addr) +{ + instrument_atomic_read(addr + BIT_WORD(nr), sizeof(long)); + return arch_test_bit_acquire(nr, addr); +} + +#endif /* _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H */ diff --git a/include/asm-generic/bitops/le.h b/include/asm-generic/bitops/le.h index 188d3eba3ace..d51beff60375 100644 --- a/include/asm-generic/bitops/le.h +++ b/include/asm-generic/bitops/le.h @@ -9,46 +9,12 @@ #define BITOP_LE_SWIZZLE 0 -static inline unsigned long find_next_zero_bit_le(const void *addr, - unsigned long size, unsigned long offset) -{ - return find_next_zero_bit(addr, size, offset); -} - -static inline unsigned long find_next_bit_le(const void *addr, - unsigned long size, unsigned long offset) -{ - return find_next_bit(addr, size, offset); -} - -static inline unsigned long find_first_zero_bit_le(const void *addr, - unsigned long size) -{ - return find_first_zero_bit(addr, size); -} - #elif defined(__BIG_ENDIAN) #define BITOP_LE_SWIZZLE ((BITS_PER_LONG-1) & ~0x7) -#ifndef find_next_zero_bit_le -extern unsigned long find_next_zero_bit_le(const void *addr, - unsigned long size, unsigned long offset); -#endif - -#ifndef find_next_bit_le -extern unsigned long find_next_bit_le(const void *addr, - unsigned long size, unsigned long offset); #endif -#ifndef find_first_zero_bit_le -#define find_first_zero_bit_le(addr, size) \ - find_next_zero_bit_le((addr), (size), 0) -#endif - -#else -#error "Please fix <asm/byteorder.h>" -#endif static inline int test_bit_le(int nr, const void *addr) { diff --git a/include/asm-generic/bitops/lock.h b/include/asm-generic/bitops/lock.h index 3ae021368f48..14d4ec8c5152 100644 --- a/include/asm-generic/bitops/lock.h +++ b/include/asm-generic/bitops/lock.h @@ -7,7 +7,7 @@ #include <asm/barrier.h> /** - * test_and_set_bit_lock - Set a bit and return its old value, for lock + * arch_test_and_set_bit_lock - Set a bit and return its old value, for lock * @nr: Bit to set * @addr: Address to count from * @@ -15,8 +15,8 @@ * the returned value is 0. * It can be used to implement bit locks. */ -static inline int test_and_set_bit_lock(unsigned int nr, - volatile unsigned long *p) +static __always_inline int +arch_test_and_set_bit_lock(unsigned int nr, volatile unsigned long *p) { long old; unsigned long mask = BIT_MASK(nr); @@ -25,26 +25,27 @@ static inline int test_and_set_bit_lock(unsigned int nr, if (READ_ONCE(*p) & mask) return 1; - old = atomic_long_fetch_or_acquire(mask, (atomic_long_t *)p); + old = raw_atomic_long_fetch_or_acquire(mask, (atomic_long_t *)p); return !!(old & mask); } /** - * clear_bit_unlock - Clear a bit in memory, for unlock + * arch_clear_bit_unlock - Clear a bit in memory, for unlock * @nr: the bit to set * @addr: the address to start counting from * * This operation is atomic and provides release barrier semantics. */ -static inline void clear_bit_unlock(unsigned int nr, volatile unsigned long *p) +static __always_inline void +arch_clear_bit_unlock(unsigned int nr, volatile unsigned long *p) { p += BIT_WORD(nr); - atomic_long_fetch_andnot_release(BIT_MASK(nr), (atomic_long_t *)p); + raw_atomic_long_fetch_andnot_release(BIT_MASK(nr), (atomic_long_t *)p); } /** - * __clear_bit_unlock - Clear a bit in memory, for unlock + * arch___clear_bit_unlock - Clear a bit in memory, for unlock * @nr: the bit to set * @addr: the address to start counting from * @@ -54,38 +55,28 @@ static inline void clear_bit_unlock(unsigned int nr, volatile unsigned long *p) * * See for example x86's implementation. */ -static inline void __clear_bit_unlock(unsigned int nr, - volatile unsigned long *p) +static inline void +arch___clear_bit_unlock(unsigned int nr, volatile unsigned long *p) { unsigned long old; p += BIT_WORD(nr); old = READ_ONCE(*p); old &= ~BIT_MASK(nr); - atomic_long_set_release((atomic_long_t *)p, old); + raw_atomic_long_set_release((atomic_long_t *)p, old); } -/** - * clear_bit_unlock_is_negative_byte - Clear a bit in memory and test if bottom - * byte is negative, for unlock. - * @nr: the bit to clear - * @addr: the address to start counting from - * - * This is a bit of a one-trick-pony for the filemap code, which clears - * PG_locked and tests PG_waiters, - */ -#ifndef clear_bit_unlock_is_negative_byte -static inline bool clear_bit_unlock_is_negative_byte(unsigned int nr, - volatile unsigned long *p) +#ifndef arch_xor_unlock_is_negative_byte +static inline bool arch_xor_unlock_is_negative_byte(unsigned long mask, + volatile unsigned long *p) { long old; - unsigned long mask = BIT_MASK(nr); - p += BIT_WORD(nr); - old = atomic_long_fetch_andnot_release(mask, (atomic_long_t *)p); + old = raw_atomic_long_fetch_xor_release(mask, (atomic_long_t *)p); return !!(old & BIT(7)); } -#define clear_bit_unlock_is_negative_byte clear_bit_unlock_is_negative_byte #endif +#include <asm-generic/bitops/instrumented-lock.h> + #endif /* _ASM_GENERIC_BITOPS_LOCK_H_ */ diff --git a/include/asm-generic/bitops/non-atomic.h b/include/asm-generic/bitops/non-atomic.h index 7e10c4b50c5d..71f8d54a5195 100644 --- a/include/asm-generic/bitops/non-atomic.h +++ b/include/asm-generic/bitops/non-atomic.h @@ -2,108 +2,19 @@ #ifndef _ASM_GENERIC_BITOPS_NON_ATOMIC_H_ #define _ASM_GENERIC_BITOPS_NON_ATOMIC_H_ -#include <asm/types.h> +#include <asm-generic/bitops/generic-non-atomic.h> -/** - * __set_bit - Set a bit in memory - * @nr: the bit to set - * @addr: the address to start counting from - * - * Unlike set_bit(), this function is non-atomic and may be reordered. - * If it's called on the same region of memory simultaneously, the effect - * may be that only one operation succeeds. - */ -static inline void __set_bit(int nr, volatile unsigned long *addr) -{ - unsigned long mask = BIT_MASK(nr); - unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); +#define arch___set_bit generic___set_bit +#define arch___clear_bit generic___clear_bit +#define arch___change_bit generic___change_bit - *p |= mask; -} +#define arch___test_and_set_bit generic___test_and_set_bit +#define arch___test_and_clear_bit generic___test_and_clear_bit +#define arch___test_and_change_bit generic___test_and_change_bit -static inline void __clear_bit(int nr, volatile unsigned long *addr) -{ - unsigned long mask = BIT_MASK(nr); - unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); +#define arch_test_bit generic_test_bit +#define arch_test_bit_acquire generic_test_bit_acquire - *p &= ~mask; -} - -/** - * __change_bit - Toggle a bit in memory - * @nr: the bit to change - * @addr: the address to start counting from - * - * Unlike change_bit(), this function is non-atomic and may be reordered. - * If it's called on the same region of memory simultaneously, the effect - * may be that only one operation succeeds. - */ -static inline void __change_bit(int nr, volatile unsigned long *addr) -{ - unsigned long mask = BIT_MASK(nr); - unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); - - *p ^= mask; -} - -/** - * __test_and_set_bit - Set a bit and return its old value - * @nr: Bit to set - * @addr: Address to count from - * - * This operation is non-atomic and can be reordered. - * If two examples of this operation race, one can appear to succeed - * but actually fail. You must protect multiple accesses with a lock. - */ -static inline int __test_and_set_bit(int nr, volatile unsigned long *addr) -{ - unsigned long mask = BIT_MASK(nr); - unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); - unsigned long old = *p; - - *p = old | mask; - return (old & mask) != 0; -} - -/** - * __test_and_clear_bit - Clear a bit and return its old value - * @nr: Bit to clear - * @addr: Address to count from - * - * This operation is non-atomic and can be reordered. - * If two examples of this operation race, one can appear to succeed - * but actually fail. You must protect multiple accesses with a lock. - */ -static inline int __test_and_clear_bit(int nr, volatile unsigned long *addr) -{ - unsigned long mask = BIT_MASK(nr); - unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); - unsigned long old = *p; - - *p = old & ~mask; - return (old & mask) != 0; -} - -/* WARNING: non atomic and it can be reordered! */ -static inline int __test_and_change_bit(int nr, - volatile unsigned long *addr) -{ - unsigned long mask = BIT_MASK(nr); - unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); - unsigned long old = *p; - - *p = old ^ mask; - return (old & mask) != 0; -} - -/** - * test_bit - Determine whether a bit is set - * @nr: bit number to test - * @addr: Address to start counting from - */ -static inline int test_bit(int nr, const volatile unsigned long *addr) -{ - return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); -} +#include <asm-generic/bitops/non-instrumented-non-atomic.h> #endif /* _ASM_GENERIC_BITOPS_NON_ATOMIC_H_ */ diff --git a/include/asm-generic/bitops/non-instrumented-non-atomic.h b/include/asm-generic/bitops/non-instrumented-non-atomic.h new file mode 100644 index 000000000000..0ddc78dfc358 --- /dev/null +++ b/include/asm-generic/bitops/non-instrumented-non-atomic.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __ASM_GENERIC_BITOPS_NON_INSTRUMENTED_NON_ATOMIC_H +#define __ASM_GENERIC_BITOPS_NON_INSTRUMENTED_NON_ATOMIC_H + +#define ___set_bit arch___set_bit +#define ___clear_bit arch___clear_bit +#define ___change_bit arch___change_bit + +#define ___test_and_set_bit arch___test_and_set_bit +#define ___test_and_clear_bit arch___test_and_clear_bit +#define ___test_and_change_bit arch___test_and_change_bit + +#define _test_bit arch_test_bit +#define _test_bit_acquire arch_test_bit_acquire + +#endif /* __ASM_GENERIC_BITOPS_NON_INSTRUMENTED_NON_ATOMIC_H */ diff --git a/include/asm-generic/bitsperlong.h b/include/asm-generic/bitsperlong.h index 3905c1c93dc2..1023e2a4bd37 100644 --- a/include/asm-generic/bitsperlong.h +++ b/include/asm-generic/bitsperlong.h @@ -23,4 +23,16 @@ #define BITS_PER_LONG_LONG 64 #endif +/* + * small_const_nbits(n) is true precisely when it is known at compile-time + * that BITMAP_SIZE(n) is 1, i.e. 1 <= n <= BITS_PER_LONG. This allows + * various bit/bitmap APIs to provide a fast inline implementation. Bitmaps + * of size 0 are very rare, and a compile-time-known-size 0 is most likely + * a sign of error. They will be handled correctly by the bit/bitmap APIs, + * but using the out-of-line functions, so that the inline implementations + * can unconditionally dereference the pointer(s). + */ +#define small_const_nbits(nbits) \ + (__builtin_constant_p(nbits) && (nbits) <= BITS_PER_LONG && (nbits) > 0) + #endif /* __ASM_GENERIC_BITS_PER_LONG */ diff --git a/include/asm-generic/bug.h b/include/asm-generic/bug.h index 20561a60db9c..09e8eccee8ed 100644 --- a/include/asm-generic/bug.h +++ b/include/asm-generic/bug.h @@ -3,6 +3,8 @@ #define _ASM_GENERIC_BUG_H #include <linux/compiler.h> +#include <linux/instrumentation.h> +#include <linux/once_lite.h> #define CUT_HERE "------------[ cut here ]------------\n" @@ -10,28 +12,46 @@ #define BUGFLAG_WARNING (1 << 0) #define BUGFLAG_ONCE (1 << 1) #define BUGFLAG_DONE (1 << 2) +#define BUGFLAG_NO_CUT_HERE (1 << 3) /* CUT_HERE already sent */ +#define BUGFLAG_ARGS (1 << 4) #define BUGFLAG_TAINT(taint) ((taint) << 8) #define BUG_GET_TAINT(bug) ((bug)->flags >> 8) #endif +#ifndef WARN_CONDITION_STR +#ifdef CONFIG_DEBUG_BUGVERBOSE_DETAILED +# define WARN_CONDITION_STR(cond_str) "[" cond_str "] " +#else +# define WARN_CONDITION_STR(cond_str) +#endif +#endif /* WARN_CONDITION_STR */ + #ifndef __ASSEMBLY__ -#include <linux/kernel.h> +#include <linux/panic.h> +#include <linux/printk.h> + +struct warn_args; +struct pt_regs; + +void __warn(const char *file, int line, void *caller, unsigned taint, + struct pt_regs *regs, struct warn_args *args); #ifdef CONFIG_BUG -#ifdef CONFIG_GENERIC_BUG -struct bug_entry { #ifndef CONFIG_GENERIC_BUG_RELATIVE_POINTERS - unsigned long bug_addr; +#define BUG_REL(type, name) type name #else - signed int bug_addr_disp; +#define BUG_REL(type, name) signed int name##_disp #endif -#ifdef CONFIG_DEBUG_BUGVERBOSE -#ifndef CONFIG_GENERIC_BUG_RELATIVE_POINTERS - const char *file; -#else - signed int file_disp; + +#ifdef CONFIG_GENERIC_BUG +struct bug_entry { + BUG_REL(unsigned long, bug_addr); +#ifdef HAVE_ARCH_BUG_FORMAT + BUG_REL(const char *, format); #endif +#ifdef CONFIG_DEBUG_BUGVERBOSE + BUG_REL(const char *, file); unsigned short line; #endif unsigned short flags; @@ -61,20 +81,8 @@ struct bug_entry { #define BUG_ON(condition) do { if (unlikely(condition)) BUG(); } while (0) #endif -#ifdef __WARN_FLAGS -#define __WARN_TAINT(taint) __WARN_FLAGS(BUGFLAG_TAINT(taint)) -#define __WARN_ONCE_TAINT(taint) __WARN_FLAGS(BUGFLAG_ONCE|BUGFLAG_TAINT(taint)) - -#define WARN_ON_ONCE(condition) ({ \ - int __ret_warn_on = !!(condition); \ - if (unlikely(__ret_warn_on)) \ - __WARN_ONCE_TAINT(TAINT_WARN); \ - unlikely(__ret_warn_on); \ -}) -#endif - /* - * WARN(), WARN_ON(), WARN_ON_ONCE, and so on can be used to report + * WARN(), WARN_ON(), WARN_ON_ONCE(), and so on can be used to report * significant kernel issues that need prompt attention if they should ever * appear at runtime. * @@ -89,33 +97,58 @@ struct bug_entry { * * Use the versions with printk format strings to provide better diagnostics. */ -#ifndef __WARN_TAINT -extern __printf(3, 4) -void warn_slowpath_fmt(const char *file, const int line, - const char *fmt, ...); extern __printf(4, 5) -void warn_slowpath_fmt_taint(const char *file, const int line, unsigned taint, - const char *fmt, ...); -extern void warn_slowpath_null(const char *file, const int line); -#define WANT_WARN_ON_SLOWPATH -#define __WARN() warn_slowpath_null(__FILE__, __LINE__) -#define __WARN_printf(arg...) warn_slowpath_fmt(__FILE__, __LINE__, arg) -#define __WARN_printf_taint(taint, arg...) \ - warn_slowpath_fmt_taint(__FILE__, __LINE__, taint, arg) -#else +void warn_slowpath_fmt(const char *file, const int line, unsigned taint, + const char *fmt, ...); extern __printf(1, 2) void __warn_printk(const char *fmt, ...); -#define __WARN() __WARN_TAINT(TAINT_WARN) -#define __WARN_printf(arg...) do { __warn_printk(arg); __WARN(); } while (0) -#define __WARN_printf_taint(taint, arg...) \ - do { __warn_printk(arg); __WARN_TAINT(taint); } while (0) + +#ifdef __WARN_FLAGS +#define __WARN() __WARN_FLAGS("", BUGFLAG_TAINT(TAINT_WARN)) + +#ifndef WARN_ON +#define WARN_ON(condition) ({ \ + int __ret_warn_on = !!(condition); \ + if (unlikely(__ret_warn_on)) \ + __WARN_FLAGS(#condition, \ + BUGFLAG_TAINT(TAINT_WARN)); \ + unlikely(__ret_warn_on); \ +}) #endif -/* used internally by panic.c */ -struct warn_args; -struct pt_regs; +#ifndef WARN_ON_ONCE +#define WARN_ON_ONCE(condition) ({ \ + int __ret_warn_on = !!(condition); \ + if (unlikely(__ret_warn_on)) \ + __WARN_FLAGS(#condition, \ + BUGFLAG_ONCE | \ + BUGFLAG_TAINT(TAINT_WARN)); \ + unlikely(__ret_warn_on); \ +}) +#endif +#endif /* __WARN_FLAGS */ -void __warn(const char *file, int line, void *caller, unsigned taint, - struct pt_regs *regs, struct warn_args *args); +#if defined(__WARN_FLAGS) && !defined(__WARN_printf) +#define __WARN_printf(taint, arg...) do { \ + instrumentation_begin(); \ + __warn_printk(arg); \ + __WARN_FLAGS("", BUGFLAG_NO_CUT_HERE | BUGFLAG_TAINT(taint));\ + instrumentation_end(); \ + } while (0) +#endif + +#ifndef __WARN_printf +#define __WARN_printf(taint, arg...) do { \ + instrumentation_begin(); \ + warn_slowpath_fmt(__FILE__, __LINE__, taint, arg); \ + instrumentation_end(); \ + } while (0) +#endif + +#ifndef __WARN +#define __WARN() __WARN_printf(TAINT_WARN, NULL) +#endif + +/* used internally by panic.c */ #ifndef WARN_ON #define WARN_ON(condition) ({ \ @@ -130,7 +163,7 @@ void __warn(const char *file, int line, void *caller, unsigned taint, #define WARN(condition, format...) ({ \ int __ret_warn_on = !!(condition); \ if (unlikely(__ret_warn_on)) \ - __WARN_printf(format); \ + __WARN_printf(TAINT_WARN, format); \ unlikely(__ret_warn_on); \ }) #endif @@ -138,52 +171,33 @@ void __warn(const char *file, int line, void *caller, unsigned taint, #define WARN_TAINT(condition, taint, format...) ({ \ int __ret_warn_on = !!(condition); \ if (unlikely(__ret_warn_on)) \ - __WARN_printf_taint(taint, format); \ + __WARN_printf(taint, format); \ unlikely(__ret_warn_on); \ }) #ifndef WARN_ON_ONCE -#define WARN_ON_ONCE(condition) ({ \ - static bool __section(.data.once) __warned; \ - int __ret_warn_once = !!(condition); \ - \ - if (unlikely(__ret_warn_once && !__warned)) { \ - __warned = true; \ - WARN_ON(1); \ - } \ - unlikely(__ret_warn_once); \ -}) +#define WARN_ON_ONCE(condition) \ + DO_ONCE_LITE_IF(condition, WARN_ON, 1) #endif -#define WARN_ONCE(condition, format...) ({ \ - static bool __section(.data.once) __warned; \ - int __ret_warn_once = !!(condition); \ - \ - if (unlikely(__ret_warn_once && !__warned)) { \ - __warned = true; \ - WARN(1, format); \ - } \ - unlikely(__ret_warn_once); \ -}) +#ifndef WARN_ONCE +#define WARN_ONCE(condition, format...) \ + DO_ONCE_LITE_IF(condition, WARN, 1, format) +#endif -#define WARN_TAINT_ONCE(condition, taint, format...) ({ \ - static bool __section(.data.once) __warned; \ - int __ret_warn_once = !!(condition); \ - \ - if (unlikely(__ret_warn_once && !__warned)) { \ - __warned = true; \ - WARN_TAINT(1, taint, format); \ - } \ - unlikely(__ret_warn_once); \ -}) +#define WARN_TAINT_ONCE(condition, taint, format...) \ + DO_ONCE_LITE_IF(condition, WARN_TAINT, 1, taint, format) #else /* !CONFIG_BUG */ #ifndef HAVE_ARCH_BUG -#define BUG() do {} while (1) +#define BUG() do { \ + do {} while (1); \ + unreachable(); \ +} while (0) #endif #ifndef HAVE_ARCH_BUG_ON -#define BUG_ON(condition) do { if (condition) BUG(); } while (0) +#define BUG_ON(condition) do { if (unlikely(condition)) BUG(); } while (0) #endif #ifndef HAVE_ARCH_WARN_ON @@ -211,9 +225,6 @@ void __warn(const char *file, int line, void *caller, unsigned taint, /* * WARN_ON_SMP() is for cases that the warning is either * meaningless for !SMP or may even cause failures. - * This is usually used for cases that we have - * WARN_ON(!spin_is_locked(&lock)) checks, as spin_is_locked() - * returns 0 for uniprocessor settings. * It can also be used with values that are only defined * on SMP: * diff --git a/include/asm-generic/bugs.h b/include/asm-generic/bugs.h deleted file mode 100644 index 69021830f078..000000000000 --- a/include/asm-generic/bugs.h +++ /dev/null @@ -1,11 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef __ASM_GENERIC_BUGS_H -#define __ASM_GENERIC_BUGS_H -/* - * This file is included by 'init/main.c' to check for - * architecture-dependent bugs. - */ - -static inline void check_bugs(void) { } - -#endif /* __ASM_GENERIC_BUGS_H */ diff --git a/include/asm-generic/cacheflush.h b/include/asm-generic/cacheflush.h index 0dd47a6db2cf..7ee8a179d103 100644 --- a/include/asm-generic/cacheflush.h +++ b/include/asm-generic/cacheflush.h @@ -1,35 +1,127 @@ /* SPDX-License-Identifier: GPL-2.0 */ -#ifndef __ASM_CACHEFLUSH_H -#define __ASM_CACHEFLUSH_H +#ifndef _ASM_GENERIC_CACHEFLUSH_H +#define _ASM_GENERIC_CACHEFLUSH_H -/* Keep includes the same across arches. */ -#include <linux/mm.h> +#include <linux/instrumented.h> + +struct mm_struct; +struct vm_area_struct; +struct page; +struct address_space; /* * The cache doesn't need to be flushed when TLB entries change when * the cache is mapped to physical memory, not virtual memory */ -#define flush_cache_all() do { } while (0) -#define flush_cache_mm(mm) do { } while (0) -#define flush_cache_dup_mm(mm) do { } while (0) -#define flush_cache_range(vma, start, end) do { } while (0) -#define flush_cache_page(vma, vmaddr, pfn) do { } while (0) +#ifndef flush_cache_all +static inline void flush_cache_all(void) +{ +} +#endif + +#ifndef flush_cache_mm +static inline void flush_cache_mm(struct mm_struct *mm) +{ +} +#endif + +#ifndef flush_cache_dup_mm +static inline void flush_cache_dup_mm(struct mm_struct *mm) +{ +} +#endif + +#ifndef flush_cache_range +static inline void flush_cache_range(struct vm_area_struct *vma, + unsigned long start, + unsigned long end) +{ +} +#endif + +#ifndef flush_cache_page +static inline void flush_cache_page(struct vm_area_struct *vma, + unsigned long vmaddr, + unsigned long pfn) +{ +} +#endif + +#ifndef ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE +static inline void flush_dcache_page(struct page *page) +{ +} + #define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 0 -#define flush_dcache_page(page) do { } while (0) -#define flush_dcache_mmap_lock(mapping) do { } while (0) -#define flush_dcache_mmap_unlock(mapping) do { } while (0) -#define flush_icache_range(start, end) do { } while (0) -#define flush_icache_page(vma,pg) do { } while (0) -#define flush_icache_user_range(vma,pg,adr,len) do { } while (0) -#define flush_cache_vmap(start, end) do { } while (0) -#define flush_cache_vunmap(start, end) do { } while (0) - -#define copy_to_user_page(vma, page, vaddr, dst, src, len) \ +#endif + +#ifndef flush_dcache_mmap_lock +static inline void flush_dcache_mmap_lock(struct address_space *mapping) +{ +} +#endif + +#ifndef flush_dcache_mmap_unlock +static inline void flush_dcache_mmap_unlock(struct address_space *mapping) +{ +} +#endif + +#ifndef flush_icache_range +static inline void flush_icache_range(unsigned long start, unsigned long end) +{ +} +#endif + +#ifndef flush_icache_user_range +#define flush_icache_user_range flush_icache_range +#endif + +#ifndef flush_icache_user_page +static inline void flush_icache_user_page(struct vm_area_struct *vma, + struct page *page, + unsigned long addr, int len) +{ +} +#endif + +#ifndef flush_cache_vmap +static inline void flush_cache_vmap(unsigned long start, unsigned long end) +{ +} +#endif + +#ifndef flush_cache_vmap_early +static inline void flush_cache_vmap_early(unsigned long start, unsigned long end) +{ +} +#endif + +#ifndef flush_cache_vunmap +static inline void flush_cache_vunmap(unsigned long start, unsigned long end) +{ +} +#endif + +#ifndef copy_to_user_page +#define copy_to_user_page(vma, page, vaddr, dst, src, len) \ do { \ + instrument_copy_to_user((void __user *)dst, src, len); \ memcpy(dst, src, len); \ - flush_icache_user_range(vma, page, vaddr, len); \ + flush_icache_user_page(vma, page, vaddr, len); \ + } while (0) +#endif + + +#ifndef copy_from_user_page +#define copy_from_user_page(vma, page, vaddr, dst, src, len) \ + do { \ + instrument_copy_from_user_before(dst, (void __user *)src, \ + len); \ + memcpy(dst, src, len); \ + instrument_copy_from_user_after(dst, (void __user *)src, len, \ + 0); \ } while (0) -#define copy_from_user_page(vma, page, vaddr, dst, src, len) \ - memcpy(dst, src, len) +#endif -#endif /* __ASM_CACHEFLUSH_H */ +#endif /* _ASM_GENERIC_CACHEFLUSH_H */ diff --git a/include/asm-generic/cfi.h b/include/asm-generic/cfi.h new file mode 100644 index 000000000000..41fac3537bf9 --- /dev/null +++ b/include/asm-generic/cfi.h @@ -0,0 +1,5 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_GENERIC_CFI_H +#define __ASM_GENERIC_CFI_H + +#endif /* __ASM_GENERIC_CFI_H */ diff --git a/include/asm-generic/checksum.h b/include/asm-generic/checksum.h index 34785c0f57b0..ad928cce268b 100644 --- a/include/asm-generic/checksum.h +++ b/include/asm-generic/checksum.h @@ -2,6 +2,8 @@ #ifndef __ASM_GENERIC_CHECKSUM_H #define __ASM_GENERIC_CHECKSUM_H +#include <linux/bitops.h> + /* * computes the checksum of a memory block at buff, length len, * and adds in "sum" (32-bit) @@ -16,29 +18,6 @@ */ extern __wsum csum_partial(const void *buff, int len, __wsum sum); -/* - * the same as csum_partial, but copies from src while it - * checksums - * - * here even more important to align src and dst on a 32-bit (or even - * better 64-bit) boundary - */ -extern __wsum csum_partial_copy(const void *src, void *dst, int len, __wsum sum); - -/* - * the same as csum_partial_copy, but copies from user space. - * - * here even more important to align src and dst on a 32-bit (or even - * better 64-bit) boundary - */ -extern __wsum csum_partial_copy_from_user(const void __user *src, void *dst, - int len, __wsum sum, int *csum_err); - -#ifndef csum_partial_copy_nocheck -#define csum_partial_copy_nocheck(src, dst, len, sum) \ - csum_partial_copy((src), (dst), (len), (sum)) -#endif - #ifndef ip_fast_csum /* * This is a version of ip_compute_csum() optimized for IP headers, @@ -54,9 +33,7 @@ extern __sum16 ip_fast_csum(const void *iph, unsigned int ihl); static inline __sum16 csum_fold(__wsum csum) { u32 sum = (__force u32)csum; - sum = (sum & 0xffff) + (sum >> 16); - sum = (sum & 0xffff) + (sum >> 16); - return (__force __sum16)~sum; + return (__force __sum16)((~sum - ror32(sum, 16)) >> 16); } #endif diff --git a/include/asm-generic/cmpxchg-local.h b/include/asm-generic/cmpxchg-local.h index f17f14f84d09..f27d66fdc00a 100644 --- a/include/asm-generic/cmpxchg-local.h +++ b/include/asm-generic/cmpxchg-local.h @@ -12,7 +12,7 @@ extern unsigned long wrong_size_cmpxchg(volatile void *ptr) * Generic version of __cmpxchg_local (disables interrupts). Takes an unsigned * long parameter, supporting various types of architectures. */ -static inline unsigned long __cmpxchg_local_generic(volatile void *ptr, +static inline unsigned long __generic_cmpxchg_local(volatile void *ptr, unsigned long old, unsigned long new, int size) { unsigned long flags, prev; @@ -26,16 +26,16 @@ static inline unsigned long __cmpxchg_local_generic(volatile void *ptr, raw_local_irq_save(flags); switch (size) { case 1: prev = *(u8 *)ptr; - if (prev == old) - *(u8 *)ptr = (u8)new; + if (prev == (old & 0xffu)) + *(u8 *)ptr = (new & 0xffu); break; case 2: prev = *(u16 *)ptr; - if (prev == old) - *(u16 *)ptr = (u16)new; + if (prev == (old & 0xffffu)) + *(u16 *)ptr = (new & 0xffffu); break; case 4: prev = *(u32 *)ptr; - if (prev == old) - *(u32 *)ptr = (u32)new; + if (prev == (old & 0xffffffffu)) + *(u32 *)ptr = (new & 0xffffffffu); break; case 8: prev = *(u64 *)ptr; if (prev == old) @@ -51,7 +51,7 @@ static inline unsigned long __cmpxchg_local_generic(volatile void *ptr, /* * Generic version of __cmpxchg64_local. Takes an u64 parameter. */ -static inline u64 __cmpxchg64_local_generic(volatile void *ptr, +static inline u64 __generic_cmpxchg64_local(volatile void *ptr, u64 old, u64 new) { u64 prev; diff --git a/include/asm-generic/cmpxchg.h b/include/asm-generic/cmpxchg.h index 9a24510cd8c1..848de25fc4bf 100644 --- a/include/asm-generic/cmpxchg.h +++ b/include/asm-generic/cmpxchg.h @@ -14,16 +14,14 @@ #include <linux/types.h> #include <linux/irqflags.h> -#ifndef xchg - /* * This function doesn't exist, so you'll get a linker error if * something tries to do an invalidly-sized xchg(). */ -extern void __xchg_called_with_bad_pointer(void); +extern void __generic_xchg_called_with_bad_pointer(void); static inline -unsigned long __xchg(unsigned long x, volatile void *ptr, int size) +unsigned long __generic_xchg(unsigned long x, volatile void *ptr, int size) { unsigned long ret, flags; @@ -34,7 +32,7 @@ unsigned long __xchg(unsigned long x, volatile void *ptr, int size) #else local_irq_save(flags); ret = *(volatile u8 *)ptr; - *(volatile u8 *)ptr = x; + *(volatile u8 *)ptr = (x & 0xffu); local_irq_restore(flags); return ret; #endif /* __xchg_u8 */ @@ -45,7 +43,7 @@ unsigned long __xchg(unsigned long x, volatile void *ptr, int size) #else local_irq_save(flags); ret = *(volatile u16 *)ptr; - *(volatile u16 *)ptr = x; + *(volatile u16 *)ptr = (x & 0xffffu); local_irq_restore(flags); return ret; #endif /* __xchg_u16 */ @@ -56,7 +54,7 @@ unsigned long __xchg(unsigned long x, volatile void *ptr, int size) #else local_irq_save(flags); ret = *(volatile u32 *)ptr; - *(volatile u32 *)ptr = x; + *(volatile u32 *)ptr = (x & 0xffffffffu); local_irq_restore(flags); return ret; #endif /* __xchg_u32 */ @@ -75,35 +73,43 @@ unsigned long __xchg(unsigned long x, volatile void *ptr, int size) #endif /* CONFIG_64BIT */ default: - __xchg_called_with_bad_pointer(); + __generic_xchg_called_with_bad_pointer(); return x; } } -#define xchg(ptr, x) ({ \ - ((__typeof__(*(ptr))) \ - __xchg((unsigned long)(x), (ptr), sizeof(*(ptr)))); \ +#define generic_xchg(ptr, x) ({ \ + ((__typeof__(*(ptr))) \ + __generic_xchg((unsigned long)(x), (ptr), sizeof(*(ptr)))); \ }) -#endif /* xchg */ - /* * Atomic compare and exchange. */ #include <asm-generic/cmpxchg-local.h> -#ifndef cmpxchg_local -#define cmpxchg_local(ptr, o, n) ({ \ - ((__typeof__(*(ptr)))__cmpxchg_local_generic((ptr), (unsigned long)(o),\ - (unsigned long)(n), sizeof(*(ptr)))); \ +#define generic_cmpxchg_local(ptr, o, n) ({ \ + ((__typeof__(*(ptr)))__generic_cmpxchg_local((ptr), (unsigned long)(o), \ + (unsigned long)(n), sizeof(*(ptr)))); \ }) + +#define generic_cmpxchg64_local(ptr, o, n) \ + __generic_cmpxchg64_local((ptr), (o), (n)) + + +#ifndef arch_xchg +#define arch_xchg generic_xchg +#endif + +#ifndef arch_cmpxchg_local +#define arch_cmpxchg_local generic_cmpxchg_local #endif -#ifndef cmpxchg64_local -#define cmpxchg64_local(ptr, o, n) __cmpxchg64_local_generic((ptr), (o), (n)) +#ifndef arch_cmpxchg64_local +#define arch_cmpxchg64_local generic_cmpxchg64_local #endif -#define cmpxchg(ptr, o, n) cmpxchg_local((ptr), (o), (n)) -#define cmpxchg64(ptr, o, n) cmpxchg64_local((ptr), (o), (n)) +#define arch_cmpxchg arch_cmpxchg_local +#define arch_cmpxchg64 arch_cmpxchg64_local #endif /* __ASM_GENERIC_CMPXCHG_H */ diff --git a/include/asm-generic/codetag.lds.h b/include/asm-generic/codetag.lds.h new file mode 100644 index 000000000000..a14f4bdafdda --- /dev/null +++ b/include/asm-generic/codetag.lds.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __ASM_GENERIC_CODETAG_LDS_H +#define __ASM_GENERIC_CODETAG_LDS_H + +#ifdef CONFIG_MEM_ALLOC_PROFILING +#define IF_MEM_ALLOC_PROFILING(...) __VA_ARGS__ +#else +#define IF_MEM_ALLOC_PROFILING(...) +#endif + +#define SECTION_WITH_BOUNDARIES(_name) \ + . = ALIGN(8); \ + __start_##_name = .; \ + KEEP(*(_name)) \ + __stop_##_name = .; + +#define CODETAG_SECTIONS() \ + IF_MEM_ALLOC_PROFILING(SECTION_WITH_BOUNDARIES(alloc_tags)) + +#define MOD_SEPARATE_CODETAG_SECTION(_name) \ + .codetag.##_name : { \ + SECTION_WITH_BOUNDARIES(_name) \ + } + +/* + * For codetags which might be used after module unload, therefore might stay + * longer in memory. Each such codetag type has its own section so that we can + * unload them individually once unused. + */ +#define MOD_SEPARATE_CODETAG_SECTIONS() \ + IF_MEM_ALLOC_PROFILING(MOD_SEPARATE_CODETAG_SECTION(alloc_tags)) + +#endif /* __ASM_GENERIC_CODETAG_LDS_H */ diff --git a/include/asm-generic/compat.h b/include/asm-generic/compat.h index a86f65bffab8..8392caea398f 100644 --- a/include/asm-generic/compat.h +++ b/include/asm-generic/compat.h @@ -2,6 +2,30 @@ #ifndef __ASM_GENERIC_COMPAT_H #define __ASM_GENERIC_COMPAT_H +#ifndef COMPAT_USER_HZ +#define COMPAT_USER_HZ 100 +#endif + +#ifndef COMPAT_RLIM_INFINITY +#define COMPAT_RLIM_INFINITY 0xffffffff +#endif + +#ifndef COMPAT_OFF_T_MAX +#define COMPAT_OFF_T_MAX 0x7fffffff +#endif + +#ifndef compat_arg_u64 +#ifndef CONFIG_CPU_BIG_ENDIAN +#define compat_arg_u64(name) u32 name##_lo, u32 name##_hi +#define compat_arg_u64_dual(name) u32, name##_lo, u32, name##_hi +#else +#define compat_arg_u64(name) u32 name##_hi, u32 name##_lo +#define compat_arg_u64_dual(name) u32, name##_hi, u32, name##_lo +#endif +#define compat_arg_u64_glue(name) (((u64)name##_lo & 0xffffffffUL) | \ + ((u64)name##_hi << 32)) +#endif /* compat_arg_u64 */ + /* These types are common across all compat ABIs */ typedef u32 compat_size_t; typedef s32 compat_ssize_t; @@ -20,6 +44,125 @@ typedef u16 compat_ushort_t; typedef u32 compat_uint_t; typedef u32 compat_ulong_t; typedef u32 compat_uptr_t; +typedef u32 compat_caddr_t; typedef u32 compat_aio_context_t; +typedef u32 compat_old_sigset_t; + +#ifndef __compat_uid_t +typedef u32 __compat_uid_t; +typedef u32 __compat_gid_t; +#endif + +#ifndef __compat_uid32_t +typedef u32 __compat_uid32_t; +typedef u32 __compat_gid32_t; +#endif + +#ifndef compat_mode_t +typedef u32 compat_mode_t; +#endif + +#ifdef CONFIG_COMPAT_FOR_U64_ALIGNMENT +typedef s64 __attribute__((aligned(4))) compat_s64; +typedef u64 __attribute__((aligned(4))) compat_u64; +#else +typedef s64 compat_s64; +typedef u64 compat_u64; +#endif + +#ifndef _COMPAT_NSIG +typedef u32 compat_sigset_word; +#define _COMPAT_NSIG _NSIG +#define _COMPAT_NSIG_BPW 32 +#endif + +#ifndef compat_dev_t +typedef u32 compat_dev_t; +#endif + +#ifndef compat_ipc_pid_t +typedef s32 compat_ipc_pid_t; +#endif + +#ifndef compat_fsid_t +typedef __kernel_fsid_t compat_fsid_t; +#endif + +#ifndef compat_statfs +struct compat_statfs { + compat_int_t f_type; + compat_int_t f_bsize; + compat_int_t f_blocks; + compat_int_t f_bfree; + compat_int_t f_bavail; + compat_int_t f_files; + compat_int_t f_ffree; + compat_fsid_t f_fsid; + compat_int_t f_namelen; + compat_int_t f_frsize; + compat_int_t f_flags; + compat_int_t f_spare[4]; +}; +#endif + +#ifndef compat_ipc64_perm +struct compat_ipc64_perm { + compat_key_t key; + __compat_uid32_t uid; + __compat_gid32_t gid; + __compat_uid32_t cuid; + __compat_gid32_t cgid; + compat_mode_t mode; + unsigned char __pad1[4 - sizeof(compat_mode_t)]; + compat_ushort_t seq; + compat_ushort_t __pad2; + compat_ulong_t unused1; + compat_ulong_t unused2; +}; + +struct compat_semid64_ds { + struct compat_ipc64_perm sem_perm; + compat_ulong_t sem_otime; + compat_ulong_t sem_otime_high; + compat_ulong_t sem_ctime; + compat_ulong_t sem_ctime_high; + compat_ulong_t sem_nsems; + compat_ulong_t __unused3; + compat_ulong_t __unused4; +}; + +struct compat_msqid64_ds { + struct compat_ipc64_perm msg_perm; + compat_ulong_t msg_stime; + compat_ulong_t msg_stime_high; + compat_ulong_t msg_rtime; + compat_ulong_t msg_rtime_high; + compat_ulong_t msg_ctime; + compat_ulong_t msg_ctime_high; + compat_ulong_t msg_cbytes; + compat_ulong_t msg_qnum; + compat_ulong_t msg_qbytes; + compat_pid_t msg_lspid; + compat_pid_t msg_lrpid; + compat_ulong_t __unused4; + compat_ulong_t __unused5; +}; + +struct compat_shmid64_ds { + struct compat_ipc64_perm shm_perm; + compat_size_t shm_segsz; + compat_ulong_t shm_atime; + compat_ulong_t shm_atime_high; + compat_ulong_t shm_dtime; + compat_ulong_t shm_dtime_high; + compat_ulong_t shm_ctime; + compat_ulong_t shm_ctime_high; + compat_pid_t shm_cpid; + compat_pid_t shm_lpid; + compat_ulong_t shm_nattch; + compat_ulong_t __unused4; + compat_ulong_t __unused5; +}; +#endif #endif diff --git a/include/asm-generic/current.h b/include/asm-generic/current.h index 3a2e224b9fa0..9c2aeecbd05a 100644 --- a/include/asm-generic/current.h +++ b/include/asm-generic/current.h @@ -2,9 +2,11 @@ #ifndef __ASM_GENERIC_CURRENT_H #define __ASM_GENERIC_CURRENT_H +#ifndef __ASSEMBLY__ #include <linux/thread_info.h> #define get_current() (current_thread_info()->task) #define current get_current() +#endif #endif /* __ASM_GENERIC_CURRENT_H */ diff --git a/include/asm-generic/delay.h b/include/asm-generic/delay.h index e448ac61430c..03b0ec7afca6 100644 --- a/include/asm-generic/delay.h +++ b/include/asm-generic/delay.h @@ -2,6 +2,9 @@ #ifndef __ASM_GENERIC_DELAY_H #define __ASM_GENERIC_DELAY_H +#include <linux/math.h> +#include <vdso/time64.h> + /* Undefined functions to get compile-time errors */ extern void __bad_udelay(void); extern void __bad_ndelay(void); @@ -12,34 +15,73 @@ extern void __const_udelay(unsigned long xloops); extern void __delay(unsigned long loops); /* - * The weird n/20000 thing suppresses a "comparison is always false due to - * limited range of data type" warning with non-const 8-bit arguments. + * The microseconds/nanosecond delay multiplicators are used to convert a + * constant microseconds/nanoseconds value to a value which can be used by the + * architectures specific implementation to transform it into loops. + */ +#define UDELAY_CONST_MULT ((unsigned long)DIV_ROUND_UP(1ULL << 32, USEC_PER_SEC)) +#define NDELAY_CONST_MULT ((unsigned long)DIV_ROUND_UP(1ULL << 32, NSEC_PER_SEC)) + +/* + * The maximum constant udelay/ndelay value picked out of thin air to prevent + * too long constant udelays/ndelays. */ +#define DELAY_CONST_MAX 20000 -/* 0x10c7 is 2**32 / 1000000 (rounded up) */ -#define udelay(n) \ - ({ \ - if (__builtin_constant_p(n)) { \ - if ((n) / 20000 >= 1) \ - __bad_udelay(); \ - else \ - __const_udelay((n) * 0x10c7ul); \ - } else { \ - __udelay(n); \ - } \ - }) - -/* 0x5 is 2**32 / 1000000000 (rounded up) */ -#define ndelay(n) \ - ({ \ - if (__builtin_constant_p(n)) { \ - if ((n) / 20000 >= 1) \ - __bad_ndelay(); \ - else \ - __const_udelay((n) * 5ul); \ - } else { \ - __ndelay(n); \ - } \ - }) +/** + * udelay - Inserting a delay based on microseconds with busy waiting + * @usec: requested delay in microseconds + * + * When delaying in an atomic context ndelay(), udelay() and mdelay() are the + * only valid variants of delaying/sleeping to go with. + * + * When inserting delays in non atomic context which are shorter than the time + * which is required to queue e.g. an hrtimer and to enter then the scheduler, + * it is also valuable to use udelay(). But it is not simple to specify a + * generic threshold for this which will fit for all systems. An approximation + * is a threshold for all delays up to 10 microseconds. + * + * When having a delay which is larger than the architecture specific + * %MAX_UDELAY_MS value, please make sure mdelay() is used. Otherwise a overflow + * risk is given. + * + * Please note that ndelay(), udelay() and mdelay() may return early for several + * reasons (https://lists.openwall.net/linux-kernel/2011/01/09/56): + * + * #. computed loops_per_jiffy too low (due to the time taken to execute the + * timer interrupt.) + * #. cache behaviour affecting the time it takes to execute the loop function. + * #. CPU clock rate changes. + */ +static __always_inline void udelay(unsigned long usec) +{ + if (__builtin_constant_p(usec)) { + if (usec >= DELAY_CONST_MAX) + __bad_udelay(); + else + __const_udelay(usec * UDELAY_CONST_MULT); + } else { + __udelay(usec); + } +} + +/** + * ndelay - Inserting a delay based on nanoseconds with busy waiting + * @nsec: requested delay in nanoseconds + * + * See udelay() for basic information about ndelay() and it's variants. + */ +static __always_inline void ndelay(unsigned long nsec) +{ + if (__builtin_constant_p(nsec)) { + if (nsec >= DELAY_CONST_MAX) + __bad_ndelay(); + else + __const_udelay(nsec * NDELAY_CONST_MULT); + } else { + __ndelay(nsec); + } +} +#define ndelay(x) ndelay(x) #endif /* __ASM_GENERIC_DELAY_H */ diff --git a/include/asm-generic/device.h b/include/asm-generic/device.h index d7c76bba640d..974517cdf736 100644 --- a/include/asm-generic/device.h +++ b/include/asm-generic/device.h @@ -1,7 +1,6 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * Arch specific extensions to struct device - * - * This file is released under the GPLv2 */ #ifndef _ASM_GENERIC_DEVICE_H #define _ASM_GENERIC_DEVICE_H diff --git a/include/asm-generic/div64.h b/include/asm-generic/div64.h index dc9726fdac8f..25e7b4b58dcf 100644 --- a/include/asm-generic/div64.h +++ b/include/asm-generic/div64.h @@ -8,12 +8,14 @@ * Optimization for constant divisors on 32-bit machines: * Copyright (C) 2006-2015 Nicolas Pitre * - * The semantics of do_div() are: + * The semantics of do_div() is, in C++ notation, observing that the name + * is a function-like macro and the n parameter has the semantics of a C++ + * reference: * - * uint32_t do_div(uint64_t *n, uint32_t base) + * uint32_t do_div(uint64_t &n, uint32_t base) * { - * uint32_t remainder = *n % base; - * *n = *n / base; + * uint32_t remainder = n % base; + * n = n / base; * return remainder; * } * @@ -28,12 +30,12 @@ /** * do_div - returns 2 values: calculate remainder and update new dividend - * @n: pointer to uint64_t dividend (will be updated) + * @n: uint64_t dividend (will be updated) * @base: uint32_t divisor * * Summary: - * ``uint32_t remainder = *n % base;`` - * ``*n = *n / base;`` + * ``uint32_t remainder = n % base;`` + * ``n = n / base;`` * * Return: (uint32_t)remainder * @@ -55,17 +57,11 @@ /* * If the divisor happens to be constant, we determine the appropriate * inverse at compile time to turn the division into a few inline - * multiplications which ought to be much faster. And yet only if compiling - * with a sufficiently recent gcc version to perform proper 64-bit constant - * propagation. + * multiplications which ought to be much faster. * * (It is unfortunate that gcc doesn't perform all this internally.) */ -#ifndef __div64_const32_is_OK -#define __div64_const32_is_OK (__GNUC__ >= 4) -#endif - #define __div64_const32(n, ___b) \ ({ \ /* \ @@ -78,7 +74,8 @@ * do the trick here). \ */ \ uint64_t ___res, ___x, ___t, ___m, ___n = (n); \ - uint32_t ___p, ___bias; \ + uint32_t ___p; \ + bool ___bias = false; \ \ /* determine MSB of b */ \ ___p = 1 << ilog2(___b); \ @@ -91,22 +88,14 @@ ___x = ~0ULL / ___b * ___b - 1; \ \ /* test our ___m with res = m * x / (p << 64) */ \ - ___res = ((___m & 0xffffffff) * (___x & 0xffffffff)) >> 32; \ - ___t = ___res += (___m & 0xffffffff) * (___x >> 32); \ - ___res += (___x & 0xffffffff) * (___m >> 32); \ - ___t = (___res < ___t) ? (1ULL << 32) : 0; \ - ___res = (___res >> 32) + ___t; \ - ___res += (___m >> 32) * (___x >> 32); \ - ___res /= ___p; \ + ___res = (___m & 0xffffffff) * (___x & 0xffffffff); \ + ___t = (___m & 0xffffffff) * (___x >> 32) + (___res >> 32); \ + ___res = (___m >> 32) * (___x >> 32) + (___t >> 32); \ + ___t = (___m >> 32) * (___x & 0xffffffff) + (___t & 0xffffffff);\ + ___res = (___res + (___t >> 32)) / ___p; \ \ - /* Now sanitize and optimize what we've got. */ \ - if (~0ULL % (___b / (___b & -___b)) == 0) { \ - /* special case, can be simplified to ... */ \ - ___n /= (___b & -___b); \ - ___m = ~0ULL / (___b / (___b & -___b)); \ - ___p = 1; \ - ___bias = 1; \ - } else if (___res != ___x / ___b) { \ + /* Now validate what we've got. */ \ + if (___res != ___x / ___b) { \ /* \ * We can't get away without a bias to compensate \ * for bit truncation errors. To avoid it we'd need an \ @@ -115,45 +104,18 @@ * \ * Instead we do m = p / b and n / b = (n * m + m) / p. \ */ \ - ___bias = 1; \ + ___bias = true; \ /* Compute m = (p << 64) / b */ \ ___m = (~0ULL / ___b) * ___p; \ ___m += ((~0ULL % ___b + 1) * ___p) / ___b; \ - } else { \ - /* \ - * Reduce m / p, and try to clear bit 31 of m when \ - * possible, otherwise that'll need extra overflow \ - * handling later. \ - */ \ - uint32_t ___bits = -(___m & -___m); \ - ___bits |= ___m >> 32; \ - ___bits = (~___bits) << 1; \ - /* \ - * If ___bits == 0 then setting bit 31 is unavoidable. \ - * Simply apply the maximum possible reduction in that \ - * case. Otherwise the MSB of ___bits indicates the \ - * best reduction we should apply. \ - */ \ - if (!___bits) { \ - ___p /= (___m & -___m); \ - ___m /= (___m & -___m); \ - } else { \ - ___p >>= ilog2(___bits); \ - ___m >>= ilog2(___bits); \ - } \ - /* No bias needed. */ \ - ___bias = 0; \ } \ \ + /* Reduce m / p to help avoid overflow handling later. */ \ + ___p /= (___m & -___m); \ + ___m /= (___m & -___m); \ + \ /* \ - * Now we have a combination of 2 conditions: \ - * \ - * 1) whether or not we need to apply a bias, and \ - * \ - * 2) whether or not there might be an overflow in the cross \ - * product determined by (___m & ((1 << 63) | (1 << 31))). \ - * \ - * Select the best way to do (m_bias + m * n) / (1 << 64). \ + * Perform (m_bias + m * n) / (1 << 64). \ * From now on there will be actual runtime code generated. \ */ \ ___res = __arch_xprod_64(___m, ___n, ___bias); \ @@ -169,43 +131,42 @@ * Semantic: retval = ((bias ? m : 0) + m * n) >> 64 * * The product is a 128-bit value, scaled down to 64 bits. - * Assuming constant propagation to optimize away unused conditional code. + * Hoping for compile-time optimization of conditional code. * Architectures may provide their own optimized assembly implementation. */ -static inline uint64_t __arch_xprod_64(const uint64_t m, uint64_t n, bool bias) +#ifdef CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE +static __always_inline +#else +static inline +#endif +uint64_t __arch_xprod_64(const uint64_t m, uint64_t n, bool bias) { uint32_t m_lo = m; uint32_t m_hi = m >> 32; uint32_t n_lo = n; uint32_t n_hi = n >> 32; - uint64_t res, tmp; - - if (!bias) { - res = ((uint64_t)m_lo * n_lo) >> 32; - } else if (!(m & ((1ULL << 63) | (1ULL << 31)))) { - /* there can't be any overflow here */ - res = (m + (uint64_t)m_lo * n_lo) >> 32; + uint64_t x, y; + + /* Determine if overflow handling can be dispensed with. */ + bool no_ovf = __builtin_constant_p(m) && + ((m >> 32) + (m & 0xffffffff) < 0x100000000); + + if (no_ovf) { + x = (uint64_t)m_lo * n_lo + (bias ? m : 0); + x >>= 32; + x += (uint64_t)m_lo * n_hi; + x += (uint64_t)m_hi * n_lo; + x >>= 32; + x += (uint64_t)m_hi * n_hi; } else { - res = m + (uint64_t)m_lo * n_lo; - tmp = (res < m) ? (1ULL << 32) : 0; - res = (res >> 32) + tmp; + x = (uint64_t)m_lo * n_lo + (bias ? m_lo : 0); + y = (uint64_t)m_lo * n_hi + (uint32_t)(x >> 32) + (bias ? m_hi : 0); + x = (uint64_t)m_hi * n_hi + (uint32_t)(y >> 32); + y = (uint64_t)m_hi * n_lo + (uint32_t)y; + x += (uint32_t)(y >> 32); } - if (!(m & ((1ULL << 63) | (1ULL << 31)))) { - /* there can't be any overflow here */ - res += (uint64_t)m_lo * n_hi; - res += (uint64_t)m_hi * n_lo; - res >>= 32; - } else { - tmp = res += (uint64_t)m_lo * n_hi; - res += (uint64_t)m_hi * n_lo; - tmp = (res < tmp) ? (1ULL << 32) : 0; - res = (res >> 32) + tmp; - } - - res += (uint64_t)m_hi * n_hi; - - return res; + return x; } #endif @@ -224,8 +185,7 @@ extern uint32_t __div64_32(uint64_t *dividend, uint32_t divisor); is_power_of_2(__base)) { \ __rem = (n) & (__base - 1); \ (n) >>= ilog2(__base); \ - } else if (__div64_const32_is_OK && \ - __builtin_constant_p(__base) && \ + } else if (__builtin_constant_p(__base) && \ __base != 0) { \ uint32_t __res_lo, __n_lo = (n); \ (n) = __div64_const32(n, __base); \ @@ -235,8 +195,9 @@ extern uint32_t __div64_32(uint64_t *dividend, uint32_t divisor); } else if (likely(((n) >> 32) == 0)) { \ __rem = (uint32_t)(n) % __base; \ (n) = (uint32_t)(n) / __base; \ - } else \ + } else { \ __rem = __div64_32(&(n), __base); \ + } \ __rem; \ }) diff --git a/include/asm-generic/dma-contiguous.h b/include/asm-generic/dma-contiguous.h deleted file mode 100644 index f24b0f9a4f05..000000000000 --- a/include/asm-generic/dma-contiguous.h +++ /dev/null @@ -1,10 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _ASM_GENERIC_DMA_CONTIGUOUS_H -#define _ASM_GENERIC_DMA_CONTIGUOUS_H - -#include <linux/types.h> - -static inline void -dma_contiguous_early_fixup(phys_addr_t base, unsigned long size) { } - -#endif diff --git a/include/asm-generic/dma-mapping.h b/include/asm-generic/dma-mapping.h index c13f46109e88..46a0016efd81 100644 --- a/include/asm-generic/dma-mapping.h +++ b/include/asm-generic/dma-mapping.h @@ -2,7 +2,7 @@ #ifndef _ASM_GENERIC_DMA_MAPPING_H #define _ASM_GENERIC_DMA_MAPPING_H -static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus) +static inline const struct dma_map_ops *get_arch_dma_ops(void) { return NULL; } diff --git a/include/asm-generic/early_ioremap.h b/include/asm-generic/early_ioremap.h index 9def22e6e2b3..5db59a1efb65 100644 --- a/include/asm-generic/early_ioremap.h +++ b/include/asm-generic/early_ioremap.h @@ -19,12 +19,6 @@ extern void *early_memremap_prot(resource_size_t phys_addr, extern void early_iounmap(void __iomem *addr, unsigned long size); extern void early_memunmap(void *addr, unsigned long size); -/* - * Weak function called by early_ioremap_reset(). It does nothing, but - * architectures may provide their own version to do any needed cleanups. - */ -extern void early_ioremap_shutdown(void); - #if defined(CONFIG_GENERIC_EARLY_IOREMAP) && defined(CONFIG_MMU) /* Arch-specific initialization */ extern void early_ioremap_init(void); @@ -41,7 +35,7 @@ extern void early_ioremap_reset(void); /* * Early copy from unmapped memory to kernel mapped memory. */ -extern void copy_from_early_mem(void *dest, phys_addr_t src, +extern int copy_from_early_mem(void *dest, phys_addr_t src, unsigned long size); #else diff --git a/include/asm-generic/error-injection.h b/include/asm-generic/error-injection.h index 95a159a4137f..b05253f68eaa 100644 --- a/include/asm-generic/error-injection.h +++ b/include/asm-generic/error-injection.h @@ -4,7 +4,6 @@ #if defined(__KERNEL__) && !defined(__ASSEMBLY__) enum { - EI_ETYPE_NONE, /* Dummy value for undefined case */ EI_ETYPE_NULL, /* Return NULL if failure */ EI_ETYPE_ERRNO, /* Return -ERRNO if failure */ EI_ETYPE_ERRNO_NULL, /* Return -ERRNO or NULL if failure */ @@ -16,20 +15,28 @@ struct error_injection_entry { int etype; }; +struct pt_regs; + #ifdef CONFIG_FUNCTION_ERROR_INJECTION /* - * Whitelist ganerating macro. Specify functions which can be - * error-injectable using this macro. + * Whitelist generating macro. Specify functions which can be error-injectable + * using this macro. If you unsure what is required for the error-injectable + * functions, please read Documentation/fault-injection/fault-injection.rst + * 'Error Injectable Functions' section. */ #define ALLOW_ERROR_INJECTION(fname, _etype) \ static struct error_injection_entry __used \ - __attribute__((__section__("_error_injection_whitelist"))) \ + __section("_error_injection_whitelist") \ _eil_addr_##fname = { \ .addr = (unsigned long)fname, \ .etype = EI_ETYPE_##_etype, \ - }; + } + +void override_function_with_return(struct pt_regs *regs); #else #define ALLOW_ERROR_INJECTION(fname, _etype) + +static inline void override_function_with_return(struct pt_regs *regs) { } #endif #endif diff --git a/include/asm-generic/exec.h b/include/asm-generic/exec.h index 32c0a216f576..f66dc71fac4f 100644 --- a/include/asm-generic/exec.h +++ b/include/asm-generic/exec.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* Generic process execution definitions. * * It should be possible to use these on really simple architectures, @@ -5,11 +6,6 @@ * * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved. * Written by David Howells (dhowells@redhat.com) - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public Licence - * as published by the Free Software Foundation; either version - * 2 of the Licence, or (at your option) any later version. */ #ifndef __ASM_GENERIC_EXEC_H #define __ASM_GENERIC_EXEC_H diff --git a/include/asm-generic/export.h b/include/asm-generic/export.h deleted file mode 100644 index 294d6ae785d4..000000000000 --- a/include/asm-generic/export.h +++ /dev/null @@ -1,96 +0,0 @@ -#ifndef __ASM_GENERIC_EXPORT_H -#define __ASM_GENERIC_EXPORT_H - -#ifndef KSYM_FUNC -#define KSYM_FUNC(x) x -#endif -#ifdef CONFIG_64BIT -#ifndef KSYM_ALIGN -#define KSYM_ALIGN 8 -#endif -#else -#ifndef KSYM_ALIGN -#define KSYM_ALIGN 4 -#endif -#endif -#ifndef KCRC_ALIGN -#define KCRC_ALIGN 4 -#endif - -.macro __put, val, name -#ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS - .long \val - ., \name - . -#elif defined(CONFIG_64BIT) - .quad \val, \name -#else - .long \val, \name -#endif -.endm - -/* - * note on .section use: @progbits vs %progbits nastiness doesn't matter, - * since we immediately emit into those sections anyway. - */ -.macro ___EXPORT_SYMBOL name,val,sec -#ifdef CONFIG_MODULES - .globl __ksymtab_\name - .section ___ksymtab\sec+\name,"a" - .balign KSYM_ALIGN -__ksymtab_\name: - __put \val, __kstrtab_\name - .previous - .section __ksymtab_strings,"a" -__kstrtab_\name: - .asciz "\name" - .previous -#ifdef CONFIG_MODVERSIONS - .section ___kcrctab\sec+\name,"a" - .balign KCRC_ALIGN -__kcrctab_\name: -#if defined(CONFIG_MODULE_REL_CRCS) - .long __crc_\name - . -#else - .long __crc_\name -#endif - .weak __crc_\name - .previous -#endif -#endif -.endm -#undef __put - -#if defined(CONFIG_TRIM_UNUSED_KSYMS) - -#include <linux/kconfig.h> -#include <generated/autoksyms.h> - -.macro __ksym_marker sym - .section ".discard.ksym","a" -__ksym_marker_\sym: - .previous -.endm - -#define __EXPORT_SYMBOL(sym, val, sec) \ - __ksym_marker sym; \ - __cond_export_sym(sym, val, sec, __is_defined(__KSYM_##sym)) -#define __cond_export_sym(sym, val, sec, conf) \ - ___cond_export_sym(sym, val, sec, conf) -#define ___cond_export_sym(sym, val, sec, enabled) \ - __cond_export_sym_##enabled(sym, val, sec) -#define __cond_export_sym_1(sym, val, sec) ___EXPORT_SYMBOL sym, val, sec -#define __cond_export_sym_0(sym, val, sec) /* nothing */ - -#else -#define __EXPORT_SYMBOL(sym, val, sec) ___EXPORT_SYMBOL sym, val, sec -#endif - -#define EXPORT_SYMBOL(name) \ - __EXPORT_SYMBOL(name, KSYM_FUNC(name),) -#define EXPORT_SYMBOL_GPL(name) \ - __EXPORT_SYMBOL(name, KSYM_FUNC(name), _gpl) -#define EXPORT_DATA_SYMBOL(name) \ - __EXPORT_SYMBOL(name, name,) -#define EXPORT_DATA_SYMBOL_GPL(name) \ - __EXPORT_SYMBOL(name, name,_gpl) - -#endif diff --git a/include/asm-generic/fb.h b/include/asm-generic/fb.h deleted file mode 100644 index f9f18101ed36..000000000000 --- a/include/asm-generic/fb.h +++ /dev/null @@ -1,13 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef __ASM_GENERIC_FB_H_ -#define __ASM_GENERIC_FB_H_ -#include <linux/fb.h> - -#define fb_pgprotect(...) do {} while (0) - -static inline int fb_is_primary_device(struct fb_info *info) -{ - return 0; -} - -#endif /* __ASM_GENERIC_FB_H_ */ diff --git a/include/asm-generic/fixmap.h b/include/asm-generic/fixmap.h index 8cc7b09c1bc7..29cab7947980 100644 --- a/include/asm-generic/fixmap.h +++ b/include/asm-generic/fixmap.h @@ -97,8 +97,5 @@ static inline unsigned long virt_to_fix(const unsigned long vaddr) #define set_fixmap_io(idx, phys) \ __set_fixmap(idx, phys, FIXMAP_PAGE_IO) -#define set_fixmap_offset_io(idx, phys) \ - __set_fixmap_offset(idx, phys, FIXMAP_PAGE_IO) - #endif /* __ASSEMBLY__ */ #endif /* __ASM_GENERIC_FIXMAP_H */ diff --git a/include/asm-generic/flat.h b/include/asm-generic/flat.h new file mode 100644 index 000000000000..1928a3596938 --- /dev/null +++ b/include/asm-generic/flat.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_GENERIC_FLAT_H +#define _ASM_GENERIC_FLAT_H + +#include <linux/uaccess.h> + +static inline int flat_get_addr_from_rp(u32 __user *rp, u32 relval, u32 flags, + u32 *addr) +{ +#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS + return copy_from_user(addr, rp, 4) ? -EFAULT : 0; +#else + return get_user(*addr, rp); +#endif +} + +static inline int flat_put_addr_at_rp(u32 __user *rp, u32 addr, u32 rel) +{ +#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS + return copy_to_user(rp, &addr, 4) ? -EFAULT : 0; +#else + return put_user(addr, rp); +#endif +} + +#endif /* _ASM_GENERIC_FLAT_H */ diff --git a/include/asm-generic/fprobe.h b/include/asm-generic/fprobe.h new file mode 100644 index 000000000000..8659a4dc6eb6 --- /dev/null +++ b/include/asm-generic/fprobe.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Generic arch dependent fprobe macros. + */ +#ifndef __ASM_GENERIC_FPROBE_H__ +#define __ASM_GENERIC_FPROBE_H__ + +#include <linux/bits.h> + +#ifdef CONFIG_64BIT +/* + * Encoding the size and the address of fprobe into one 64bit entry. + * The 32bit architectures should use 2 entries to store those info. + */ + +#define ARCH_DEFINE_ENCODE_FPROBE_HEADER + +#define FPROBE_HEADER_MSB_SIZE_SHIFT (BITS_PER_LONG - FPROBE_DATA_SIZE_BITS) +#define FPROBE_HEADER_MSB_MASK \ + GENMASK(FPROBE_HEADER_MSB_SIZE_SHIFT - 1, 0) + +/* + * By default, this expects the MSBs in the address of kprobe is 0xf. + * If any arch needs another fixed pattern (e.g. s390 is zero filled), + * override this. + */ +#define FPROBE_HEADER_MSB_PATTERN \ + GENMASK(BITS_PER_LONG - 1, FPROBE_HEADER_MSB_SIZE_SHIFT) + +#define arch_fprobe_header_encodable(fp) \ + (((unsigned long)(fp) & ~FPROBE_HEADER_MSB_MASK) == \ + FPROBE_HEADER_MSB_PATTERN) + +#define arch_encode_fprobe_header(fp, size) \ + (((unsigned long)(fp) & FPROBE_HEADER_MSB_MASK) | \ + ((unsigned long)(size) << FPROBE_HEADER_MSB_SIZE_SHIFT)) + +#define arch_decode_fprobe_header_size(val) \ + ((unsigned long)(val) >> FPROBE_HEADER_MSB_SIZE_SHIFT) + +#define arch_decode_fprobe_header_fp(val) \ + ((struct fprobe *)(((unsigned long)(val) & FPROBE_HEADER_MSB_MASK) | \ + FPROBE_HEADER_MSB_PATTERN)) +#endif /* CONFIG_64BIT */ + +#endif /* __ASM_GENERIC_FPROBE_H__ */ diff --git a/include/asm-generic/ftrace.h b/include/asm-generic/ftrace.h index 51abba9ea7ad..3a23028d69d2 100644 --- a/include/asm-generic/ftrace.h +++ b/include/asm-generic/ftrace.h @@ -1,9 +1,6 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * linux/include/asm-generic/ftrace.h - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #ifndef __ASM_GENERIC_FTRACE_H__ #define __ASM_GENERIC_FTRACE_H__ diff --git a/include/asm-generic/futex.h b/include/asm-generic/futex.h index fcb61b4659b3..2a19215baae5 100644 --- a/include/asm-generic/futex.h +++ b/include/asm-generic/futex.h @@ -6,15 +6,22 @@ #include <linux/uaccess.h> #include <asm/errno.h> +#ifndef futex_atomic_cmpxchg_inatomic #ifndef CONFIG_SMP /* * The following implementation only for uniprocessor machines. * It relies on preempt_disable() ensuring mutual exclusion. * */ +#define futex_atomic_cmpxchg_inatomic(uval, uaddr, oldval, newval) \ + futex_atomic_cmpxchg_inatomic_local(uval, uaddr, oldval, newval) +#define arch_futex_atomic_op_inuser(op, oparg, oval, uaddr) \ + futex_atomic_op_inuser_local(op, oparg, oval, uaddr) +#endif /* CONFIG_SMP */ +#endif /** - * arch_futex_atomic_op_inuser() - Atomic arithmetic operation with constant + * futex_atomic_op_inuser_local() - Atomic arithmetic operation with constant * argument and comparison of the previous * futex value with another constant. * @@ -23,16 +30,17 @@ * * Return: * 0 - On success - * <0 - On error + * -EFAULT - User access resulted in a page fault + * -EAGAIN - Atomic operation was unable to complete due to contention + * -ENOSYS - Operation not supported */ static inline int -arch_futex_atomic_op_inuser(int op, u32 oparg, int *oval, u32 __user *uaddr) +futex_atomic_op_inuser_local(int op, u32 oparg, int *oval, u32 __user *uaddr) { int oldval, ret; u32 tmp; preempt_disable(); - pagefault_disable(); ret = -EFAULT; if (unlikely(get_user(oldval, uaddr) != 0)) @@ -65,7 +73,6 @@ arch_futex_atomic_op_inuser(int op, u32 oparg, int *oval, u32 __user *uaddr) ret = -EFAULT; out_pagefault_enable: - pagefault_enable(); preempt_enable(); if (ret == 0) @@ -75,7 +82,7 @@ out_pagefault_enable: } /** - * futex_atomic_cmpxchg_inatomic() - Compare and exchange the content of the + * futex_atomic_cmpxchg_inatomic_local() - Compare and exchange the content of the * uaddr with newval if the current value is * oldval. * @uval: pointer to store content of @uaddr @@ -85,10 +92,11 @@ out_pagefault_enable: * * Return: * 0 - On success - * <0 - On error + * -EFAULT - User access resulted in a page fault + * -EAGAIN - Atomic operation was unable to complete due to contention */ static inline int -futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, +futex_atomic_cmpxchg_inatomic_local(u32 *uval, u32 __user *uaddr, u32 oldval, u32 newval) { u32 val; @@ -110,38 +118,4 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, return 0; } -#else -static inline int -arch_futex_atomic_op_inuser(int op, u32 oparg, int *oval, u32 __user *uaddr) -{ - int oldval = 0, ret; - - pagefault_disable(); - - switch (op) { - case FUTEX_OP_SET: - case FUTEX_OP_ADD: - case FUTEX_OP_OR: - case FUTEX_OP_ANDN: - case FUTEX_OP_XOR: - default: - ret = -ENOSYS; - } - - pagefault_enable(); - - if (!ret) - *oval = oldval; - - return ret; -} - -static inline int -futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, - u32 oldval, u32 newval) -{ - return -ENOSYS; -} - -#endif /* CONFIG_SMP */ #endif diff --git a/include/asm-generic/getorder.h b/include/asm-generic/getorder.h index c64bea7a52be..f2979e3a96b6 100644 --- a/include/asm-generic/getorder.h +++ b/include/asm-generic/getorder.h @@ -7,24 +7,6 @@ #include <linux/compiler.h> #include <linux/log2.h> -/* - * Runtime evaluation of get_order() - */ -static inline __attribute_const__ -int __get_order(unsigned long size) -{ - int order; - - size--; - size >>= PAGE_SHIFT; -#if BITS_PER_LONG == 32 - order = fls(size); -#else - order = fls64(size); -#endif - return order; -} - /** * get_order - Determine the allocation order of a memory size * @size: The size for which to get the order @@ -43,19 +25,27 @@ int __get_order(unsigned long size) * to hold an object of the specified size. * * The result is undefined if the size is 0. - * - * This function may be used to initialise variables with compile time - * evaluations of constants. */ -#define get_order(n) \ -( \ - __builtin_constant_p(n) ? ( \ - ((n) == 0UL) ? BITS_PER_LONG - PAGE_SHIFT : \ - (((n) < (1UL << PAGE_SHIFT)) ? 0 : \ - ilog2((n) - 1) - PAGE_SHIFT + 1) \ - ) : \ - __get_order(n) \ -) +static __always_inline __attribute_const__ int get_order(unsigned long size) +{ + if (__builtin_constant_p(size)) { + if (!size) + return BITS_PER_LONG - PAGE_SHIFT; + + if (size < (1UL << PAGE_SHIFT)) + return 0; + + return ilog2((size) - 1) - PAGE_SHIFT + 1; + } + + size--; + size >>= PAGE_SHIFT; +#if BITS_PER_LONG == 32 + return fls(size); +#else + return fls64(size); +#endif +} #endif /* __ASSEMBLY__ */ diff --git a/include/asm-generic/gpio.h b/include/asm-generic/gpio.h deleted file mode 100644 index 19eadac415c4..000000000000 --- a/include/asm-generic/gpio.h +++ /dev/null @@ -1,172 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _ASM_GENERIC_GPIO_H -#define _ASM_GENERIC_GPIO_H - -#include <linux/kernel.h> -#include <linux/types.h> -#include <linux/errno.h> -#include <linux/of.h> - -#ifdef CONFIG_GPIOLIB - -#include <linux/compiler.h> -#include <linux/gpio/driver.h> -#include <linux/gpio/consumer.h> - -/* Platforms may implement their GPIO interface with library code, - * at a small performance cost for non-inlined operations and some - * extra memory (for code and for per-GPIO table entries). - * - * While the GPIO programming interface defines valid GPIO numbers - * to be in the range 0..MAX_INT, this library restricts them to the - * smaller range 0..ARCH_NR_GPIOS-1. - * - * ARCH_NR_GPIOS is somewhat arbitrary; it usually reflects the sum of - * builtin/SoC GPIOs plus a number of GPIOs on expanders; the latter is - * actually an estimate of a board-specific value. - */ - -#ifndef ARCH_NR_GPIOS -#if defined(CONFIG_ARCH_NR_GPIO) && CONFIG_ARCH_NR_GPIO > 0 -#define ARCH_NR_GPIOS CONFIG_ARCH_NR_GPIO -#else -#define ARCH_NR_GPIOS 512 -#endif -#endif - -/* - * "valid" GPIO numbers are nonnegative and may be passed to - * setup routines like gpio_request(). only some valid numbers - * can successfully be requested and used. - * - * Invalid GPIO numbers are useful for indicating no-such-GPIO in - * platform data and other tables. - */ - -static inline bool gpio_is_valid(int number) -{ - return number >= 0 && number < ARCH_NR_GPIOS; -} - -struct device; -struct gpio; -struct seq_file; -struct module; -struct device_node; -struct gpio_desc; - -/* caller holds gpio_lock *OR* gpio is marked as requested */ -static inline struct gpio_chip *gpio_to_chip(unsigned gpio) -{ - return gpiod_to_chip(gpio_to_desc(gpio)); -} - -/* Always use the library code for GPIO management calls, - * or when sleeping may be involved. - */ -extern int gpio_request(unsigned gpio, const char *label); -extern void gpio_free(unsigned gpio); - -static inline int gpio_direction_input(unsigned gpio) -{ - return gpiod_direction_input(gpio_to_desc(gpio)); -} -static inline int gpio_direction_output(unsigned gpio, int value) -{ - return gpiod_direction_output_raw(gpio_to_desc(gpio), value); -} - -static inline int gpio_set_debounce(unsigned gpio, unsigned debounce) -{ - return gpiod_set_debounce(gpio_to_desc(gpio), debounce); -} - -static inline int gpio_get_value_cansleep(unsigned gpio) -{ - return gpiod_get_raw_value_cansleep(gpio_to_desc(gpio)); -} -static inline void gpio_set_value_cansleep(unsigned gpio, int value) -{ - return gpiod_set_raw_value_cansleep(gpio_to_desc(gpio), value); -} - - -/* A platform's <asm/gpio.h> code may want to inline the I/O calls when - * the GPIO is constant and refers to some always-present controller, - * giving direct access to chip registers and tight bitbanging loops. - */ -static inline int __gpio_get_value(unsigned gpio) -{ - return gpiod_get_raw_value(gpio_to_desc(gpio)); -} -static inline void __gpio_set_value(unsigned gpio, int value) -{ - return gpiod_set_raw_value(gpio_to_desc(gpio), value); -} - -static inline int __gpio_cansleep(unsigned gpio) -{ - return gpiod_cansleep(gpio_to_desc(gpio)); -} - -static inline int __gpio_to_irq(unsigned gpio) -{ - return gpiod_to_irq(gpio_to_desc(gpio)); -} - -extern int gpio_request_one(unsigned gpio, unsigned long flags, const char *label); -extern int gpio_request_array(const struct gpio *array, size_t num); -extern void gpio_free_array(const struct gpio *array, size_t num); - -/* - * A sysfs interface can be exported by individual drivers if they want, - * but more typically is configured entirely from userspace. - */ -static inline int gpio_export(unsigned gpio, bool direction_may_change) -{ - return gpiod_export(gpio_to_desc(gpio), direction_may_change); -} - -static inline int gpio_export_link(struct device *dev, const char *name, - unsigned gpio) -{ - return gpiod_export_link(dev, name, gpio_to_desc(gpio)); -} - -static inline void gpio_unexport(unsigned gpio) -{ - gpiod_unexport(gpio_to_desc(gpio)); -} - -#else /* !CONFIG_GPIOLIB */ - -static inline bool gpio_is_valid(int number) -{ - /* only non-negative numbers are valid */ - return number >= 0; -} - -/* platforms that don't directly support access to GPIOs through I2C, SPI, - * or other blocking infrastructure can use these wrappers. - */ - -static inline int gpio_cansleep(unsigned gpio) -{ - return 0; -} - -static inline int gpio_get_value_cansleep(unsigned gpio) -{ - might_sleep(); - return __gpio_get_value(gpio); -} - -static inline void gpio_set_value_cansleep(unsigned gpio, int value) -{ - might_sleep(); - __gpio_set_value(gpio, value); -} - -#endif /* !CONFIG_GPIOLIB */ - -#endif /* _ASM_GENERIC_GPIO_H */ diff --git a/include/asm-generic/hardirq.h b/include/asm-generic/hardirq.h index d14214dfc10b..7317e8258b48 100644 --- a/include/asm-generic/hardirq.h +++ b/include/asm-generic/hardirq.h @@ -7,9 +7,13 @@ typedef struct { unsigned int __softirq_pending; +#ifdef ARCH_WANTS_NMI_IRQSTAT + unsigned int __nmi_count; +#endif } ____cacheline_aligned irq_cpustat_t; -#include <linux/irq_cpustat.h> /* Standard mappings for irq_cpustat_t above */ +DECLARE_PER_CPU_ALIGNED(irq_cpustat_t, irq_stat); + #include <linux/irq.h> #ifndef ack_bad_irq diff --git a/include/asm-generic/hugetlb.h b/include/asm-generic/hugetlb.h index 71d7b77eea50..e1a2e1b7c8e7 100644 --- a/include/asm-generic/hugetlb.h +++ b/include/asm-generic/hugetlb.h @@ -2,10 +2,8 @@ #ifndef _ASM_GENERIC_HUGETLB_H #define _ASM_GENERIC_HUGETLB_H -static inline pte_t mk_huge_pte(struct page *page, pgprot_t pgprot) -{ - return mk_pte(page, pgprot); -} +#include <linux/swap.h> +#include <linux/swapops.h> static inline unsigned long huge_pte_write(pte_t pte) { @@ -19,9 +17,16 @@ static inline unsigned long huge_pte_dirty(pte_t pte) static inline pte_t huge_pte_mkwrite(pte_t pte) { - return pte_mkwrite(pte); + return pte_mkwrite_novma(pte); } +#ifndef __HAVE_ARCH_HUGE_PTE_WRPROTECT +static inline pte_t huge_pte_wrprotect(pte_t pte) +{ + return pte_wrprotect(pte); +} +#endif + static inline pte_t huge_pte_mkdirty(pte_t pte) { return pte_mkdirty(pte); @@ -32,26 +37,38 @@ static inline pte_t huge_pte_modify(pte_t pte, pgprot_t newprot) return pte_modify(pte, newprot); } -#ifndef __HAVE_ARCH_HUGE_PTE_CLEAR -static inline void huge_pte_clear(struct mm_struct *mm, unsigned long addr, - pte_t *ptep, unsigned long sz) +#ifndef __HAVE_ARCH_HUGE_PTE_MKUFFD_WP +static inline pte_t huge_pte_mkuffd_wp(pte_t pte) { - pte_clear(mm, addr, ptep); + return huge_pte_wrprotect(pte_mkuffd_wp(pte)); +} +#endif + +#ifndef __HAVE_ARCH_HUGE_PTE_CLEAR_UFFD_WP +static inline pte_t huge_pte_clear_uffd_wp(pte_t pte) +{ + return pte_clear_uffd_wp(pte); +} +#endif + +#ifndef __HAVE_ARCH_HUGE_PTE_UFFD_WP +static inline int huge_pte_uffd_wp(pte_t pte) +{ + return pte_uffd_wp(pte); } #endif -#ifndef __HAVE_ARCH_HUGETLB_FREE_PGD_RANGE -static inline void hugetlb_free_pgd_range(struct mmu_gather *tlb, - unsigned long addr, unsigned long end, - unsigned long floor, unsigned long ceiling) +#ifndef __HAVE_ARCH_HUGE_PTE_CLEAR +static inline void huge_pte_clear(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, unsigned long sz) { - free_pgd_range(tlb, addr, end, floor, ceiling); + pte_clear(mm, addr, ptep); } #endif #ifndef __HAVE_ARCH_HUGE_SET_HUGE_PTE_AT static inline void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, - pte_t *ptep, pte_t pte) + pte_t *ptep, pte_t pte, unsigned long sz) { set_pte_at(mm, addr, ptep, pte); } @@ -59,17 +76,17 @@ static inline void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, #ifndef __HAVE_ARCH_HUGE_PTEP_GET_AND_CLEAR static inline pte_t huge_ptep_get_and_clear(struct mm_struct *mm, - unsigned long addr, pte_t *ptep) + unsigned long addr, pte_t *ptep, unsigned long sz) { return ptep_get_and_clear(mm, addr, ptep); } #endif #ifndef __HAVE_ARCH_HUGE_PTEP_CLEAR_FLUSH -static inline void huge_ptep_clear_flush(struct vm_area_struct *vma, +static inline pte_t huge_ptep_clear_flush(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep) { - ptep_clear_flush(vma, addr, ptep); + return ptep_clear_flush(vma, addr, ptep); } #endif @@ -80,28 +97,6 @@ static inline int huge_pte_none(pte_t pte) } #endif -#ifndef __HAVE_ARCH_HUGE_PTE_WRPROTECT -static inline pte_t huge_pte_wrprotect(pte_t pte) -{ - return pte_wrprotect(pte); -} -#endif - -#ifndef __HAVE_ARCH_PREPARE_HUGEPAGE_RANGE -static inline int prepare_hugepage_range(struct file *file, - unsigned long addr, unsigned long len) -{ - struct hstate *h = hstate_file(file); - - if (len & ~huge_page_mask(h)) - return -EINVAL; - if (addr & ~huge_page_mask(h)) - return -EINVAL; - - return 0; -} -#endif - #ifndef __HAVE_ARCH_HUGE_PTEP_SET_WRPROTECT static inline void huge_ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, pte_t *ptep) @@ -120,10 +115,17 @@ static inline int huge_ptep_set_access_flags(struct vm_area_struct *vma, #endif #ifndef __HAVE_ARCH_HUGE_PTEP_GET -static inline pte_t huge_ptep_get(pte_t *ptep) +static inline pte_t huge_ptep_get(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { - return *ptep; + return ptep_get(ptep); } #endif +#ifndef __HAVE_ARCH_GIGANTIC_PAGE_RUNTIME_SUPPORTED +static inline bool gigantic_page_runtime_supported(void) +{ + return IS_ENABLED(CONFIG_ARCH_HAS_GIGANTIC_PAGE); +} +#endif /* __HAVE_ARCH_GIGANTIC_PAGE_RUNTIME_SUPPORTED */ + #endif /* _ASM_GENERIC_HUGETLB_H */ diff --git a/include/asm-generic/ide_iops.h b/include/asm-generic/ide_iops.h deleted file mode 100644 index 81dfa3ee5e06..000000000000 --- a/include/asm-generic/ide_iops.h +++ /dev/null @@ -1,39 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* Generic I/O and MEMIO string operations. */ - -#define __ide_insw insw -#define __ide_insl insl -#define __ide_outsw outsw -#define __ide_outsl outsl - -static __inline__ void __ide_mm_insw(void __iomem *port, void *addr, u32 count) -{ - while (count--) { - *(u16 *)addr = readw(port); - addr += 2; - } -} - -static __inline__ void __ide_mm_insl(void __iomem *port, void *addr, u32 count) -{ - while (count--) { - *(u32 *)addr = readl(port); - addr += 4; - } -} - -static __inline__ void __ide_mm_outsw(void __iomem *port, void *addr, u32 count) -{ - while (count--) { - writew(*(u16 *)addr, port); - addr += 2; - } -} - -static __inline__ void __ide_mm_outsl(void __iomem * port, void *addr, u32 count) -{ - while (count--) { - writel(*(u32 *)addr, port); - addr += 4; - } -} diff --git a/include/asm-generic/io.h b/include/asm-generic/io.h index d356f802945a..ca5a1ce6f0f8 100644 --- a/include/asm-generic/io.h +++ b/include/asm-generic/io.h @@ -1,30 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* Generic I/O port emulation. * * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved. * Written by David Howells (dhowells@redhat.com) - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public Licence - * as published by the Free Software Foundation; either version - * 2 of the Licence, or (at your option) any later version. */ #ifndef __ASM_GENERIC_IO_H #define __ASM_GENERIC_IO_H #include <asm/page.h> /* I/O is all done through memory accesses */ #include <linux/string.h> /* for memset() and memcpy() */ +#include <linux/sizes.h> #include <linux/types.h> +#include <linux/instruction_pointer.h> #ifdef CONFIG_GENERIC_IOMAP #include <asm-generic/iomap.h> #endif +#include <asm/mmiowb.h> #include <asm-generic/pci_iomap.h> -#ifndef mmiowb -#define mmiowb() do {} while (0) -#endif - #ifndef __io_br #define __io_br() barrier() #endif @@ -32,9 +27,9 @@ /* prevent prefetching of coherent DMA data ahead of a dma-complete */ #ifndef __io_ar #ifdef rmb -#define __io_ar() rmb() +#define __io_ar(v) rmb() #else -#define __io_ar() barrier() +#define __io_ar(v) barrier() #endif #endif @@ -49,7 +44,7 @@ /* serialize device access against a spin_unlock, usually handled there. */ #ifndef __io_aw -#define __io_aw() barrier() +#define __io_aw() mmiowb_set_pending() #endif #ifndef __io_pbw @@ -65,9 +60,49 @@ #endif #ifndef __io_par -#define __io_par() __io_ar() +#define __io_par(v) __io_ar(v) #endif +/* + * "__DISABLE_TRACE_MMIO__" flag can be used to disable MMIO tracing for + * specific kernel drivers in case of excessive/unwanted logging. + * + * Usage: Add a #define flag at the beginning of the driver file. + * Ex: #define __DISABLE_TRACE_MMIO__ + * #include <...> + * ... + */ +#if IS_ENABLED(CONFIG_TRACE_MMIO_ACCESS) && !(defined(__DISABLE_TRACE_MMIO__)) +#include <linux/tracepoint-defs.h> + +#define rwmmio_tracepoint_enabled(tracepoint) tracepoint_enabled(tracepoint) +DECLARE_TRACEPOINT(rwmmio_write); +DECLARE_TRACEPOINT(rwmmio_post_write); +DECLARE_TRACEPOINT(rwmmio_read); +DECLARE_TRACEPOINT(rwmmio_post_read); + +void log_write_mmio(u64 val, u8 width, volatile void __iomem *addr, + unsigned long caller_addr, unsigned long caller_addr0); +void log_post_write_mmio(u64 val, u8 width, volatile void __iomem *addr, + unsigned long caller_addr, unsigned long caller_addr0); +void log_read_mmio(u8 width, const volatile void __iomem *addr, + unsigned long caller_addr, unsigned long caller_addr0); +void log_post_read_mmio(u64 val, u8 width, const volatile void __iomem *addr, + unsigned long caller_addr, unsigned long caller_addr0); + +#else + +#define rwmmio_tracepoint_enabled(tracepoint) false +static inline void log_write_mmio(u64 val, u8 width, volatile void __iomem *addr, + unsigned long caller_addr, unsigned long caller_addr0) {} +static inline void log_post_write_mmio(u64 val, u8 width, volatile void __iomem *addr, + unsigned long caller_addr, unsigned long caller_addr0) {} +static inline void log_read_mmio(u8 width, const volatile void __iomem *addr, + unsigned long caller_addr, unsigned long caller_addr0) {} +static inline void log_post_read_mmio(u64 val, u8 width, const volatile void __iomem *addr, + unsigned long caller_addr, unsigned long caller_addr0) {} + +#endif /* CONFIG_TRACE_MMIO_ACCESS */ /* * __raw_{read,write}{b,w,l,q}() access memory in native endianness. @@ -156,9 +191,13 @@ static inline u8 readb(const volatile void __iomem *addr) { u8 val; + if (rwmmio_tracepoint_enabled(rwmmio_read)) + log_read_mmio(8, addr, _THIS_IP_, _RET_IP_); __io_br(); val = __raw_readb(addr); - __io_ar(); + __io_ar(val); + if (rwmmio_tracepoint_enabled(rwmmio_post_read)) + log_post_read_mmio(val, 8, addr, _THIS_IP_, _RET_IP_); return val; } #endif @@ -169,9 +208,13 @@ static inline u16 readw(const volatile void __iomem *addr) { u16 val; + if (rwmmio_tracepoint_enabled(rwmmio_read)) + log_read_mmio(16, addr, _THIS_IP_, _RET_IP_); __io_br(); - val = __le16_to_cpu(__raw_readw(addr)); - __io_ar(); + val = __le16_to_cpu((__le16 __force)__raw_readw(addr)); + __io_ar(val); + if (rwmmio_tracepoint_enabled(rwmmio_post_read)) + log_post_read_mmio(val, 16, addr, _THIS_IP_, _RET_IP_); return val; } #endif @@ -182,9 +225,13 @@ static inline u32 readl(const volatile void __iomem *addr) { u32 val; + if (rwmmio_tracepoint_enabled(rwmmio_read)) + log_read_mmio(32, addr, _THIS_IP_, _RET_IP_); __io_br(); - val = __le32_to_cpu(__raw_readl(addr)); - __io_ar(); + val = __le32_to_cpu((__le32 __force)__raw_readl(addr)); + __io_ar(val); + if (rwmmio_tracepoint_enabled(rwmmio_post_read)) + log_post_read_mmio(val, 32, addr, _THIS_IP_, _RET_IP_); return val; } #endif @@ -196,9 +243,13 @@ static inline u64 readq(const volatile void __iomem *addr) { u64 val; + if (rwmmio_tracepoint_enabled(rwmmio_read)) + log_read_mmio(64, addr, _THIS_IP_, _RET_IP_); __io_br(); - val = __le64_to_cpu(__raw_readq(addr)); - __io_ar(); + val = __le64_to_cpu((__le64 __force)__raw_readq(addr)); + __io_ar(val); + if (rwmmio_tracepoint_enabled(rwmmio_post_read)) + log_post_read_mmio(val, 64, addr, _THIS_IP_, _RET_IP_); return val; } #endif @@ -208,9 +259,13 @@ static inline u64 readq(const volatile void __iomem *addr) #define writeb writeb static inline void writeb(u8 value, volatile void __iomem *addr) { + if (rwmmio_tracepoint_enabled(rwmmio_write)) + log_write_mmio(value, 8, addr, _THIS_IP_, _RET_IP_); __io_bw(); __raw_writeb(value, addr); __io_aw(); + if (rwmmio_tracepoint_enabled(rwmmio_post_write)) + log_post_write_mmio(value, 8, addr, _THIS_IP_, _RET_IP_); } #endif @@ -218,9 +273,13 @@ static inline void writeb(u8 value, volatile void __iomem *addr) #define writew writew static inline void writew(u16 value, volatile void __iomem *addr) { + if (rwmmio_tracepoint_enabled(rwmmio_write)) + log_write_mmio(value, 16, addr, _THIS_IP_, _RET_IP_); __io_bw(); - __raw_writew(cpu_to_le16(value), addr); + __raw_writew((u16 __force)cpu_to_le16(value), addr); __io_aw(); + if (rwmmio_tracepoint_enabled(rwmmio_post_write)) + log_post_write_mmio(value, 16, addr, _THIS_IP_, _RET_IP_); } #endif @@ -228,9 +287,13 @@ static inline void writew(u16 value, volatile void __iomem *addr) #define writel writel static inline void writel(u32 value, volatile void __iomem *addr) { + if (rwmmio_tracepoint_enabled(rwmmio_write)) + log_write_mmio(value, 32, addr, _THIS_IP_, _RET_IP_); __io_bw(); - __raw_writel(__cpu_to_le32(value), addr); + __raw_writel((u32 __force)__cpu_to_le32(value), addr); __io_aw(); + if (rwmmio_tracepoint_enabled(rwmmio_post_write)) + log_post_write_mmio(value, 32, addr, _THIS_IP_, _RET_IP_); } #endif @@ -239,9 +302,13 @@ static inline void writel(u32 value, volatile void __iomem *addr) #define writeq writeq static inline void writeq(u64 value, volatile void __iomem *addr) { + if (rwmmio_tracepoint_enabled(rwmmio_write)) + log_write_mmio(value, 64, addr, _THIS_IP_, _RET_IP_); __io_bw(); - __raw_writeq(__cpu_to_le64(value), addr); + __raw_writeq((u64 __force)__cpu_to_le64(value), addr); __io_aw(); + if (rwmmio_tracepoint_enabled(rwmmio_post_write)) + log_post_write_mmio(value, 64, addr, _THIS_IP_, _RET_IP_); } #endif #endif /* CONFIG_64BIT */ @@ -255,7 +322,14 @@ static inline void writeq(u64 value, volatile void __iomem *addr) #define readb_relaxed readb_relaxed static inline u8 readb_relaxed(const volatile void __iomem *addr) { - return __raw_readb(addr); + u8 val; + + if (rwmmio_tracepoint_enabled(rwmmio_read)) + log_read_mmio(8, addr, _THIS_IP_, _RET_IP_); + val = __raw_readb(addr); + if (rwmmio_tracepoint_enabled(rwmmio_post_read)) + log_post_read_mmio(val, 8, addr, _THIS_IP_, _RET_IP_); + return val; } #endif @@ -263,7 +337,14 @@ static inline u8 readb_relaxed(const volatile void __iomem *addr) #define readw_relaxed readw_relaxed static inline u16 readw_relaxed(const volatile void __iomem *addr) { - return __le16_to_cpu(__raw_readw(addr)); + u16 val; + + if (rwmmio_tracepoint_enabled(rwmmio_read)) + log_read_mmio(16, addr, _THIS_IP_, _RET_IP_); + val = __le16_to_cpu((__le16 __force)__raw_readw(addr)); + if (rwmmio_tracepoint_enabled(rwmmio_post_read)) + log_post_read_mmio(val, 16, addr, _THIS_IP_, _RET_IP_); + return val; } #endif @@ -271,7 +352,14 @@ static inline u16 readw_relaxed(const volatile void __iomem *addr) #define readl_relaxed readl_relaxed static inline u32 readl_relaxed(const volatile void __iomem *addr) { - return __le32_to_cpu(__raw_readl(addr)); + u32 val; + + if (rwmmio_tracepoint_enabled(rwmmio_read)) + log_read_mmio(32, addr, _THIS_IP_, _RET_IP_); + val = __le32_to_cpu((__le32 __force)__raw_readl(addr)); + if (rwmmio_tracepoint_enabled(rwmmio_post_read)) + log_post_read_mmio(val, 32, addr, _THIS_IP_, _RET_IP_); + return val; } #endif @@ -279,7 +367,14 @@ static inline u32 readl_relaxed(const volatile void __iomem *addr) #define readq_relaxed readq_relaxed static inline u64 readq_relaxed(const volatile void __iomem *addr) { - return __le64_to_cpu(__raw_readq(addr)); + u64 val; + + if (rwmmio_tracepoint_enabled(rwmmio_read)) + log_read_mmio(64, addr, _THIS_IP_, _RET_IP_); + val = __le64_to_cpu((__le64 __force)__raw_readq(addr)); + if (rwmmio_tracepoint_enabled(rwmmio_post_read)) + log_post_read_mmio(val, 64, addr, _THIS_IP_, _RET_IP_); + return val; } #endif @@ -287,7 +382,11 @@ static inline u64 readq_relaxed(const volatile void __iomem *addr) #define writeb_relaxed writeb_relaxed static inline void writeb_relaxed(u8 value, volatile void __iomem *addr) { + if (rwmmio_tracepoint_enabled(rwmmio_write)) + log_write_mmio(value, 8, addr, _THIS_IP_, _RET_IP_); __raw_writeb(value, addr); + if (rwmmio_tracepoint_enabled(rwmmio_post_write)) + log_post_write_mmio(value, 8, addr, _THIS_IP_, _RET_IP_); } #endif @@ -295,7 +394,11 @@ static inline void writeb_relaxed(u8 value, volatile void __iomem *addr) #define writew_relaxed writew_relaxed static inline void writew_relaxed(u16 value, volatile void __iomem *addr) { - __raw_writew(cpu_to_le16(value), addr); + if (rwmmio_tracepoint_enabled(rwmmio_write)) + log_write_mmio(value, 16, addr, _THIS_IP_, _RET_IP_); + __raw_writew((u16 __force)cpu_to_le16(value), addr); + if (rwmmio_tracepoint_enabled(rwmmio_post_write)) + log_post_write_mmio(value, 16, addr, _THIS_IP_, _RET_IP_); } #endif @@ -303,7 +406,11 @@ static inline void writew_relaxed(u16 value, volatile void __iomem *addr) #define writel_relaxed writel_relaxed static inline void writel_relaxed(u32 value, volatile void __iomem *addr) { - __raw_writel(__cpu_to_le32(value), addr); + if (rwmmio_tracepoint_enabled(rwmmio_write)) + log_write_mmio(value, 32, addr, _THIS_IP_, _RET_IP_); + __raw_writel((u32 __force)__cpu_to_le32(value), addr); + if (rwmmio_tracepoint_enabled(rwmmio_post_write)) + log_post_write_mmio(value, 32, addr, _THIS_IP_, _RET_IP_); } #endif @@ -311,7 +418,11 @@ static inline void writel_relaxed(u32 value, volatile void __iomem *addr) #define writeq_relaxed writeq_relaxed static inline void writeq_relaxed(u64 value, volatile void __iomem *addr) { - __raw_writeq(__cpu_to_le64(value), addr); + if (rwmmio_tracepoint_enabled(rwmmio_write)) + log_write_mmio(value, 64, addr, _THIS_IP_, _RET_IP_); + __raw_writeq((u64 __force)__cpu_to_le64(value), addr); + if (rwmmio_tracepoint_enabled(rwmmio_post_write)) + log_post_write_mmio(value, 64, addr, _THIS_IP_, _RET_IP_); } #endif @@ -455,81 +566,135 @@ static inline void writesq(volatile void __iomem *addr, const void *buffer, #define IO_SPACE_LIMIT 0xffff #endif -#include <linux/logic_pio.h> - /* * {in,out}{b,w,l}() access little endian I/O. {in,out}{b,w,l}_p() can be * implemented on hardware that needs an additional delay for I/O accesses to * take effect. */ -#ifndef inb -#define inb inb -static inline u8 inb(unsigned long addr) +#if !defined(inb) && !defined(_inb) +#define _inb _inb +#ifdef CONFIG_HAS_IOPORT +static inline u8 _inb(unsigned long addr) { u8 val; __io_pbr(); val = __raw_readb(PCI_IOBASE + addr); - __io_par(); + __io_par(val); return val; } +#else +u8 _inb(unsigned long addr) + __compiletime_error("inb()) requires CONFIG_HAS_IOPORT"); +#endif #endif -#ifndef inw -#define inw inw -static inline u16 inw(unsigned long addr) +#if !defined(inw) && !defined(_inw) +#define _inw _inw +#ifdef CONFIG_HAS_IOPORT +static inline u16 _inw(unsigned long addr) { u16 val; __io_pbr(); - val = __le16_to_cpu(__raw_readw(PCI_IOBASE + addr)); - __io_par(); + val = __le16_to_cpu((__le16 __force)__raw_readw(PCI_IOBASE + addr)); + __io_par(val); return val; } +#else +u16 _inw(unsigned long addr) + __compiletime_error("inw() requires CONFIG_HAS_IOPORT"); +#endif #endif -#ifndef inl -#define inl inl -static inline u32 inl(unsigned long addr) +#if !defined(inl) && !defined(_inl) +#define _inl _inl +#ifdef CONFIG_HAS_IOPORT +static inline u32 _inl(unsigned long addr) { u32 val; __io_pbr(); - val = __le32_to_cpu(__raw_readl(PCI_IOBASE + addr)); - __io_par(); + val = __le32_to_cpu((__le32 __force)__raw_readl(PCI_IOBASE + addr)); + __io_par(val); return val; } +#else +u32 _inl(unsigned long addr) + __compiletime_error("inl() requires CONFIG_HAS_IOPORT"); +#endif #endif -#ifndef outb -#define outb outb -static inline void outb(u8 value, unsigned long addr) +#if !defined(outb) && !defined(_outb) +#define _outb _outb +#ifdef CONFIG_HAS_IOPORT +static inline void _outb(u8 value, unsigned long addr) { __io_pbw(); __raw_writeb(value, PCI_IOBASE + addr); __io_paw(); } +#else +void _outb(u8 value, unsigned long addr) + __compiletime_error("outb() requires CONFIG_HAS_IOPORT"); +#endif #endif -#ifndef outw -#define outw outw -static inline void outw(u16 value, unsigned long addr) +#if !defined(outw) && !defined(_outw) +#define _outw _outw +#ifdef CONFIG_HAS_IOPORT +static inline void _outw(u16 value, unsigned long addr) { __io_pbw(); - __raw_writew(cpu_to_le16(value), PCI_IOBASE + addr); + __raw_writew((u16 __force)cpu_to_le16(value), PCI_IOBASE + addr); __io_paw(); } +#else +void _outw(u16 value, unsigned long addr) + __compiletime_error("outw() requires CONFIG_HAS_IOPORT"); +#endif #endif -#ifndef outl -#define outl outl -static inline void outl(u32 value, unsigned long addr) +#if !defined(outl) && !defined(_outl) +#define _outl _outl +#ifdef CONFIG_HAS_IOPORT +static inline void _outl(u32 value, unsigned long addr) { __io_pbw(); - __raw_writel(cpu_to_le32(value), PCI_IOBASE + addr); + __raw_writel((u32 __force)cpu_to_le32(value), PCI_IOBASE + addr); __io_paw(); } +#else +void _outl(u32 value, unsigned long addr) + __compiletime_error("outl() requires CONFIG_HAS_IOPORT"); +#endif +#endif + +#include <linux/logic_pio.h> + +#ifndef inb +#define inb _inb +#endif + +#ifndef inw +#define inw _inw +#endif + +#ifndef inl +#define inl _inl +#endif + +#ifndef outb +#define outb _outb +#endif + +#ifndef outw +#define outw _outw +#endif + +#ifndef outl +#define outl _outl #endif #ifndef inb_p @@ -587,53 +752,83 @@ static inline void outl_p(u32 value, unsigned long addr) #ifndef insb #define insb insb +#ifdef CONFIG_HAS_IOPORT static inline void insb(unsigned long addr, void *buffer, unsigned int count) { readsb(PCI_IOBASE + addr, buffer, count); } +#else +void insb(unsigned long addr, void *buffer, unsigned int count) + __compiletime_error("insb() requires HAS_IOPORT"); +#endif #endif #ifndef insw #define insw insw +#ifdef CONFIG_HAS_IOPORT static inline void insw(unsigned long addr, void *buffer, unsigned int count) { readsw(PCI_IOBASE + addr, buffer, count); } +#else +void insw(unsigned long addr, void *buffer, unsigned int count) + __compiletime_error("insw() requires HAS_IOPORT"); +#endif #endif #ifndef insl #define insl insl +#ifdef CONFIG_HAS_IOPORT static inline void insl(unsigned long addr, void *buffer, unsigned int count) { readsl(PCI_IOBASE + addr, buffer, count); } +#else +void insl(unsigned long addr, void *buffer, unsigned int count) + __compiletime_error("insl() requires HAS_IOPORT"); +#endif #endif #ifndef outsb #define outsb outsb +#ifdef CONFIG_HAS_IOPORT static inline void outsb(unsigned long addr, const void *buffer, unsigned int count) { writesb(PCI_IOBASE + addr, buffer, count); } +#else +void outsb(unsigned long addr, const void *buffer, unsigned int count) + __compiletime_error("outsb() requires HAS_IOPORT"); +#endif #endif #ifndef outsw #define outsw outsw +#ifdef CONFIG_HAS_IOPORT static inline void outsw(unsigned long addr, const void *buffer, unsigned int count) { writesw(PCI_IOBASE + addr, buffer, count); } +#else +void outsw(unsigned long addr, const void *buffer, unsigned int count) + __compiletime_error("outsw() requires HAS_IOPORT"); +#endif #endif #ifndef outsl #define outsl outsl +#ifdef CONFIG_HAS_IOPORT static inline void outsl(unsigned long addr, const void *buffer, unsigned int count) { writesl(PCI_IOBASE + addr, buffer, count); } +#else +void outsl(unsigned long addr, const void *buffer, unsigned int count) + __compiletime_error("outsl() requires HAS_IOPORT"); +#endif #endif #ifndef insb_p @@ -891,21 +1086,8 @@ static inline void iowrite64_rep(volatile void __iomem *addr, #ifdef __KERNEL__ -#include <linux/vmalloc.h> #define __io_virt(x) ((void __force *)(x)) -#ifndef CONFIG_GENERIC_IOMAP -struct pci_dev; -extern void __iomem *pci_iomap(struct pci_dev *dev, int bar, unsigned long max); - -#ifndef pci_iounmap -#define pci_iounmap pci_iounmap -static inline void pci_iounmap(struct pci_dev *dev, void __iomem *p) -{ -} -#endif -#endif /* CONFIG_GENERIC_IOMAP */ - /* * Change virtual addresses to physical addresses and vv. * These are pretty trivial @@ -929,39 +1111,19 @@ static inline void *phys_to_virt(unsigned long address) /** * DOC: ioremap() and ioremap_*() variants * - * If you have an IOMMU your architecture is expected to have both ioremap() - * and iounmap() implemented otherwise the asm-generic helpers will provide a - * direct mapping. + * Architectures with an MMU are expected to provide ioremap() and iounmap() + * themselves or rely on GENERIC_IOREMAP. For NOMMU architectures we provide + * a default nop-op implementation that expect that the physical address used + * for MMIO are already marked as uncached, and can be used as kernel virtual + * addresses. * - * There are ioremap_*() call variants, if you have no IOMMU we naturally will - * default to direct mapping for all of them, you can override these defaults. - * If you have an IOMMU you are highly encouraged to provide your own - * ioremap variant implementation as there currently is no safe architecture - * agnostic default. To avoid possible improper behaviour default asm-generic - * ioremap_*() variants all return NULL when an IOMMU is available. If you've - * defined your own ioremap_*() variant you must then declare your own - * ioremap_*() variant as defined to itself to avoid the default NULL return. + * ioremap_wc() and ioremap_wt() can provide more relaxed caching attributes + * for specific drivers if the architecture choses to implement them. If they + * are not implemented we fall back to plain ioremap. Conversely, ioremap_np() + * can provide stricter non-posted write semantics if the architecture + * implements them. */ - -#ifdef CONFIG_MMU - -#ifndef ioremap_uc -#define ioremap_uc ioremap_uc -static inline void __iomem *ioremap_uc(phys_addr_t offset, size_t size) -{ - return NULL; -} -#endif - -#else /* !CONFIG_MMU */ - -/* - * Change "struct page" to physical address. - * - * This implementation is for the no-MMU case only... if you have an MMU - * you'll need to provide your own definitions. - */ - +#ifndef CONFIG_MMU #ifndef ioremap #define ioremap ioremap static inline void __iomem *ioremap(phys_addr_t offset, size_t size) @@ -970,53 +1132,70 @@ static inline void __iomem *ioremap(phys_addr_t offset, size_t size) } #endif -#ifndef __ioremap -#define __ioremap __ioremap -static inline void __iomem *__ioremap(phys_addr_t offset, size_t size, - unsigned long flags) +#ifndef iounmap +#define iounmap iounmap +static inline void iounmap(volatile void __iomem *addr) { - return ioremap(offset, size); } #endif +#elif defined(CONFIG_GENERIC_IOREMAP) +#include <linux/pgtable.h> -#ifndef iounmap -#define iounmap iounmap +void __iomem *generic_ioremap_prot(phys_addr_t phys_addr, size_t size, + pgprot_t prot); + +void __iomem *ioremap_prot(phys_addr_t phys_addr, size_t size, + pgprot_t prot); +void iounmap(volatile void __iomem *addr); +void generic_iounmap(volatile void __iomem *addr); -static inline void iounmap(void __iomem *addr) +#ifndef ioremap +#define ioremap ioremap +static inline void __iomem *ioremap(phys_addr_t addr, size_t size) { + /* _PAGE_IOREMAP needs to be supplied by the architecture */ + return ioremap_prot(addr, size, __pgprot(_PAGE_IOREMAP)); } #endif -#endif /* CONFIG_MMU */ -#ifndef ioremap_nocache -void __iomem *ioremap(phys_addr_t phys_addr, size_t size); -#define ioremap_nocache ioremap_nocache -static inline void __iomem *ioremap_nocache(phys_addr_t offset, size_t size) -{ - return ioremap(offset, size); -} +#endif /* !CONFIG_MMU || CONFIG_GENERIC_IOREMAP */ + +#ifndef ioremap_wc +#define ioremap_wc ioremap #endif +#ifndef ioremap_wt +#define ioremap_wt ioremap +#endif + +/* + * ioremap_uc is special in that we do require an explicit architecture + * implementation. In general you do not want to use this function in a + * driver and use plain ioremap, which is uncached by default. Similarly + * architectures should not implement it unless they have a very good + * reason. + */ #ifndef ioremap_uc #define ioremap_uc ioremap_uc static inline void __iomem *ioremap_uc(phys_addr_t offset, size_t size) { - return ioremap_nocache(offset, size); -} -#endif - -#ifndef ioremap_wc -#define ioremap_wc ioremap_wc -static inline void __iomem *ioremap_wc(phys_addr_t offset, size_t size) -{ - return ioremap_nocache(offset, size); + return NULL; } #endif -#ifndef ioremap_wt -#define ioremap_wt ioremap_wt -static inline void __iomem *ioremap_wt(phys_addr_t offset, size_t size) +/* + * ioremap_np needs an explicit architecture implementation, as it + * requests stronger semantics than regular ioremap(). Portable drivers + * should instead use one of the higher-level abstractions, like + * devm_ioremap_resource(), to choose the correct variant for any given + * device and bus. Portable drivers with a good reason to want non-posted + * write semantics should always provide an ioremap() fallback in case + * ioremap_np() is not available. + */ +#ifndef ioremap_np +#define ioremap_np ioremap_np +static inline void __iomem *ioremap_np(phys_addr_t offset, size_t size) { - return ioremap_nocache(offset, size); + return NULL; } #endif @@ -1029,6 +1208,7 @@ static inline void __iomem *ioport_map(unsigned long port, unsigned int nr) port &= IO_SPACE_LIMIT; return (port > MMIO_UPPER_LIMIT) ? NULL : PCI_IOBASE + port; } +#define ARCH_HAS_GENERIC_IOPORT_MAP #endif #ifndef ioport_unmap @@ -1043,15 +1223,10 @@ extern void ioport_unmap(void __iomem *p); #endif /* CONFIG_GENERIC_IOMAP */ #endif /* CONFIG_HAS_IOPORT_MAP */ -/* - * Convert a virtual cached pointer to an uncached pointer - */ -#ifndef xlate_dev_kmem_ptr -#define xlate_dev_kmem_ptr xlate_dev_kmem_ptr -static inline void *xlate_dev_kmem_ptr(void *addr) -{ - return addr; -} +#ifndef CONFIG_GENERIC_IOMAP +#ifndef pci_iounmap +#define ARCH_WANTS_GENERIC_PCI_IOUNMAP +#endif #endif #ifndef xlate_dev_mem_ptr @@ -1069,72 +1244,44 @@ static inline void unxlate_dev_mem_ptr(phys_addr_t phys, void *addr) } #endif -#ifdef CONFIG_VIRT_TO_BUS -#ifndef virt_to_bus -static inline unsigned long virt_to_bus(void *address) -{ - return (unsigned long)address; -} - -static inline void *bus_to_virt(unsigned long address) -{ - return (void *)address; -} -#endif -#endif - #ifndef memset_io -#define memset_io memset_io /** - * memset_io Set a range of I/O memory to a constant value + * memset_io - Set a range of I/O memory to a constant value * @addr: The beginning of the I/O-memory range to set * @val: The value to set the memory to * @count: The number of bytes to set * * Set a range of I/O memory to a given value. */ -static inline void memset_io(volatile void __iomem *addr, int value, - size_t size) -{ - memset(__io_virt(addr), value, size); -} +void memset_io(volatile void __iomem *addr, int val, size_t count); #endif #ifndef memcpy_fromio -#define memcpy_fromio memcpy_fromio /** - * memcpy_fromio Copy a block of data from I/O memory + * memcpy_fromio - Copy a block of data from I/O memory * @dst: The (RAM) destination for the copy * @src: The (I/O memory) source for the data * @count: The number of bytes to copy * * Copy a block of data from I/O memory. */ -static inline void memcpy_fromio(void *buffer, - const volatile void __iomem *addr, - size_t size) -{ - memcpy(buffer, __io_virt(addr), size); -} +void memcpy_fromio(void *dst, const volatile void __iomem *src, size_t count); #endif #ifndef memcpy_toio -#define memcpy_toio memcpy_toio /** - * memcpy_toio Copy a block of data into I/O memory + * memcpy_toio - Copy a block of data into I/O memory * @dst: The (I/O memory) destination for the copy * @src: The (RAM) source for the data * @count: The number of bytes to copy * * Copy a block of data to I/O memory. */ -static inline void memcpy_toio(volatile void __iomem *addr, const void *buffer, - size_t size) -{ - memcpy(__io_virt(addr), buffer, size); -} +void memcpy_toio(volatile void __iomem *dst, const void *src, size_t count); #endif +extern int devmem_is_allowed(unsigned long pfn); + #endif /* __KERNEL__ */ #endif /* __ASM_GENERIC_IO_H */ diff --git a/include/asm-generic/iomap.h b/include/asm-generic/iomap.h index 5b63b94ef6b5..9f3f25d7fc58 100644 --- a/include/asm-generic/iomap.h +++ b/include/asm-generic/iomap.h @@ -26,25 +26,27 @@ * in the low address range. Architectures for which this is not * true can't use this generic implementation. */ -extern unsigned int ioread8(void __iomem *); -extern unsigned int ioread16(void __iomem *); -extern unsigned int ioread16be(void __iomem *); -extern unsigned int ioread32(void __iomem *); -extern unsigned int ioread32be(void __iomem *); -#ifdef CONFIG_64BIT -extern u64 ioread64(void __iomem *); -extern u64 ioread64be(void __iomem *); -#endif +extern unsigned int ioread8(const void __iomem *); +extern unsigned int ioread16(const void __iomem *); +extern unsigned int ioread16be(const void __iomem *); +extern unsigned int ioread32(const void __iomem *); +extern unsigned int ioread32be(const void __iomem *); + +extern u64 __ioread64_lo_hi(const void __iomem *addr); +extern u64 __ioread64_hi_lo(const void __iomem *addr); +extern u64 __ioread64be_lo_hi(const void __iomem *addr); +extern u64 __ioread64be_hi_lo(const void __iomem *addr); extern void iowrite8(u8, void __iomem *); extern void iowrite16(u16, void __iomem *); extern void iowrite16be(u16, void __iomem *); extern void iowrite32(u32, void __iomem *); extern void iowrite32be(u32, void __iomem *); -#ifdef CONFIG_64BIT -extern void iowrite64(u64, void __iomem *); -extern void iowrite64be(u64, void __iomem *); -#endif + +extern void __iowrite64_lo_hi(u64 val, void __iomem *addr); +extern void __iowrite64_hi_lo(u64 val, void __iomem *addr); +extern void __iowrite64be_lo_hi(u64 val, void __iomem *addr); +extern void __iowrite64be_hi_lo(u64 val, void __iomem *addr); /* * "string" versions of the above. Note that they @@ -57,9 +59,9 @@ extern void iowrite64be(u64, void __iomem *); * memory across multiple ports, use "memcpy_toio()" * and friends. */ -extern void ioread8_rep(void __iomem *port, void *buf, unsigned long count); -extern void ioread16_rep(void __iomem *port, void *buf, unsigned long count); -extern void ioread32_rep(void __iomem *port, void *buf, unsigned long count); +extern void ioread8_rep(const void __iomem *port, void *buf, unsigned long count); +extern void ioread16_rep(const void __iomem *port, void *buf, unsigned long count); +extern void ioread32_rep(const void __iomem *port, void *buf, unsigned long count); extern void iowrite8_rep(void __iomem *port, const void *buf, unsigned long count); extern void iowrite16_rep(void __iomem *port, const void *buf, unsigned long count); @@ -71,22 +73,21 @@ extern void __iomem *ioport_map(unsigned long port, unsigned int nr); extern void ioport_unmap(void __iomem *); #endif -#ifndef ARCH_HAS_IOREMAP_WC -#define ioremap_wc ioremap_nocache +#ifndef ioremap_wc +#define ioremap_wc ioremap #endif -#ifndef ARCH_HAS_IOREMAP_WT -#define ioremap_wt ioremap_nocache +#ifndef ioremap_wt +#define ioremap_wt ioremap #endif -#ifdef CONFIG_PCI -/* Destroy a virtual mapping cookie for a PCI BAR (memory or IO) */ -struct pci_dev; -extern void pci_iounmap(struct pci_dev *dev, void __iomem *); -#elif defined(CONFIG_GENERIC_IOMAP) -struct pci_dev; -static inline void pci_iounmap(struct pci_dev *dev, void __iomem *addr) -{ } +#ifndef ioremap_np +/* See the comment in asm-generic/io.h about ioremap_np(). */ +#define ioremap_np ioremap_np +static inline void __iomem *ioremap_np(phys_addr_t offset, size_t size) +{ + return NULL; +} #endif #include <asm-generic/pci_iomap.h> diff --git a/include/asm-generic/irq_regs.h b/include/asm-generic/irq_regs.h index 6bf9355fa7eb..2e7c6e89d42e 100644 --- a/include/asm-generic/irq_regs.h +++ b/include/asm-generic/irq_regs.h @@ -1,12 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* Fallback per-CPU frame pointer holder * * Copyright (C) 2006 Red Hat, Inc. All Rights Reserved. * Written by David Howells (dhowells@redhat.com) - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. */ #ifndef _ASM_GENERIC_IRQ_REGS_H diff --git a/include/asm-generic/kmap_size.h b/include/asm-generic/kmap_size.h new file mode 100644 index 000000000000..6e36b2443ece --- /dev/null +++ b/include/asm-generic/kmap_size.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_GENERIC_KMAP_SIZE_H +#define _ASM_GENERIC_KMAP_SIZE_H + +/* For debug this provides guard pages between the maps */ +#ifdef CONFIG_DEBUG_KMAP_LOCAL +# define KM_MAX_IDX 33 +#else +# define KM_MAX_IDX 16 +#endif + +#endif diff --git a/include/asm-generic/kmap_types.h b/include/asm-generic/kmap_types.h deleted file mode 100644 index 9f95b7b63d19..000000000000 --- a/include/asm-generic/kmap_types.h +++ /dev/null @@ -1,11 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _ASM_GENERIC_KMAP_TYPES_H -#define _ASM_GENERIC_KMAP_TYPES_H - -#ifdef __WITH_KM_FENCE -# define KM_TYPE_NR 41 -#else -# define KM_TYPE_NR 20 -#endif - -#endif diff --git a/include/asm-generic/kprobes.h b/include/asm-generic/kprobes.h index 4a982089c95c..060eab094e5a 100644 --- a/include/asm-generic/kprobes.h +++ b/include/asm-generic/kprobes.h @@ -10,11 +10,11 @@ */ # define __NOKPROBE_SYMBOL(fname) \ static unsigned long __used \ - __attribute__((__section__("_kprobe_blacklist"))) \ + __section("_kprobe_blacklist") \ _kbl_addr_##fname = (unsigned long)fname; # define NOKPROBE_SYMBOL(fname) __NOKPROBE_SYMBOL(fname) /* Use this to forbid a kprobes attach on very low level functions */ -# define __kprobes __attribute__((__section__(".kprobes.text"))) +# define __kprobes __section(".kprobes.text") # define nokprobe_inline __always_inline #else # define NOKPROBE_SYMBOL(fname) diff --git a/include/asm-generic/kvm_types.h b/include/asm-generic/kvm_types.h new file mode 100644 index 000000000000..2a82daf110f1 --- /dev/null +++ b/include/asm-generic/kvm_types.h @@ -0,0 +1,5 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_GENERIC_KVM_TYPES_H +#define _ASM_GENERIC_KVM_TYPES_H + +#endif diff --git a/include/asm-generic/local.h b/include/asm-generic/local.h index fca7f1d84818..7f97018df66f 100644 --- a/include/asm-generic/local.h +++ b/include/asm-generic/local.h @@ -42,6 +42,7 @@ typedef struct #define local_inc_return(l) atomic_long_inc_return(&(l)->a) #define local_cmpxchg(l, o, n) atomic_long_cmpxchg((&(l)->a), (o), (n)) +#define local_try_cmpxchg(l, po, n) atomic_long_try_cmpxchg((&(l)->a), (po), (n)) #define local_xchg(l, n) atomic_long_xchg((&(l)->a), (n)) #define local_add_unless(l, _a, u) atomic_long_add_unless((&(l)->a), (_a), (u)) #define local_inc_not_zero(l) atomic_long_inc_not_zero(&(l)->a) diff --git a/include/asm-generic/local64.h b/include/asm-generic/local64.h index 765be0b7d883..14963a7a6253 100644 --- a/include/asm-generic/local64.h +++ b/include/asm-generic/local64.h @@ -42,7 +42,16 @@ typedef struct { #define local64_sub_return(i, l) local_sub_return((i), (&(l)->a)) #define local64_inc_return(l) local_inc_return(&(l)->a) -#define local64_cmpxchg(l, o, n) local_cmpxchg((&(l)->a), (o), (n)) +static inline s64 local64_cmpxchg(local64_t *l, s64 old, s64 new) +{ + return local_cmpxchg(&l->a, old, new); +} + +static inline bool local64_try_cmpxchg(local64_t *l, s64 *old, s64 new) +{ + return local_try_cmpxchg(&l->a, (long *)old, new); +} + #define local64_xchg(l, n) local_xchg((&(l)->a), (n)) #define local64_add_unless(l, _a, u) local_add_unless((&(l)->a), (_a), (u)) #define local64_inc_not_zero(l) local_inc_not_zero(&(l)->a) @@ -81,6 +90,7 @@ typedef struct { #define local64_inc_return(l) atomic64_inc_return(&(l)->a) #define local64_cmpxchg(l, o, n) atomic64_cmpxchg((&(l)->a), (o), (n)) +#define local64_try_cmpxchg(l, po, n) atomic64_try_cmpxchg((&(l)->a), (po), (n)) #define local64_xchg(l, n) atomic64_xchg((&(l)->a), (n)) #define local64_add_unless(l, _a, u) atomic64_add_unless((&(l)->a), (_a), (u)) #define local64_inc_not_zero(l) atomic64_inc_not_zero(&(l)->a) diff --git a/include/asm-generic/logic_io.h b/include/asm-generic/logic_io.h new file mode 100644 index 000000000000..8a59b6e567df --- /dev/null +++ b/include/asm-generic/logic_io.h @@ -0,0 +1,78 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 Intel Corporation + * Author: johannes@sipsolutions.net + */ +#ifndef _LOGIC_IO_H +#define _LOGIC_IO_H +#include <linux/types.h> + +/* include this file into asm/io.h */ + +#ifdef CONFIG_INDIRECT_IOMEM + +#ifdef CONFIG_INDIRECT_IOMEM_FALLBACK +/* + * If you want emulated IO memory to fall back to 'normal' IO memory + * if a region wasn't registered as emulated, then you need to have + * all of the real_* functions implemented. + */ +#if !defined(real_ioremap) || !defined(real_iounmap) || \ + !defined(real_raw_readb) || !defined(real_raw_writeb) || \ + !defined(real_raw_readw) || !defined(real_raw_writew) || \ + !defined(real_raw_readl) || !defined(real_raw_writel) || \ + (defined(CONFIG_64BIT) && \ + (!defined(real_raw_readq) || !defined(real_raw_writeq))) || \ + !defined(real_memset_io) || \ + !defined(real_memcpy_fromio) || \ + !defined(real_memcpy_toio) +#error "Must provide fallbacks for real IO memory access" +#endif /* defined ... */ +#endif /* CONFIG_INDIRECT_IOMEM_FALLBACK */ + +#define ioremap ioremap +void __iomem *ioremap(phys_addr_t offset, size_t size); + +#define iounmap iounmap +void iounmap(void volatile __iomem *addr); + +#define __raw_readb __raw_readb +u8 __raw_readb(const volatile void __iomem *addr); + +#define __raw_readw __raw_readw +u16 __raw_readw(const volatile void __iomem *addr); + +#define __raw_readl __raw_readl +u32 __raw_readl(const volatile void __iomem *addr); + +#ifdef CONFIG_64BIT +#define __raw_readq __raw_readq +u64 __raw_readq(const volatile void __iomem *addr); +#endif /* CONFIG_64BIT */ + +#define __raw_writeb __raw_writeb +void __raw_writeb(u8 value, volatile void __iomem *addr); + +#define __raw_writew __raw_writew +void __raw_writew(u16 value, volatile void __iomem *addr); + +#define __raw_writel __raw_writel +void __raw_writel(u32 value, volatile void __iomem *addr); + +#ifdef CONFIG_64BIT +#define __raw_writeq __raw_writeq +void __raw_writeq(u64 value, volatile void __iomem *addr); +#endif /* CONFIG_64BIT */ + +#define memset_io memset_io +void memset_io(volatile void __iomem *addr, int value, size_t size); + +#define memcpy_fromio memcpy_fromio +void memcpy_fromio(void *buffer, const volatile void __iomem *addr, + size_t size); + +#define memcpy_toio memcpy_toio +void memcpy_toio(volatile void __iomem *addr, const void *buffer, size_t size); + +#endif /* CONFIG_INDIRECT_IOMEM */ +#endif /* _LOGIC_IO_H */ diff --git a/include/asm-generic/mcs_spinlock.h b/include/asm-generic/mcs_spinlock.h index 10cd4ffc6ba2..39c94012b88a 100644 --- a/include/asm-generic/mcs_spinlock.h +++ b/include/asm-generic/mcs_spinlock.h @@ -1,6 +1,12 @@ #ifndef __ASM_MCS_SPINLOCK_H #define __ASM_MCS_SPINLOCK_H +struct mcs_spinlock { + struct mcs_spinlock *next; + int locked; /* 1 if lock acquired */ + int count; /* nesting count, see qspinlock.c */ +}; + /* * Architectures can define their own: * diff --git a/include/asm-generic/memory_model.h b/include/asm-generic/memory_model.h index 7637fb46ba4f..efa6610acbc7 100644 --- a/include/asm-generic/memory_model.h +++ b/include/asm-generic/memory_model.h @@ -6,47 +6,39 @@ #ifndef __ASSEMBLY__ +/* + * supports 3 memory models. + */ #if defined(CONFIG_FLATMEM) #ifndef ARCH_PFN_OFFSET #define ARCH_PFN_OFFSET (0UL) #endif -#elif defined(CONFIG_DISCONTIGMEM) - -#ifndef arch_pfn_to_nid -#define arch_pfn_to_nid(pfn) pfn_to_nid(pfn) -#endif - -#ifndef arch_local_page_offset -#define arch_local_page_offset(pfn, nid) \ - ((pfn) - NODE_DATA(nid)->node_start_pfn) -#endif - -#endif /* CONFIG_DISCONTIGMEM */ - -/* - * supports 3 memory models. - */ -#if defined(CONFIG_FLATMEM) - #define __pfn_to_page(pfn) (mem_map + ((pfn) - ARCH_PFN_OFFSET)) #define __page_to_pfn(page) ((unsigned long)((page) - mem_map) + \ ARCH_PFN_OFFSET) -#elif defined(CONFIG_DISCONTIGMEM) -#define __pfn_to_page(pfn) \ -({ unsigned long __pfn = (pfn); \ - unsigned long __nid = arch_pfn_to_nid(__pfn); \ - NODE_DATA(__nid)->node_mem_map + arch_local_page_offset(__pfn, __nid);\ -}) +/* avoid <linux/mm.h> include hell */ +extern unsigned long max_mapnr; -#define __page_to_pfn(pg) \ -({ const struct page *__pg = (pg); \ - struct pglist_data *__pgdat = NODE_DATA(page_to_nid(__pg)); \ - (unsigned long)(__pg - __pgdat->node_mem_map) + \ - __pgdat->node_start_pfn; \ -}) +#ifndef pfn_valid +static inline int pfn_valid(unsigned long pfn) +{ + unsigned long pfn_offset = ARCH_PFN_OFFSET; + + return pfn >= pfn_offset && (pfn - pfn_offset) < max_mapnr; +} +#define pfn_valid pfn_valid + +#ifndef for_each_valid_pfn +#define for_each_valid_pfn(pfn, start_pfn, end_pfn) \ + for ((pfn) = max_t(unsigned long, (start_pfn), ARCH_PFN_OFFSET); \ + (pfn) < min_t(unsigned long, (end_pfn), \ + ARCH_PFN_OFFSET + max_mapnr); \ + (pfn)++) +#endif /* for_each_valid_pfn */ +#endif /* valid_pfn */ #elif defined(CONFIG_SPARSEMEM_VMEMMAP) @@ -61,7 +53,7 @@ */ #define __page_to_pfn(pg) \ ({ const struct page *__pg = (pg); \ - int __sec = page_to_section(__pg); \ + int __sec = memdesc_section(__pg->flags); \ (unsigned long)(__pg - __section_mem_map_addr(__nr_to_section(__sec))); \ }) @@ -70,7 +62,7 @@ struct mem_section *__sec = __pfn_to_section(__pfn); \ __section_mem_map_addr(__sec) + __pfn; \ }) -#endif /* CONFIG_FLATMEM/DISCONTIGMEM/SPARSEMEM */ +#endif /* CONFIG_FLATMEM/SPARSEMEM */ /* * Convert a physical address to a Page Frame Number and back @@ -81,6 +73,19 @@ #define page_to_pfn __page_to_pfn #define pfn_to_page __pfn_to_page +#ifdef CONFIG_DEBUG_VIRTUAL +#define page_to_phys(page) \ +({ \ + unsigned long __pfn = page_to_pfn(page); \ + \ + WARN_ON_ONCE(!pfn_valid(__pfn)); \ + PFN_PHYS(__pfn); \ +}) +#else +#define page_to_phys(page) PFN_PHYS(page_to_pfn(page)) +#endif /* CONFIG_DEBUG_VIRTUAL */ +#define phys_to_page(phys) pfn_to_page(PHYS_PFN(phys)) + #endif /* __ASSEMBLY__ */ #endif diff --git a/include/asm-generic/mm-arch-hooks.h b/include/asm-generic/mm-arch-hooks.h deleted file mode 100644 index 5ff0e5193f85..000000000000 --- a/include/asm-generic/mm-arch-hooks.h +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Architecture specific mm hooks - */ - -#ifndef _ASM_GENERIC_MM_ARCH_HOOKS_H -#define _ASM_GENERIC_MM_ARCH_HOOKS_H - -/* - * This file should be included through arch/../include/asm/Kbuild for - * the architecture which doesn't need specific mm hooks. - * - * In that case, the generic hooks defined in include/linux/mm-arch-hooks.h - * are used. - */ - -#endif /* _ASM_GENERIC_MM_ARCH_HOOKS_H */ diff --git a/include/asm-generic/mm_hooks.h b/include/asm-generic/mm_hooks.h index 8ac4e68a12f0..6eea3b3c1e65 100644 --- a/include/asm-generic/mm_hooks.h +++ b/include/asm-generic/mm_hooks.h @@ -1,8 +1,8 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* - * Define generic no-op hooks for arch_dup_mmap, arch_exit_mmap - * and arch_unmap to be included in asm-FOO/mmu_context.h for any - * arch FOO which doesn't need to hook these. + * Define generic no-op hooks for arch_dup_mmap and arch_exit_mmap + * to be included in asm-FOO/mmu_context.h for any arch FOO which + * doesn't need to hook these. */ #ifndef _ASM_GENERIC_MM_HOOKS_H #define _ASM_GENERIC_MM_HOOKS_H @@ -17,17 +17,6 @@ static inline void arch_exit_mmap(struct mm_struct *mm) { } -static inline void arch_unmap(struct mm_struct *mm, - struct vm_area_struct *vma, - unsigned long start, unsigned long end) -{ -} - -static inline void arch_bprm_mm_init(struct mm_struct *mm, - struct vm_area_struct *vma) -{ -} - static inline bool arch_vma_access_permitted(struct vm_area_struct *vma, bool write, bool execute, bool foreign) { diff --git a/include/asm-generic/mmiowb.h b/include/asm-generic/mmiowb.h new file mode 100644 index 000000000000..5698fca3bf56 --- /dev/null +++ b/include/asm-generic/mmiowb.h @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_GENERIC_MMIOWB_H +#define __ASM_GENERIC_MMIOWB_H + +/* + * Generic implementation of mmiowb() tracking for spinlocks. + * + * If your architecture doesn't ensure that writes to an I/O peripheral + * within two spinlocked sections on two different CPUs are seen by the + * peripheral in the order corresponding to the lock handover, then you + * need to follow these FIVE easy steps: + * + * 1. Implement mmiowb() (and arch_mmiowb_state() if you're fancy) + * in asm/mmiowb.h, then #include this file + * 2. Ensure your I/O write accessors call mmiowb_set_pending() + * 3. Select ARCH_HAS_MMIOWB + * 4. Untangle the resulting mess of header files + * 5. Complain to your architects + */ +#ifdef CONFIG_MMIOWB + +#include <linux/compiler.h> +#include <asm-generic/mmiowb_types.h> + +#ifndef arch_mmiowb_state +#include <asm/percpu.h> +#include <asm/smp.h> + +DECLARE_PER_CPU(struct mmiowb_state, __mmiowb_state); +#define __mmiowb_state() raw_cpu_ptr(&__mmiowb_state) +#else +#define __mmiowb_state() arch_mmiowb_state() +#endif /* arch_mmiowb_state */ + +static inline void mmiowb_set_pending(void) +{ + struct mmiowb_state *ms = __mmiowb_state(); + + if (likely(ms->nesting_count)) + ms->mmiowb_pending = ms->nesting_count; +} + +static inline void mmiowb_spin_lock(void) +{ + struct mmiowb_state *ms = __mmiowb_state(); + ms->nesting_count++; +} + +static inline void mmiowb_spin_unlock(void) +{ + struct mmiowb_state *ms = __mmiowb_state(); + + if (unlikely(ms->mmiowb_pending)) { + ms->mmiowb_pending = 0; + mmiowb(); + } + + ms->nesting_count--; +} +#else +#define mmiowb_set_pending() do { } while (0) +#define mmiowb_spin_lock() do { } while (0) +#define mmiowb_spin_unlock() do { } while (0) +#endif /* CONFIG_MMIOWB */ +#endif /* __ASM_GENERIC_MMIOWB_H */ diff --git a/include/asm-generic/mmiowb_types.h b/include/asm-generic/mmiowb_types.h new file mode 100644 index 000000000000..8eb0095655e7 --- /dev/null +++ b/include/asm-generic/mmiowb_types.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_GENERIC_MMIOWB_TYPES_H +#define __ASM_GENERIC_MMIOWB_TYPES_H + +#include <linux/types.h> + +struct mmiowb_state { + u16 nesting_count; + u16 mmiowb_pending; +}; + +#endif /* __ASM_GENERIC_MMIOWB_TYPES_H */ diff --git a/include/asm-generic/mmu_context.h b/include/asm-generic/mmu_context.h index 6be9106fb6fb..91727065bacb 100644 --- a/include/asm-generic/mmu_context.h +++ b/include/asm-generic/mmu_context.h @@ -3,44 +3,74 @@ #define __ASM_GENERIC_MMU_CONTEXT_H /* - * Generic hooks for NOMMU architectures, which do not need to do - * anything special here. + * Generic hooks to implement no-op functionality. */ -#include <asm-generic/mm_hooks.h> - struct task_struct; struct mm_struct; +/* + * enter_lazy_tlb - Called when "tsk" is about to enter lazy TLB mode. + * + * @mm: the currently active mm context which is becoming lazy + * @tsk: task which is entering lazy tlb + * + * tsk->mm will be NULL + */ +#ifndef enter_lazy_tlb static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk) { } +#endif +/** + * init_new_context - Initialize context of a new mm_struct. + * @tsk: task struct for the mm + * @mm: the new mm struct + * @return: 0 on success, -errno on failure + */ +#ifndef init_new_context static inline int init_new_context(struct task_struct *tsk, struct mm_struct *mm) { return 0; } +#endif +/** + * destroy_context - Undo init_new_context when the mm is going away + * @mm: old mm struct + */ +#ifndef destroy_context static inline void destroy_context(struct mm_struct *mm) { } +#endif -static inline void deactivate_mm(struct task_struct *task, - struct mm_struct *mm) -{ -} - -static inline void switch_mm(struct mm_struct *prev, - struct mm_struct *next, - struct task_struct *tsk) +/** + * activate_mm - called after exec switches the current task to a new mm, to switch to it + * @prev_mm: previous mm of this task + * @next_mm: new mm + */ +#ifndef activate_mm +static inline void activate_mm(struct mm_struct *prev_mm, + struct mm_struct *next_mm) { + switch_mm(prev_mm, next_mm, current); } +#endif -static inline void activate_mm(struct mm_struct *prev_mm, - struct mm_struct *next_mm) +/** + * dectivate_mm - called when an mm is released after exit or exec switches away from it + * @tsk: the task + * @mm: the old mm + */ +#ifndef deactivate_mm +static inline void deactivate_mm(struct task_struct *tsk, + struct mm_struct *mm) { } +#endif #endif /* __ASM_GENERIC_MMU_CONTEXT_H */ diff --git a/include/asm-generic/mmzone.h b/include/asm-generic/mmzone.h new file mode 100644 index 000000000000..2ab5193e8394 --- /dev/null +++ b/include/asm-generic/mmzone.h @@ -0,0 +1,5 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_GENERIC_MMZONE_H +#define _ASM_GENERIC_MMZONE_H + +#endif diff --git a/include/asm-generic/module.h b/include/asm-generic/module.h index 98e1541b72b7..a8622501b975 100644 --- a/include/asm-generic/module.h +++ b/include/asm-generic/module.h @@ -19,12 +19,8 @@ struct mod_arch_specific #define Elf_Dyn Elf64_Dyn #define Elf_Ehdr Elf64_Ehdr #define Elf_Addr Elf64_Addr -#ifdef CONFIG_MODULES_USE_ELF_REL #define Elf_Rel Elf64_Rel -#endif -#ifdef CONFIG_MODULES_USE_ELF_RELA #define Elf_Rela Elf64_Rela -#endif #define ELF_R_TYPE(X) ELF64_R_TYPE(X) #define ELF_R_SYM(X) ELF64_R_SYM(X) @@ -36,12 +32,8 @@ struct mod_arch_specific #define Elf_Dyn Elf32_Dyn #define Elf_Ehdr Elf32_Ehdr #define Elf_Addr Elf32_Addr -#ifdef CONFIG_MODULES_USE_ELF_REL #define Elf_Rel Elf32_Rel -#endif -#ifdef CONFIG_MODULES_USE_ELF_RELA #define Elf_Rela Elf32_Rela -#endif #define ELF_R_TYPE(X) ELF32_R_TYPE(X) #define ELF_R_SYM(X) ELF32_R_SYM(X) #endif diff --git a/include/asm-generic/module.lds.h b/include/asm-generic/module.lds.h new file mode 100644 index 000000000000..f210d5c1b78b --- /dev/null +++ b/include/asm-generic/module.lds.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __ASM_GENERIC_MODULE_LDS_H +#define __ASM_GENERIC_MODULE_LDS_H + +/* + * <asm/module.lds.h> can specify arch-specific sections for linking modules. + * Empty for the asm-generic header. + */ + +#endif /* __ASM_GENERIC_MODULE_LDS_H */ diff --git a/include/asm-generic/mshyperv.h b/include/asm-generic/mshyperv.h new file mode 100644 index 000000000000..ecedab554c80 --- /dev/null +++ b/include/asm-generic/mshyperv.h @@ -0,0 +1,373 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* + * Linux-specific definitions for managing interactions with Microsoft's + * Hyper-V hypervisor. The definitions in this file are architecture + * independent. See arch/<arch>/include/asm/mshyperv.h for definitions + * that are specific to architecture <arch>. + * + * Definitions that are derived from Hyper-V code or headers should not go in + * this file, but should instead go in the relevant files in include/hyperv. + * + * Copyright (C) 2019, Microsoft, Inc. + * + * Author : Michael Kelley <mikelley@microsoft.com> + */ + +#ifndef _ASM_GENERIC_MSHYPERV_H +#define _ASM_GENERIC_MSHYPERV_H + +#include <linux/types.h> +#include <linux/atomic.h> +#include <linux/bitops.h> +#include <acpi/acpi_numa.h> +#include <linux/cpumask.h> +#include <linux/nmi.h> +#include <asm/ptrace.h> +#include <hyperv/hvhdk.h> + +#define VTPM_BASE_ADDRESS 0xfed40000 + +enum hv_partition_type { + HV_PARTITION_TYPE_GUEST, + HV_PARTITION_TYPE_ROOT, + HV_PARTITION_TYPE_L1VH, +}; + +struct ms_hyperv_info { + u32 features; + u32 priv_high; + u32 ext_features; + u32 misc_features; + u32 hints; + u32 nested_features; + u32 max_vp_index; + u32 max_lp_index; + u8 vtl; + union { + u32 isolation_config_a; + struct { + u32 paravisor_present : 1; + u32 reserved_a1 : 31; + }; + }; + union { + u32 isolation_config_b; + struct { + u32 cvm_type : 4; + u32 reserved_b1 : 1; + u32 shared_gpa_boundary_active : 1; + u32 shared_gpa_boundary_bits : 6; + u32 reserved_b2 : 20; + }; + }; + u64 shared_gpa_boundary; + bool msi_ext_dest_id; + bool confidential_vmbus_available; +}; +extern struct ms_hyperv_info ms_hyperv; +extern bool hv_nested; +extern u64 hv_current_partition_id; +extern enum hv_partition_type hv_curr_partition_type; + +extern void * __percpu *hyperv_pcpu_input_arg; +extern void * __percpu *hyperv_pcpu_output_arg; + +u64 hv_do_hypercall(u64 control, void *inputaddr, void *outputaddr); +u64 hv_do_fast_hypercall8(u16 control, u64 input8); +u64 hv_do_fast_hypercall16(u16 control, u64 input1, u64 input2); + +bool hv_isolation_type_snp(void); +bool hv_isolation_type_tdx(void); + +/* + * On architectures where Hyper-V doesn't support AEOI (e.g., ARM64), + * it doesn't provide a recommendation flag and AEOI must be disabled. + */ +static inline bool hv_recommend_using_aeoi(void) +{ +#ifdef HV_DEPRECATING_AEOI_RECOMMENDED + return !(ms_hyperv.hints & HV_DEPRECATING_AEOI_RECOMMENDED); +#else + return false; +#endif +} + +static inline struct hv_proximity_domain_info hv_numa_node_to_pxm_info(int node) +{ + struct hv_proximity_domain_info pxm_info = {}; + + if (node != NUMA_NO_NODE) { + pxm_info.domain_id = node_to_pxm(node); + pxm_info.flags.proximity_info_valid = 1; + pxm_info.flags.proximity_preferred = 1; + } + + return pxm_info; +} + +/* Helper functions that provide a consistent pattern for checking Hyper-V hypercall status. */ +static inline int hv_result(u64 status) +{ + return status & HV_HYPERCALL_RESULT_MASK; +} + +static inline bool hv_result_success(u64 status) +{ + return hv_result(status) == HV_STATUS_SUCCESS; +} + +static inline unsigned int hv_repcomp(u64 status) +{ + /* Bits [43:32] of status have 'Reps completed' data. */ + return (status & HV_HYPERCALL_REP_COMP_MASK) >> + HV_HYPERCALL_REP_COMP_OFFSET; +} + +/* + * Rep hypercalls. Callers of this functions are supposed to ensure that + * rep_count, varhead_size, and rep_start comply with Hyper-V hypercall + * definition. + */ +static inline u64 hv_do_rep_hypercall_ex(u16 code, u16 rep_count, + u16 varhead_size, u16 rep_start, + void *input, void *output) +{ + u64 control = code; + u64 status; + u16 rep_comp; + + control |= (u64)varhead_size << HV_HYPERCALL_VARHEAD_OFFSET; + control |= (u64)rep_count << HV_HYPERCALL_REP_COMP_OFFSET; + control |= (u64)rep_start << HV_HYPERCALL_REP_START_OFFSET; + + do { + status = hv_do_hypercall(control, input, output); + if (!hv_result_success(status)) + return status; + + rep_comp = hv_repcomp(status); + + control &= ~HV_HYPERCALL_REP_START_MASK; + control |= (u64)rep_comp << HV_HYPERCALL_REP_START_OFFSET; + + touch_nmi_watchdog(); + } while (rep_comp < rep_count); + + return status; +} + +/* For the typical case where rep_start is 0 */ +static inline u64 hv_do_rep_hypercall(u16 code, u16 rep_count, u16 varhead_size, + void *input, void *output) +{ + return hv_do_rep_hypercall_ex(code, rep_count, varhead_size, 0, + input, output); +} + +/* Generate the guest OS identifier as described in the Hyper-V TLFS */ +static inline u64 hv_generate_guest_id(u64 kernel_version) +{ + u64 guest_id; + + guest_id = (((u64)HV_LINUX_VENDOR_ID) << 48); + guest_id |= (kernel_version << 16); + + return guest_id; +} + +int hv_get_hypervisor_version(union hv_hypervisor_version_info *info); + +void hv_setup_vmbus_handler(void (*handler)(void)); +void hv_remove_vmbus_handler(void); +void hv_setup_stimer0_handler(void (*handler)(void)); +void hv_remove_stimer0_handler(void); + +void hv_setup_kexec_handler(void (*handler)(void)); +void hv_remove_kexec_handler(void); +void hv_setup_crash_handler(void (*handler)(struct pt_regs *regs)); +void hv_remove_crash_handler(void); +void hv_setup_mshv_handler(void (*handler)(void)); + +#if IS_ENABLED(CONFIG_HYPERV) +/* + * Hypervisor's notion of virtual processor ID is different from + * Linux' notion of CPU ID. This information can only be retrieved + * in the context of the calling CPU. Setup a map for easy access + * to this information. + */ +extern u32 *hv_vp_index; +extern u32 hv_max_vp_index; + +extern u64 (*hv_read_reference_counter)(void); + +/* Sentinel value for an uninitialized entry in hv_vp_index array */ +#define VP_INVAL U32_MAX + +int __init hv_common_init(void); +void __init hv_get_partition_id(void); +void __init hv_common_free(void); +void __init ms_hyperv_late_init(void); +int hv_common_cpu_init(unsigned int cpu); +int hv_common_cpu_die(unsigned int cpu); +void hv_identify_partition_type(void); + +/** + * hv_cpu_number_to_vp_number() - Map CPU to VP. + * @cpu_number: CPU number in Linux terms + * + * This function returns the mapping between the Linux processor + * number and the hypervisor's virtual processor number, useful + * in making hypercalls and such that talk about specific + * processors. + * + * Return: Virtual processor number in Hyper-V terms + */ +static inline int hv_cpu_number_to_vp_number(int cpu_number) +{ + return hv_vp_index[cpu_number]; +} + +static inline int __cpumask_to_vpset(struct hv_vpset *vpset, + const struct cpumask *cpus, + bool (*func)(int cpu)) +{ + int cpu, vcpu, vcpu_bank, vcpu_offset, nr_bank = 1; + int max_vcpu_bank = hv_max_vp_index / HV_VCPUS_PER_SPARSE_BANK; + + /* vpset.valid_bank_mask can represent up to HV_MAX_SPARSE_VCPU_BANKS banks */ + if (max_vcpu_bank >= HV_MAX_SPARSE_VCPU_BANKS) + return 0; + + /* + * Clear all banks up to the maximum possible bank as hv_tlb_flush_ex + * structs are not cleared between calls, we risk flushing unneeded + * vCPUs otherwise. + */ + for (vcpu_bank = 0; vcpu_bank <= max_vcpu_bank; vcpu_bank++) + vpset->bank_contents[vcpu_bank] = 0; + + /* + * Some banks may end up being empty but this is acceptable. + */ + for_each_cpu(cpu, cpus) { + if (func && func(cpu)) + continue; + vcpu = hv_cpu_number_to_vp_number(cpu); + if (vcpu == VP_INVAL) + return -1; + vcpu_bank = vcpu / HV_VCPUS_PER_SPARSE_BANK; + vcpu_offset = vcpu % HV_VCPUS_PER_SPARSE_BANK; + __set_bit(vcpu_offset, (unsigned long *) + &vpset->bank_contents[vcpu_bank]); + if (vcpu_bank >= nr_bank) + nr_bank = vcpu_bank + 1; + } + vpset->valid_bank_mask = GENMASK_ULL(nr_bank - 1, 0); + return nr_bank; +} + +/* + * Convert a Linux cpumask into a Hyper-V VPset. In the _skip variant, + * 'func' is called for each CPU present in cpumask. If 'func' returns + * true, that CPU is skipped -- i.e., that CPU from cpumask is *not* + * added to the Hyper-V VPset. If 'func' is NULL, no CPUs are + * skipped. + */ +static inline int cpumask_to_vpset(struct hv_vpset *vpset, + const struct cpumask *cpus) +{ + return __cpumask_to_vpset(vpset, cpus, NULL); +} + +static inline int cpumask_to_vpset_skip(struct hv_vpset *vpset, + const struct cpumask *cpus, + bool (*func)(int cpu)) +{ + return __cpumask_to_vpset(vpset, cpus, func); +} + +#define _hv_status_fmt(fmt) "%s: Hyper-V status: %#x = %s: " fmt +#define hv_status_printk(level, status, fmt, ...) \ +do { \ + u64 __status = (status); \ + pr_##level(_hv_status_fmt(fmt), __func__, hv_result(__status), \ + hv_result_to_string(__status), ##__VA_ARGS__); \ +} while (0) +#define hv_status_err(status, fmt, ...) \ + hv_status_printk(err, status, fmt, ##__VA_ARGS__) +#define hv_status_debug(status, fmt, ...) \ + hv_status_printk(debug, status, fmt, ##__VA_ARGS__) + +const char *hv_result_to_string(u64 hv_status); +int hv_result_to_errno(u64 status); +void hyperv_report_panic(struct pt_regs *regs, long err, bool in_die); +bool hv_is_hyperv_initialized(void); +bool hv_is_hibernation_supported(void); +enum hv_isolation_type hv_get_isolation_type(void); +bool hv_is_isolation_supported(void); +bool hv_isolation_type_snp(void); +u64 hv_ghcb_hypercall(u64 control, void *input, void *output, u32 input_size); +u64 hv_tdx_hypercall(u64 control, u64 param1, u64 param2); +void hv_enable_coco_interrupt(unsigned int cpu, unsigned int vector, bool set); +void hv_para_set_sint_proxy(bool enable); +u64 hv_para_get_synic_register(unsigned int reg); +void hv_para_set_synic_register(unsigned int reg, u64 val); +void hyperv_cleanup(void); +bool hv_query_ext_cap(u64 cap_query); +void hv_setup_dma_ops(struct device *dev, bool coherent); +#else /* CONFIG_HYPERV */ +static inline void hv_identify_partition_type(void) {} +static inline bool hv_is_hyperv_initialized(void) { return false; } +static inline bool hv_is_hibernation_supported(void) { return false; } +static inline void hyperv_cleanup(void) {} +static inline void ms_hyperv_late_init(void) {} +static inline bool hv_is_isolation_supported(void) { return false; } +static inline enum hv_isolation_type hv_get_isolation_type(void) +{ + return HV_ISOLATION_TYPE_NONE; +} +#endif /* CONFIG_HYPERV */ + +#if IS_ENABLED(CONFIG_MSHV_ROOT) +static inline bool hv_root_partition(void) +{ + return hv_curr_partition_type == HV_PARTITION_TYPE_ROOT; +} +static inline bool hv_l1vh_partition(void) +{ + return hv_curr_partition_type == HV_PARTITION_TYPE_L1VH; +} +static inline bool hv_parent_partition(void) +{ + return hv_root_partition() || hv_l1vh_partition(); +} +int hv_call_deposit_pages(int node, u64 partition_id, u32 num_pages); +int hv_call_add_logical_proc(int node, u32 lp_index, u32 acpi_id); +int hv_call_create_vp(int node, u64 partition_id, u32 vp_index, u32 flags); + +#else /* CONFIG_MSHV_ROOT */ +static inline bool hv_root_partition(void) { return false; } +static inline bool hv_l1vh_partition(void) { return false; } +static inline bool hv_parent_partition(void) { return false; } +static inline int hv_call_deposit_pages(int node, u64 partition_id, u32 num_pages) +{ + return -EOPNOTSUPP; +} +static inline int hv_call_add_logical_proc(int node, u32 lp_index, u32 acpi_id) +{ + return -EOPNOTSUPP; +} +static inline int hv_call_create_vp(int node, u64 partition_id, u32 vp_index, u32 flags) +{ + return -EOPNOTSUPP; +} +#endif /* CONFIG_MSHV_ROOT */ + +#if IS_ENABLED(CONFIG_HYPERV_VTL_MODE) +u8 __init get_vtl(void); +#else +static inline u8 get_vtl(void) { return 0; } +#endif + +#endif diff --git a/include/asm-generic/msi.h b/include/asm-generic/msi.h index e6795f088bdd..92cca4b23f13 100644 --- a/include/asm-generic/msi.h +++ b/include/asm-generic/msi.h @@ -4,6 +4,8 @@ #include <linux/types.h> +#ifdef CONFIG_GENERIC_MSI_IRQ + #ifndef NUM_MSI_ALLOC_SCRATCHPAD_REGS # define NUM_MSI_ALLOC_SCRATCHPAD_REGS 2 #endif @@ -22,12 +24,19 @@ struct msi_desc; typedef struct msi_alloc_info { struct msi_desc *desc; irq_hw_number_t hwirq; + unsigned long flags; union { unsigned long ul; void *ptr; } scratchpad[NUM_MSI_ALLOC_SCRATCHPAD_REGS]; } msi_alloc_info_t; +/* Device generating MSIs is proxying for another device */ +#define MSI_ALLOC_FLAGS_PROXY_DEVICE (1UL << 0) +#define MSI_ALLOC_FLAGS_FIXED_MSG_DATA (1UL << 1) + #define GENERIC_MSI_DOMAIN_OPS 1 +#endif /* CONFIG_GENERIC_MSI_IRQ */ + #endif diff --git a/include/asm-generic/nommu_context.h b/include/asm-generic/nommu_context.h new file mode 100644 index 000000000000..4f916f9e16cd --- /dev/null +++ b/include/asm-generic/nommu_context.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_GENERIC_NOMMU_H +#define __ASM_GENERIC_NOMMU_H + +/* + * Generic hooks for NOMMU architectures, which do not need to do + * anything special here. + */ +#include <asm-generic/mm_hooks.h> + +static inline void switch_mm(struct mm_struct *prev, + struct mm_struct *next, + struct task_struct *tsk) +{ +} + +#include <asm-generic/mmu_context.h> + +#endif /* __ASM_GENERIC_NOMMU_H */ diff --git a/include/asm-generic/numa.h b/include/asm-generic/numa.h new file mode 100644 index 000000000000..e063d6487f66 --- /dev/null +++ b/include/asm-generic/numa.h @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_GENERIC_NUMA_H +#define __ASM_GENERIC_NUMA_H + +#ifdef CONFIG_NUMA + +#define NR_NODE_MEMBLKS (MAX_NUMNODES * 2) + +int __node_distance(int from, int to); +#define node_distance(a, b) __node_distance(a, b) + +extern nodemask_t numa_nodes_parsed __initdata; + +extern bool numa_off; + +/* Mappings between node number and cpus on that node. */ +extern cpumask_var_t node_to_cpumask_map[MAX_NUMNODES]; +void numa_clear_node(unsigned int cpu); + +#ifdef CONFIG_DEBUG_PER_CPU_MAPS +const struct cpumask *cpumask_of_node(int node); +#else +/* Returns a pointer to the cpumask of CPUs on Node 'node'. */ +static inline const struct cpumask *cpumask_of_node(int node) +{ + if (node == NUMA_NO_NODE) + return cpu_all_mask; + + return node_to_cpumask_map[node]; +} +#endif + +void __init arch_numa_init(void); +int __init numa_add_memblk(int nodeid, u64 start, u64 end); +void __init early_map_cpu_to_node(unsigned int cpu, int nid); +int early_cpu_to_node(int cpu); +void numa_store_cpu_info(unsigned int cpu); +void numa_add_cpu(unsigned int cpu); +void numa_remove_cpu(unsigned int cpu); + +#else /* CONFIG_NUMA */ + +static inline void numa_store_cpu_info(unsigned int cpu) { } +static inline void numa_add_cpu(unsigned int cpu) { } +static inline void numa_remove_cpu(unsigned int cpu) { } +static inline void arch_numa_init(void) { } +static inline void early_map_cpu_to_node(unsigned int cpu, int nid) { } +static inline int early_cpu_to_node(int cpu) { return 0; } + +#endif /* CONFIG_NUMA */ + +#ifdef CONFIG_NUMA_EMU +void debug_cpumask_set_cpu(unsigned int cpu, int node, bool enable); +#endif + +#endif /* __ASM_GENERIC_NUMA_H */ diff --git a/include/asm-generic/page.h b/include/asm-generic/page.h deleted file mode 100644 index 27bf3377b0cb..000000000000 --- a/include/asm-generic/page.h +++ /dev/null @@ -1,101 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef __ASM_GENERIC_PAGE_H -#define __ASM_GENERIC_PAGE_H -/* - * Generic page.h implementation, for NOMMU architectures. - * This provides the dummy definitions for the memory management. - */ - -#ifdef CONFIG_MMU -#error need to prove a real asm/page.h -#endif - - -/* PAGE_SHIFT determines the page size */ - -#define PAGE_SHIFT 12 -#ifdef __ASSEMBLY__ -#define PAGE_SIZE (1 << PAGE_SHIFT) -#else -#define PAGE_SIZE (1UL << PAGE_SHIFT) -#endif -#define PAGE_MASK (~(PAGE_SIZE-1)) - -#include <asm/setup.h> - -#ifndef __ASSEMBLY__ - -#define clear_page(page) memset((page), 0, PAGE_SIZE) -#define copy_page(to,from) memcpy((to), (from), PAGE_SIZE) - -#define clear_user_page(page, vaddr, pg) clear_page(page) -#define copy_user_page(to, from, vaddr, pg) copy_page(to, from) - -/* - * These are used to make use of C type-checking.. - */ -typedef struct { - unsigned long pte; -} pte_t; -typedef struct { - unsigned long pmd[16]; -} pmd_t; -typedef struct { - unsigned long pgd; -} pgd_t; -typedef struct { - unsigned long pgprot; -} pgprot_t; -typedef struct page *pgtable_t; - -#define pte_val(x) ((x).pte) -#define pmd_val(x) ((&x)->pmd[0]) -#define pgd_val(x) ((x).pgd) -#define pgprot_val(x) ((x).pgprot) - -#define __pte(x) ((pte_t) { (x) } ) -#define __pmd(x) ((pmd_t) { (x) } ) -#define __pgd(x) ((pgd_t) { (x) } ) -#define __pgprot(x) ((pgprot_t) { (x) } ) - -extern unsigned long memory_start; -extern unsigned long memory_end; - -#endif /* !__ASSEMBLY__ */ - -#ifdef CONFIG_KERNEL_RAM_BASE_ADDRESS -#define PAGE_OFFSET (CONFIG_KERNEL_RAM_BASE_ADDRESS) -#else -#define PAGE_OFFSET (0) -#endif - -#ifndef ARCH_PFN_OFFSET -#define ARCH_PFN_OFFSET (PAGE_OFFSET >> PAGE_SHIFT) -#endif - -#ifndef __ASSEMBLY__ - -#define __va(x) ((void *)((unsigned long) (x))) -#define __pa(x) ((unsigned long) (x)) - -#define virt_to_pfn(kaddr) (__pa(kaddr) >> PAGE_SHIFT) -#define pfn_to_virt(pfn) __va((pfn) << PAGE_SHIFT) - -#define virt_to_page(addr) pfn_to_page(virt_to_pfn(addr)) -#define page_to_virt(page) pfn_to_virt(page_to_pfn(page)) - -#ifndef page_to_phys -#define page_to_phys(page) ((dma_addr_t)page_to_pfn(page) << PAGE_SHIFT) -#endif - -#define pfn_valid(pfn) ((pfn) >= ARCH_PFN_OFFSET && ((pfn) - ARCH_PFN_OFFSET) < max_mapnr) - -#define virt_addr_valid(kaddr) (((void *)(kaddr) >= (void *)PAGE_OFFSET) && \ - ((void *)(kaddr) < (void *)memory_end)) - -#endif /* __ASSEMBLY__ */ - -#include <asm-generic/memory_model.h> -#include <asm-generic/getorder.h> - -#endif /* __ASM_GENERIC_PAGE_H */ diff --git a/include/asm-generic/param.h b/include/asm-generic/param.h index 8d3009dd28ff..8348c116aa3b 100644 --- a/include/asm-generic/param.h +++ b/include/asm-generic/param.h @@ -6,6 +6,6 @@ # undef HZ # define HZ CONFIG_HZ /* Internal kernel timer frequency */ -# define USER_HZ 100 /* some user interfaces are */ +# define USER_HZ __USER_HZ /* some user interfaces are */ # define CLOCKS_PER_SEC (USER_HZ) /* in "ticks" like times() */ #endif /* __ASM_GENERIC_PARAM_H */ diff --git a/include/asm-generic/pci.h b/include/asm-generic/pci.h index 6bb3cd3d695a..6869f1061528 100644 --- a/include/asm-generic/pci.h +++ b/include/asm-generic/pci.h @@ -1,17 +1,30 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * linux/include/asm-generic/pci.h - * - * Copyright (C) 2003 Russell King - */ -#ifndef _ASM_GENERIC_PCI_H -#define _ASM_GENERIC_PCI_H +/* SPDX-License-Identifier: GPL-2.0-only */ -#ifndef HAVE_ARCH_PCI_GET_LEGACY_IDE_IRQ -static inline int pci_get_legacy_ide_irq(struct pci_dev *dev, int channel) +#ifndef __ASM_GENERIC_PCI_H +#define __ASM_GENERIC_PCI_H + +#ifndef PCIBIOS_MIN_IO +#define PCIBIOS_MIN_IO 0 +#endif + +#ifndef PCIBIOS_MIN_MEM +#define PCIBIOS_MIN_MEM 0 +#endif + +#ifndef pcibios_assign_all_busses +/* For bootloaders that do not initialize the PCI bus */ +#define pcibios_assign_all_busses() 1 +#endif + +/* Enable generic resource mapping code in drivers/pci/ */ +#define ARCH_GENERIC_PCI_MMAP_RESOURCE + +#ifdef CONFIG_PCI_DOMAINS +static inline int pci_proc_domain(struct pci_bus *bus) { - return channel ? 15 : 14; + /* always show the domain in /proc */ + return 1; } -#endif /* HAVE_ARCH_PCI_GET_LEGACY_IDE_IRQ */ +#endif /* CONFIG_PCI_DOMAINS */ -#endif /* _ASM_GENERIC_PCI_H */ +#endif /* __ASM_GENERIC_PCI_H */ diff --git a/include/asm-generic/pci_iomap.h b/include/asm-generic/pci_iomap.h index d4f16dcc2ed7..8fbb0a55545d 100644 --- a/include/asm-generic/pci_iomap.h +++ b/include/asm-generic/pci_iomap.h @@ -18,12 +18,15 @@ extern void __iomem *pci_iomap_range(struct pci_dev *dev, int bar, extern void __iomem *pci_iomap_wc_range(struct pci_dev *dev, int bar, unsigned long offset, unsigned long maxlen); +extern void pci_iounmap(struct pci_dev *dev, void __iomem *); /* Create a virtual mapping cookie for a port on a given PCI device. * Do not call this directly, it exists to make it easier for architectures * to override */ #ifdef CONFIG_NO_GENERIC_PCI_IOPORT_MAP extern void __iomem *__pci_ioport_map(struct pci_dev *dev, unsigned long port, unsigned int nr); +#elif !defined(CONFIG_HAS_IOPORT_MAP) +#define __pci_ioport_map(dev, port, nr) NULL #else #define __pci_ioport_map(dev, port, nr) ioport_map((port), (nr)) #endif @@ -50,6 +53,8 @@ static inline void __iomem *pci_iomap_wc_range(struct pci_dev *dev, int bar, { return NULL; } +static inline void pci_iounmap(struct pci_dev *dev, void __iomem *addr) +{ } #endif -#endif /* __ASM_GENERIC_IO_H */ +#endif /* __ASM_GENERIC_PCI_IOMAP_H */ diff --git a/include/asm-generic/percpu.h b/include/asm-generic/percpu.h index c2de013b2cf4..6628670bcb90 100644 --- a/include/asm-generic/percpu.h +++ b/include/asm-generic/percpu.h @@ -2,10 +2,25 @@ #ifndef _ASM_GENERIC_PERCPU_H_ #define _ASM_GENERIC_PERCPU_H_ +#ifndef __ASSEMBLER__ + #include <linux/compiler.h> #include <linux/threads.h> #include <linux/percpu-defs.h> +/* + * __percpu_qual is the qualifier for the percpu named address space. + * + * Most arches use generic named address space for percpu variables but + * some arches define percpu variables in different named address space + * (on the x86 arch, percpu variable may be declared as being relative + * to the %fs or %gs segments using __seg_fs or __seg_gs named address + * space qualifier). + */ +#ifndef __percpu_qual +# define __percpu_qual +#endif + #ifdef CONFIG_SMP /* @@ -74,7 +89,7 @@ do { \ #define raw_cpu_generic_add_return(pcp, val) \ ({ \ - typeof(&(pcp)) __p = raw_cpu_ptr(&(pcp)); \ + TYPEOF_UNQUAL(pcp) *__p = raw_cpu_ptr(&(pcp)); \ \ *__p += val; \ *__p; \ @@ -82,58 +97,66 @@ do { \ #define raw_cpu_generic_xchg(pcp, nval) \ ({ \ - typeof(&(pcp)) __p = raw_cpu_ptr(&(pcp)); \ - typeof(pcp) __ret; \ + TYPEOF_UNQUAL(pcp) *__p = raw_cpu_ptr(&(pcp)); \ + TYPEOF_UNQUAL(pcp) __ret; \ __ret = *__p; \ *__p = nval; \ __ret; \ }) -#define raw_cpu_generic_cmpxchg(pcp, oval, nval) \ +#define __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, _cmpxchg) \ ({ \ - typeof(&(pcp)) __p = raw_cpu_ptr(&(pcp)); \ - typeof(pcp) __ret; \ - __ret = *__p; \ - if (__ret == (oval)) \ + TYPEOF_UNQUAL(pcp) __val, __old = *(ovalp); \ + __val = _cmpxchg(pcp, __old, nval); \ + if (__val != __old) \ + *(ovalp) = __val; \ + __val == __old; \ +}) + +#define raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval) \ +({ \ + TYPEOF_UNQUAL(pcp) *__p = raw_cpu_ptr(&(pcp)); \ + TYPEOF_UNQUAL(pcp) __val = *__p, ___old = *(ovalp); \ + bool __ret; \ + if (__val == ___old) { \ *__p = nval; \ + __ret = true; \ + } else { \ + *(ovalp) = __val; \ + __ret = false; \ + } \ __ret; \ }) -#define raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \ +#define raw_cpu_generic_cmpxchg(pcp, oval, nval) \ ({ \ - typeof(&(pcp1)) __p1 = raw_cpu_ptr(&(pcp1)); \ - typeof(&(pcp2)) __p2 = raw_cpu_ptr(&(pcp2)); \ - int __ret = 0; \ - if (*__p1 == (oval1) && *__p2 == (oval2)) { \ - *__p1 = nval1; \ - *__p2 = nval2; \ - __ret = 1; \ - } \ - (__ret); \ + TYPEOF_UNQUAL(pcp) __old = (oval); \ + raw_cpu_generic_try_cmpxchg(pcp, &__old, nval); \ + __old; \ }) #define __this_cpu_generic_read_nopreempt(pcp) \ ({ \ - typeof(pcp) __ret; \ + TYPEOF_UNQUAL(pcp) ___ret; \ preempt_disable_notrace(); \ - __ret = READ_ONCE(*raw_cpu_ptr(&(pcp))); \ + ___ret = READ_ONCE(*raw_cpu_ptr(&(pcp))); \ preempt_enable_notrace(); \ - __ret; \ + ___ret; \ }) #define __this_cpu_generic_read_noirq(pcp) \ ({ \ - typeof(pcp) __ret; \ - unsigned long __flags; \ - raw_local_irq_save(__flags); \ - __ret = raw_cpu_generic_read(pcp); \ - raw_local_irq_restore(__flags); \ - __ret; \ + TYPEOF_UNQUAL(pcp) ___ret; \ + unsigned long ___flags; \ + raw_local_irq_save(___flags); \ + ___ret = raw_cpu_generic_read(pcp); \ + raw_local_irq_restore(___flags); \ + ___ret; \ }) #define this_cpu_generic_read(pcp) \ ({ \ - typeof(pcp) __ret; \ + TYPEOF_UNQUAL(pcp) __ret; \ if (__native_word(pcp)) \ __ret = __this_cpu_generic_read_nopreempt(pcp); \ else \ @@ -152,7 +175,7 @@ do { \ #define this_cpu_generic_add_return(pcp, val) \ ({ \ - typeof(pcp) __ret; \ + TYPEOF_UNQUAL(pcp) __ret; \ unsigned long __flags; \ raw_local_irq_save(__flags); \ __ret = raw_cpu_generic_add_return(pcp, val); \ @@ -162,7 +185,7 @@ do { \ #define this_cpu_generic_xchg(pcp, nval) \ ({ \ - typeof(pcp) __ret; \ + TYPEOF_UNQUAL(pcp) __ret; \ unsigned long __flags; \ raw_local_irq_save(__flags); \ __ret = raw_cpu_generic_xchg(pcp, nval); \ @@ -170,23 +193,22 @@ do { \ __ret; \ }) -#define this_cpu_generic_cmpxchg(pcp, oval, nval) \ +#define this_cpu_generic_try_cmpxchg(pcp, ovalp, nval) \ ({ \ - typeof(pcp) __ret; \ + bool __ret; \ unsigned long __flags; \ raw_local_irq_save(__flags); \ - __ret = raw_cpu_generic_cmpxchg(pcp, oval, nval); \ + __ret = raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval); \ raw_local_irq_restore(__flags); \ __ret; \ }) -#define this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \ +#define this_cpu_generic_cmpxchg(pcp, oval, nval) \ ({ \ - int __ret; \ + TYPEOF_UNQUAL(pcp) __ret; \ unsigned long __flags; \ raw_local_irq_save(__flags); \ - __ret = raw_cpu_generic_cmpxchg_double(pcp1, pcp2, \ - oval1, oval2, nval1, nval2); \ + __ret = raw_cpu_generic_cmpxchg(pcp, oval, nval); \ raw_local_irq_restore(__flags); \ __ret; \ }) @@ -282,6 +304,62 @@ do { \ #define raw_cpu_xchg_8(pcp, nval) raw_cpu_generic_xchg(pcp, nval) #endif +#ifndef raw_cpu_try_cmpxchg_1 +#ifdef raw_cpu_cmpxchg_1 +#define raw_cpu_try_cmpxchg_1(pcp, ovalp, nval) \ + __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg_1) +#else +#define raw_cpu_try_cmpxchg_1(pcp, ovalp, nval) \ + raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval) +#endif +#endif +#ifndef raw_cpu_try_cmpxchg_2 +#ifdef raw_cpu_cmpxchg_2 +#define raw_cpu_try_cmpxchg_2(pcp, ovalp, nval) \ + __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg_2) +#else +#define raw_cpu_try_cmpxchg_2(pcp, ovalp, nval) \ + raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval) +#endif +#endif +#ifndef raw_cpu_try_cmpxchg_4 +#ifdef raw_cpu_cmpxchg_4 +#define raw_cpu_try_cmpxchg_4(pcp, ovalp, nval) \ + __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg_4) +#else +#define raw_cpu_try_cmpxchg_4(pcp, ovalp, nval) \ + raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval) +#endif +#endif +#ifndef raw_cpu_try_cmpxchg_8 +#ifdef raw_cpu_cmpxchg_8 +#define raw_cpu_try_cmpxchg_8(pcp, ovalp, nval) \ + __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg_8) +#else +#define raw_cpu_try_cmpxchg_8(pcp, ovalp, nval) \ + raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval) +#endif +#endif + +#ifndef raw_cpu_try_cmpxchg64 +#ifdef raw_cpu_cmpxchg64 +#define raw_cpu_try_cmpxchg64(pcp, ovalp, nval) \ + __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg64) +#else +#define raw_cpu_try_cmpxchg64(pcp, ovalp, nval) \ + raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval) +#endif +#endif +#ifndef raw_cpu_try_cmpxchg128 +#ifdef raw_cpu_cmpxchg128 +#define raw_cpu_try_cmpxchg128(pcp, ovalp, nval) \ + __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg128) +#else +#define raw_cpu_try_cmpxchg128(pcp, ovalp, nval) \ + raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval) +#endif +#endif + #ifndef raw_cpu_cmpxchg_1 #define raw_cpu_cmpxchg_1(pcp, oval, nval) \ raw_cpu_generic_cmpxchg(pcp, oval, nval) @@ -299,21 +377,13 @@ do { \ raw_cpu_generic_cmpxchg(pcp, oval, nval) #endif -#ifndef raw_cpu_cmpxchg_double_1 -#define raw_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \ - raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) -#endif -#ifndef raw_cpu_cmpxchg_double_2 -#define raw_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \ - raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) -#endif -#ifndef raw_cpu_cmpxchg_double_4 -#define raw_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \ - raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) +#ifndef raw_cpu_cmpxchg64 +#define raw_cpu_cmpxchg64(pcp, oval, nval) \ + raw_cpu_generic_cmpxchg(pcp, oval, nval) #endif -#ifndef raw_cpu_cmpxchg_double_8 -#define raw_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \ - raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) +#ifndef raw_cpu_cmpxchg128 +#define raw_cpu_cmpxchg128(pcp, oval, nval) \ + raw_cpu_generic_cmpxchg(pcp, oval, nval) #endif #ifndef this_cpu_read_1 @@ -407,6 +477,62 @@ do { \ #define this_cpu_xchg_8(pcp, nval) this_cpu_generic_xchg(pcp, nval) #endif +#ifndef this_cpu_try_cmpxchg_1 +#ifdef this_cpu_cmpxchg_1 +#define this_cpu_try_cmpxchg_1(pcp, ovalp, nval) \ + __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg_1) +#else +#define this_cpu_try_cmpxchg_1(pcp, ovalp, nval) \ + this_cpu_generic_try_cmpxchg(pcp, ovalp, nval) +#endif +#endif +#ifndef this_cpu_try_cmpxchg_2 +#ifdef this_cpu_cmpxchg_2 +#define this_cpu_try_cmpxchg_2(pcp, ovalp, nval) \ + __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg_2) +#else +#define this_cpu_try_cmpxchg_2(pcp, ovalp, nval) \ + this_cpu_generic_try_cmpxchg(pcp, ovalp, nval) +#endif +#endif +#ifndef this_cpu_try_cmpxchg_4 +#ifdef this_cpu_cmpxchg_4 +#define this_cpu_try_cmpxchg_4(pcp, ovalp, nval) \ + __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg_4) +#else +#define this_cpu_try_cmpxchg_4(pcp, ovalp, nval) \ + this_cpu_generic_try_cmpxchg(pcp, ovalp, nval) +#endif +#endif +#ifndef this_cpu_try_cmpxchg_8 +#ifdef this_cpu_cmpxchg_8 +#define this_cpu_try_cmpxchg_8(pcp, ovalp, nval) \ + __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg_8) +#else +#define this_cpu_try_cmpxchg_8(pcp, ovalp, nval) \ + this_cpu_generic_try_cmpxchg(pcp, ovalp, nval) +#endif +#endif + +#ifndef this_cpu_try_cmpxchg64 +#ifdef this_cpu_cmpxchg64 +#define this_cpu_try_cmpxchg64(pcp, ovalp, nval) \ + __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg64) +#else +#define this_cpu_try_cmpxchg64(pcp, ovalp, nval) \ + this_cpu_generic_try_cmpxchg(pcp, ovalp, nval) +#endif +#endif +#ifndef this_cpu_try_cmpxchg128 +#ifdef this_cpu_cmpxchg128 +#define this_cpu_try_cmpxchg128(pcp, ovalp, nval) \ + __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg128) +#else +#define this_cpu_try_cmpxchg128(pcp, ovalp, nval) \ + this_cpu_generic_try_cmpxchg(pcp, ovalp, nval) +#endif +#endif + #ifndef this_cpu_cmpxchg_1 #define this_cpu_cmpxchg_1(pcp, oval, nval) \ this_cpu_generic_cmpxchg(pcp, oval, nval) @@ -424,21 +550,14 @@ do { \ this_cpu_generic_cmpxchg(pcp, oval, nval) #endif -#ifndef this_cpu_cmpxchg_double_1 -#define this_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \ - this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) -#endif -#ifndef this_cpu_cmpxchg_double_2 -#define this_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \ - this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) -#endif -#ifndef this_cpu_cmpxchg_double_4 -#define this_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \ - this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) +#ifndef this_cpu_cmpxchg64 +#define this_cpu_cmpxchg64(pcp, oval, nval) \ + this_cpu_generic_cmpxchg(pcp, oval, nval) #endif -#ifndef this_cpu_cmpxchg_double_8 -#define this_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \ - this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) +#ifndef this_cpu_cmpxchg128 +#define this_cpu_cmpxchg128(pcp, oval, nval) \ + this_cpu_generic_cmpxchg(pcp, oval, nval) #endif +#endif /* __ASSEMBLER__ */ #endif /* _ASM_GENERIC_PERCPU_H_ */ diff --git a/include/asm-generic/pgalloc.h b/include/asm-generic/pgalloc.h index 948714c1535a..57137d3ac159 100644 --- a/include/asm-generic/pgalloc.h +++ b/include/asm-generic/pgalloc.h @@ -1,13 +1,315 @@ /* SPDX-License-Identifier: GPL-2.0 */ #ifndef __ASM_GENERIC_PGALLOC_H #define __ASM_GENERIC_PGALLOC_H + +#ifdef CONFIG_MMU + +#define GFP_PGTABLE_KERNEL (GFP_KERNEL | __GFP_ZERO) +#define GFP_PGTABLE_USER (GFP_PGTABLE_KERNEL | __GFP_ACCOUNT) + +/** + * __pte_alloc_one_kernel - allocate memory for a PTE-level kernel page table + * @mm: the mm_struct of the current context + * + * This function is intended for architectures that need + * anything beyond simple page allocation. + * + * Return: pointer to the allocated memory or %NULL on error + */ +static inline pte_t *__pte_alloc_one_kernel_noprof(struct mm_struct *mm) +{ + struct ptdesc *ptdesc = pagetable_alloc_noprof(GFP_PGTABLE_KERNEL, 0); + + if (!ptdesc) + return NULL; + if (!pagetable_pte_ctor(mm, ptdesc)) { + pagetable_free(ptdesc); + return NULL; + } + + ptdesc_set_kernel(ptdesc); + + return ptdesc_address(ptdesc); +} +#define __pte_alloc_one_kernel(...) alloc_hooks(__pte_alloc_one_kernel_noprof(__VA_ARGS__)) + +#ifndef __HAVE_ARCH_PTE_ALLOC_ONE_KERNEL +/** + * pte_alloc_one_kernel - allocate memory for a PTE-level kernel page table + * @mm: the mm_struct of the current context + * + * Return: pointer to the allocated memory or %NULL on error + */ +static inline pte_t *pte_alloc_one_kernel_noprof(struct mm_struct *mm) +{ + return __pte_alloc_one_kernel_noprof(mm); +} +#define pte_alloc_one_kernel(...) alloc_hooks(pte_alloc_one_kernel_noprof(__VA_ARGS__)) +#endif + +/** + * pte_free_kernel - free PTE-level kernel page table memory + * @mm: the mm_struct of the current context + * @pte: pointer to the memory containing the page table + */ +static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) +{ + pagetable_dtor_free(virt_to_ptdesc(pte)); +} + +/** + * __pte_alloc_one - allocate memory for a PTE-level user page table + * @mm: the mm_struct of the current context + * @gfp: GFP flags to use for the allocation + * + * Allocate memory for a page table and ptdesc and runs pagetable_pte_ctor(). + * + * This function is intended for architectures that need + * anything beyond simple page allocation or must have custom GFP flags. + * + * Return: `struct page` referencing the ptdesc or %NULL on error + */ +static inline pgtable_t __pte_alloc_one_noprof(struct mm_struct *mm, gfp_t gfp) +{ + struct ptdesc *ptdesc; + + ptdesc = pagetable_alloc_noprof(gfp, 0); + if (!ptdesc) + return NULL; + if (!pagetable_pte_ctor(mm, ptdesc)) { + pagetable_free(ptdesc); + return NULL; + } + + return ptdesc_page(ptdesc); +} +#define __pte_alloc_one(...) alloc_hooks(__pte_alloc_one_noprof(__VA_ARGS__)) + +#ifndef __HAVE_ARCH_PTE_ALLOC_ONE +/** + * pte_alloc_one - allocate a page for PTE-level user page table + * @mm: the mm_struct of the current context + * + * Allocate memory for a page table and ptdesc and runs pagetable_pte_ctor(). + * + * Return: `struct page` referencing the ptdesc or %NULL on error + */ +static inline pgtable_t pte_alloc_one_noprof(struct mm_struct *mm) +{ + return __pte_alloc_one_noprof(mm, GFP_PGTABLE_USER); +} +#define pte_alloc_one(...) alloc_hooks(pte_alloc_one_noprof(__VA_ARGS__)) +#endif + /* - * an empty file is enough for a nommu architecture + * Should really implement gc for free page table pages. This could be + * done with a reference count in struct page. */ -#ifdef CONFIG_MMU -#error need to implement an architecture specific asm/pgalloc.h + +/** + * pte_free - free PTE-level user page table memory + * @mm: the mm_struct of the current context + * @pte_page: the `struct page` referencing the ptdesc + */ +static inline void pte_free(struct mm_struct *mm, struct page *pte_page) +{ + struct ptdesc *ptdesc = page_ptdesc(pte_page); + + pagetable_dtor_free(ptdesc); +} + + +#if CONFIG_PGTABLE_LEVELS > 2 + +#ifndef __HAVE_ARCH_PMD_ALLOC_ONE +/** + * pmd_alloc_one - allocate memory for a PMD-level page table + * @mm: the mm_struct of the current context + * + * Allocate memory for a page table and ptdesc and runs pagetable_pmd_ctor(). + * + * Allocations use %GFP_PGTABLE_USER in user context and + * %GFP_PGTABLE_KERNEL in kernel context. + * + * Return: pointer to the allocated memory or %NULL on error + */ +static inline pmd_t *pmd_alloc_one_noprof(struct mm_struct *mm, unsigned long addr) +{ + struct ptdesc *ptdesc; + gfp_t gfp = GFP_PGTABLE_USER; + + if (mm == &init_mm) + gfp = GFP_PGTABLE_KERNEL; + ptdesc = pagetable_alloc_noprof(gfp, 0); + if (!ptdesc) + return NULL; + if (!pagetable_pmd_ctor(mm, ptdesc)) { + pagetable_free(ptdesc); + return NULL; + } + + if (mm == &init_mm) + ptdesc_set_kernel(ptdesc); + + return ptdesc_address(ptdesc); +} +#define pmd_alloc_one(...) alloc_hooks(pmd_alloc_one_noprof(__VA_ARGS__)) +#endif + +#ifndef __HAVE_ARCH_PMD_FREE +static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) +{ + struct ptdesc *ptdesc = virt_to_ptdesc(pmd); + + BUG_ON((unsigned long)pmd & (PAGE_SIZE-1)); + pagetable_dtor_free(ptdesc); +} +#endif + +#endif /* CONFIG_PGTABLE_LEVELS > 2 */ + +#if CONFIG_PGTABLE_LEVELS > 3 + +static inline pud_t *__pud_alloc_one_noprof(struct mm_struct *mm, unsigned long addr) +{ + gfp_t gfp = GFP_PGTABLE_USER; + struct ptdesc *ptdesc; + + if (mm == &init_mm) + gfp = GFP_PGTABLE_KERNEL; + + ptdesc = pagetable_alloc_noprof(gfp, 0); + if (!ptdesc) + return NULL; + + pagetable_pud_ctor(ptdesc); + + if (mm == &init_mm) + ptdesc_set_kernel(ptdesc); + + return ptdesc_address(ptdesc); +} +#define __pud_alloc_one(...) alloc_hooks(__pud_alloc_one_noprof(__VA_ARGS__)) + +#ifndef __HAVE_ARCH_PUD_ALLOC_ONE +/** + * pud_alloc_one - allocate memory for a PUD-level page table + * @mm: the mm_struct of the current context + * + * Allocate memory for a page table using %GFP_PGTABLE_USER for user context + * and %GFP_PGTABLE_KERNEL for kernel context. + * + * Return: pointer to the allocated memory or %NULL on error + */ +static inline pud_t *pud_alloc_one_noprof(struct mm_struct *mm, unsigned long addr) +{ + return __pud_alloc_one_noprof(mm, addr); +} +#define pud_alloc_one(...) alloc_hooks(pud_alloc_one_noprof(__VA_ARGS__)) +#endif + +static inline void __pud_free(struct mm_struct *mm, pud_t *pud) +{ + struct ptdesc *ptdesc = virt_to_ptdesc(pud); + + BUG_ON((unsigned long)pud & (PAGE_SIZE-1)); + pagetable_dtor_free(ptdesc); +} + +#ifndef __HAVE_ARCH_PUD_FREE +static inline void pud_free(struct mm_struct *mm, pud_t *pud) +{ + __pud_free(mm, pud); +} +#endif + +#endif /* CONFIG_PGTABLE_LEVELS > 3 */ + +#if CONFIG_PGTABLE_LEVELS > 4 + +static inline p4d_t *__p4d_alloc_one_noprof(struct mm_struct *mm, unsigned long addr) +{ + gfp_t gfp = GFP_PGTABLE_USER; + struct ptdesc *ptdesc; + + if (mm == &init_mm) + gfp = GFP_PGTABLE_KERNEL; + + ptdesc = pagetable_alloc_noprof(gfp, 0); + if (!ptdesc) + return NULL; + + pagetable_p4d_ctor(ptdesc); + + if (mm == &init_mm) + ptdesc_set_kernel(ptdesc); + + return ptdesc_address(ptdesc); +} +#define __p4d_alloc_one(...) alloc_hooks(__p4d_alloc_one_noprof(__VA_ARGS__)) + +#ifndef __HAVE_ARCH_P4D_ALLOC_ONE +static inline p4d_t *p4d_alloc_one_noprof(struct mm_struct *mm, unsigned long addr) +{ + return __p4d_alloc_one_noprof(mm, addr); +} +#define p4d_alloc_one(...) alloc_hooks(p4d_alloc_one_noprof(__VA_ARGS__)) +#endif + +static inline void __p4d_free(struct mm_struct *mm, p4d_t *p4d) +{ + struct ptdesc *ptdesc = virt_to_ptdesc(p4d); + + BUG_ON((unsigned long)p4d & (PAGE_SIZE-1)); + pagetable_dtor_free(ptdesc); +} + +#ifndef __HAVE_ARCH_P4D_FREE +static inline void p4d_free(struct mm_struct *mm, p4d_t *p4d) +{ + if (!mm_p4d_folded(mm)) + __p4d_free(mm, p4d); +} +#endif + +#endif /* CONFIG_PGTABLE_LEVELS > 4 */ + +static inline pgd_t *__pgd_alloc_noprof(struct mm_struct *mm, unsigned int order) +{ + gfp_t gfp = GFP_PGTABLE_USER; + struct ptdesc *ptdesc; + + if (mm == &init_mm) + gfp = GFP_PGTABLE_KERNEL; + + ptdesc = pagetable_alloc_noprof(gfp, order); + if (!ptdesc) + return NULL; + + pagetable_pgd_ctor(ptdesc); + + if (mm == &init_mm) + ptdesc_set_kernel(ptdesc); + + return ptdesc_address(ptdesc); +} +#define __pgd_alloc(...) alloc_hooks(__pgd_alloc_noprof(__VA_ARGS__)) + +static inline void __pgd_free(struct mm_struct *mm, pgd_t *pgd) +{ + struct ptdesc *ptdesc = virt_to_ptdesc(pgd); + + BUG_ON((unsigned long)pgd & (PAGE_SIZE-1)); + pagetable_dtor_free(ptdesc); +} + +#ifndef __HAVE_ARCH_PGD_FREE +static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd) +{ + __pgd_free(mm, pgd); +} #endif -#define check_pgt_cache() do { } while (0) +#endif /* CONFIG_MMU */ #endif /* __ASM_GENERIC_PGALLOC_H */ diff --git a/include/asm-generic/pgtable-nop4d-hack.h b/include/asm-generic/pgtable-nop4d-hack.h deleted file mode 100644 index 829bdb0d6327..000000000000 --- a/include/asm-generic/pgtable-nop4d-hack.h +++ /dev/null @@ -1,64 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _PGTABLE_NOP4D_HACK_H -#define _PGTABLE_NOP4D_HACK_H - -#ifndef __ASSEMBLY__ -#include <asm-generic/5level-fixup.h> - -#define __PAGETABLE_PUD_FOLDED 1 - -/* - * Having the pud type consist of a pgd gets the size right, and allows - * us to conceptually access the pgd entry that this pud is folded into - * without casting. - */ -typedef struct { pgd_t pgd; } pud_t; - -#define PUD_SHIFT PGDIR_SHIFT -#define PTRS_PER_PUD 1 -#define PUD_SIZE (1UL << PUD_SHIFT) -#define PUD_MASK (~(PUD_SIZE-1)) - -/* - * The "pgd_xxx()" functions here are trivial for a folded two-level - * setup: the pud is never bad, and a pud always exists (as it's folded - * into the pgd entry) - */ -static inline int pgd_none(pgd_t pgd) { return 0; } -static inline int pgd_bad(pgd_t pgd) { return 0; } -static inline int pgd_present(pgd_t pgd) { return 1; } -static inline void pgd_clear(pgd_t *pgd) { } -#define pud_ERROR(pud) (pgd_ERROR((pud).pgd)) - -#define pgd_populate(mm, pgd, pud) do { } while (0) -#define pgd_populate_safe(mm, pgd, pud) do { } while (0) -/* - * (puds are folded into pgds so this doesn't get actually called, - * but the define is needed for a generic inline function.) - */ -#define set_pgd(pgdptr, pgdval) set_pud((pud_t *)(pgdptr), (pud_t) { pgdval }) - -static inline pud_t *pud_offset(pgd_t *pgd, unsigned long address) -{ - return (pud_t *)pgd; -} - -#define pud_val(x) (pgd_val((x).pgd)) -#define __pud(x) ((pud_t) { __pgd(x) }) - -#define pgd_page(pgd) (pud_page((pud_t){ pgd })) -#define pgd_page_vaddr(pgd) (pud_page_vaddr((pud_t){ pgd })) - -/* - * allocating and freeing a pud is trivial: the 1-entry pud is - * inside the pgd, so has no extra memory associated with it. - */ -#define pud_alloc_one(mm, address) NULL -#define pud_free(mm, x) do { } while (0) -#define __pud_free_tlb(tlb, x, a) do { } while (0) - -#undef pud_addr_end -#define pud_addr_end(addr, end) (end) - -#endif /* __ASSEMBLY__ */ -#endif /* _PGTABLE_NOP4D_HACK_H */ diff --git a/include/asm-generic/pgtable-nop4d.h b/include/asm-generic/pgtable-nop4d.h index aebab905e6cd..03b7dae47dd4 100644 --- a/include/asm-generic/pgtable-nop4d.h +++ b/include/asm-generic/pgtable-nop4d.h @@ -9,7 +9,6 @@ typedef struct { pgd_t pgd; } p4d_t; #define P4D_SHIFT PGDIR_SHIFT -#define MAX_PTRS_PER_P4D 1 #define PTRS_PER_P4D 1 #define P4D_SIZE (1UL << P4D_SHIFT) #define P4D_MASK (~(P4D_SIZE-1)) @@ -42,7 +41,7 @@ static inline p4d_t *p4d_offset(pgd_t *pgd, unsigned long address) #define __p4d(x) ((p4d_t) { __pgd(x) }) #define pgd_page(pgd) (p4d_page((p4d_t){ pgd })) -#define pgd_page_vaddr(pgd) (p4d_page_vaddr((p4d_t){ pgd })) +#define pgd_page_vaddr(pgd) ((unsigned long)(p4d_pgtable((p4d_t){ pgd }))) /* * allocating and freeing a p4d is trivial: the 1-entry p4d is @@ -50,7 +49,7 @@ static inline p4d_t *p4d_offset(pgd_t *pgd, unsigned long address) */ #define p4d_alloc_one(mm, address) NULL #define p4d_free(mm, x) do { } while (0) -#define __p4d_free_tlb(tlb, x, a) do { } while (0) +#define p4d_free_tlb(tlb, x, a) do { } while (0) #undef p4d_addr_end #define p4d_addr_end(addr, end) (end) diff --git a/include/asm-generic/pgtable-nopmd.h b/include/asm-generic/pgtable-nopmd.h index b85b8271a73d..8ffd64e7a24c 100644 --- a/include/asm-generic/pgtable-nopmd.h +++ b/include/asm-generic/pgtable-nopmd.h @@ -30,6 +30,8 @@ typedef struct { pud_t pud; } pmd_t; static inline int pud_none(pud_t pud) { return 0; } static inline int pud_bad(pud_t pud) { return 0; } static inline int pud_present(pud_t pud) { return 1; } +static inline int pud_user(pud_t pud) { return 0; } +static inline int pud_leaf(pud_t pud) { return 0; } static inline void pud_clear(pud_t *pud) { } #define pmd_ERROR(pmd) (pud_ERROR((pmd).pud)) @@ -45,12 +47,13 @@ static inline pmd_t * pmd_offset(pud_t * pud, unsigned long address) { return (pmd_t *)pud; } +#define pmd_offset pmd_offset #define pmd_val(x) (pud_val((x).pud)) #define __pmd(x) ((pmd_t) { __pud(x) } ) #define pud_page(pud) (pmd_page((pmd_t){ pud })) -#define pud_page_vaddr(pud) (pmd_page_vaddr((pmd_t){ pud })) +#define pud_pgtable(pud) ((pmd_t *)(pmd_page_vaddr((pmd_t){ pud }))) /* * allocating and freeing a pmd is trivial: the 1-entry pmd is @@ -60,7 +63,7 @@ static inline pmd_t * pmd_offset(pud_t * pud, unsigned long address) static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) { } -#define __pmd_free_tlb(tlb, x, a) do { } while (0) +#define pmd_free_tlb(tlb, x, a) do { } while (0) #undef pmd_addr_end #define pmd_addr_end(addr, end) (end) diff --git a/include/asm-generic/pgtable-nopud.h b/include/asm-generic/pgtable-nopud.h index c77a1d301155..eb70c6d7ceff 100644 --- a/include/asm-generic/pgtable-nopud.h +++ b/include/asm-generic/pgtable-nopud.h @@ -4,9 +4,6 @@ #ifndef __ASSEMBLY__ -#ifdef __ARCH_USE_5LEVEL_HACK -#include <asm-generic/pgtable-nop4d-hack.h> -#else #include <asm-generic/pgtable-nop4d.h> #define __PAGETABLE_PUD_FOLDED 1 @@ -46,12 +43,13 @@ static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address) { return (pud_t *)p4d; } +#define pud_offset pud_offset #define pud_val(x) (p4d_val((x).p4d)) #define __pud(x) ((pud_t) { __p4d(x) }) #define p4d_page(p4d) (pud_page((pud_t){ p4d })) -#define p4d_page_vaddr(p4d) (pud_page_vaddr((pud_t){ p4d })) +#define p4d_pgtable(p4d) ((pud_t *)(pud_pgtable((pud_t){ p4d }))) /* * allocating and freeing a pud is trivial: the 1-entry pud is @@ -59,11 +57,10 @@ static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address) */ #define pud_alloc_one(mm, address) NULL #define pud_free(mm, x) do { } while (0) -#define __pud_free_tlb(tlb, x, a) do { } while (0) +#define pud_free_tlb(tlb, x, a) do { } while (0) #undef pud_addr_end #define pud_addr_end(addr, end) (end) #endif /* __ASSEMBLY__ */ -#endif /* !__ARCH_USE_5LEVEL_HACK */ #endif /* _PGTABLE_NOPUD_H */ diff --git a/include/asm-generic/pgtable.h b/include/asm-generic/pgtable.h deleted file mode 100644 index 05e61e6c843f..000000000000 --- a/include/asm-generic/pgtable.h +++ /dev/null @@ -1,1189 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _ASM_GENERIC_PGTABLE_H -#define _ASM_GENERIC_PGTABLE_H - -#include <linux/pfn.h> - -#ifndef __ASSEMBLY__ -#ifdef CONFIG_MMU - -#include <linux/mm_types.h> -#include <linux/bug.h> -#include <linux/errno.h> - -#if 5 - defined(__PAGETABLE_P4D_FOLDED) - defined(__PAGETABLE_PUD_FOLDED) - \ - defined(__PAGETABLE_PMD_FOLDED) != CONFIG_PGTABLE_LEVELS -#error CONFIG_PGTABLE_LEVELS is not consistent with __PAGETABLE_{P4D,PUD,PMD}_FOLDED -#endif - -/* - * On almost all architectures and configurations, 0 can be used as the - * upper ceiling to free_pgtables(): on many architectures it has the same - * effect as using TASK_SIZE. However, there is one configuration which - * must impose a more careful limit, to avoid freeing kernel pgtables. - */ -#ifndef USER_PGTABLES_CEILING -#define USER_PGTABLES_CEILING 0UL -#endif - -#ifndef __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS -extern int ptep_set_access_flags(struct vm_area_struct *vma, - unsigned long address, pte_t *ptep, - pte_t entry, int dirty); -#endif - -#ifndef __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS -#ifdef CONFIG_TRANSPARENT_HUGEPAGE -extern int pmdp_set_access_flags(struct vm_area_struct *vma, - unsigned long address, pmd_t *pmdp, - pmd_t entry, int dirty); -extern int pudp_set_access_flags(struct vm_area_struct *vma, - unsigned long address, pud_t *pudp, - pud_t entry, int dirty); -#else -static inline int pmdp_set_access_flags(struct vm_area_struct *vma, - unsigned long address, pmd_t *pmdp, - pmd_t entry, int dirty) -{ - BUILD_BUG(); - return 0; -} -static inline int pudp_set_access_flags(struct vm_area_struct *vma, - unsigned long address, pud_t *pudp, - pud_t entry, int dirty) -{ - BUILD_BUG(); - return 0; -} -#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ -#endif - -#ifndef __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG -static inline int ptep_test_and_clear_young(struct vm_area_struct *vma, - unsigned long address, - pte_t *ptep) -{ - pte_t pte = *ptep; - int r = 1; - if (!pte_young(pte)) - r = 0; - else - set_pte_at(vma->vm_mm, address, ptep, pte_mkold(pte)); - return r; -} -#endif - -#ifndef __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG -#ifdef CONFIG_TRANSPARENT_HUGEPAGE -static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma, - unsigned long address, - pmd_t *pmdp) -{ - pmd_t pmd = *pmdp; - int r = 1; - if (!pmd_young(pmd)) - r = 0; - else - set_pmd_at(vma->vm_mm, address, pmdp, pmd_mkold(pmd)); - return r; -} -#else -static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma, - unsigned long address, - pmd_t *pmdp) -{ - BUILD_BUG(); - return 0; -} -#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ -#endif - -#ifndef __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH -int ptep_clear_flush_young(struct vm_area_struct *vma, - unsigned long address, pte_t *ptep); -#endif - -#ifndef __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH -#ifdef CONFIG_TRANSPARENT_HUGEPAGE -extern int pmdp_clear_flush_young(struct vm_area_struct *vma, - unsigned long address, pmd_t *pmdp); -#else -/* - * Despite relevant to THP only, this API is called from generic rmap code - * under PageTransHuge(), hence needs a dummy implementation for !THP - */ -static inline int pmdp_clear_flush_young(struct vm_area_struct *vma, - unsigned long address, pmd_t *pmdp) -{ - BUILD_BUG(); - return 0; -} -#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ -#endif - -#ifndef __HAVE_ARCH_PTEP_GET_AND_CLEAR -static inline pte_t ptep_get_and_clear(struct mm_struct *mm, - unsigned long address, - pte_t *ptep) -{ - pte_t pte = *ptep; - pte_clear(mm, address, ptep); - return pte; -} -#endif - -#ifdef CONFIG_TRANSPARENT_HUGEPAGE -#ifndef __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR -static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, - unsigned long address, - pmd_t *pmdp) -{ - pmd_t pmd = *pmdp; - pmd_clear(pmdp); - return pmd; -} -#endif /* __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR */ -#ifndef __HAVE_ARCH_PUDP_HUGE_GET_AND_CLEAR -static inline pud_t pudp_huge_get_and_clear(struct mm_struct *mm, - unsigned long address, - pud_t *pudp) -{ - pud_t pud = *pudp; - - pud_clear(pudp); - return pud; -} -#endif /* __HAVE_ARCH_PUDP_HUGE_GET_AND_CLEAR */ -#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ - -#ifdef CONFIG_TRANSPARENT_HUGEPAGE -#ifndef __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR_FULL -static inline pmd_t pmdp_huge_get_and_clear_full(struct mm_struct *mm, - unsigned long address, pmd_t *pmdp, - int full) -{ - return pmdp_huge_get_and_clear(mm, address, pmdp); -} -#endif - -#ifndef __HAVE_ARCH_PUDP_HUGE_GET_AND_CLEAR_FULL -static inline pud_t pudp_huge_get_and_clear_full(struct mm_struct *mm, - unsigned long address, pud_t *pudp, - int full) -{ - return pudp_huge_get_and_clear(mm, address, pudp); -} -#endif -#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ - -#ifndef __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL -static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm, - unsigned long address, pte_t *ptep, - int full) -{ - pte_t pte; - pte = ptep_get_and_clear(mm, address, ptep); - return pte; -} -#endif - -/* - * Some architectures may be able to avoid expensive synchronization - * primitives when modifications are made to PTE's which are already - * not present, or in the process of an address space destruction. - */ -#ifndef __HAVE_ARCH_PTE_CLEAR_NOT_PRESENT_FULL -static inline void pte_clear_not_present_full(struct mm_struct *mm, - unsigned long address, - pte_t *ptep, - int full) -{ - pte_clear(mm, address, ptep); -} -#endif - -#ifndef __HAVE_ARCH_PTEP_CLEAR_FLUSH -extern pte_t ptep_clear_flush(struct vm_area_struct *vma, - unsigned long address, - pte_t *ptep); -#endif - -#ifndef __HAVE_ARCH_PMDP_HUGE_CLEAR_FLUSH -extern pmd_t pmdp_huge_clear_flush(struct vm_area_struct *vma, - unsigned long address, - pmd_t *pmdp); -extern pud_t pudp_huge_clear_flush(struct vm_area_struct *vma, - unsigned long address, - pud_t *pudp); -#endif - -#ifndef __HAVE_ARCH_PTEP_SET_WRPROTECT -struct mm_struct; -static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long address, pte_t *ptep) -{ - pte_t old_pte = *ptep; - set_pte_at(mm, address, ptep, pte_wrprotect(old_pte)); -} -#endif - -#ifndef pte_savedwrite -#define pte_savedwrite pte_write -#endif - -#ifndef pte_mk_savedwrite -#define pte_mk_savedwrite pte_mkwrite -#endif - -#ifndef pte_clear_savedwrite -#define pte_clear_savedwrite pte_wrprotect -#endif - -#ifndef pmd_savedwrite -#define pmd_savedwrite pmd_write -#endif - -#ifndef pmd_mk_savedwrite -#define pmd_mk_savedwrite pmd_mkwrite -#endif - -#ifndef pmd_clear_savedwrite -#define pmd_clear_savedwrite pmd_wrprotect -#endif - -#ifndef __HAVE_ARCH_PMDP_SET_WRPROTECT -#ifdef CONFIG_TRANSPARENT_HUGEPAGE -static inline void pmdp_set_wrprotect(struct mm_struct *mm, - unsigned long address, pmd_t *pmdp) -{ - pmd_t old_pmd = *pmdp; - set_pmd_at(mm, address, pmdp, pmd_wrprotect(old_pmd)); -} -#else -static inline void pmdp_set_wrprotect(struct mm_struct *mm, - unsigned long address, pmd_t *pmdp) -{ - BUILD_BUG(); -} -#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ -#endif -#ifndef __HAVE_ARCH_PUDP_SET_WRPROTECT -#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD -static inline void pudp_set_wrprotect(struct mm_struct *mm, - unsigned long address, pud_t *pudp) -{ - pud_t old_pud = *pudp; - - set_pud_at(mm, address, pudp, pud_wrprotect(old_pud)); -} -#else -static inline void pudp_set_wrprotect(struct mm_struct *mm, - unsigned long address, pud_t *pudp) -{ - BUILD_BUG(); -} -#endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ -#endif - -#ifndef pmdp_collapse_flush -#ifdef CONFIG_TRANSPARENT_HUGEPAGE -extern pmd_t pmdp_collapse_flush(struct vm_area_struct *vma, - unsigned long address, pmd_t *pmdp); -#else -static inline pmd_t pmdp_collapse_flush(struct vm_area_struct *vma, - unsigned long address, - pmd_t *pmdp) -{ - BUILD_BUG(); - return *pmdp; -} -#define pmdp_collapse_flush pmdp_collapse_flush -#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ -#endif - -#ifndef __HAVE_ARCH_PGTABLE_DEPOSIT -extern void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, - pgtable_t pgtable); -#endif - -#ifndef __HAVE_ARCH_PGTABLE_WITHDRAW -extern pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp); -#endif - -#ifdef CONFIG_TRANSPARENT_HUGEPAGE -/* - * This is an implementation of pmdp_establish() that is only suitable for an - * architecture that doesn't have hardware dirty/accessed bits. In this case we - * can't race with CPU which sets these bits and non-atomic aproach is fine. - */ -static inline pmd_t generic_pmdp_establish(struct vm_area_struct *vma, - unsigned long address, pmd_t *pmdp, pmd_t pmd) -{ - pmd_t old_pmd = *pmdp; - set_pmd_at(vma->vm_mm, address, pmdp, pmd); - return old_pmd; -} -#endif - -#ifndef __HAVE_ARCH_PMDP_INVALIDATE -extern pmd_t pmdp_invalidate(struct vm_area_struct *vma, unsigned long address, - pmd_t *pmdp); -#endif - -#ifndef __HAVE_ARCH_PTE_SAME -static inline int pte_same(pte_t pte_a, pte_t pte_b) -{ - return pte_val(pte_a) == pte_val(pte_b); -} -#endif - -#ifndef __HAVE_ARCH_PTE_UNUSED -/* - * Some architectures provide facilities to virtualization guests - * so that they can flag allocated pages as unused. This allows the - * host to transparently reclaim unused pages. This function returns - * whether the pte's page is unused. - */ -static inline int pte_unused(pte_t pte) -{ - return 0; -} -#endif - -#ifndef pte_access_permitted -#define pte_access_permitted(pte, write) \ - (pte_present(pte) && (!(write) || pte_write(pte))) -#endif - -#ifndef pmd_access_permitted -#define pmd_access_permitted(pmd, write) \ - (pmd_present(pmd) && (!(write) || pmd_write(pmd))) -#endif - -#ifndef pud_access_permitted -#define pud_access_permitted(pud, write) \ - (pud_present(pud) && (!(write) || pud_write(pud))) -#endif - -#ifndef p4d_access_permitted -#define p4d_access_permitted(p4d, write) \ - (p4d_present(p4d) && (!(write) || p4d_write(p4d))) -#endif - -#ifndef pgd_access_permitted -#define pgd_access_permitted(pgd, write) \ - (pgd_present(pgd) && (!(write) || pgd_write(pgd))) -#endif - -#ifndef __HAVE_ARCH_PMD_SAME -static inline int pmd_same(pmd_t pmd_a, pmd_t pmd_b) -{ - return pmd_val(pmd_a) == pmd_val(pmd_b); -} - -static inline int pud_same(pud_t pud_a, pud_t pud_b) -{ - return pud_val(pud_a) == pud_val(pud_b); -} -#endif - -#ifndef __HAVE_ARCH_P4D_SAME -static inline int p4d_same(p4d_t p4d_a, p4d_t p4d_b) -{ - return p4d_val(p4d_a) == p4d_val(p4d_b); -} -#endif - -#ifndef __HAVE_ARCH_PGD_SAME -static inline int pgd_same(pgd_t pgd_a, pgd_t pgd_b) -{ - return pgd_val(pgd_a) == pgd_val(pgd_b); -} -#endif - -/* - * Use set_p*_safe(), and elide TLB flushing, when confident that *no* - * TLB flush will be required as a result of the "set". For example, use - * in scenarios where it is known ahead of time that the routine is - * setting non-present entries, or re-setting an existing entry to the - * same value. Otherwise, use the typical "set" helpers and flush the - * TLB. - */ -#define set_pte_safe(ptep, pte) \ -({ \ - WARN_ON_ONCE(pte_present(*ptep) && !pte_same(*ptep, pte)); \ - set_pte(ptep, pte); \ -}) - -#define set_pmd_safe(pmdp, pmd) \ -({ \ - WARN_ON_ONCE(pmd_present(*pmdp) && !pmd_same(*pmdp, pmd)); \ - set_pmd(pmdp, pmd); \ -}) - -#define set_pud_safe(pudp, pud) \ -({ \ - WARN_ON_ONCE(pud_present(*pudp) && !pud_same(*pudp, pud)); \ - set_pud(pudp, pud); \ -}) - -#define set_p4d_safe(p4dp, p4d) \ -({ \ - WARN_ON_ONCE(p4d_present(*p4dp) && !p4d_same(*p4dp, p4d)); \ - set_p4d(p4dp, p4d); \ -}) - -#define set_pgd_safe(pgdp, pgd) \ -({ \ - WARN_ON_ONCE(pgd_present(*pgdp) && !pgd_same(*pgdp, pgd)); \ - set_pgd(pgdp, pgd); \ -}) - -#ifndef __HAVE_ARCH_DO_SWAP_PAGE -/* - * Some architectures support metadata associated with a page. When a - * page is being swapped out, this metadata must be saved so it can be - * restored when the page is swapped back in. SPARC M7 and newer - * processors support an ADI (Application Data Integrity) tag for the - * page as metadata for the page. arch_do_swap_page() can restore this - * metadata when a page is swapped back in. - */ -static inline void arch_do_swap_page(struct mm_struct *mm, - struct vm_area_struct *vma, - unsigned long addr, - pte_t pte, pte_t oldpte) -{ - -} -#endif - -#ifndef __HAVE_ARCH_UNMAP_ONE -/* - * Some architectures support metadata associated with a page. When a - * page is being swapped out, this metadata must be saved so it can be - * restored when the page is swapped back in. SPARC M7 and newer - * processors support an ADI (Application Data Integrity) tag for the - * page as metadata for the page. arch_unmap_one() can save this - * metadata on a swap-out of a page. - */ -static inline int arch_unmap_one(struct mm_struct *mm, - struct vm_area_struct *vma, - unsigned long addr, - pte_t orig_pte) -{ - return 0; -} -#endif - -#ifndef __HAVE_ARCH_PGD_OFFSET_GATE -#define pgd_offset_gate(mm, addr) pgd_offset(mm, addr) -#endif - -#ifndef __HAVE_ARCH_MOVE_PTE -#define move_pte(pte, prot, old_addr, new_addr) (pte) -#endif - -#ifndef pte_accessible -# define pte_accessible(mm, pte) ((void)(pte), 1) -#endif - -#ifndef flush_tlb_fix_spurious_fault -#define flush_tlb_fix_spurious_fault(vma, address) flush_tlb_page(vma, address) -#endif - -#ifndef pgprot_noncached -#define pgprot_noncached(prot) (prot) -#endif - -#ifndef pgprot_writecombine -#define pgprot_writecombine pgprot_noncached -#endif - -#ifndef pgprot_writethrough -#define pgprot_writethrough pgprot_noncached -#endif - -#ifndef pgprot_device -#define pgprot_device pgprot_noncached -#endif - -#ifndef pgprot_modify -#define pgprot_modify pgprot_modify -static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot) -{ - if (pgprot_val(oldprot) == pgprot_val(pgprot_noncached(oldprot))) - newprot = pgprot_noncached(newprot); - if (pgprot_val(oldprot) == pgprot_val(pgprot_writecombine(oldprot))) - newprot = pgprot_writecombine(newprot); - if (pgprot_val(oldprot) == pgprot_val(pgprot_device(oldprot))) - newprot = pgprot_device(newprot); - return newprot; -} -#endif - -/* - * When walking page tables, get the address of the next boundary, - * or the end address of the range if that comes earlier. Although no - * vma end wraps to 0, rounded up __boundary may wrap to 0 throughout. - */ - -#define pgd_addr_end(addr, end) \ -({ unsigned long __boundary = ((addr) + PGDIR_SIZE) & PGDIR_MASK; \ - (__boundary - 1 < (end) - 1)? __boundary: (end); \ -}) - -#ifndef p4d_addr_end -#define p4d_addr_end(addr, end) \ -({ unsigned long __boundary = ((addr) + P4D_SIZE) & P4D_MASK; \ - (__boundary - 1 < (end) - 1)? __boundary: (end); \ -}) -#endif - -#ifndef pud_addr_end -#define pud_addr_end(addr, end) \ -({ unsigned long __boundary = ((addr) + PUD_SIZE) & PUD_MASK; \ - (__boundary - 1 < (end) - 1)? __boundary: (end); \ -}) -#endif - -#ifndef pmd_addr_end -#define pmd_addr_end(addr, end) \ -({ unsigned long __boundary = ((addr) + PMD_SIZE) & PMD_MASK; \ - (__boundary - 1 < (end) - 1)? __boundary: (end); \ -}) -#endif - -/* - * When walking page tables, we usually want to skip any p?d_none entries; - * and any p?d_bad entries - reporting the error before resetting to none. - * Do the tests inline, but report and clear the bad entry in mm/memory.c. - */ -void pgd_clear_bad(pgd_t *); -void p4d_clear_bad(p4d_t *); -void pud_clear_bad(pud_t *); -void pmd_clear_bad(pmd_t *); - -static inline int pgd_none_or_clear_bad(pgd_t *pgd) -{ - if (pgd_none(*pgd)) - return 1; - if (unlikely(pgd_bad(*pgd))) { - pgd_clear_bad(pgd); - return 1; - } - return 0; -} - -static inline int p4d_none_or_clear_bad(p4d_t *p4d) -{ - if (p4d_none(*p4d)) - return 1; - if (unlikely(p4d_bad(*p4d))) { - p4d_clear_bad(p4d); - return 1; - } - return 0; -} - -static inline int pud_none_or_clear_bad(pud_t *pud) -{ - if (pud_none(*pud)) - return 1; - if (unlikely(pud_bad(*pud))) { - pud_clear_bad(pud); - return 1; - } - return 0; -} - -static inline int pmd_none_or_clear_bad(pmd_t *pmd) -{ - if (pmd_none(*pmd)) - return 1; - if (unlikely(pmd_bad(*pmd))) { - pmd_clear_bad(pmd); - return 1; - } - return 0; -} - -static inline pte_t __ptep_modify_prot_start(struct mm_struct *mm, - unsigned long addr, - pte_t *ptep) -{ - /* - * Get the current pte state, but zero it out to make it - * non-present, preventing the hardware from asynchronously - * updating it. - */ - return ptep_get_and_clear(mm, addr, ptep); -} - -static inline void __ptep_modify_prot_commit(struct mm_struct *mm, - unsigned long addr, - pte_t *ptep, pte_t pte) -{ - /* - * The pte is non-present, so there's no hardware state to - * preserve. - */ - set_pte_at(mm, addr, ptep, pte); -} - -#ifndef __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION -/* - * Start a pte protection read-modify-write transaction, which - * protects against asynchronous hardware modifications to the pte. - * The intention is not to prevent the hardware from making pte - * updates, but to prevent any updates it may make from being lost. - * - * This does not protect against other software modifications of the - * pte; the appropriate pte lock must be held over the transation. - * - * Note that this interface is intended to be batchable, meaning that - * ptep_modify_prot_commit may not actually update the pte, but merely - * queue the update to be done at some later time. The update must be - * actually committed before the pte lock is released, however. - */ -static inline pte_t ptep_modify_prot_start(struct mm_struct *mm, - unsigned long addr, - pte_t *ptep) -{ - return __ptep_modify_prot_start(mm, addr, ptep); -} - -/* - * Commit an update to a pte, leaving any hardware-controlled bits in - * the PTE unmodified. - */ -static inline void ptep_modify_prot_commit(struct mm_struct *mm, - unsigned long addr, - pte_t *ptep, pte_t pte) -{ - __ptep_modify_prot_commit(mm, addr, ptep, pte); -} -#endif /* __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION */ -#endif /* CONFIG_MMU */ - -/* - * No-op macros that just return the current protection value. Defined here - * because these macros can be used used even if CONFIG_MMU is not defined. - */ -#ifndef pgprot_encrypted -#define pgprot_encrypted(prot) (prot) -#endif - -#ifndef pgprot_decrypted -#define pgprot_decrypted(prot) (prot) -#endif - -/* - * A facility to provide lazy MMU batching. This allows PTE updates and - * page invalidations to be delayed until a call to leave lazy MMU mode - * is issued. Some architectures may benefit from doing this, and it is - * beneficial for both shadow and direct mode hypervisors, which may batch - * the PTE updates which happen during this window. Note that using this - * interface requires that read hazards be removed from the code. A read - * hazard could result in the direct mode hypervisor case, since the actual - * write to the page tables may not yet have taken place, so reads though - * a raw PTE pointer after it has been modified are not guaranteed to be - * up to date. This mode can only be entered and left under the protection of - * the page table locks for all page tables which may be modified. In the UP - * case, this is required so that preemption is disabled, and in the SMP case, - * it must synchronize the delayed page table writes properly on other CPUs. - */ -#ifndef __HAVE_ARCH_ENTER_LAZY_MMU_MODE -#define arch_enter_lazy_mmu_mode() do {} while (0) -#define arch_leave_lazy_mmu_mode() do {} while (0) -#define arch_flush_lazy_mmu_mode() do {} while (0) -#endif - -/* - * A facility to provide batching of the reload of page tables and - * other process state with the actual context switch code for - * paravirtualized guests. By convention, only one of the batched - * update (lazy) modes (CPU, MMU) should be active at any given time, - * entry should never be nested, and entry and exits should always be - * paired. This is for sanity of maintaining and reasoning about the - * kernel code. In this case, the exit (end of the context switch) is - * in architecture-specific code, and so doesn't need a generic - * definition. - */ -#ifndef __HAVE_ARCH_START_CONTEXT_SWITCH -#define arch_start_context_switch(prev) do {} while (0) -#endif - -#ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY -#ifndef CONFIG_ARCH_ENABLE_THP_MIGRATION -static inline pmd_t pmd_swp_mksoft_dirty(pmd_t pmd) -{ - return pmd; -} - -static inline int pmd_swp_soft_dirty(pmd_t pmd) -{ - return 0; -} - -static inline pmd_t pmd_swp_clear_soft_dirty(pmd_t pmd) -{ - return pmd; -} -#endif -#else /* !CONFIG_HAVE_ARCH_SOFT_DIRTY */ -static inline int pte_soft_dirty(pte_t pte) -{ - return 0; -} - -static inline int pmd_soft_dirty(pmd_t pmd) -{ - return 0; -} - -static inline pte_t pte_mksoft_dirty(pte_t pte) -{ - return pte; -} - -static inline pmd_t pmd_mksoft_dirty(pmd_t pmd) -{ - return pmd; -} - -static inline pte_t pte_clear_soft_dirty(pte_t pte) -{ - return pte; -} - -static inline pmd_t pmd_clear_soft_dirty(pmd_t pmd) -{ - return pmd; -} - -static inline pte_t pte_swp_mksoft_dirty(pte_t pte) -{ - return pte; -} - -static inline int pte_swp_soft_dirty(pte_t pte) -{ - return 0; -} - -static inline pte_t pte_swp_clear_soft_dirty(pte_t pte) -{ - return pte; -} - -static inline pmd_t pmd_swp_mksoft_dirty(pmd_t pmd) -{ - return pmd; -} - -static inline int pmd_swp_soft_dirty(pmd_t pmd) -{ - return 0; -} - -static inline pmd_t pmd_swp_clear_soft_dirty(pmd_t pmd) -{ - return pmd; -} -#endif - -#ifndef __HAVE_PFNMAP_TRACKING -/* - * Interfaces that can be used by architecture code to keep track of - * memory type of pfn mappings specified by the remap_pfn_range, - * vmf_insert_pfn. - */ - -/* - * track_pfn_remap is called when a _new_ pfn mapping is being established - * by remap_pfn_range() for physical range indicated by pfn and size. - */ -static inline int track_pfn_remap(struct vm_area_struct *vma, pgprot_t *prot, - unsigned long pfn, unsigned long addr, - unsigned long size) -{ - return 0; -} - -/* - * track_pfn_insert is called when a _new_ single pfn is established - * by vmf_insert_pfn(). - */ -static inline void track_pfn_insert(struct vm_area_struct *vma, pgprot_t *prot, - pfn_t pfn) -{ -} - -/* - * track_pfn_copy is called when vma that is covering the pfnmap gets - * copied through copy_page_range(). - */ -static inline int track_pfn_copy(struct vm_area_struct *vma) -{ - return 0; -} - -/* - * untrack_pfn is called while unmapping a pfnmap for a region. - * untrack can be called for a specific region indicated by pfn and size or - * can be for the entire vma (in which case pfn, size are zero). - */ -static inline void untrack_pfn(struct vm_area_struct *vma, - unsigned long pfn, unsigned long size) -{ -} - -/* - * untrack_pfn_moved is called while mremapping a pfnmap for a new region. - */ -static inline void untrack_pfn_moved(struct vm_area_struct *vma) -{ -} -#else -extern int track_pfn_remap(struct vm_area_struct *vma, pgprot_t *prot, - unsigned long pfn, unsigned long addr, - unsigned long size); -extern void track_pfn_insert(struct vm_area_struct *vma, pgprot_t *prot, - pfn_t pfn); -extern int track_pfn_copy(struct vm_area_struct *vma); -extern void untrack_pfn(struct vm_area_struct *vma, unsigned long pfn, - unsigned long size); -extern void untrack_pfn_moved(struct vm_area_struct *vma); -#endif - -#ifdef __HAVE_COLOR_ZERO_PAGE -static inline int is_zero_pfn(unsigned long pfn) -{ - extern unsigned long zero_pfn; - unsigned long offset_from_zero_pfn = pfn - zero_pfn; - return offset_from_zero_pfn <= (zero_page_mask >> PAGE_SHIFT); -} - -#define my_zero_pfn(addr) page_to_pfn(ZERO_PAGE(addr)) - -#else -static inline int is_zero_pfn(unsigned long pfn) -{ - extern unsigned long zero_pfn; - return pfn == zero_pfn; -} - -static inline unsigned long my_zero_pfn(unsigned long addr) -{ - extern unsigned long zero_pfn; - return zero_pfn; -} -#endif - -#ifdef CONFIG_MMU - -#ifndef CONFIG_TRANSPARENT_HUGEPAGE -static inline int pmd_trans_huge(pmd_t pmd) -{ - return 0; -} -#ifndef pmd_write -static inline int pmd_write(pmd_t pmd) -{ - BUG(); - return 0; -} -#endif /* pmd_write */ -#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ - -#ifndef pud_write -static inline int pud_write(pud_t pud) -{ - BUG(); - return 0; -} -#endif /* pud_write */ - -#if !defined(CONFIG_TRANSPARENT_HUGEPAGE) || \ - (defined(CONFIG_TRANSPARENT_HUGEPAGE) && \ - !defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)) -static inline int pud_trans_huge(pud_t pud) -{ - return 0; -} -#endif - -#ifndef pmd_read_atomic -static inline pmd_t pmd_read_atomic(pmd_t *pmdp) -{ - /* - * Depend on compiler for an atomic pmd read. NOTE: this is - * only going to work, if the pmdval_t isn't larger than - * an unsigned long. - */ - return *pmdp; -} -#endif - -#ifndef arch_needs_pgtable_deposit -#define arch_needs_pgtable_deposit() (false) -#endif -/* - * This function is meant to be used by sites walking pagetables with - * the mmap_sem hold in read mode to protect against MADV_DONTNEED and - * transhuge page faults. MADV_DONTNEED can convert a transhuge pmd - * into a null pmd and the transhuge page fault can convert a null pmd - * into an hugepmd or into a regular pmd (if the hugepage allocation - * fails). While holding the mmap_sem in read mode the pmd becomes - * stable and stops changing under us only if it's not null and not a - * transhuge pmd. When those races occurs and this function makes a - * difference vs the standard pmd_none_or_clear_bad, the result is - * undefined so behaving like if the pmd was none is safe (because it - * can return none anyway). The compiler level barrier() is critically - * important to compute the two checks atomically on the same pmdval. - * - * For 32bit kernels with a 64bit large pmd_t this automatically takes - * care of reading the pmd atomically to avoid SMP race conditions - * against pmd_populate() when the mmap_sem is hold for reading by the - * caller (a special atomic read not done by "gcc" as in the generic - * version above, is also needed when THP is disabled because the page - * fault can populate the pmd from under us). - */ -static inline int pmd_none_or_trans_huge_or_clear_bad(pmd_t *pmd) -{ - pmd_t pmdval = pmd_read_atomic(pmd); - /* - * The barrier will stabilize the pmdval in a register or on - * the stack so that it will stop changing under the code. - * - * When CONFIG_TRANSPARENT_HUGEPAGE=y on x86 32bit PAE, - * pmd_read_atomic is allowed to return a not atomic pmdval - * (for example pointing to an hugepage that has never been - * mapped in the pmd). The below checks will only care about - * the low part of the pmd with 32bit PAE x86 anyway, with the - * exception of pmd_none(). So the important thing is that if - * the low part of the pmd is found null, the high part will - * be also null or the pmd_none() check below would be - * confused. - */ -#ifdef CONFIG_TRANSPARENT_HUGEPAGE - barrier(); -#endif - /* - * !pmd_present() checks for pmd migration entries - * - * The complete check uses is_pmd_migration_entry() in linux/swapops.h - * But using that requires moving current function and pmd_trans_unstable() - * to linux/swapops.h to resovle dependency, which is too much code move. - * - * !pmd_present() is equivalent to is_pmd_migration_entry() currently, - * because !pmd_present() pages can only be under migration not swapped - * out. - * - * pmd_none() is preseved for future condition checks on pmd migration - * entries and not confusing with this function name, although it is - * redundant with !pmd_present(). - */ - if (pmd_none(pmdval) || pmd_trans_huge(pmdval) || - (IS_ENABLED(CONFIG_ARCH_ENABLE_THP_MIGRATION) && !pmd_present(pmdval))) - return 1; - if (unlikely(pmd_bad(pmdval))) { - pmd_clear_bad(pmd); - return 1; - } - return 0; -} - -/* - * This is a noop if Transparent Hugepage Support is not built into - * the kernel. Otherwise it is equivalent to - * pmd_none_or_trans_huge_or_clear_bad(), and shall only be called in - * places that already verified the pmd is not none and they want to - * walk ptes while holding the mmap sem in read mode (write mode don't - * need this). If THP is not enabled, the pmd can't go away under the - * code even if MADV_DONTNEED runs, but if THP is enabled we need to - * run a pmd_trans_unstable before walking the ptes after - * split_huge_page_pmd returns (because it may have run when the pmd - * become null, but then a page fault can map in a THP and not a - * regular page). - */ -static inline int pmd_trans_unstable(pmd_t *pmd) -{ -#ifdef CONFIG_TRANSPARENT_HUGEPAGE - return pmd_none_or_trans_huge_or_clear_bad(pmd); -#else - return 0; -#endif -} - -#ifndef CONFIG_NUMA_BALANCING -/* - * Technically a PTE can be PROTNONE even when not doing NUMA balancing but - * the only case the kernel cares is for NUMA balancing and is only ever set - * when the VMA is accessible. For PROT_NONE VMAs, the PTEs are not marked - * _PAGE_PROTNONE so by by default, implement the helper as "always no". It - * is the responsibility of the caller to distinguish between PROT_NONE - * protections and NUMA hinting fault protections. - */ -static inline int pte_protnone(pte_t pte) -{ - return 0; -} - -static inline int pmd_protnone(pmd_t pmd) -{ - return 0; -} -#endif /* CONFIG_NUMA_BALANCING */ - -#endif /* CONFIG_MMU */ - -#ifdef CONFIG_HAVE_ARCH_HUGE_VMAP - -#ifndef __PAGETABLE_P4D_FOLDED -int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot); -int p4d_clear_huge(p4d_t *p4d); -#else -static inline int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot) -{ - return 0; -} -static inline int p4d_clear_huge(p4d_t *p4d) -{ - return 0; -} -#endif /* !__PAGETABLE_P4D_FOLDED */ - -int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot); -int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot); -int pud_clear_huge(pud_t *pud); -int pmd_clear_huge(pmd_t *pmd); -int p4d_free_pud_page(p4d_t *p4d, unsigned long addr); -int pud_free_pmd_page(pud_t *pud, unsigned long addr); -int pmd_free_pte_page(pmd_t *pmd, unsigned long addr); -#else /* !CONFIG_HAVE_ARCH_HUGE_VMAP */ -static inline int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot) -{ - return 0; -} -static inline int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot) -{ - return 0; -} -static inline int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot) -{ - return 0; -} -static inline int p4d_clear_huge(p4d_t *p4d) -{ - return 0; -} -static inline int pud_clear_huge(pud_t *pud) -{ - return 0; -} -static inline int pmd_clear_huge(pmd_t *pmd) -{ - return 0; -} -static inline int p4d_free_pud_page(p4d_t *p4d, unsigned long addr) -{ - return 0; -} -static inline int pud_free_pmd_page(pud_t *pud, unsigned long addr) -{ - return 0; -} -static inline int pmd_free_pte_page(pmd_t *pmd, unsigned long addr) -{ - return 0; -} -#endif /* CONFIG_HAVE_ARCH_HUGE_VMAP */ - -#ifndef __HAVE_ARCH_FLUSH_PMD_TLB_RANGE -#ifdef CONFIG_TRANSPARENT_HUGEPAGE -/* - * ARCHes with special requirements for evicting THP backing TLB entries can - * implement this. Otherwise also, it can help optimize normal TLB flush in - * THP regime. stock flush_tlb_range() typically has optimization to nuke the - * entire TLB TLB if flush span is greater than a threshold, which will - * likely be true for a single huge page. Thus a single thp flush will - * invalidate the entire TLB which is not desitable. - * e.g. see arch/arc: flush_pmd_tlb_range - */ -#define flush_pmd_tlb_range(vma, addr, end) flush_tlb_range(vma, addr, end) -#define flush_pud_tlb_range(vma, addr, end) flush_tlb_range(vma, addr, end) -#else -#define flush_pmd_tlb_range(vma, addr, end) BUILD_BUG() -#define flush_pud_tlb_range(vma, addr, end) BUILD_BUG() -#endif -#endif - -struct file; -int phys_mem_access_prot_allowed(struct file *file, unsigned long pfn, - unsigned long size, pgprot_t *vma_prot); - -#ifndef CONFIG_X86_ESPFIX64 -static inline void init_espfix_bsp(void) { } -#endif - -#ifndef __HAVE_ARCH_PFN_MODIFY_ALLOWED -static inline bool pfn_modify_allowed(unsigned long pfn, pgprot_t prot) -{ - return true; -} - -static inline bool arch_has_pfn_modify_check(void) -{ - return false; -} -#endif /* !_HAVE_ARCH_PFN_MODIFY_ALLOWED */ - -/* - * Architecture PAGE_KERNEL_* fallbacks - * - * Some architectures don't define certain PAGE_KERNEL_* flags. This is either - * because they really don't support them, or the port needs to be updated to - * reflect the required functionality. Below are a set of relatively safe - * fallbacks, as best effort, which we can count on in lieu of the architectures - * not defining them on their own yet. - */ - -#ifndef PAGE_KERNEL_RO -# define PAGE_KERNEL_RO PAGE_KERNEL -#endif - -#ifndef PAGE_KERNEL_EXEC -# define PAGE_KERNEL_EXEC PAGE_KERNEL -#endif - -#endif /* !__ASSEMBLY__ */ - -#ifndef io_remap_pfn_range -#define io_remap_pfn_range remap_pfn_range -#endif - -#ifndef has_transparent_hugepage -#ifdef CONFIG_TRANSPARENT_HUGEPAGE -#define has_transparent_hugepage() 1 -#else -#define has_transparent_hugepage() 0 -#endif -#endif - -/* - * On some architectures it depends on the mm if the p4d/pud or pmd - * layer of the page table hierarchy is folded or not. - */ -#ifndef mm_p4d_folded -#define mm_p4d_folded(mm) __is_defined(__PAGETABLE_P4D_FOLDED) -#endif - -#ifndef mm_pud_folded -#define mm_pud_folded(mm) __is_defined(__PAGETABLE_PUD_FOLDED) -#endif - -#ifndef mm_pmd_folded -#define mm_pmd_folded(mm) __is_defined(__PAGETABLE_PMD_FOLDED) -#endif - -#endif /* _ASM_GENERIC_PGTABLE_H */ diff --git a/include/asm-generic/pgtable_uffd.h b/include/asm-generic/pgtable_uffd.h new file mode 100644 index 000000000000..0d85791efdf7 --- /dev/null +++ b/include/asm-generic/pgtable_uffd.h @@ -0,0 +1,83 @@ +#ifndef _ASM_GENERIC_PGTABLE_UFFD_H +#define _ASM_GENERIC_PGTABLE_UFFD_H + +/* + * Some platforms can customize the uffd-wp bit, making it unavailable + * even if the architecture provides the resource. + * Adding this API allows architectures to add their own checks for the + * devices on which the kernel is running. + * Note: When overriding it, please make sure the + * CONFIG_HAVE_ARCH_USERFAULTFD_WP is part of this macro. + */ +#ifndef pgtable_supports_uffd_wp +#define pgtable_supports_uffd_wp() IS_ENABLED(CONFIG_HAVE_ARCH_USERFAULTFD_WP) +#endif + +static inline bool uffd_supports_wp_marker(void) +{ + return pgtable_supports_uffd_wp() && IS_ENABLED(CONFIG_PTE_MARKER_UFFD_WP); +} + +#ifndef CONFIG_HAVE_ARCH_USERFAULTFD_WP +static __always_inline int pte_uffd_wp(pte_t pte) +{ + return 0; +} + +static __always_inline int pmd_uffd_wp(pmd_t pmd) +{ + return 0; +} + +static __always_inline pte_t pte_mkuffd_wp(pte_t pte) +{ + return pte; +} + +static __always_inline pmd_t pmd_mkuffd_wp(pmd_t pmd) +{ + return pmd; +} + +static __always_inline pte_t pte_clear_uffd_wp(pte_t pte) +{ + return pte; +} + +static __always_inline pmd_t pmd_clear_uffd_wp(pmd_t pmd) +{ + return pmd; +} + +static __always_inline pte_t pte_swp_mkuffd_wp(pte_t pte) +{ + return pte; +} + +static __always_inline int pte_swp_uffd_wp(pte_t pte) +{ + return 0; +} + +static __always_inline pte_t pte_swp_clear_uffd_wp(pte_t pte) +{ + return pte; +} + +static inline pmd_t pmd_swp_mkuffd_wp(pmd_t pmd) +{ + return pmd; +} + +static inline int pmd_swp_uffd_wp(pmd_t pmd) +{ + return 0; +} + +static inline pmd_t pmd_swp_clear_uffd_wp(pmd_t pmd) +{ + return pmd; +} +#endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */ + +#endif /* _ASM_GENERIC_PGTABLE_UFFD_H */ diff --git a/include/asm-generic/preempt.h b/include/asm-generic/preempt.h index c3046c920063..51f8f3881523 100644 --- a/include/asm-generic/preempt.h +++ b/include/asm-generic/preempt.h @@ -29,7 +29,7 @@ static __always_inline void preempt_count_set(int pc) } while (0) #define init_idle_preempt_count(p, cpu) do { \ - task_thread_info(p)->preempt_count = PREEMPT_ENABLED; \ + task_thread_info(p)->preempt_count = PREEMPT_DISABLED; \ } while (0) static __always_inline void set_preempt_need_resched(void) @@ -78,11 +78,23 @@ static __always_inline bool should_resched(int preempt_offset) tif_need_resched()); } -#ifdef CONFIG_PREEMPT +#ifdef CONFIG_PREEMPTION extern asmlinkage void preempt_schedule(void); -#define __preempt_schedule() preempt_schedule() extern asmlinkage void preempt_schedule_notrace(void); + +#if defined(CONFIG_PREEMPT_DYNAMIC) && defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) + +void dynamic_preempt_schedule(void); +void dynamic_preempt_schedule_notrace(void); +#define __preempt_schedule() dynamic_preempt_schedule() +#define __preempt_schedule_notrace() dynamic_preempt_schedule_notrace() + +#else /* !CONFIG_PREEMPT_DYNAMIC || !CONFIG_HAVE_PREEMPT_DYNAMIC_KEY*/ + +#define __preempt_schedule() preempt_schedule() #define __preempt_schedule_notrace() preempt_schedule_notrace() -#endif /* CONFIG_PREEMPT */ + +#endif /* CONFIG_PREEMPT_DYNAMIC && CONFIG_HAVE_PREEMPT_DYNAMIC_KEY*/ +#endif /* CONFIG_PREEMPTION */ #endif /* __ASM_PREEMPT_H */ diff --git a/include/asm-generic/ptrace.h b/include/asm-generic/ptrace.h deleted file mode 100644 index 82e674f6b337..000000000000 --- a/include/asm-generic/ptrace.h +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Common low level (register) ptrace helpers - * - * Copyright 2004-2011 Analog Devices Inc. - * - * Licensed under the GPL-2 or later. - */ - -#ifndef __ASM_GENERIC_PTRACE_H__ -#define __ASM_GENERIC_PTRACE_H__ - -#ifndef __ASSEMBLY__ - -/* Helpers for working with the instruction pointer */ -#ifndef GET_IP -#define GET_IP(regs) ((regs)->pc) -#endif -#ifndef SET_IP -#define SET_IP(regs, val) (GET_IP(regs) = (val)) -#endif - -static inline unsigned long instruction_pointer(struct pt_regs *regs) -{ - return GET_IP(regs); -} -static inline void instruction_pointer_set(struct pt_regs *regs, - unsigned long val) -{ - SET_IP(regs, val); -} - -#ifndef profile_pc -#define profile_pc(regs) instruction_pointer(regs) -#endif - -/* Helpers for working with the user stack pointer */ -#ifndef GET_USP -#define GET_USP(regs) ((regs)->usp) -#endif -#ifndef SET_USP -#define SET_USP(regs, val) (GET_USP(regs) = (val)) -#endif - -static inline unsigned long user_stack_pointer(struct pt_regs *regs) -{ - return GET_USP(regs); -} -static inline void user_stack_pointer_set(struct pt_regs *regs, - unsigned long val) -{ - SET_USP(regs, val); -} - -/* Helpers for working with the frame pointer */ -#ifndef GET_FP -#define GET_FP(regs) ((regs)->fp) -#endif -#ifndef SET_FP -#define SET_FP(regs, val) (GET_FP(regs) = (val)) -#endif - -static inline unsigned long frame_pointer(struct pt_regs *regs) -{ - return GET_FP(regs); -} -static inline void frame_pointer_set(struct pt_regs *regs, - unsigned long val) -{ - SET_FP(regs, val); -} - -#endif /* __ASSEMBLY__ */ - -#endif diff --git a/include/asm-generic/qrwlock.h b/include/asm-generic/qrwlock.h index 36254d2da8e0..75b8f4601b28 100644 --- a/include/asm-generic/qrwlock.h +++ b/include/asm-generic/qrwlock.h @@ -1,15 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * Queue read/write lock * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. + * These use generic atomic and locking routines, but depend on a fair spinlock + * implementation in order to be fair themselves. The implementation in + * asm-generic/spinlock.h meets these requirements. * * (C) Copyright 2013-2014 Hewlett-Packard Development Company, L.P. * @@ -24,6 +19,8 @@ #include <asm-generic/qrwlock_types.h> +/* Must be included from asm/spinlock.h after defining arch_spin_is_locked. */ + /* * Writer states & reader shift and bias. */ @@ -40,13 +37,13 @@ extern void queued_read_lock_slowpath(struct qrwlock *lock); extern void queued_write_lock_slowpath(struct qrwlock *lock); /** - * queued_read_trylock - try to acquire read lock of a queue rwlock - * @lock : Pointer to queue rwlock structure + * queued_read_trylock - try to acquire read lock of a queued rwlock + * @lock : Pointer to queued rwlock structure * Return: 1 if lock acquired, 0 if failed */ static inline int queued_read_trylock(struct qrwlock *lock) { - u32 cnts; + int cnts; cnts = atomic_read(&lock->cnts); if (likely(!(cnts & _QW_WMASK))) { @@ -59,13 +56,13 @@ static inline int queued_read_trylock(struct qrwlock *lock) } /** - * queued_write_trylock - try to acquire write lock of a queue rwlock - * @lock : Pointer to queue rwlock structure + * queued_write_trylock - try to acquire write lock of a queued rwlock + * @lock : Pointer to queued rwlock structure * Return: 1 if lock acquired, 0 if failed */ static inline int queued_write_trylock(struct qrwlock *lock) { - u32 cnts; + int cnts; cnts = atomic_read(&lock->cnts); if (unlikely(cnts)) @@ -75,12 +72,12 @@ static inline int queued_write_trylock(struct qrwlock *lock) _QW_LOCKED)); } /** - * queued_read_lock - acquire read lock of a queue rwlock - * @lock: Pointer to queue rwlock structure + * queued_read_lock - acquire read lock of a queued rwlock + * @lock: Pointer to queued rwlock structure */ static inline void queued_read_lock(struct qrwlock *lock) { - u32 cnts; + int cnts; cnts = atomic_add_return_acquire(_QR_BIAS, &lock->cnts); if (likely(!(cnts & _QW_WMASK))) @@ -91,12 +88,12 @@ static inline void queued_read_lock(struct qrwlock *lock) } /** - * queued_write_lock - acquire write lock of a queue rwlock - * @lock : Pointer to queue rwlock structure + * queued_write_lock - acquire write lock of a queued rwlock + * @lock : Pointer to queued rwlock structure */ static inline void queued_write_lock(struct qrwlock *lock) { - u32 cnts = 0; + int cnts = 0; /* Optimize for the unfair lock case where the fair flag is 0. */ if (likely(atomic_try_cmpxchg_acquire(&lock->cnts, &cnts, _QW_LOCKED))) return; @@ -105,8 +102,8 @@ static inline void queued_write_lock(struct qrwlock *lock) } /** - * queued_read_unlock - release read lock of a queue rwlock - * @lock : Pointer to queue rwlock structure + * queued_read_unlock - release read lock of a queued rwlock + * @lock : Pointer to queued rwlock structure */ static inline void queued_read_unlock(struct qrwlock *lock) { @@ -117,23 +114,34 @@ static inline void queued_read_unlock(struct qrwlock *lock) } /** - * queued_write_unlock - release write lock of a queue rwlock - * @lock : Pointer to queue rwlock structure + * queued_write_unlock - release write lock of a queued rwlock + * @lock : Pointer to queued rwlock structure */ static inline void queued_write_unlock(struct qrwlock *lock) { smp_store_release(&lock->wlocked, 0); } +/** + * queued_rwlock_is_contended - check if the lock is contended + * @lock : Pointer to queued rwlock structure + * Return: 1 if lock contended, 0 otherwise + */ +static inline int queued_rwlock_is_contended(struct qrwlock *lock) +{ + return arch_spin_is_locked(&lock->wait_lock); +} + /* * Remapping rwlock architecture specific functions to the corresponding - * queue rwlock functions. + * queued rwlock functions. */ -#define arch_read_lock(l) queued_read_lock(l) -#define arch_write_lock(l) queued_write_lock(l) -#define arch_read_trylock(l) queued_read_trylock(l) -#define arch_write_trylock(l) queued_write_trylock(l) -#define arch_read_unlock(l) queued_read_unlock(l) -#define arch_write_unlock(l) queued_write_unlock(l) +#define arch_read_lock(l) queued_read_lock(l) +#define arch_write_lock(l) queued_write_lock(l) +#define arch_read_trylock(l) queued_read_trylock(l) +#define arch_write_trylock(l) queued_write_trylock(l) +#define arch_read_unlock(l) queued_read_unlock(l) +#define arch_write_unlock(l) queued_write_unlock(l) +#define arch_rwlock_is_contended(l) queued_rwlock_is_contended(l) #endif /* __ASM_GENERIC_QRWLOCK_H */ diff --git a/include/asm-generic/qrwlock_types.h b/include/asm-generic/qrwlock_types.h index c36f1d5a2572..12392c14c4d0 100644 --- a/include/asm-generic/qrwlock_types.h +++ b/include/asm-generic/qrwlock_types.h @@ -7,7 +7,7 @@ #include <asm/spinlock_types.h> /* - * The queue read/write lock data structure + * The queued read/write lock data structure */ typedef struct qrwlock { diff --git a/include/asm-generic/qspinlock.h b/include/asm-generic/qspinlock.h index 7541fa707f5b..bf47cca2c375 100644 --- a/include/asm-generic/qspinlock.h +++ b/include/asm-generic/qspinlock.h @@ -1,15 +1,35 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * Queued spinlock * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. + * A 'generic' spinlock implementation that is based on MCS locks. For an + * architecture that's looking for a 'generic' spinlock, please first consider + * ticket-lock.h and only come looking here when you've considered all the + * constraints below and can show your hardware does actually perform better + * with qspinlock. * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. + * qspinlock relies on atomic_*_release()/atomic_*_acquire() to be RCsc (or no + * weaker than RCtso if you're power), where regular code only expects atomic_t + * to be RCpc. + * + * qspinlock relies on a far greater (compared to asm-generic/spinlock.h) set + * of atomic operations to behave well together, please audit them carefully to + * ensure they all have forward progress. Many atomic operations may default to + * cmpxchg() loops which will not have good forward progress properties on + * LL/SC architectures. + * + * One notable example is atomic_fetch_or_acquire(), which x86 cannot (cheaply) + * do. Carefully read the patches that introduced + * queued_fetch_set_pending_acquire(). + * + * qspinlock also heavily relies on mixed size atomic operations, in specific + * it requires architectures to have xchg16; something which many LL/SC + * architectures need to implement as a 32bit and+or in order to satisfy the + * forward progress guarantees mentioned above. + * + * Further reading on mixed size atomics that might be relevant: + * + * http://www.cl.cam.ac.uk/~pes20/popl17/mixed-size.pdf * * (C) Copyright 2013-2015 Hewlett-Packard Development Company, L.P. * (C) Copyright 2015 Hewlett-Packard Enterprise Development LP @@ -20,7 +40,9 @@ #define __ASM_GENERIC_QSPINLOCK_H #include <asm-generic/qspinlock_types.h> +#include <linux/atomic.h> +#ifndef queued_spin_is_locked /** * queued_spin_is_locked - is the spinlock locked? * @lock: Pointer to queued spinlock structure @@ -34,6 +56,7 @@ static __always_inline int queued_spin_is_locked(struct qspinlock *lock) */ return atomic_read(&lock->val); } +#endif /** * queued_spin_value_unlocked - is the spinlock structure unlocked? @@ -47,7 +70,7 @@ static __always_inline int queued_spin_is_locked(struct qspinlock *lock) */ static __always_inline int queued_spin_value_unlocked(struct qspinlock lock) { - return !atomic_read(&lock.val); + return !lock.val.counter; } /** @@ -66,7 +89,7 @@ static __always_inline int queued_spin_is_contended(struct qspinlock *lock) */ static __always_inline int queued_spin_trylock(struct qspinlock *lock) { - u32 val = atomic_read(&lock->val); + int val = atomic_read(&lock->val); if (unlikely(val)) return 0; @@ -76,19 +99,21 @@ static __always_inline int queued_spin_trylock(struct qspinlock *lock) extern void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val); +#ifndef queued_spin_lock /** * queued_spin_lock - acquire a queued spinlock * @lock: Pointer to queued spinlock structure */ static __always_inline void queued_spin_lock(struct qspinlock *lock) { - u32 val = 0; + int val = 0; if (likely(atomic_try_cmpxchg_acquire(&lock->val, &val, _Q_LOCKED_VAL))) return; queued_spin_lock_slowpath(lock, val); } +#endif #ifndef queued_spin_unlock /** @@ -111,6 +136,7 @@ static __always_inline bool virt_spin_lock(struct qspinlock *lock) } #endif +#ifndef __no_arch_spinlock_redefine /* * Remapping spinlock architecture specific functions to the corresponding * queued spinlock functions. @@ -121,5 +147,6 @@ static __always_inline bool virt_spin_lock(struct qspinlock *lock) #define arch_spin_lock(l) queued_spin_lock(l) #define arch_spin_trylock(l) queued_spin_trylock(l) #define arch_spin_unlock(l) queued_spin_unlock(l) +#endif #endif /* __ASM_GENERIC_QSPINLOCK_H */ diff --git a/include/asm-generic/qspinlock_types.h b/include/asm-generic/qspinlock_types.h index d10f1e7d6ba8..2fd1fb89ec36 100644 --- a/include/asm-generic/qspinlock_types.h +++ b/include/asm-generic/qspinlock_types.h @@ -1,16 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * Queued spinlock * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * * (C) Copyright 2013-2015 Hewlett-Packard Development Company, L.P. * * Authors: Waiman Long <waiman.long@hp.com> @@ -18,15 +9,7 @@ #ifndef __ASM_GENERIC_QSPINLOCK_TYPES_H #define __ASM_GENERIC_QSPINLOCK_TYPES_H -/* - * Including atomic.h with PARAVIRT on will cause compilation errors because - * of recursive header file incluson via paravirt_types.h. So don't include - * it if PARAVIRT is on. - */ -#ifndef CONFIG_PARAVIRT #include <linux/types.h> -#include <linux/atomic.h> -#endif typedef struct qspinlock { union { diff --git a/include/asm-generic/rqspinlock.h b/include/asm-generic/rqspinlock.h new file mode 100644 index 000000000000..0f2dcbbfee2f --- /dev/null +++ b/include/asm-generic/rqspinlock.h @@ -0,0 +1,254 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Resilient Queued Spin Lock + * + * (C) Copyright 2024-2025 Meta Platforms, Inc. and affiliates. + * + * Authors: Kumar Kartikeya Dwivedi <memxor@gmail.com> + */ +#ifndef __ASM_GENERIC_RQSPINLOCK_H +#define __ASM_GENERIC_RQSPINLOCK_H + +#include <linux/types.h> +#include <vdso/time64.h> +#include <linux/percpu.h> +#ifdef CONFIG_QUEUED_SPINLOCKS +#include <asm/qspinlock.h> +#endif + +struct rqspinlock { + union { + atomic_t val; + u32 locked; + }; +}; + +/* Even though this is same as struct rqspinlock, we need to emit a distinct + * type in BTF for BPF programs. + */ +struct bpf_res_spin_lock { + u32 val; +}; + +struct qspinlock; +#ifdef CONFIG_QUEUED_SPINLOCKS +typedef struct qspinlock rqspinlock_t; +#else +typedef struct rqspinlock rqspinlock_t; +#endif + +extern int resilient_tas_spin_lock(rqspinlock_t *lock); +#ifdef CONFIG_QUEUED_SPINLOCKS +extern int resilient_queued_spin_lock_slowpath(rqspinlock_t *lock, u32 val); +#endif + +#ifndef resilient_virt_spin_lock_enabled +static __always_inline bool resilient_virt_spin_lock_enabled(void) +{ + return false; +} +#endif + +#ifndef resilient_virt_spin_lock +static __always_inline int resilient_virt_spin_lock(rqspinlock_t *lock) +{ + return 0; +} +#endif + +/* + * Default timeout for waiting loops is 0.25 seconds + */ +#define RES_DEF_TIMEOUT (NSEC_PER_SEC / 4) + +/* + * Choose 31 as it makes rqspinlock_held cacheline-aligned. + */ +#define RES_NR_HELD 31 + +struct rqspinlock_held { + int cnt; + void *locks[RES_NR_HELD]; +}; + +DECLARE_PER_CPU_ALIGNED(struct rqspinlock_held, rqspinlock_held_locks); + +static __always_inline void grab_held_lock_entry(void *lock) +{ + int cnt = this_cpu_inc_return(rqspinlock_held_locks.cnt); + + if (unlikely(cnt > RES_NR_HELD)) { + /* Still keep the inc so we decrement later. */ + return; + } + + /* + * Implied compiler barrier in per-CPU operations; otherwise we can have + * the compiler reorder inc with write to table, allowing interrupts to + * overwrite and erase our write to the table (as on interrupt exit it + * will be reset to NULL). + * + * It is fine for cnt inc to be reordered wrt remote readers though, + * they won't observe our entry until the cnt update is visible, that's + * all. + */ + this_cpu_write(rqspinlock_held_locks.locks[cnt - 1], lock); +} + +/* + * We simply don't support out-of-order unlocks, and keep the logic simple here. + * The verifier prevents BPF programs from unlocking out-of-order, and the same + * holds for in-kernel users. + * + * It is possible to run into misdetection scenarios of AA deadlocks on the same + * CPU, and missed ABBA deadlocks on remote CPUs if this function pops entries + * out of order (due to lock A, lock B, unlock A, unlock B) pattern. The correct + * logic to preserve right entries in the table would be to walk the array of + * held locks and swap and clear out-of-order entries, but that's too + * complicated and we don't have a compelling use case for out of order unlocking. + */ +static __always_inline void release_held_lock_entry(void) +{ + struct rqspinlock_held *rqh = this_cpu_ptr(&rqspinlock_held_locks); + + if (unlikely(rqh->cnt > RES_NR_HELD)) + goto dec; + WRITE_ONCE(rqh->locks[rqh->cnt - 1], NULL); +dec: + /* + * Reordering of clearing above with inc and its write in + * grab_held_lock_entry that came before us (in same acquisition + * attempt) is ok, we either see a valid entry or NULL when it's + * visible. + * + * But this helper is invoked when we unwind upon failing to acquire the + * lock. Unlike the unlock path which constitutes a release store after + * we clear the entry, we need to emit a write barrier here. Otherwise, + * we may have a situation as follows: + * + * <error> for lock B + * release_held_lock_entry + * + * grab_held_lock_entry + * try_cmpxchg_acquire for lock A + * + * Lack of any ordering means reordering may occur such that dec, inc + * are done before entry is overwritten. This permits a remote lock + * holder of lock B (which this CPU failed to acquire) to now observe it + * as being attempted on this CPU, and may lead to misdetection (if this + * CPU holds a lock it is attempting to acquire, leading to false ABBA + * diagnosis). + * + * The case of unlock is treated differently due to NMI reentrancy, see + * comments in res_spin_unlock. + * + * In theory we don't have a problem if the dec and WRITE_ONCE above get + * reordered with each other, we either notice an empty NULL entry on + * top (if dec succeeds WRITE_ONCE), or a potentially stale entry which + * cannot be observed (if dec precedes WRITE_ONCE). + * + * Emit the write barrier _before_ the dec, this permits dec-inc + * reordering but that is harmless as we'd have new entry set to NULL + * already, i.e. they cannot precede the NULL store above. + */ + smp_wmb(); + this_cpu_dec(rqspinlock_held_locks.cnt); +} + +#ifdef CONFIG_QUEUED_SPINLOCKS + +/** + * res_spin_lock - acquire a queued spinlock + * @lock: Pointer to queued spinlock structure + * + * Return: + * * 0 - Lock was acquired successfully. + * * -EDEADLK - Lock acquisition failed because of AA/ABBA deadlock. + * * -ETIMEDOUT - Lock acquisition failed because of timeout. + */ +static __always_inline int res_spin_lock(rqspinlock_t *lock) +{ + int val = 0; + + /* + * Grab the deadlock detection entry before doing the cmpxchg, so that + * reentrancy due to NMIs between the succeeding cmpxchg and creation of + * held lock entry can correctly detect an acquisition attempt in the + * interrupted context. + * + * cmpxchg lock A + * <NMI> + * res_spin_lock(A) --> missed AA, leads to timeout + * </NMI> + * grab_held_lock_entry(A) + */ + grab_held_lock_entry(lock); + + if (likely(atomic_try_cmpxchg_acquire(&lock->val, &val, _Q_LOCKED_VAL))) + return 0; + return resilient_queued_spin_lock_slowpath(lock, val); +} + +#else + +#define res_spin_lock(lock) resilient_tas_spin_lock(lock) + +#endif /* CONFIG_QUEUED_SPINLOCKS */ + +static __always_inline void res_spin_unlock(rqspinlock_t *lock) +{ + struct rqspinlock_held *rqh = this_cpu_ptr(&rqspinlock_held_locks); + + /* + * Release barrier, ensures correct ordering. Perform release store + * instead of queued_spin_unlock, since we use this function for the TAS + * fallback as well. When we have CONFIG_QUEUED_SPINLOCKS=n, we clear + * the full 4-byte lockword. + * + * Perform the smp_store_release before clearing the lock entry so that + * NMIs landing in the unlock path can correctly detect AA issues. The + * opposite order shown below may lead to missed AA checks: + * + * WRITE_ONCE(rqh->locks[rqh->cnt - 1], NULL) + * <NMI> + * res_spin_lock(A) --> missed AA, leads to timeout + * </NMI> + * smp_store_release(A->locked, 0) + */ + smp_store_release(&lock->locked, 0); + if (likely(rqh->cnt <= RES_NR_HELD)) + WRITE_ONCE(rqh->locks[rqh->cnt - 1], NULL); + this_cpu_dec(rqspinlock_held_locks.cnt); +} + +#ifdef CONFIG_QUEUED_SPINLOCKS +#define raw_res_spin_lock_init(lock) ({ *(lock) = (rqspinlock_t)__ARCH_SPIN_LOCK_UNLOCKED; }) +#else +#define raw_res_spin_lock_init(lock) ({ *(lock) = (rqspinlock_t){0}; }) +#endif + +#define raw_res_spin_lock(lock) \ + ({ \ + int __ret; \ + preempt_disable(); \ + __ret = res_spin_lock(lock); \ + if (__ret) \ + preempt_enable(); \ + __ret; \ + }) + +#define raw_res_spin_unlock(lock) ({ res_spin_unlock(lock); preempt_enable(); }) + +#define raw_res_spin_lock_irqsave(lock, flags) \ + ({ \ + int __ret; \ + local_irq_save(flags); \ + __ret = raw_res_spin_lock(lock); \ + if (__ret) \ + local_irq_restore(flags); \ + __ret; \ + }) + +#define raw_res_spin_unlock_irqrestore(lock, flags) ({ raw_res_spin_unlock(lock); local_irq_restore(flags); }) + +#endif /* __ASM_GENERIC_RQSPINLOCK_H */ diff --git a/include/asm-generic/runtime-const.h b/include/asm-generic/runtime-const.h new file mode 100644 index 000000000000..670499459514 --- /dev/null +++ b/include/asm-generic/runtime-const.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_RUNTIME_CONST_H +#define _ASM_RUNTIME_CONST_H + +/* + * This is the fallback for when the architecture doesn't + * support the runtime const operations. + * + * We just use the actual symbols as-is. + */ +#define runtime_const_ptr(sym) (sym) +#define runtime_const_shift_right_32(val, sym) ((u32)(val)>>(sym)) +#define runtime_const_init(type,sym) do { } while (0) + +#endif diff --git a/include/asm-generic/rwonce.h b/include/asm-generic/rwonce.h new file mode 100644 index 000000000000..52b969c7cef9 --- /dev/null +++ b/include/asm-generic/rwonce.h @@ -0,0 +1,98 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Prevent the compiler from merging or refetching reads or writes. The + * compiler is also forbidden from reordering successive instances of + * READ_ONCE and WRITE_ONCE, but only when the compiler is aware of some + * particular ordering. One way to make the compiler aware of ordering is to + * put the two invocations of READ_ONCE or WRITE_ONCE in different C + * statements. + * + * These two macros will also work on aggregate data types like structs or + * unions. + * + * Their two major use cases are: (1) Mediating communication between + * process-level code and irq/NMI handlers, all running on the same CPU, + * and (2) Ensuring that the compiler does not fold, spindle, or otherwise + * mutilate accesses that either do not require ordering or that interact + * with an explicit memory barrier or atomic instruction that provides the + * required ordering. + */ +#ifndef __ASM_GENERIC_RWONCE_H +#define __ASM_GENERIC_RWONCE_H + +#ifndef __ASSEMBLY__ + +#include <linux/compiler_types.h> +#include <linux/kasan-checks.h> +#include <linux/kcsan-checks.h> + +/* + * Yes, this permits 64-bit accesses on 32-bit architectures. These will + * actually be atomic in some cases (namely Armv7 + LPAE), but for others we + * rely on the access being split into 2x32-bit accesses for a 32-bit quantity + * (e.g. a virtual address) and a strong prevailing wind. + */ +#define compiletime_assert_rwonce_type(t) \ + compiletime_assert(__native_word(t) || sizeof(t) == sizeof(long long), \ + "Unsupported access size for {READ,WRITE}_ONCE().") + +/* + * Use __READ_ONCE() instead of READ_ONCE() if you do not require any + * atomicity. Note that this may result in tears! + */ +#ifndef __READ_ONCE +#define __READ_ONCE(x) (*(const volatile __unqual_scalar_typeof(x) *)&(x)) +#endif + +#define READ_ONCE(x) \ +({ \ + compiletime_assert_rwonce_type(x); \ + __READ_ONCE(x); \ +}) + +#define __WRITE_ONCE(x, val) \ +do { \ + *(volatile typeof(x) *)&(x) = (val); \ +} while (0) + +#define WRITE_ONCE(x, val) \ +do { \ + compiletime_assert_rwonce_type(x); \ + __WRITE_ONCE(x, val); \ +} while (0) + +static __no_sanitize_or_inline +unsigned long __read_once_word_nocheck(const void *addr) +{ + return __READ_ONCE(*(unsigned long *)addr); +} + +/* + * Use READ_ONCE_NOCHECK() instead of READ_ONCE() if you need to load a + * word from memory atomically but without telling KASAN/KCSAN. This is + * usually used by unwinding code when walking the stack of a running process. + */ +#define READ_ONCE_NOCHECK(x) \ +({ \ + compiletime_assert(sizeof(x) == sizeof(unsigned long), \ + "Unsupported access size for READ_ONCE_NOCHECK()."); \ + (typeof(x))__read_once_word_nocheck(&(x)); \ +}) + +static __no_sanitize_or_inline +unsigned long read_word_at_a_time(const void *addr) +{ + /* open-coded instrument_read(addr, 1) */ + kasan_check_read(addr, 1); + kcsan_check_read(addr, 1); + + /* + * This load can race with concurrent stores to out-of-bounds memory, + * but READ_ONCE() can't be used because it requires higher alignment + * than plain loads in arm64 builds with LTO. + */ + return *(unsigned long *)addr; +} + +#endif /* __ASSEMBLY__ */ +#endif /* __ASM_GENERIC_RWONCE_H */ diff --git a/include/asm-generic/rwsem.h b/include/asm-generic/rwsem.h deleted file mode 100644 index 93e67a055a4d..000000000000 --- a/include/asm-generic/rwsem.h +++ /dev/null @@ -1,140 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _ASM_GENERIC_RWSEM_H -#define _ASM_GENERIC_RWSEM_H - -#ifndef _LINUX_RWSEM_H -#error "Please don't include <asm/rwsem.h> directly, use <linux/rwsem.h> instead." -#endif - -#ifdef __KERNEL__ - -/* - * R/W semaphores originally for PPC using the stuff in lib/rwsem.c. - * Adapted largely from include/asm-i386/rwsem.h - * by Paul Mackerras <paulus@samba.org>. - */ - -/* - * the semaphore definition - */ -#ifdef CONFIG_64BIT -# define RWSEM_ACTIVE_MASK 0xffffffffL -#else -# define RWSEM_ACTIVE_MASK 0x0000ffffL -#endif - -#define RWSEM_UNLOCKED_VALUE 0x00000000L -#define RWSEM_ACTIVE_BIAS 0x00000001L -#define RWSEM_WAITING_BIAS (-RWSEM_ACTIVE_MASK-1) -#define RWSEM_ACTIVE_READ_BIAS RWSEM_ACTIVE_BIAS -#define RWSEM_ACTIVE_WRITE_BIAS (RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS) - -/* - * lock for reading - */ -static inline void __down_read(struct rw_semaphore *sem) -{ - if (unlikely(atomic_long_inc_return_acquire(&sem->count) <= 0)) - rwsem_down_read_failed(sem); -} - -static inline int __down_read_killable(struct rw_semaphore *sem) -{ - if (unlikely(atomic_long_inc_return_acquire(&sem->count) <= 0)) { - if (IS_ERR(rwsem_down_read_failed_killable(sem))) - return -EINTR; - } - - return 0; -} - -static inline int __down_read_trylock(struct rw_semaphore *sem) -{ - long tmp; - - while ((tmp = atomic_long_read(&sem->count)) >= 0) { - if (tmp == atomic_long_cmpxchg_acquire(&sem->count, tmp, - tmp + RWSEM_ACTIVE_READ_BIAS)) { - return 1; - } - } - return 0; -} - -/* - * lock for writing - */ -static inline void __down_write(struct rw_semaphore *sem) -{ - long tmp; - - tmp = atomic_long_add_return_acquire(RWSEM_ACTIVE_WRITE_BIAS, - &sem->count); - if (unlikely(tmp != RWSEM_ACTIVE_WRITE_BIAS)) - rwsem_down_write_failed(sem); -} - -static inline int __down_write_killable(struct rw_semaphore *sem) -{ - long tmp; - - tmp = atomic_long_add_return_acquire(RWSEM_ACTIVE_WRITE_BIAS, - &sem->count); - if (unlikely(tmp != RWSEM_ACTIVE_WRITE_BIAS)) - if (IS_ERR(rwsem_down_write_failed_killable(sem))) - return -EINTR; - return 0; -} - -static inline int __down_write_trylock(struct rw_semaphore *sem) -{ - long tmp; - - tmp = atomic_long_cmpxchg_acquire(&sem->count, RWSEM_UNLOCKED_VALUE, - RWSEM_ACTIVE_WRITE_BIAS); - return tmp == RWSEM_UNLOCKED_VALUE; -} - -/* - * unlock after reading - */ -static inline void __up_read(struct rw_semaphore *sem) -{ - long tmp; - - tmp = atomic_long_dec_return_release(&sem->count); - if (unlikely(tmp < -1 && (tmp & RWSEM_ACTIVE_MASK) == 0)) - rwsem_wake(sem); -} - -/* - * unlock after writing - */ -static inline void __up_write(struct rw_semaphore *sem) -{ - if (unlikely(atomic_long_sub_return_release(RWSEM_ACTIVE_WRITE_BIAS, - &sem->count) < 0)) - rwsem_wake(sem); -} - -/* - * downgrade write lock to read lock - */ -static inline void __downgrade_write(struct rw_semaphore *sem) -{ - long tmp; - - /* - * When downgrading from exclusive to shared ownership, - * anything inside the write-locked region cannot leak - * into the read side. In contrast, anything in the - * read-locked region is ok to be re-ordered into the - * write side. As such, rely on RELEASE semantics. - */ - tmp = atomic_long_add_return_release(-RWSEM_WAITING_BIAS, &sem->count); - if (tmp < 0) - rwsem_downgrade_wake(sem); -} - -#endif /* __KERNEL__ */ -#endif /* _ASM_GENERIC_RWSEM_H */ diff --git a/include/asm-generic/seccomp.h b/include/asm-generic/seccomp.h index e74072d23e69..6b6f42bc58f9 100644 --- a/include/asm-generic/seccomp.h +++ b/include/asm-generic/seccomp.h @@ -1,12 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * include/asm-generic/seccomp.h * * Copyright (C) 2014 Linaro Limited * Author: AKASHI Takahiro <takahiro.akashi@linaro.org> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #ifndef _ASM_GENERIC_SECCOMP_H #define _ASM_GENERIC_SECCOMP_H @@ -36,7 +33,7 @@ static inline const int *get_compat_mode1_syscalls(void) static const int mode1_syscalls_32[] = { __NR_seccomp_read_32, __NR_seccomp_write_32, __NR_seccomp_exit_32, __NR_seccomp_sigreturn_32, - 0, /* null terminated */ + -1, /* negative terminated */ }; return mode1_syscalls_32; } diff --git a/include/asm-generic/sections.h b/include/asm-generic/sections.h index d79abca81a52..0755bc39b0d8 100644 --- a/include/asm-generic/sections.h +++ b/include/asm-generic/sections.h @@ -39,7 +39,7 @@ extern char __init_begin[], __init_end[]; extern char _sinittext[], _einittext[]; extern char __start_ro_after_init[], __end_ro_after_init[]; extern char _end[]; -extern char __per_cpu_load[], __per_cpu_start[], __per_cpu_end[]; +extern char __per_cpu_start[], __per_cpu_end[]; extern char __kprobes_text_start[], __kprobes_text_end[]; extern char __entry_text_start[], __entry_text_end[]; extern char __start_rodata[], __end_rodata[]; @@ -53,29 +53,29 @@ extern char __ctors_start[], __ctors_end[]; /* Start and end of .opd section - used for function descriptors. */ extern char __start_opd[], __end_opd[]; +/* Start and end of instrumentation protected text section */ +extern char __noinstr_text_start[], __noinstr_text_end[]; + extern __visible const void __nosave_begin, __nosave_end; /* Function descriptor handling (if any). Override in asm/sections.h */ -#ifndef dereference_function_descriptor -#define dereference_function_descriptor(p) (p) -#define dereference_kernel_function_descriptor(p) (p) -#endif - -/* random extra sections (if any). Override - * in asm/sections.h */ -#ifndef arch_is_kernel_text -static inline int arch_is_kernel_text(unsigned long addr) -{ - return 0; -} +#ifdef CONFIG_HAVE_FUNCTION_DESCRIPTORS +void *dereference_function_descriptor(void *ptr); +void *dereference_kernel_function_descriptor(void *ptr); +#else +#define dereference_function_descriptor(p) ((void *)(p)) +#define dereference_kernel_function_descriptor(p) ((void *)(p)) + +/* An address is simply the address of the function. */ +typedef struct { + unsigned long addr; +} func_desc_t; #endif -#ifndef arch_is_kernel_data -static inline int arch_is_kernel_data(unsigned long addr) +static inline bool have_function_descriptors(void) { - return 0; + return IS_ENABLED(CONFIG_HAVE_FUNCTION_DESCRIPTORS); } -#endif /** * memory_contains - checks if an object is contained within a memory region @@ -97,7 +97,7 @@ static inline bool memory_contains(void *begin, void *end, void *virt, /** * memory_intersects - checks if the region occupied by an object intersects * with another memory region - * @begin: virtual address of the beginning of the memory regien + * @begin: virtual address of the beginning of the memory region * @end: virtual address of the end of the memory region * @virt: virtual address of the memory object * @size: size of the memory object @@ -110,7 +110,10 @@ static inline bool memory_intersects(void *begin, void *end, void *virt, { void *vend = virt + size; - return (virt >= begin && virt < end) || (vend >= begin && vend < end); + if (virt < end && vend > begin) + return true; + + return false; } /** @@ -142,6 +145,28 @@ static inline bool init_section_intersects(void *virt, size_t size) } /** + * is_kernel_core_data - checks if the pointer address is located in the + * .data or .bss section + * + * @addr: address to check + * + * Returns: true if the address is located in .data or .bss, false otherwise. + * Note: On some archs it may return true for core RODATA, and false + * for others. But will always be true for core RW data. + */ +static inline bool is_kernel_core_data(unsigned long addr) +{ + if (addr >= (unsigned long)_sdata && addr < (unsigned long)_edata) + return true; + + if (addr >= (unsigned long)__bss_start && + addr < (unsigned long)__bss_stop) + return true; + + return false; +} + +/** * is_kernel_rodata - checks if the pointer address is located in the * .rodata section * @@ -155,4 +180,56 @@ static inline bool is_kernel_rodata(unsigned long addr) addr < (unsigned long)__end_rodata; } +static inline bool is_kernel_ro_after_init(unsigned long addr) +{ + return addr >= (unsigned long)__start_ro_after_init && + addr < (unsigned long)__end_ro_after_init; +} +/** + * is_kernel_inittext - checks if the pointer address is located in the + * .init.text section + * + * @addr: address to check + * + * Returns: true if the address is located in .init.text, false otherwise. + */ +static inline bool is_kernel_inittext(unsigned long addr) +{ + return addr >= (unsigned long)_sinittext && + addr < (unsigned long)_einittext; +} + +/** + * __is_kernel_text - checks if the pointer address is located in the + * .text section + * + * @addr: address to check + * + * Returns: true if the address is located in .text, false otherwise. + * Note: an internal helper, only check the range of _stext to _etext. + */ +static inline bool __is_kernel_text(unsigned long addr) +{ + return addr >= (unsigned long)_stext && + addr < (unsigned long)_etext; +} + +/** + * __is_kernel - checks if the pointer address is located in the kernel range + * + * @addr: address to check + * + * Returns: true if the address is located in the kernel range, false otherwise. + * Note: an internal helper, check the range of _stext to _end, + * and range from __init_begin to __init_end, which can be outside + * of the _stext to _end range. + */ +static inline bool __is_kernel(unsigned long addr) +{ + return ((addr >= (unsigned long)_stext && + addr < (unsigned long)_end) || + (addr >= (unsigned long)__init_begin && + addr < (unsigned long)__init_end)); +} + #endif /* _ASM_GENERIC_SECTIONS_H_ */ diff --git a/include/asm-generic/segment.h b/include/asm-generic/segment.h deleted file mode 100644 index 5580eace622c..000000000000 --- a/include/asm-generic/segment.h +++ /dev/null @@ -1,9 +0,0 @@ -#ifndef __ASM_GENERIC_SEGMENT_H -#define __ASM_GENERIC_SEGMENT_H -/* - * Only here because we have some old header files that expect it... - * - * New architectures probably don't want to have their own version. - */ - -#endif /* __ASM_GENERIC_SEGMENT_H */ diff --git a/include/asm-generic/shmparam.h b/include/asm-generic/shmparam.h new file mode 100644 index 000000000000..b8f9035ffc2c --- /dev/null +++ b/include/asm-generic/shmparam.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_GENERIC_SHMPARAM_H +#define __ASM_GENERIC_SHMPARAM_H + +#define SHMLBA PAGE_SIZE /* attach addr a multiple of this */ + +#endif /* _ASM_GENERIC_SHMPARAM_H */ diff --git a/include/asm-generic/signal.h b/include/asm-generic/signal.h index c53984fa9761..663dd6d0795d 100644 --- a/include/asm-generic/signal.h +++ b/include/asm-generic/signal.h @@ -5,8 +5,6 @@ #include <uapi/asm-generic/signal.h> #ifndef __ASSEMBLY__ -#ifdef SA_RESTORER -#endif #include <asm/sigcontext.h> #undef __HAVE_ARCH_SIG_BITOPS diff --git a/include/asm-generic/simd.h b/include/asm-generic/simd.h index d0343d58a74a..70c8716ad32a 100644 --- a/include/asm-generic/simd.h +++ b/include/asm-generic/simd.h @@ -1,6 +1,11 @@ /* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_GENERIC_SIMD_H +#define _ASM_GENERIC_SIMD_H -#include <linux/hardirq.h> +#include <linux/compiler_attributes.h> +#include <linux/preempt.h> +#include <linux/sched.h> +#include <linux/types.h> /* * may_use_simd - whether it is allowable at this time to issue SIMD @@ -13,3 +18,5 @@ static __must_check inline bool may_use_simd(void) { return !in_interrupt(); } + +#endif /* _ASM_GENERIC_SIMD_H */ diff --git a/include/asm-generic/sizes.h b/include/asm-generic/sizes.h deleted file mode 100644 index 1dcfad9629ef..000000000000 --- a/include/asm-generic/sizes.h +++ /dev/null @@ -1,2 +0,0 @@ -/* This is a placeholder, to be removed over time */ -#include <linux/sizes.h> diff --git a/include/asm-generic/softirq_stack.h b/include/asm-generic/softirq_stack.h new file mode 100644 index 000000000000..2a67aed9ac52 --- /dev/null +++ b/include/asm-generic/softirq_stack.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef __ASM_GENERIC_SOFTIRQ_STACK_H +#define __ASM_GENERIC_SOFTIRQ_STACK_H + +#ifdef CONFIG_SOFTIRQ_ON_OWN_STACK +void do_softirq_own_stack(void); +#else +static inline void do_softirq_own_stack(void) +{ + __do_softirq(); +} +#endif + +#endif diff --git a/include/asm-generic/spinlock.h b/include/asm-generic/spinlock.h index adaf6acab172..970590baf61b 100644 --- a/include/asm-generic/spinlock.h +++ b/include/asm-generic/spinlock.h @@ -1,12 +1,9 @@ /* SPDX-License-Identifier: GPL-2.0 */ + #ifndef __ASM_GENERIC_SPINLOCK_H #define __ASM_GENERIC_SPINLOCK_H -/* - * You need to implement asm/spinlock.h for SMP support. The generic - * version does not handle SMP. - */ -#ifdef CONFIG_SMP -#error need an architecture specific asm/spinlock.h -#endif + +#include <asm-generic/ticket_spinlock.h> +#include <asm/qrwlock.h> #endif /* __ASM_GENERIC_SPINLOCK_H */ diff --git a/include/asm-generic/spinlock_types.h b/include/asm-generic/spinlock_types.h new file mode 100644 index 000000000000..f534aa5de394 --- /dev/null +++ b/include/asm-generic/spinlock_types.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __ASM_GENERIC_SPINLOCK_TYPES_H +#define __ASM_GENERIC_SPINLOCK_TYPES_H + +#include <asm-generic/qspinlock_types.h> +#include <asm-generic/qrwlock_types.h> + +#endif /* __ASM_GENERIC_SPINLOCK_TYPES_H */ diff --git a/include/asm-generic/switch_to.h b/include/asm-generic/switch_to.h index 986acc9d34bb..5897d100a6e6 100644 --- a/include/asm-generic/switch_to.h +++ b/include/asm-generic/switch_to.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* Generic task switch macro wrapper. * * It should be possible to use these on really simple architectures, @@ -5,11 +6,6 @@ * * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved. * Written by David Howells (dhowells@redhat.com) - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public Licence - * as published by the Free Software Foundation; either version - * 2 of the Licence, or (at your option) any later version. */ #ifndef __ASM_GENERIC_SWITCH_TO_H #define __ASM_GENERIC_SWITCH_TO_H diff --git a/include/asm-generic/syscall.h b/include/asm-generic/syscall.h index 0c938a4354f6..c5a3ad53beec 100644 --- a/include/asm-generic/syscall.h +++ b/include/asm-generic/syscall.h @@ -1,14 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * Access to user system call parameters and results * * Copyright (C) 2008-2009 Red Hat, Inc. All rights reserved. * - * This copyrighted material is made available to anyone wishing to use, - * modify, copy, or redistribute it subject to the terms and conditions - * of the GNU General Public License v.2. - * * This file is a stub providing documentation for what functions - * asm-ARCH/syscall.h files need to define. Most arch definitions + * arch/ARCH/include/asm/syscall.h files need to define. Most arch definitions * will be simple inlines. * * All of these functions expect to be called with no locks, @@ -41,14 +38,28 @@ struct pt_regs; int syscall_get_nr(struct task_struct *task, struct pt_regs *regs); /** + * syscall_set_nr - change the system call a task is executing + * @task: task of interest, must be blocked + * @regs: task_pt_regs() of @task + * @nr: system call number + * + * Changes the system call number @task is about to execute. + * + * It's only valid to call this when @task is stopped for tracing on + * entry to a system call, due to %SYSCALL_WORK_SYSCALL_TRACE or + * %SYSCALL_WORK_SYSCALL_AUDIT. + */ +void syscall_set_nr(struct task_struct *task, struct pt_regs *regs, int nr); + +/** * syscall_rollback - roll back registers after an aborted system call * @task: task of interest, must be in system call exit tracing * @regs: task_pt_regs() of @task * * It's only valid to call this when @task is stopped for system - * call exit tracing (due to TIF_SYSCALL_TRACE or TIF_SYSCALL_AUDIT), - * after tracehook_report_syscall_entry() returned nonzero to prevent - * the system call from taking place. + * call exit tracing (due to %SYSCALL_WORK_SYSCALL_TRACE or + * %SYSCALL_WORK_SYSCALL_AUDIT), after ptrace_report_syscall_entry() + * returned nonzero to prevent the system call from taking place. * * This rolls back the register state in @regs so it's as if the * system call instruction was a no-op. The registers containing @@ -66,7 +77,8 @@ void syscall_rollback(struct task_struct *task, struct pt_regs *regs); * Returns 0 if the system call succeeded, or -ERRORCODE if it failed. * * It's only valid to call this when @task is stopped for tracing on exit - * from a system call, due to %TIF_SYSCALL_TRACE or %TIF_SYSCALL_AUDIT. + * from a system call, due to %SYSCALL_WORK_SYSCALL_TRACE or + * %SYSCALL_WORK_SYSCALL_AUDIT. */ long syscall_get_error(struct task_struct *task, struct pt_regs *regs); @@ -79,7 +91,8 @@ long syscall_get_error(struct task_struct *task, struct pt_regs *regs); * This value is meaningless if syscall_get_error() returned nonzero. * * It's only valid to call this when @task is stopped for tracing on exit - * from a system call, due to %TIF_SYSCALL_TRACE or %TIF_SYSCALL_AUDIT. + * from a system call, due to %SYSCALL_WORK_SYSCALL_TRACE or + * %SYSCALL_WORK_SYSCALL_AUDIT. */ long syscall_get_return_value(struct task_struct *task, struct pt_regs *regs); @@ -96,7 +109,8 @@ long syscall_get_return_value(struct task_struct *task, struct pt_regs *regs); * code; the user sees a failed system call with this errno code. * * It's only valid to call this when @task is stopped for tracing on exit - * from a system call, due to %TIF_SYSCALL_TRACE or %TIF_SYSCALL_AUDIT. + * from a system call, due to %SYSCALL_WORK_SYSCALL_TRACE or + * %SYSCALL_WORK_SYSCALL_AUDIT. */ void syscall_set_return_value(struct task_struct *task, struct pt_regs *regs, int error, long val); @@ -105,53 +119,46 @@ void syscall_set_return_value(struct task_struct *task, struct pt_regs *regs, * syscall_get_arguments - extract system call parameter values * @task: task of interest, must be blocked * @regs: task_pt_regs() of @task - * @i: argument index [0,5] - * @n: number of arguments; n+i must be [1,6]. * @args: array filled with argument values * - * Fetches @n arguments to the system call starting with the @i'th argument - * (from 0 through 5). Argument @i is stored in @args[0], and so on. - * An arch inline version is probably optimal when @i and @n are constants. + * Fetches 6 arguments to the system call. First argument is stored in +* @args[0], and so on. * * It's only valid to call this when @task is stopped for tracing on - * entry to a system call, due to %TIF_SYSCALL_TRACE or %TIF_SYSCALL_AUDIT. - * It's invalid to call this with @i + @n > 6; we only support system calls - * taking up to 6 arguments. + * entry to a system call, due to %SYSCALL_WORK_SYSCALL_TRACE or + * %SYSCALL_WORK_SYSCALL_AUDIT. */ void syscall_get_arguments(struct task_struct *task, struct pt_regs *regs, - unsigned int i, unsigned int n, unsigned long *args); + unsigned long *args); /** * syscall_set_arguments - change system call parameter value * @task: task of interest, must be in system call entry tracing * @regs: task_pt_regs() of @task - * @i: argument index [0,5] - * @n: number of arguments; n+i must be [1,6]. * @args: array of argument values to store * - * Changes @n arguments to the system call starting with the @i'th argument. - * Argument @i gets value @args[0], and so on. - * An arch inline version is probably optimal when @i and @n are constants. + * Changes 6 arguments to the system call. + * The first argument gets value @args[0], and so on. * * It's only valid to call this when @task is stopped for tracing on - * entry to a system call, due to %TIF_SYSCALL_TRACE or %TIF_SYSCALL_AUDIT. - * It's invalid to call this with @i + @n > 6; we only support system calls - * taking up to 6 arguments. + * entry to a system call, due to %SYSCALL_WORK_SYSCALL_TRACE or + * %SYSCALL_WORK_SYSCALL_AUDIT. */ void syscall_set_arguments(struct task_struct *task, struct pt_regs *regs, - unsigned int i, unsigned int n, const unsigned long *args); /** * syscall_get_arch - return the AUDIT_ARCH for the current system call + * @task: task of interest, must be blocked * * Returns the AUDIT_ARCH_* based on the system call convention in use. * - * It's only valid to call this when current is stopped on entry to a system - * call, due to %TIF_SYSCALL_TRACE, %TIF_SYSCALL_AUDIT, or %TIF_SECCOMP. + * It's only valid to call this when @task is stopped on entry to a system + * call, due to %SYSCALL_WORK_SYSCALL_TRACE, %SYSCALL_WORK_SYSCALL_AUDIT, or + * %SYSCALL_WORK_SECCOMP. * * Architectures which permit CONFIG_HAVE_ARCH_SECCOMP_FILTER must * provide an implementation of this. */ -int syscall_get_arch(void); +int syscall_get_arch(struct task_struct *task); #endif /* _ASM_SYSCALL_H */ diff --git a/include/asm-generic/syscalls.h b/include/asm-generic/syscalls.h index 933ca6581aba..fabcefe8a80a 100644 --- a/include/asm-generic/syscalls.h +++ b/include/asm-generic/syscalls.h @@ -19,7 +19,7 @@ asmlinkage long sys_mmap2(unsigned long addr, unsigned long len, #ifndef sys_mmap asmlinkage long sys_mmap(unsigned long addr, unsigned long len, unsigned long prot, unsigned long flags, - unsigned long fd, off_t pgoff); + unsigned long fd, unsigned long off); #endif #ifndef sys_rt_sigreturn diff --git a/include/asm-generic/termios-base.h b/include/asm-generic/termios-base.h deleted file mode 100644 index 59c5a3bd4a6e..000000000000 --- a/include/asm-generic/termios-base.h +++ /dev/null @@ -1,78 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* termios.h: generic termios/termio user copying/translation - */ - -#ifndef _ASM_GENERIC_TERMIOS_BASE_H -#define _ASM_GENERIC_TERMIOS_BASE_H - -#include <linux/uaccess.h> - -#ifndef __ARCH_TERMIO_GETPUT - -/* - * Translate a "termio" structure into a "termios". Ugh. - */ -static inline int user_termio_to_kernel_termios(struct ktermios *termios, - struct termio __user *termio) -{ - unsigned short tmp; - - if (get_user(tmp, &termio->c_iflag) < 0) - goto fault; - termios->c_iflag = (0xffff0000 & termios->c_iflag) | tmp; - - if (get_user(tmp, &termio->c_oflag) < 0) - goto fault; - termios->c_oflag = (0xffff0000 & termios->c_oflag) | tmp; - - if (get_user(tmp, &termio->c_cflag) < 0) - goto fault; - termios->c_cflag = (0xffff0000 & termios->c_cflag) | tmp; - - if (get_user(tmp, &termio->c_lflag) < 0) - goto fault; - termios->c_lflag = (0xffff0000 & termios->c_lflag) | tmp; - - if (get_user(termios->c_line, &termio->c_line) < 0) - goto fault; - - if (copy_from_user(termios->c_cc, termio->c_cc, NCC) != 0) - goto fault; - - return 0; - - fault: - return -EFAULT; -} - -/* - * Translate a "termios" structure into a "termio". Ugh. - */ -static inline int kernel_termios_to_user_termio(struct termio __user *termio, - struct ktermios *termios) -{ - if (put_user(termios->c_iflag, &termio->c_iflag) < 0 || - put_user(termios->c_oflag, &termio->c_oflag) < 0 || - put_user(termios->c_cflag, &termio->c_cflag) < 0 || - put_user(termios->c_lflag, &termio->c_lflag) < 0 || - put_user(termios->c_line, &termio->c_line) < 0 || - copy_to_user(termio->c_cc, termios->c_cc, NCC) != 0) - return -EFAULT; - - return 0; -} - -#ifndef user_termios_to_kernel_termios -#define user_termios_to_kernel_termios(k, u) copy_from_user(k, u, sizeof(struct termios)) -#endif - -#ifndef kernel_termios_to_user_termios -#define kernel_termios_to_user_termios(u, k) copy_to_user(u, k, sizeof(struct termios)) -#endif - -#define user_termios_to_kernel_termios_1(k, u) copy_from_user(k, u, sizeof(struct termios)) -#define kernel_termios_to_user_termios_1(u, k) copy_to_user(u, k, sizeof(struct termios)) - -#endif /* __ARCH_TERMIO_GETPUT */ - -#endif /* _ASM_GENERIC_TERMIOS_BASE_H */ diff --git a/include/asm-generic/termios.h b/include/asm-generic/termios.h deleted file mode 100644 index b1398d0d4a1d..000000000000 --- a/include/asm-generic/termios.h +++ /dev/null @@ -1,108 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _ASM_GENERIC_TERMIOS_H -#define _ASM_GENERIC_TERMIOS_H - - -#include <linux/uaccess.h> -#include <uapi/asm-generic/termios.h> - -/* intr=^C quit=^\ erase=del kill=^U - eof=^D vtime=\0 vmin=\1 sxtc=\0 - start=^Q stop=^S susp=^Z eol=\0 - reprint=^R discard=^U werase=^W lnext=^V - eol2=\0 -*/ -#define INIT_C_CC "\003\034\177\025\004\0\1\0\021\023\032\0\022\017\027\026\0" - -/* - * Translate a "termio" structure into a "termios". Ugh. - */ -static inline int user_termio_to_kernel_termios(struct ktermios *termios, - const struct termio __user *termio) -{ - unsigned short tmp; - - if (get_user(tmp, &termio->c_iflag) < 0) - goto fault; - termios->c_iflag = (0xffff0000 & termios->c_iflag) | tmp; - - if (get_user(tmp, &termio->c_oflag) < 0) - goto fault; - termios->c_oflag = (0xffff0000 & termios->c_oflag) | tmp; - - if (get_user(tmp, &termio->c_cflag) < 0) - goto fault; - termios->c_cflag = (0xffff0000 & termios->c_cflag) | tmp; - - if (get_user(tmp, &termio->c_lflag) < 0) - goto fault; - termios->c_lflag = (0xffff0000 & termios->c_lflag) | tmp; - - if (get_user(termios->c_line, &termio->c_line) < 0) - goto fault; - - if (copy_from_user(termios->c_cc, termio->c_cc, NCC) != 0) - goto fault; - - return 0; - - fault: - return -EFAULT; -} - -/* - * Translate a "termios" structure into a "termio". Ugh. - */ -static inline int kernel_termios_to_user_termio(struct termio __user *termio, - struct ktermios *termios) -{ - if (put_user(termios->c_iflag, &termio->c_iflag) < 0 || - put_user(termios->c_oflag, &termio->c_oflag) < 0 || - put_user(termios->c_cflag, &termio->c_cflag) < 0 || - put_user(termios->c_lflag, &termio->c_lflag) < 0 || - put_user(termios->c_line, &termio->c_line) < 0 || - copy_to_user(termio->c_cc, termios->c_cc, NCC) != 0) - return -EFAULT; - - return 0; -} - -#ifdef TCGETS2 -static inline int user_termios_to_kernel_termios(struct ktermios *k, - struct termios2 __user *u) -{ - return copy_from_user(k, u, sizeof(struct termios2)); -} - -static inline int kernel_termios_to_user_termios(struct termios2 __user *u, - struct ktermios *k) -{ - return copy_to_user(u, k, sizeof(struct termios2)); -} - -static inline int user_termios_to_kernel_termios_1(struct ktermios *k, - struct termios __user *u) -{ - return copy_from_user(k, u, sizeof(struct termios)); -} - -static inline int kernel_termios_to_user_termios_1(struct termios __user *u, - struct ktermios *k) -{ - return copy_to_user(u, k, sizeof(struct termios)); -} -#else /* TCGETS2 */ -static inline int user_termios_to_kernel_termios(struct ktermios *k, - struct termios __user *u) -{ - return copy_from_user(k, u, sizeof(struct termios)); -} - -static inline int kernel_termios_to_user_termios(struct termios __user *u, - struct ktermios *k) -{ - return copy_to_user(u, k, sizeof(struct termios)); -} -#endif /* TCGETS2 */ - -#endif /* _ASM_GENERIC_TERMIOS_H */ diff --git a/include/asm-generic/text-patching.h b/include/asm-generic/text-patching.h new file mode 100644 index 000000000000..2245c641b741 --- /dev/null +++ b/include/asm-generic/text-patching.h @@ -0,0 +1,5 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_GENERIC_TEXT_PATCHING_H +#define _ASM_GENERIC_TEXT_PATCHING_H + +#endif /* _ASM_GENERIC_TEXT_PATCHING_H */ diff --git a/include/asm-generic/thread_info_tif.h b/include/asm-generic/thread_info_tif.h new file mode 100644 index 000000000000..da1610a78f92 --- /dev/null +++ b/include/asm-generic/thread_info_tif.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_GENERIC_THREAD_INFO_TIF_H_ +#define _ASM_GENERIC_THREAD_INFO_TIF_H_ + +#include <vdso/bits.h> + +/* Bits 16-31 are reserved for architecture specific purposes */ + +#define TIF_NOTIFY_RESUME 0 // callback before returning to user +#define _TIF_NOTIFY_RESUME BIT(TIF_NOTIFY_RESUME) + +#define TIF_SIGPENDING 1 // signal pending +#define _TIF_SIGPENDING BIT(TIF_SIGPENDING) + +#define TIF_NOTIFY_SIGNAL 2 // signal notifications exist +#define _TIF_NOTIFY_SIGNAL BIT(TIF_NOTIFY_SIGNAL) + +#define TIF_MEMDIE 3 // is terminating due to OOM killer +#define _TIF_MEMDIE BIT(TIF_MEMDIE) + +#define TIF_NEED_RESCHED 4 // rescheduling necessary +#define _TIF_NEED_RESCHED BIT(TIF_NEED_RESCHED) + +#ifdef HAVE_TIF_NEED_RESCHED_LAZY +# define TIF_NEED_RESCHED_LAZY 5 // Lazy rescheduling needed +# define _TIF_NEED_RESCHED_LAZY BIT(TIF_NEED_RESCHED_LAZY) +#endif + +#ifdef HAVE_TIF_POLLING_NRFLAG +# define TIF_POLLING_NRFLAG 6 // idle is polling for TIF_NEED_RESCHED +# define _TIF_POLLING_NRFLAG BIT(TIF_POLLING_NRFLAG) +#endif + +#define TIF_USER_RETURN_NOTIFY 7 // notify kernel of userspace return +#define _TIF_USER_RETURN_NOTIFY BIT(TIF_USER_RETURN_NOTIFY) + +#define TIF_UPROBE 8 // breakpointed or singlestepping +#define _TIF_UPROBE BIT(TIF_UPROBE) + +#define TIF_PATCH_PENDING 9 // pending live patching update +#define _TIF_PATCH_PENDING BIT(TIF_PATCH_PENDING) + +#ifdef HAVE_TIF_RESTORE_SIGMASK +# define TIF_RESTORE_SIGMASK 10 // Restore signal mask in do_signal() */ +# define _TIF_RESTORE_SIGMASK BIT(TIF_RESTORE_SIGMASK) +#endif + +#define TIF_RSEQ 11 // Run RSEQ fast path +#define _TIF_RSEQ BIT(TIF_RSEQ) + +#endif /* _ASM_GENERIC_THREAD_INFO_TIF_H_ */ diff --git a/include/asm-generic/ticket_spinlock.h b/include/asm-generic/ticket_spinlock.h new file mode 100644 index 000000000000..325779970d8a --- /dev/null +++ b/include/asm-generic/ticket_spinlock.h @@ -0,0 +1,105 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* + * 'Generic' ticket-lock implementation. + * + * It relies on atomic_fetch_add() having well defined forward progress + * guarantees under contention. If your architecture cannot provide this, stick + * to a test-and-set lock. + * + * It also relies on atomic_fetch_add() being safe vs smp_store_release() on a + * sub-word of the value. This is generally true for anything LL/SC although + * you'd be hard pressed to find anything useful in architecture specifications + * about this. If your architecture cannot do this you might be better off with + * a test-and-set. + * + * It further assumes atomic_*_release() + atomic_*_acquire() is RCpc and hence + * uses atomic_fetch_add() which is RCsc to create an RCsc hot path, along with + * a full fence after the spin to upgrade the otherwise-RCpc + * atomic_cond_read_acquire(). + * + * The implementation uses smp_cond_load_acquire() to spin, so if the + * architecture has WFE like instructions to sleep instead of poll for word + * modifications be sure to implement that (see ARM64 for example). + * + */ + +#ifndef __ASM_GENERIC_TICKET_SPINLOCK_H +#define __ASM_GENERIC_TICKET_SPINLOCK_H + +#include <linux/atomic.h> +#include <asm-generic/spinlock_types.h> + +static __always_inline void ticket_spin_lock(arch_spinlock_t *lock) +{ + u32 val = atomic_fetch_add(1<<16, &lock->val); + u16 ticket = val >> 16; + + if (ticket == (u16)val) + return; + + /* + * atomic_cond_read_acquire() is RCpc, but rather than defining a + * custom cond_read_rcsc() here we just emit a full fence. We only + * need the prior reads before subsequent writes ordering from + * smb_mb(), but as atomic_cond_read_acquire() just emits reads and we + * have no outstanding writes due to the atomic_fetch_add() the extra + * orderings are free. + */ + atomic_cond_read_acquire(&lock->val, ticket == (u16)VAL); + smp_mb(); +} + +static __always_inline bool ticket_spin_trylock(arch_spinlock_t *lock) +{ + u32 old = atomic_read(&lock->val); + + if ((old >> 16) != (old & 0xffff)) + return false; + + return atomic_try_cmpxchg(&lock->val, &old, old + (1<<16)); /* SC, for RCsc */ +} + +static __always_inline void ticket_spin_unlock(arch_spinlock_t *lock) +{ + u16 *ptr = (u16 *)lock + IS_ENABLED(CONFIG_CPU_BIG_ENDIAN); + u32 val = atomic_read(&lock->val); + + smp_store_release(ptr, (u16)val + 1); +} + +static __always_inline int ticket_spin_value_unlocked(arch_spinlock_t lock) +{ + u32 val = lock.val.counter; + + return ((val >> 16) == (val & 0xffff)); +} + +static __always_inline int ticket_spin_is_locked(arch_spinlock_t *lock) +{ + arch_spinlock_t val = READ_ONCE(*lock); + + return !ticket_spin_value_unlocked(val); +} + +static __always_inline int ticket_spin_is_contended(arch_spinlock_t *lock) +{ + u32 val = atomic_read(&lock->val); + + return (s16)((val >> 16) - (val & 0xffff)) > 1; +} + +#ifndef __no_arch_spinlock_redefine +/* + * Remapping spinlock architecture specific functions to the corresponding + * ticket spinlock functions. + */ +#define arch_spin_is_locked(l) ticket_spin_is_locked(l) +#define arch_spin_is_contended(l) ticket_spin_is_contended(l) +#define arch_spin_value_unlocked(l) ticket_spin_value_unlocked(l) +#define arch_spin_lock(l) ticket_spin_lock(l) +#define arch_spin_trylock(l) ticket_spin_trylock(l) +#define arch_spin_unlock(l) ticket_spin_unlock(l) +#endif + +#endif /* __ASM_GENERIC_TICKET_SPINLOCK_H */ diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h index 6be86c1c5c58..1fff717cae51 100644 --- a/include/asm-generic/tlb.h +++ b/include/asm-generic/tlb.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* include/asm-generic/tlb.h * * Generic TLB shootdown code @@ -6,65 +7,261 @@ * Based on code from mm/memory.c Copyright Linus Torvalds and others. * * Copyright 2011 Red Hat, Inc., Peter Zijlstra - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. */ #ifndef _ASM_GENERIC__TLB_H #define _ASM_GENERIC__TLB_H #include <linux/mmu_notifier.h> #include <linux/swap.h> -#include <asm/pgalloc.h> +#include <linux/hugetlb_inline.h> #include <asm/tlbflush.h> +#include <asm/cacheflush.h> + +/* + * Blindly accessing user memory from NMI context can be dangerous + * if we're in the middle of switching the current user task or switching + * the loaded mm. + */ +#ifndef nmi_uaccess_okay +# define nmi_uaccess_okay() true +#endif #ifdef CONFIG_MMU -#ifdef CONFIG_HAVE_RCU_TABLE_FREE /* - * Semi RCU freeing of the page directories. + * Generic MMU-gather implementation. + * + * The mmu_gather data structure is used by the mm code to implement the + * correct and efficient ordering of freeing pages and TLB invalidations. + * + * This correct ordering is: + * + * 1) unhook page + * 2) TLB invalidate page + * 3) free page + * + * That is, we must never free a page before we have ensured there are no live + * translations left to it. Otherwise it might be possible to observe (or + * worse, change) the page content after it has been reused. + * + * The mmu_gather API consists of: + * + * - tlb_gather_mmu() / tlb_gather_mmu_fullmm() / tlb_finish_mmu() + * + * start and finish a mmu_gather + * + * Finish in particular will issue a (final) TLB invalidate and free + * all (remaining) queued pages. + * + * - tlb_start_vma() / tlb_end_vma(); marks the start / end of a VMA + * + * Defaults to flushing at tlb_end_vma() to reset the range; helps when + * there's large holes between the VMAs. + * + * - tlb_free_vmas() + * + * tlb_free_vmas() marks the start of unlinking of one or more vmas + * and freeing page-tables. + * + * - tlb_remove_table() + * + * tlb_remove_table() is the basic primitive to free page-table directories + * (__p*_free_tlb()). In it's most primitive form it is an alias for + * tlb_remove_page() below, for when page directories are pages and have no + * additional constraints. + * + * See also MMU_GATHER_TABLE_FREE and MMU_GATHER_RCU_TABLE_FREE. + * + * - tlb_remove_page() / tlb_remove_page_size() + * - __tlb_remove_folio_pages() / __tlb_remove_page_size() + * - __tlb_remove_folio_pages_size() + * + * __tlb_remove_folio_pages_size() is the basic primitive that queues pages + * for freeing. It will return a boolean indicating if the queue is (now) + * full and a call to tlb_flush_mmu() is required. + * + * tlb_remove_page() and tlb_remove_page_size() imply the call to + * tlb_flush_mmu() when required and has no return value. + * + * __tlb_remove_folio_pages() is similar to __tlb_remove_page_size(), + * however, instead of removing a single page, assume PAGE_SIZE and remove + * the given number of consecutive pages that are all part of the + * same (large) folio. + * + * - tlb_change_page_size() + * + * call before __tlb_remove_page*() to set the current page-size; implies a + * possible tlb_flush_mmu() call. + * + * - tlb_flush_mmu() / tlb_flush_mmu_tlbonly() + * + * tlb_flush_mmu_tlbonly() - does the TLB invalidate (and resets + * related state, like the range) + * + * tlb_flush_mmu() - in addition to the above TLB invalidate, also frees + * whatever pages are still batched. + * + * - mmu_gather::fullmm + * + * A flag set by tlb_gather_mmu_fullmm() to indicate we're going to free + * the entire mm; this allows a number of optimizations. + * + * - We can ignore tlb_{start,end}_vma(); because we don't + * care about ranges. Everything will be shot down. + * + * - (RISC) architectures that use ASIDs can cycle to a new ASID + * and delay the invalidation until ASID space runs out. + * + * - mmu_gather::need_flush_all + * + * A flag that can be set by the arch code if it wants to force + * flush the entire TLB irrespective of the range. For instance + * x86-PAE needs this when changing top-level entries. + * + * And allows the architecture to provide and implement tlb_flush(): + * + * tlb_flush() may, in addition to the above mentioned mmu_gather fields, make + * use of: + * + * - mmu_gather::start / mmu_gather::end + * + * which provides the range that needs to be flushed to cover the pages to + * be freed. + * + * - mmu_gather::freed_tables + * + * set when we freed page table pages + * + * - tlb_get_unmap_shift() / tlb_get_unmap_size() + * + * returns the smallest TLB entry size unmapped in this range. + * + * If an architecture does not provide tlb_flush() a default implementation + * based on flush_tlb_range() will be used, unless MMU_GATHER_NO_RANGE is + * specified, in which case we'll default to flush_tlb_mm(). + * + * Additionally there are a few opt-in features: + * + * MMU_GATHER_PAGE_SIZE + * + * This ensures we call tlb_flush() every time tlb_change_page_size() actually + * changes the size and provides mmu_gather::page_size to tlb_flush(). + * + * This might be useful if your architecture has size specific TLB + * invalidation instructions. + * + * MMU_GATHER_TABLE_FREE + * + * This provides tlb_remove_table(), to be used instead of tlb_remove_page() + * for page directores (__p*_free_tlb()). + * + * Useful if your architecture has non-page page directories. + * + * When used, an architecture is expected to provide __tlb_remove_table() or + * use the generic __tlb_remove_table(), which does the actual freeing of these + * pages. * - * This is needed by some architectures to implement software pagetable walkers. + * MMU_GATHER_RCU_TABLE_FREE * - * gup_fast() and other software pagetable walkers do a lockless page-table - * walk and therefore needs some synchronization with the freeing of the page - * directories. The chosen means to accomplish that is by disabling IRQs over - * the walk. + * Like MMU_GATHER_TABLE_FREE, and adds semi-RCU semantics to the free (see + * comment below). * - * Architectures that use IPIs to flush TLBs will then automagically DTRT, - * since we unlink the page, flush TLBs, free the page. Since the disabling of - * IRQs delays the completion of the TLB flush we can never observe an already - * freed page. + * Useful if your architecture doesn't use IPIs for remote TLB invalidates + * and therefore doesn't naturally serialize with software page-table walkers. * - * Architectures that do not have this (PPC) need to delay the freeing by some - * other means, this is that means. + * MMU_GATHER_NO_FLUSH_CACHE * - * What we do is batch the freed directory pages (tables) and RCU free them. - * We use the sched RCU variant, as that guarantees that IRQ/preempt disabling - * holds off grace periods. + * Indicates the architecture has flush_cache_range() but it needs *NOT* be called + * before unmapping a VMA. * - * However, in order to batch these pages we need to allocate storage, this - * allocation is deep inside the MM code and can thus easily fail on memory - * pressure. To guarantee progress we fall back to single table freeing, see - * the implementation of tlb_remove_table_one(). + * NOTE: strictly speaking we shouldn't have this knob and instead rely on + * flush_cache_range() being a NOP, except Sparc64 seems to be + * different here. * + * MMU_GATHER_MERGE_VMAS + * + * Indicates the architecture wants to merge ranges over VMAs; typical when + * multiple range invalidates are more expensive than a full invalidate. + * + * MMU_GATHER_NO_RANGE + * + * Use this if your architecture lacks an efficient flush_tlb_range(). This + * option implies MMU_GATHER_MERGE_VMAS above. + * + * MMU_GATHER_NO_GATHER + * + * If the option is set the mmu_gather will not track individual pages for + * delayed page free anymore. A platform that enables the option needs to + * provide its own implementation of the __tlb_remove_page_size() function to + * free pages. + * + * This is useful if your architecture already flushes TLB entries in the + * various ptep_get_and_clear() functions. */ + +#ifdef CONFIG_MMU_GATHER_TABLE_FREE + struct mmu_table_batch { +#ifdef CONFIG_MMU_GATHER_RCU_TABLE_FREE struct rcu_head rcu; +#endif unsigned int nr; - void *tables[0]; + void *tables[]; }; #define MAX_TABLE_BATCH \ ((PAGE_SIZE - sizeof(struct mmu_table_batch)) / sizeof(void *)) -extern void tlb_table_flush(struct mmu_gather *tlb); +#ifndef __HAVE_ARCH_TLB_REMOVE_TABLE +static inline void __tlb_remove_table(void *table) +{ + struct ptdesc *ptdesc = (struct ptdesc *)table; + + pagetable_dtor_free(ptdesc); +} +#endif + extern void tlb_remove_table(struct mmu_gather *tlb, void *table); +#else /* !CONFIG_MMU_GATHER_TABLE_FREE */ + +static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page); +/* + * Without MMU_GATHER_TABLE_FREE the architecture is assumed to have page based + * page directories and we can use the normal page batching to free them. + */ +static inline void tlb_remove_table(struct mmu_gather *tlb, void *table) +{ + struct ptdesc *ptdesc = (struct ptdesc *)table; + + pagetable_dtor(ptdesc); + tlb_remove_page(tlb, ptdesc_page(ptdesc)); +} +#endif /* CONFIG_MMU_GATHER_TABLE_FREE */ + +#ifdef CONFIG_MMU_GATHER_RCU_TABLE_FREE +/* + * This allows an architecture that does not use the linux page-tables for + * hardware to skip the TLBI when freeing page tables. + */ +#ifndef tlb_needs_table_invalidate +#define tlb_needs_table_invalidate() (true) #endif +void tlb_remove_table_sync_one(void); + +#else + +#ifdef tlb_needs_table_invalidate +#error tlb_needs_table_invalidate() requires MMU_GATHER_RCU_TABLE_FREE +#endif + +static inline void tlb_remove_table_sync_one(void) { } + +#endif /* CONFIG_MMU_GATHER_RCU_TABLE_FREE */ + + +#ifndef CONFIG_MMU_GATHER_NO_GATHER /* * If we can't allocate a page to make a big batch of page pointers * to work on, then just handle a few from the on-stack structure. @@ -75,7 +272,7 @@ struct mmu_gather_batch { struct mmu_gather_batch *next; unsigned int nr; unsigned int max; - struct page *pages[0]; + struct encoded_page *encoded_pages[]; }; #define MAX_GATHER_BATCH \ @@ -89,14 +286,45 @@ struct mmu_gather_batch { */ #define MAX_GATHER_BATCH_COUNT (10000UL/MAX_GATHER_BATCH) -/* struct mmu_gather is an opaque type used by the mm code for passing around +extern bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page, + bool delay_rmap, int page_size); +bool __tlb_remove_folio_pages(struct mmu_gather *tlb, struct page *page, + unsigned int nr_pages, bool delay_rmap); + +#ifdef CONFIG_SMP +/* + * This both sets 'delayed_rmap', and returns true. It would be an inline + * function, except we define it before the 'struct mmu_gather'. + */ +#define tlb_delay_rmap(tlb) (((tlb)->delayed_rmap = 1), true) +extern void tlb_flush_rmaps(struct mmu_gather *tlb, struct vm_area_struct *vma); +#endif + +#endif + +/* + * We have a no-op version of the rmap removal that doesn't + * delay anything. That is used on S390, which flushes remote + * TLBs synchronously, and on UP, which doesn't have any + * remote TLBs to flush and is not preemptible due to this + * all happening under the page table lock. + */ +#ifndef tlb_delay_rmap +#define tlb_delay_rmap(tlb) (false) +static inline void tlb_flush_rmaps(struct mmu_gather *tlb, struct vm_area_struct *vma) { } +#endif + +/* + * struct mmu_gather is an opaque type used by the mm code for passing around * any data needed by arch specific code for tlb_remove_page. */ struct mmu_gather { struct mm_struct *mm; -#ifdef CONFIG_HAVE_RCU_TABLE_FREE + +#ifdef CONFIG_MMU_GATHER_TABLE_FREE struct mmu_table_batch *batch; #endif + unsigned long start; unsigned long end; /* @@ -117,6 +345,11 @@ struct mmu_gather { unsigned int freed_tables : 1; /* + * Do we have pending delayed rmap removals? + */ + unsigned int delayed_rmap : 1; + + /* * at which levels have we cleared entries? */ unsigned int cleared_ptes : 1; @@ -124,23 +357,27 @@ struct mmu_gather { unsigned int cleared_puds : 1; unsigned int cleared_p4ds : 1; + /* + * tracks VM_EXEC | VM_HUGETLB in tlb_start_vma + */ + unsigned int vma_exec : 1; + unsigned int vma_huge : 1; + unsigned int vma_pfn : 1; + + unsigned int batch_count; + +#ifndef CONFIG_MMU_GATHER_NO_GATHER struct mmu_gather_batch *active; struct mmu_gather_batch local; struct page *__pages[MMU_GATHER_BUNDLE]; - unsigned int batch_count; - int page_size; -}; -#define HAVE_GENERIC_MMU_GATHER +#ifdef CONFIG_MMU_GATHER_PAGE_SIZE + unsigned int page_size; +#endif +#endif +}; -void arch_tlb_gather_mmu(struct mmu_gather *tlb, - struct mm_struct *mm, unsigned long start, unsigned long end); void tlb_flush_mmu(struct mmu_gather *tlb); -void arch_tlb_finish_mmu(struct mmu_gather *tlb, - unsigned long start, unsigned long end, bool force); -void tlb_flush_mmu_free(struct mmu_gather *tlb); -extern bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page, - int page_size); static inline void __tlb_adjust_range(struct mmu_gather *tlb, unsigned long address, @@ -163,54 +400,126 @@ static inline void __tlb_reset_range(struct mmu_gather *tlb) tlb->cleared_pmds = 0; tlb->cleared_puds = 0; tlb->cleared_p4ds = 0; + /* + * Do not reset mmu_gather::vma_* fields here, we do not + * call into tlb_start_vma() again to set them if there is an + * intermediate flush. + */ +} + +#ifdef CONFIG_MMU_GATHER_NO_RANGE + +#if defined(tlb_flush) +#error MMU_GATHER_NO_RANGE relies on default tlb_flush() +#endif + +/* + * When an architecture does not have efficient means of range flushing TLBs + * there is no point in doing intermediate flushes on tlb_end_vma() to keep the + * range small. We equally don't have to worry about page granularity or other + * things. + * + * All we need to do is issue a full flush for any !0 range. + */ +static inline void tlb_flush(struct mmu_gather *tlb) +{ + if (tlb->end) + flush_tlb_mm(tlb->mm); +} + +#else /* CONFIG_MMU_GATHER_NO_RANGE */ + +#ifndef tlb_flush +/* + * When an architecture does not provide its own tlb_flush() implementation + * but does have a reasonably efficient flush_vma_range() implementation + * use that. + */ +static inline void tlb_flush(struct mmu_gather *tlb) +{ + if (tlb->fullmm || tlb->need_flush_all) { + flush_tlb_mm(tlb->mm); + } else if (tlb->end) { + struct vm_area_struct vma = { + .vm_mm = tlb->mm, + .vm_flags = (tlb->vma_exec ? VM_EXEC : 0) | + (tlb->vma_huge ? VM_HUGETLB : 0), + }; + + flush_tlb_range(&vma, tlb->start, tlb->end); + } +} +#endif + +#endif /* CONFIG_MMU_GATHER_NO_RANGE */ + +static inline void +tlb_update_vma_flags(struct mmu_gather *tlb, struct vm_area_struct *vma) +{ + /* + * flush_tlb_range() implementations that look at VM_HUGETLB (tile, + * mips-4k) flush only large pages. + * + * flush_tlb_range() implementations that flush I-TLB also flush D-TLB + * (tile, xtensa, arm), so it's ok to just add VM_EXEC to an existing + * range. + * + * We rely on tlb_end_vma() to issue a flush, such that when we reset + * these values the batch is empty. + */ + tlb->vma_huge = is_vm_hugetlb_page(vma); + tlb->vma_exec = !!(vma->vm_flags & VM_EXEC); + + /* + * Track if there's at least one VM_PFNMAP/VM_MIXEDMAP vma + * in the tracked range, see tlb_free_vmas(). + */ + tlb->vma_pfn |= !!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)); } static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb) { - if (!tlb->end) + /* + * Anything calling __tlb_adjust_range() also sets at least one of + * these bits. + */ + if (!(tlb->freed_tables || tlb->cleared_ptes || tlb->cleared_pmds || + tlb->cleared_puds || tlb->cleared_p4ds)) return; tlb_flush(tlb); - mmu_notifier_invalidate_range(tlb->mm, tlb->start, tlb->end); __tlb_reset_range(tlb); } static inline void tlb_remove_page_size(struct mmu_gather *tlb, struct page *page, int page_size) { - if (__tlb_remove_page_size(tlb, page, page_size)) + if (__tlb_remove_page_size(tlb, page, false, page_size)) tlb_flush_mmu(tlb); } -static inline bool __tlb_remove_page(struct mmu_gather *tlb, struct page *page) +static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page) { - return __tlb_remove_page_size(tlb, page, PAGE_SIZE); + return tlb_remove_page_size(tlb, page, PAGE_SIZE); } -/* tlb_remove_page - * Similar to __tlb_remove_page but will call tlb_flush_mmu() itself when - * required. - */ -static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page) +static inline void tlb_remove_ptdesc(struct mmu_gather *tlb, struct ptdesc *pt) { - return tlb_remove_page_size(tlb, page, PAGE_SIZE); + tlb_remove_table(tlb, pt); } -#ifndef tlb_remove_check_page_size_change -#define tlb_remove_check_page_size_change tlb_remove_check_page_size_change -static inline void tlb_remove_check_page_size_change(struct mmu_gather *tlb, +static inline void tlb_change_page_size(struct mmu_gather *tlb, unsigned int page_size) { - /* - * We don't care about page size change, just update - * mmu_gather page size here so that debug checks - * doesn't throw false warning. - */ -#ifdef CONFIG_DEBUG_VM +#ifdef CONFIG_MMU_GATHER_PAGE_SIZE + if (tlb->page_size && tlb->page_size != page_size) { + if (!tlb->fullmm && !tlb->need_flush_all) + tlb_flush_mmu(tlb); + } + tlb->page_size = page_size; #endif } -#endif static inline unsigned long tlb_get_unmap_shift(struct mmu_gather *tlb) { @@ -236,22 +545,89 @@ static inline unsigned long tlb_get_unmap_size(struct mmu_gather *tlb) * case where we're doing a full MM flush. When we're doing a munmap, * the vmas are adjusted to only cover the region to be torn down. */ -#ifndef tlb_start_vma -#define tlb_start_vma(tlb, vma) do { } while (0) +static inline void tlb_start_vma(struct mmu_gather *tlb, struct vm_area_struct *vma) +{ + if (tlb->fullmm) + return; + + tlb_update_vma_flags(tlb, vma); +#ifndef CONFIG_MMU_GATHER_NO_FLUSH_CACHE + flush_cache_range(vma, vma->vm_start, vma->vm_end); #endif +} -#define __tlb_end_vma(tlb, vma) \ - do { \ - if (!tlb->fullmm) \ - tlb_flush_mmu_tlbonly(tlb); \ - } while (0) +static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma) +{ + if (tlb->fullmm || IS_ENABLED(CONFIG_MMU_GATHER_MERGE_VMAS)) + return; -#ifndef tlb_end_vma -#define tlb_end_vma __tlb_end_vma -#endif + /* + * Do a TLB flush and reset the range at VMA boundaries; this avoids + * the ranges growing with the unused space between consecutive VMAs, + * but also the mmu_gather::vma_* flags from tlb_start_vma() rely on + * this. + */ + tlb_flush_mmu_tlbonly(tlb); +} + +static inline void tlb_free_vmas(struct mmu_gather *tlb) +{ + if (tlb->fullmm) + return; + + /* + * VM_PFNMAP is more fragile because the core mm will not track the + * page mapcount -- there might not be page-frames for these PFNs + * after all. + * + * Specifically() there is a race between munmap() and + * unmap_mapping_range(), where munmap() will unlink the VMA, such + * that unmap_mapping_range() will no longer observe the VMA and + * no-op, without observing the TLBI, returning prematurely. + * + * So if we're about to unlink such a VMA, and we have pending + * TLBI for such a vma, flush things now. + */ + if (tlb->vma_pfn) + tlb_flush_mmu_tlbonly(tlb); +} + +/* + * tlb_flush_{pte|pmd|pud|p4d}_range() adjust the tlb->start and tlb->end, + * and set corresponding cleared_*. + */ +static inline void tlb_flush_pte_range(struct mmu_gather *tlb, + unsigned long address, unsigned long size) +{ + __tlb_adjust_range(tlb, address, size); + tlb->cleared_ptes = 1; +} + +static inline void tlb_flush_pmd_range(struct mmu_gather *tlb, + unsigned long address, unsigned long size) +{ + __tlb_adjust_range(tlb, address, size); + tlb->cleared_pmds = 1; +} + +static inline void tlb_flush_pud_range(struct mmu_gather *tlb, + unsigned long address, unsigned long size) +{ + __tlb_adjust_range(tlb, address, size); + tlb->cleared_puds = 1; +} + +static inline void tlb_flush_p4d_range(struct mmu_gather *tlb, + unsigned long address, unsigned long size) +{ + __tlb_adjust_range(tlb, address, size); + tlb->cleared_p4ds = 1; +} #ifndef __tlb_remove_tlb_entry -#define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0) +static inline void __tlb_remove_tlb_entry(struct mmu_gather *tlb, pte_t *ptep, unsigned long address) +{ +} #endif /** @@ -263,19 +639,41 @@ static inline unsigned long tlb_get_unmap_size(struct mmu_gather *tlb) */ #define tlb_remove_tlb_entry(tlb, ptep, address) \ do { \ - __tlb_adjust_range(tlb, address, PAGE_SIZE); \ - tlb->cleared_ptes = 1; \ + tlb_flush_pte_range(tlb, address, PAGE_SIZE); \ __tlb_remove_tlb_entry(tlb, ptep, address); \ } while (0) +/** + * tlb_remove_tlb_entries - remember unmapping of multiple consecutive ptes for + * later tlb invalidation. + * + * Similar to tlb_remove_tlb_entry(), but remember unmapping of multiple + * consecutive ptes instead of only a single one. + */ +static inline void tlb_remove_tlb_entries(struct mmu_gather *tlb, + pte_t *ptep, unsigned int nr, unsigned long address) +{ + tlb_flush_pte_range(tlb, address, PAGE_SIZE * nr); + for (;;) { + __tlb_remove_tlb_entry(tlb, ptep, address); + if (--nr == 0) + break; + ptep++; + address += PAGE_SIZE; + } +} + #define tlb_remove_huge_tlb_entry(h, tlb, ptep, address) \ do { \ unsigned long _sz = huge_page_size(h); \ - __tlb_adjust_range(tlb, address, _sz); \ - if (_sz == PMD_SIZE) \ - tlb->cleared_pmds = 1; \ - else if (_sz == PUD_SIZE) \ - tlb->cleared_puds = 1; \ + if (_sz >= P4D_SIZE) \ + tlb_flush_p4d_range(tlb, address, _sz); \ + else if (_sz >= PUD_SIZE) \ + tlb_flush_pud_range(tlb, address, _sz); \ + else if (_sz >= PMD_SIZE) \ + tlb_flush_pmd_range(tlb, address, _sz); \ + else \ + tlb_flush_pte_range(tlb, address, _sz); \ __tlb_remove_tlb_entry(tlb, ptep, address); \ } while (0) @@ -289,8 +687,7 @@ static inline unsigned long tlb_get_unmap_size(struct mmu_gather *tlb) #define tlb_remove_pmd_tlb_entry(tlb, pmdp, address) \ do { \ - __tlb_adjust_range(tlb, address, HPAGE_PMD_SIZE); \ - tlb->cleared_pmds = 1; \ + tlb_flush_pmd_range(tlb, address, HPAGE_PMD_SIZE); \ __tlb_remove_pmd_tlb_entry(tlb, pmdp, address); \ } while (0) @@ -304,8 +701,7 @@ static inline unsigned long tlb_get_unmap_size(struct mmu_gather *tlb) #define tlb_remove_pud_tlb_entry(tlb, pudp, address) \ do { \ - __tlb_adjust_range(tlb, address, HPAGE_PUD_SIZE); \ - tlb->cleared_puds = 1; \ + tlb_flush_pud_range(tlb, address, HPAGE_PUD_SIZE); \ __tlb_remove_pud_tlb_entry(tlb, pudp, address); \ } while (0) @@ -330,9 +726,8 @@ static inline unsigned long tlb_get_unmap_size(struct mmu_gather *tlb) #ifndef pte_free_tlb #define pte_free_tlb(tlb, ptep, address) \ do { \ - __tlb_adjust_range(tlb, address, PAGE_SIZE); \ + tlb_flush_pmd_range(tlb, address, PAGE_SIZE); \ tlb->freed_tables = 1; \ - tlb->cleared_pmds = 1; \ __pte_free_tlb(tlb, ptep, address); \ } while (0) #endif @@ -340,26 +735,21 @@ static inline unsigned long tlb_get_unmap_size(struct mmu_gather *tlb) #ifndef pmd_free_tlb #define pmd_free_tlb(tlb, pmdp, address) \ do { \ - __tlb_adjust_range(tlb, address, PAGE_SIZE); \ + tlb_flush_pud_range(tlb, address, PAGE_SIZE); \ tlb->freed_tables = 1; \ - tlb->cleared_puds = 1; \ __pmd_free_tlb(tlb, pmdp, address); \ } while (0) #endif -#ifndef __ARCH_HAS_4LEVEL_HACK #ifndef pud_free_tlb #define pud_free_tlb(tlb, pudp, address) \ do { \ - __tlb_adjust_range(tlb, address, PAGE_SIZE); \ + tlb_flush_p4d_range(tlb, address, PAGE_SIZE); \ tlb->freed_tables = 1; \ - tlb->cleared_p4ds = 1; \ __pud_free_tlb(tlb, pudp, address); \ } while (0) #endif -#endif -#ifndef __ARCH_HAS_5LEVEL_HACK #ifndef p4d_free_tlb #define p4d_free_tlb(tlb, pudp, address) \ do { \ @@ -368,10 +758,21 @@ static inline unsigned long tlb_get_unmap_size(struct mmu_gather *tlb) __p4d_free_tlb(tlb, pudp, address); \ } while (0) #endif + +#ifndef pte_needs_flush +static inline bool pte_needs_flush(pte_t oldpte, pte_t newpte) +{ + return true; +} +#endif + +#ifndef huge_pmd_needs_flush +static inline bool huge_pmd_needs_flush(pmd_t oldpmd, pmd_t newpmd) +{ + return true; +} #endif #endif /* CONFIG_MMU */ -#define tlb_migrate_finish(mm) do {} while (0) - #endif /* _ASM_GENERIC__TLB_H */ diff --git a/include/asm-generic/topology.h b/include/asm-generic/topology.h index 238873739550..4dbe715be65b 100644 --- a/include/asm-generic/topology.h +++ b/include/asm-generic/topology.h @@ -45,10 +45,10 @@ #endif #ifndef cpumask_of_node - #ifdef CONFIG_NEED_MULTIPLE_NODES + #ifdef CONFIG_NUMA #define cpumask_of_node(node) ((node) == 0 ? cpu_online_mask : cpu_none_mask) #else - #define cpumask_of_node(node) ((void)node, cpu_online_mask) + #define cpumask_of_node(node) ((void)(node), cpu_online_mask) #endif #endif #ifndef pcibus_to_node diff --git a/include/asm-generic/uaccess.h b/include/asm-generic/uaccess.h index d82c78a79da5..b276f783494c 100644 --- a/include/asm-generic/uaccess.h +++ b/include/asm-generic/uaccess.h @@ -8,45 +8,92 @@ * address space, e.g. all NOMMU machines. */ #include <linux/string.h> +#include <asm-generic/access_ok.h> -#include <asm/segment.h> +#ifdef CONFIG_UACCESS_MEMCPY +#include <linux/unaligned.h> -#define MAKE_MM_SEG(s) ((mm_segment_t) { (s) }) - -#ifndef KERNEL_DS -#define KERNEL_DS MAKE_MM_SEG(~0UL) -#endif +static __always_inline int +__get_user_fn(size_t size, const void __user *from, void *to) +{ + BUILD_BUG_ON(!__builtin_constant_p(size)); -#ifndef USER_DS -#define USER_DS MAKE_MM_SEG(TASK_SIZE - 1) -#endif + switch (size) { + case 1: + *(u8 *)to = *((u8 __force *)from); + return 0; + case 2: + *(u16 *)to = get_unaligned((u16 __force *)from); + return 0; + case 4: + *(u32 *)to = get_unaligned((u32 __force *)from); + return 0; + case 8: + *(u64 *)to = get_unaligned((u64 __force *)from); + return 0; + default: + BUILD_BUG(); + return 0; + } -#ifndef get_fs -#define get_ds() (KERNEL_DS) -#define get_fs() (current_thread_info()->addr_limit) +} +#define __get_user_fn(sz, u, k) __get_user_fn(sz, u, k) -static inline void set_fs(mm_segment_t fs) +static __always_inline int +__put_user_fn(size_t size, void __user *to, void *from) { - current_thread_info()->addr_limit = fs; + BUILD_BUG_ON(!__builtin_constant_p(size)); + + switch (size) { + case 1: + *(u8 __force *)to = *(u8 *)from; + return 0; + case 2: + put_unaligned(*(u16 *)from, (u16 __force *)to); + return 0; + case 4: + put_unaligned(*(u32 *)from, (u32 __force *)to); + return 0; + case 8: + put_unaligned(*(u64 *)from, (u64 __force *)to); + return 0; + default: + BUILD_BUG(); + return 0; + } } -#endif +#define __put_user_fn(sz, u, k) __put_user_fn(sz, u, k) -#ifndef segment_eq -#define segment_eq(a, b) ((a).seg == (b).seg) -#endif +#define __get_kernel_nofault(dst, src, type, err_label) \ +do { \ + *((type *)dst) = get_unaligned((type *)(src)); \ + if (0) /* make sure the label looks used to the compiler */ \ + goto err_label; \ +} while (0) -#define access_ok(addr, size) __access_ok((unsigned long)(addr),(size)) +#define __put_kernel_nofault(dst, src, type, err_label) \ +do { \ + put_unaligned(*((type *)src), (type *)(dst)); \ + if (0) /* make sure the label looks used to the compiler */ \ + goto err_label; \ +} while (0) -/* - * The architecture should really override this if possible, at least - * doing a check on the get_fs() - */ -#ifndef __access_ok -static inline int __access_ok(unsigned long addr, unsigned long size) +static inline __must_check unsigned long +raw_copy_from_user(void *to, const void __user * from, unsigned long n) { - return 1; + memcpy(to, (const void __force *)from, n); + return 0; } -#endif + +static inline __must_check unsigned long +raw_copy_to_user(void __user *to, const void *from, unsigned long n) +{ + memcpy((void __force *)to, from, n); + return 0; +} +#define INLINE_COPY_FROM_USER +#define INLINE_COPY_TO_USER +#endif /* CONFIG_UACCESS_MEMCPY */ /* * These are the main single-value transfer routines. They automatically @@ -158,50 +205,6 @@ static inline int __get_user_fn(size_t size, const void __user *ptr, void *x) extern int __get_user_bad(void) __attribute__((noreturn)); /* - * Copy a null terminated string from userspace. - */ -#ifndef __strncpy_from_user -static inline long -__strncpy_from_user(char *dst, const char __user *src, long count) -{ - char *tmp; - strncpy(dst, (const char __force *)src, count); - for (tmp = dst; *tmp && count > 0; tmp++, count--) - ; - return (tmp - dst); -} -#endif - -static inline long -strncpy_from_user(char *dst, const char __user *src, long count) -{ - if (!access_ok(src, 1)) - return -EFAULT; - return __strncpy_from_user(dst, src, count); -} - -/* - * Return the size of a string (including the ending 0) - * - * Return 0 on exception, a value greater than N if too long - */ -#ifndef __strnlen_user -#define __strnlen_user(s, n) (strnlen((s), (n)) + 1) -#endif - -/* - * Unlike strnlen, strnlen_user includes the nul terminator in - * its returned count. Callers should check for a returned value - * greater than N as an indication the string is too long. - */ -static inline long strnlen_user(const char __user *src, long n) -{ - if (!access_ok(src, 1)) - return 0; - return __strnlen_user(src, n); -} - -/* * Zero Userspace */ #ifndef __clear_user @@ -225,4 +228,8 @@ clear_user(void __user *to, unsigned long n) #include <asm/extable.h> +__must_check long strncpy_from_user(char *dst, const char __user *src, + long count); +__must_check long strnlen_user(const char __user *src, long n); + #endif /* __ASM_GENERIC_UACCESS_H */ diff --git a/include/asm-generic/unaligned.h b/include/asm-generic/unaligned.h deleted file mode 100644 index 374c940e9be1..000000000000 --- a/include/asm-generic/unaligned.h +++ /dev/null @@ -1,36 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef __ASM_GENERIC_UNALIGNED_H -#define __ASM_GENERIC_UNALIGNED_H - -/* - * This is the most generic implementation of unaligned accesses - * and should work almost anywhere. - */ -#include <asm/byteorder.h> - -/* Set by the arch if it can handle unaligned accesses in hardware. */ -#ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS -# include <linux/unaligned/access_ok.h> -#endif - -#if defined(__LITTLE_ENDIAN) -# ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS -# include <linux/unaligned/le_struct.h> -# include <linux/unaligned/be_byteshift.h> -# endif -# include <linux/unaligned/generic.h> -# define get_unaligned __get_unaligned_le -# define put_unaligned __put_unaligned_le -#elif defined(__BIG_ENDIAN) -# ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS -# include <linux/unaligned/be_struct.h> -# include <linux/unaligned/le_byteshift.h> -# endif -# include <linux/unaligned/generic.h> -# define get_unaligned __get_unaligned_be -# define put_unaligned __put_unaligned_be -#else -# error need to define endianess -#endif - -#endif /* __ASM_GENERIC_UNALIGNED_H */ diff --git a/include/asm-generic/unwind_user.h b/include/asm-generic/unwind_user.h new file mode 100644 index 000000000000..b8882b909944 --- /dev/null +++ b/include/asm-generic/unwind_user.h @@ -0,0 +1,5 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_GENERIC_UNWIND_USER_H +#define _ASM_GENERIC_UNWIND_USER_H + +#endif /* _ASM_GENERIC_UNWIND_USER_H */ diff --git a/include/asm-generic/vdso/vsyscall.h b/include/asm-generic/vdso/vsyscall.h new file mode 100644 index 000000000000..5c6d9799f4e7 --- /dev/null +++ b/include/asm-generic/vdso/vsyscall.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_GENERIC_VSYSCALL_H +#define __ASM_GENERIC_VSYSCALL_H + +#ifndef __ASSEMBLY__ + +#ifndef __arch_get_vdso_u_time_data +static __always_inline const struct vdso_time_data *__arch_get_vdso_u_time_data(void) +{ + return &vdso_u_time_data; +} +#endif + +#ifndef __arch_get_vdso_u_rng_data +static __always_inline const struct vdso_rng_data *__arch_get_vdso_u_rng_data(void) +{ + return &vdso_u_rng_data; +} +#endif + +#ifndef __arch_update_vdso_clock +static __always_inline void __arch_update_vdso_clock(struct vdso_clock *vc) +{ +} +#endif /* __arch_update_vdso_clock */ + +#ifndef __arch_sync_vdso_time_data +static __always_inline void __arch_sync_vdso_time_data(struct vdso_time_data *vdata) +{ +} +#endif /* __arch_sync_vdso_time_data */ + +#endif /* !__ASSEMBLY__ */ + +#endif /* __ASM_GENERIC_VSYSCALL_H */ diff --git a/include/asm-generic/vermagic.h b/include/asm-generic/vermagic.h new file mode 100644 index 000000000000..084274a1219e --- /dev/null +++ b/include/asm-generic/vermagic.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _ASM_GENERIC_VERMAGIC_H +#define _ASM_GENERIC_VERMAGIC_H + +#define MODULE_ARCH_VERMAGIC "" + +#endif /* _ASM_GENERIC_VERMAGIC_H */ diff --git a/include/asm-generic/vga.h b/include/asm-generic/vga.h index adf91a783b5c..5dcaf4ae904a 100644 --- a/include/asm-generic/vga.h +++ b/include/asm-generic/vga.h @@ -1,25 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -/* - * Access to VGA videoram - * - * (c) 1998 Martin Mares <mj@ucw.cz> - */ #ifndef __ASM_GENERIC_VGA_H #define __ASM_GENERIC_VGA_H - -/* - * On most architectures that support VGA, we can just - * recalculate addresses and then access the videoram - * directly without any black magic. - * - * Everyone else needs to ioremap the address and use - * proper I/O accesses. - */ -#ifndef VGA_MAP_MEM -#define VGA_MAP_MEM(x, s) (unsigned long)phys_to_virt(x) -#endif - -#define vga_readb(x) (*(x)) -#define vga_writeb(x, y) (*(y) = (x)) - -#endif /* _ASM_GENERIC_VGA_H */ +#endif /* __ASM_GENERIC_VGA_H */ diff --git a/include/asm-generic/video.h b/include/asm-generic/video.h new file mode 100644 index 000000000000..b1da2309d943 --- /dev/null +++ b/include/asm-generic/video.h @@ -0,0 +1,136 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __ASM_GENERIC_VIDEO_H_ +#define __ASM_GENERIC_VIDEO_H_ + +/* + * Only include this header file from your architecture's <asm/fb.h>. + */ + +#include <linux/io.h> +#include <linux/mm_types.h> +#include <linux/pgtable.h> +#include <linux/types.h> + +struct device; + +#ifndef pgprot_framebuffer +#define pgprot_framebuffer pgprot_framebuffer +static inline pgprot_t pgprot_framebuffer(pgprot_t prot, + unsigned long vm_start, unsigned long vm_end, + unsigned long offset) +{ + return pgprot_writecombine(prot); +} +#endif + +#ifndef video_is_primary_device +#define video_is_primary_device video_is_primary_device +static inline bool video_is_primary_device(struct device *dev) +{ + return false; +} +#endif + +/* + * I/O helpers for the framebuffer. Prefer these functions over their + * regular counterparts. The regular I/O functions provide in-order + * access and swap bytes to/from little-endian ordering. Neither is + * required for framebuffers. Instead, the helpers read and write + * raw framebuffer data. Independent operations can be reordered for + * improved performance. + */ + +#ifndef fb_readb +static inline u8 fb_readb(const volatile void __iomem *addr) +{ + return __raw_readb(addr); +} +#define fb_readb fb_readb +#endif + +#ifndef fb_readw +static inline u16 fb_readw(const volatile void __iomem *addr) +{ + return __raw_readw(addr); +} +#define fb_readw fb_readw +#endif + +#ifndef fb_readl +static inline u32 fb_readl(const volatile void __iomem *addr) +{ + return __raw_readl(addr); +} +#define fb_readl fb_readl +#endif + +#ifndef fb_readq +#if defined(__raw_readq) +static inline u64 fb_readq(const volatile void __iomem *addr) +{ + return __raw_readq(addr); +} +#define fb_readq fb_readq +#endif +#endif + +#ifndef fb_writeb +static inline void fb_writeb(u8 b, volatile void __iomem *addr) +{ + __raw_writeb(b, addr); +} +#define fb_writeb fb_writeb +#endif + +#ifndef fb_writew +static inline void fb_writew(u16 b, volatile void __iomem *addr) +{ + __raw_writew(b, addr); +} +#define fb_writew fb_writew +#endif + +#ifndef fb_writel +static inline void fb_writel(u32 b, volatile void __iomem *addr) +{ + __raw_writel(b, addr); +} +#define fb_writel fb_writel +#endif + +#ifndef fb_writeq +#if defined(__raw_writeq) +static inline void fb_writeq(u64 b, volatile void __iomem *addr) +{ + __raw_writeq(b, addr); +} +#define fb_writeq fb_writeq +#endif +#endif + +#ifndef fb_memcpy_fromio +static inline void fb_memcpy_fromio(void *to, const volatile void __iomem *from, size_t n) +{ + memcpy_fromio(to, from, n); +} +#define fb_memcpy_fromio fb_memcpy_fromio +#endif + +#ifndef fb_memcpy_toio +static inline void fb_memcpy_toio(volatile void __iomem *to, const void *from, size_t n) +{ + memcpy_toio(to, from, n); +} +#define fb_memcpy_toio fb_memcpy_toio +#endif + +#ifndef fb_memset +static inline void fb_memset_io(volatile void __iomem *addr, int c, size_t n) +{ + memset_io(addr, c, n); +} +#define fb_memset fb_memset_io +#endif + +#endif /* __ASM_GENERIC_VIDEO_H_ */ diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h index 3d7a6a9c2370..8ca130af301f 100644 --- a/include/asm-generic/vmlinux.lds.h +++ b/include/asm-generic/vmlinux.lds.h @@ -3,7 +3,7 @@ * linker scripts. * * A minimal linker scripts has following content: - * [This is a sample, architectures may have special requiriements] + * [This is a sample, architectures may have special requirements] * * OUTPUT_FORMAT(...) * OUTPUT_ARCH(...) @@ -23,18 +23,18 @@ * _etext = .; * * _sdata = .; - * RO_DATA_SECTION(PAGE_SIZE) - * RW_DATA_SECTION(...) + * RO_DATA(PAGE_SIZE) + * RW_DATA(...) * _edata = .; * * EXCEPTION_TABLE(...) - * NOTES * * BSS_SECTION(0, 0, 0) * _end = .; * * STABS_DEBUG * DWARF_DEBUG + * ELF_DETAILS * * DISCARDS // must be the last * } @@ -50,166 +50,278 @@ * [__nosave_begin, __nosave_end] for the nosave data */ +#include <asm-generic/codetag.lds.h> + #ifndef LOAD_OFFSET #define LOAD_OFFSET 0 #endif -/* Align . to a 8 byte boundary equals to maximum function alignment. */ -#define ALIGN_FUNCTION() . = ALIGN(8) +/* + * Only some architectures want to have the .notes segment visible in + * a separate PT_NOTE ELF Program Header. When this happens, it needs + * to be visible in both the kernel text's PT_LOAD and the PT_NOTE + * Program Headers. In this case, though, the PT_LOAD needs to be made + * the default again so that all the following sections don't also end + * up in the PT_NOTE Program Header. + */ +#ifdef EMITS_PT_NOTE +#define NOTES_HEADERS :text :note +#define NOTES_HEADERS_RESTORE __restore_ph : { *(.__restore_ph) } :text +#else +#define NOTES_HEADERS +#define NOTES_HEADERS_RESTORE +#endif + +/* + * Some architectures have non-executable read-only exception tables. + * They can be added to the RO_DATA segment by specifying their desired + * alignment. + */ +#ifdef RO_EXCEPTION_TABLE_ALIGN +#define RO_EXCEPTION_TABLE EXCEPTION_TABLE(RO_EXCEPTION_TABLE_ALIGN) +#else +#define RO_EXCEPTION_TABLE +#endif + +/* Align . function alignment. */ +#define ALIGN_FUNCTION() . = ALIGN(CONFIG_FUNCTION_ALIGNMENT) /* - * LD_DEAD_CODE_DATA_ELIMINATION option enables -fdata-sections, which - * generates .data.identifier sections, which need to be pulled in with - * .data. We don't want to pull in .data..other sections, which Linux - * has defined. Same for text and bss. + * Support -ffunction-sections by matching .text and .text.*, + * but exclude '.text..*', .text.startup[.*], and .text.exit[.*]. + * + * .text.startup and .text.startup.* are matched later by INIT_TEXT, and + * .text.exit and .text.exit.* are matched later by EXIT_TEXT, so they must be + * explicitly excluded here. + * + * Other .text.* sections that are typically grouped separately, such as + * .text.unlikely or .text.hot, must be matched explicitly before using + * TEXT_MAIN. * - * RODATA_MAIN is not used because existing code already defines .rodata.x - * sections to be brought in with rodata. + * NOTE: builds *with* and *without* -ffunction-sections are both supported by + * this single macro. Even with -ffunction-sections, there may be some objects + * NOT compiled with the flag due to the use of a specific Makefile override + * like cflags-y or AUTOFDO_PROFILE_foo.o. So this single catchall rule is + * needed to support mixed object builds. + * + * One implication is that functions named startup(), exit(), split(), + * unlikely(), hot(), and unknown() are not allowed in the kernel due to the + * ambiguity of their section names with -ffunction-sections. For example, + * .text.startup could be __attribute__((constructor)) code in a *non* + * ffunction-sections object, which should be placed in .init.text; or it could + * be an actual function named startup() in an ffunction-sections object, which + * should be placed in .text. The build will detect and complain about any such + * ambiguously named functions. + */ +#define TEXT_MAIN \ + .text \ + .text.[_0-9A-Za-df-rt-z]* \ + .text.s[_0-9A-Za-su-z]* .text.s .text.s.* \ + .text.st[_0-9A-Zb-z]* .text.st .text.st.* \ + .text.sta[_0-9A-Za-qs-z]* .text.sta .text.sta.* \ + .text.star[_0-9A-Za-su-z]* .text.star .text.star.* \ + .text.start[_0-9A-Za-tv-z]* .text.start .text.start.* \ + .text.startu[_0-9A-Za-oq-z]* .text.startu .text.startu.* \ + .text.startup[_0-9A-Za-z]* \ + .text.e[_0-9A-Za-wy-z]* .text.e .text.e.* \ + .text.ex[_0-9A-Za-hj-z]* .text.ex .text.ex.* \ + .text.exi[_0-9A-Za-su-z]* .text.exi .text.exi.* \ + .text.exit[_0-9A-Za-z]* + +/* + * Support -fdata-sections by matching .data, .data.*, and others, + * but exclude '.data..*'. */ -#ifdef CONFIG_LD_DEAD_CODE_DATA_ELIMINATION -#define TEXT_MAIN .text .text.[0-9a-zA-Z_]* -#define DATA_MAIN .data .data.[0-9a-zA-Z_]* .data..LPBX* +#define DATA_MAIN .data .data.[0-9a-zA-Z_]* .data.rel.* .data..L* .data..compoundliteral* .data.$__unnamed_* .data.$L* #define SDATA_MAIN .sdata .sdata.[0-9a-zA-Z_]* -#define RODATA_MAIN .rodata .rodata.[0-9a-zA-Z_]* -#define BSS_MAIN .bss .bss.[0-9a-zA-Z_]* +#define RODATA_MAIN .rodata .rodata.[0-9a-zA-Z_]* .rodata..L* +#define BSS_MAIN .bss .bss.[0-9a-zA-Z_]* .bss..L* .bss..compoundliteral* #define SBSS_MAIN .sbss .sbss.[0-9a-zA-Z_]* -#else -#define TEXT_MAIN .text -#define DATA_MAIN .data -#define SDATA_MAIN .sdata -#define RODATA_MAIN .rodata -#define BSS_MAIN .bss -#define SBSS_MAIN .sbss -#endif /* - * Align to a 32 byte boundary equal to the - * alignment gcc 4.5 uses for a struct + * GCC 4.5 and later have a 32 bytes section alignment for structures. + * Except GCC 4.9, that feels the need to align on 64 bytes. */ #define STRUCT_ALIGNMENT 32 #define STRUCT_ALIGN() . = ALIGN(STRUCT_ALIGNMENT) +/* + * The order of the sched class addresses are important, as they are + * used to determine the order of the priority of each sched class in + * relation to each other. + */ +#define SCHED_DATA \ + STRUCT_ALIGN(); \ + __sched_class_highest = .; \ + *(__stop_sched_class) \ + *(__dl_sched_class) \ + *(__rt_sched_class) \ + *(__fair_sched_class) \ + *(__ext_sched_class) \ + *(__idle_sched_class) \ + __sched_class_lowest = .; + /* The actual configuration determine if the init/exit sections * are handled as text/data or they can be discarded (which * often happens at runtime) */ -#ifdef CONFIG_HOTPLUG_CPU -#define CPU_KEEP(sec) *(.cpu##sec) -#define CPU_DISCARD(sec) + +#ifndef CONFIG_HAVE_DYNAMIC_FTRACE_NO_PATCHABLE +#define KEEP_PATCHABLE KEEP(*(__patchable_function_entries)) +#define PATCHABLE_DISCARDS #else -#define CPU_KEEP(sec) -#define CPU_DISCARD(sec) *(.cpu##sec) +#define KEEP_PATCHABLE +#define PATCHABLE_DISCARDS *(__patchable_function_entries) #endif -#if defined(CONFIG_MEMORY_HOTPLUG) -#define MEM_KEEP(sec) *(.mem##sec) -#define MEM_DISCARD(sec) +#ifndef CONFIG_ARCH_SUPPORTS_CFI +/* + * Simply points to ftrace_stub, but with the proper protocol. + * Defined by the linker script in linux/vmlinux.lds.h + */ +#define FTRACE_STUB_HACK ftrace_stub_graph = ftrace_stub; #else -#define MEM_KEEP(sec) -#define MEM_DISCARD(sec) *(.mem##sec) +#define FTRACE_STUB_HACK #endif -#ifdef CONFIG_FTRACE_MCOUNT_RECORD +#ifdef CONFIG_DYNAMIC_FTRACE +/* + * The ftrace call sites are logged to a section whose name depends on the + * compiler option used. A given kernel image will only use one, AKA + * FTRACE_CALLSITE_SECTION. We capture all of them here to avoid header + * dependencies for FTRACE_CALLSITE_SECTION's definition. + * + * ftrace_ops_list_func will be defined as arch_ftrace_ops_list_func + * as some archs will have a different prototype for that function + * but ftrace_ops_list_func() will have a single prototype. + */ #define MCOUNT_REC() . = ALIGN(8); \ __start_mcount_loc = .; \ KEEP(*(__mcount_loc)) \ - __stop_mcount_loc = .; + KEEP_PATCHABLE \ + __stop_mcount_loc = .; \ + FTRACE_STUB_HACK \ + ftrace_ops_list_func = arch_ftrace_ops_list_func; #else -#define MCOUNT_REC() +# ifdef CONFIG_FUNCTION_TRACER +# define MCOUNT_REC() FTRACE_STUB_HACK \ + ftrace_ops_list_func = arch_ftrace_ops_list_func; +# else +# define MCOUNT_REC() +# endif #endif +#define BOUNDED_SECTION_PRE_LABEL(_sec_, _label_, _BEGIN_, _END_) \ + _BEGIN_##_label_ = .; \ + KEEP(*(_sec_)) \ + _END_##_label_ = .; + +#define BOUNDED_SECTION_POST_LABEL(_sec_, _label_, _BEGIN_, _END_) \ + _label_##_BEGIN_ = .; \ + KEEP(*(_sec_)) \ + _label_##_END_ = .; + +#define BOUNDED_SECTION_BY(_sec_, _label_) \ + BOUNDED_SECTION_PRE_LABEL(_sec_, _label_, __start, __stop) + +#define BOUNDED_SECTION(_sec) BOUNDED_SECTION_BY(_sec, _sec) + +#define HEADERED_SECTION_PRE_LABEL(_sec_, _label_, _BEGIN_, _END_, _HDR_) \ + _HDR_##_label_ = .; \ + KEEP(*(.gnu.linkonce.##_sec_)) \ + BOUNDED_SECTION_PRE_LABEL(_sec_, _label_, _BEGIN_, _END_) + +#define HEADERED_SECTION_POST_LABEL(_sec_, _label_, _BEGIN_, _END_, _HDR_) \ + _label_##_HDR_ = .; \ + KEEP(*(.gnu.linkonce.##_sec_)) \ + BOUNDED_SECTION_POST_LABEL(_sec_, _label_, _BEGIN_, _END_) + +#define HEADERED_SECTION_BY(_sec_, _label_) \ + HEADERED_SECTION_PRE_LABEL(_sec_, _label_, __start, __stop) + +#define HEADERED_SECTION(_sec) HEADERED_SECTION_BY(_sec, _sec) + #ifdef CONFIG_TRACE_BRANCH_PROFILING -#define LIKELY_PROFILE() __start_annotated_branch_profile = .; \ - KEEP(*(_ftrace_annotated_branch)) \ - __stop_annotated_branch_profile = .; +#define LIKELY_PROFILE() \ + BOUNDED_SECTION_BY(_ftrace_annotated_branch, _annotated_branch_profile) #else #define LIKELY_PROFILE() #endif #ifdef CONFIG_PROFILE_ALL_BRANCHES -#define BRANCH_PROFILE() __start_branch_profile = .; \ - KEEP(*(_ftrace_branch)) \ - __stop_branch_profile = .; +#define BRANCH_PROFILE() \ + BOUNDED_SECTION_BY(_ftrace_branch, _branch_profile) #else #define BRANCH_PROFILE() #endif #ifdef CONFIG_KPROBES -#define KPROBE_BLACKLIST() . = ALIGN(8); \ - __start_kprobe_blacklist = .; \ - KEEP(*(_kprobe_blacklist)) \ - __stop_kprobe_blacklist = .; +#define KPROBE_BLACKLIST() \ + . = ALIGN(8); \ + BOUNDED_SECTION(_kprobe_blacklist) #else #define KPROBE_BLACKLIST() #endif #ifdef CONFIG_FUNCTION_ERROR_INJECTION -#define ERROR_INJECT_WHITELIST() STRUCT_ALIGN(); \ - __start_error_injection_whitelist = .; \ - KEEP(*(_error_injection_whitelist)) \ - __stop_error_injection_whitelist = .; +#define ERROR_INJECT_WHITELIST() \ + STRUCT_ALIGN(); \ + BOUNDED_SECTION(_error_injection_whitelist) #else #define ERROR_INJECT_WHITELIST() #endif #ifdef CONFIG_EVENT_TRACING -#define FTRACE_EVENTS() . = ALIGN(8); \ - __start_ftrace_events = .; \ - KEEP(*(_ftrace_events)) \ - __stop_ftrace_events = .; \ - __start_ftrace_eval_maps = .; \ - KEEP(*(_ftrace_eval_map)) \ - __stop_ftrace_eval_maps = .; +#define FTRACE_EVENTS() \ + . = ALIGN(8); \ + BOUNDED_SECTION(_ftrace_events) \ + BOUNDED_SECTION_BY(_ftrace_eval_map, _ftrace_eval_maps) #else #define FTRACE_EVENTS() #endif #ifdef CONFIG_TRACING -#define TRACE_PRINTKS() __start___trace_bprintk_fmt = .; \ - KEEP(*(__trace_printk_fmt)) /* Trace_printk fmt' pointer */ \ - __stop___trace_bprintk_fmt = .; -#define TRACEPOINT_STR() __start___tracepoint_str = .; \ - KEEP(*(__tracepoint_str)) /* Trace_printk fmt' pointer */ \ - __stop___tracepoint_str = .; +#define TRACE_PRINTKS() BOUNDED_SECTION_BY(__trace_printk_fmt, ___trace_bprintk_fmt) +#define TRACEPOINT_STR() BOUNDED_SECTION_BY(__tracepoint_str, ___tracepoint_str) #else #define TRACE_PRINTKS() #define TRACEPOINT_STR() #endif #ifdef CONFIG_FTRACE_SYSCALLS -#define TRACE_SYSCALLS() . = ALIGN(8); \ - __start_syscalls_metadata = .; \ - KEEP(*(__syscalls_metadata)) \ - __stop_syscalls_metadata = .; +#define TRACE_SYSCALLS() \ + . = ALIGN(8); \ + BOUNDED_SECTION_BY(__syscalls_metadata, _syscalls_metadata) #else #define TRACE_SYSCALLS() #endif #ifdef CONFIG_BPF_EVENTS -#define BPF_RAW_TP() STRUCT_ALIGN(); \ - __start__bpf_raw_tp = .; \ - KEEP(*(__bpf_raw_tp_map)) \ - __stop__bpf_raw_tp = .; +#define BPF_RAW_TP() STRUCT_ALIGN(); \ + BOUNDED_SECTION_BY(__bpf_raw_tp_map, __bpf_raw_tp) #else #define BPF_RAW_TP() #endif #ifdef CONFIG_SERIAL_EARLYCON -#define EARLYCON_TABLE() . = ALIGN(8); \ - __earlycon_table = .; \ - KEEP(*(__earlycon_table)) \ - __earlycon_table_end = .; +#define EARLYCON_TABLE() \ + . = ALIGN(8); \ + BOUNDED_SECTION_POST_LABEL(__earlycon_table, __earlycon_table, , _end) #else #define EARLYCON_TABLE() #endif #ifdef CONFIG_SECURITY -#define LSM_TABLE() . = ALIGN(8); \ - __start_lsm_info = .; \ - KEEP(*(.lsm_info.init)) \ - __end_lsm_info = .; +#define LSM_TABLE() \ + . = ALIGN(8); \ + BOUNDED_SECTION_PRE_LABEL(.lsm_info.init, _lsm_info, __start, __end) + +#define EARLY_LSM_TABLE() \ + . = ALIGN(8); \ + BOUNDED_SECTION_PRE_LABEL(.early_lsm_info.init, _early_lsm_info, __start, __end) #else #define LSM_TABLE() +#define EARLY_LSM_TABLE() #endif #define ___OF_TABLE(cfg, name) _OF_TABLE_##cfg(name) @@ -232,13 +344,21 @@ #ifdef CONFIG_ACPI #define ACPI_PROBE_TABLE(name) \ . = ALIGN(8); \ - __##name##_acpi_probe_table = .; \ - KEEP(*(__##name##_acpi_probe_table)) \ - __##name##_acpi_probe_table_end = .; + BOUNDED_SECTION_POST_LABEL(__##name##_acpi_probe_table, \ + __##name##_acpi_probe_table,, _end) #else #define ACPI_PROBE_TABLE(name) #endif +#ifdef CONFIG_THERMAL +#define THERMAL_TABLE(name) \ + . = ALIGN(8); \ + BOUNDED_SECTION_POST_LABEL(__##name##_thermal_table, \ + __##name##_thermal_table,, _end) +#else +#define THERMAL_TABLE(name) +#endif + #define KERNEL_DTB() \ STRUCT_ALIGN(); \ __dtb_start = .; \ @@ -251,26 +371,27 @@ #define DATA_DATA \ *(.xiptext) \ *(DATA_MAIN) \ + *(.data..decrypted) \ *(.ref.data) \ *(.data..shared_aligned) /* percpu related */ \ - MEM_KEEP(init.data*) \ - MEM_KEEP(exit.data*) \ - *(.data.unlikely) \ + *(.data..unlikely) \ __start_once = .; \ - *(.data.once) \ + *(.data..once) \ __end_once = .; \ + *(.data..do_once) \ STRUCT_ALIGN(); \ *(__tracepoints) \ /* implement dynamic printk debug */ \ . = ALIGN(8); \ - __start___verbose = .; \ - KEEP(*(__verbose)) \ - __stop___verbose = .; \ + BOUNDED_SECTION_BY(__dyndbg_classes, ___dyndbg_classes) \ + BOUNDED_SECTION_BY(__dyndbg, ___dyndbg) \ + CODETAG_SECTIONS() \ LIKELY_PROFILE() \ BRANCH_PROFILE() \ TRACE_PRINTKS() \ BPF_RAW_TP() \ - TRACEPOINT_STR() + TRACEPOINT_STR() \ + KUNIT_TABLE() /* * Data section helpers @@ -282,9 +403,15 @@ . = ALIGN(PAGE_SIZE); \ __nosave_end = .; +#define CACHE_HOT_DATA(align) \ + . = ALIGN(align); \ + *(SORT_BY_ALIGNMENT(.data..hot.*)) \ + . = ALIGN(align); + #define PAGE_ALIGNED_DATA(page_align) \ . = ALIGN(page_align); \ - *(.data..page_aligned) + *(.data..page_aligned) \ + . = ALIGN(page_align); #define READ_MOSTLY_DATA(align) \ . = ALIGN(align); \ @@ -297,19 +424,25 @@ #define INIT_TASK_DATA(align) \ . = ALIGN(align); \ - __start_init_task = .; \ + __start_init_stack = .; \ init_thread_union = .; \ init_stack = .; \ - KEEP(*(.data..init_task)) \ KEEP(*(.data..init_thread_info)) \ - . = __start_init_task + THREAD_SIZE; \ - __end_init_task = .; + . = __start_init_stack + THREAD_SIZE; \ + __end_init_stack = .; #define JUMP_TABLE_DATA \ . = ALIGN(8); \ - __start___jump_table = .; \ - KEEP(*(__jump_table)) \ - __stop___jump_table = .; + BOUNDED_SECTION_BY(__jump_table, ___jump_table) + +#ifdef CONFIG_HAVE_STATIC_CALL_INLINE +#define STATIC_CALL_DATA \ + . = ALIGN(8); \ + BOUNDED_SECTION_BY(.static_call_sites, _static_call_sites) \ + BOUNDED_SECTION_BY(.static_call_tramp_key, _static_call_tramp_key) +#else +#define STATIC_CALL_DATA +#endif /* * Allow architectures to handle ro_after_init data on their @@ -317,26 +450,40 @@ */ #ifndef RO_AFTER_INIT_DATA #define RO_AFTER_INIT_DATA \ + . = ALIGN(8); \ __start_ro_after_init = .; \ *(.data..ro_after_init) \ JUMP_TABLE_DATA \ + STATIC_CALL_DATA \ __end_ro_after_init = .; #endif /* + * .kcfi_traps contains a list KCFI trap locations. + */ +#ifndef KCFI_TRAPS +#ifdef CONFIG_ARCH_USES_CFI_TRAPS +#define KCFI_TRAPS \ + __kcfi_traps : AT(ADDR(__kcfi_traps) - LOAD_OFFSET) { \ + BOUNDED_SECTION_BY(.kcfi_traps, ___kcfi_traps) \ + } +#else +#define KCFI_TRAPS +#endif +#endif + +/* * Read only Data */ -#define RO_DATA_SECTION(align) \ +#define RO_DATA(align) \ . = ALIGN((align)); \ .rodata : AT(ADDR(.rodata) - LOAD_OFFSET) { \ __start_rodata = .; \ - *(.rodata) *(.rodata.*) \ + *(.rodata) *(.rodata.*) *(.data.rel.ro*) \ + SCHED_DATA \ RO_AFTER_INIT_DATA /* Read only after init */ \ - KEEP(*(__vermagic)) /* Kernel version magic */ \ . = ALIGN(8); \ - __start___tracepoints_ptrs = .; \ - KEEP(*(__tracepoints_ptrs)) /* Tracepoints: pointer array */ \ - __stop___tracepoints_ptrs = .; \ + BOUNDED_SECTION_BY(__tracepoints_ptrs, ___tracepoints_ptrs) \ *(__tracepoints_strings)/* Tracepoints: strings */ \ } \ \ @@ -346,41 +493,21 @@ \ /* PCI quirks */ \ .pci_fixup : AT(ADDR(.pci_fixup) - LOAD_OFFSET) { \ - __start_pci_fixups_early = .; \ - KEEP(*(.pci_fixup_early)) \ - __end_pci_fixups_early = .; \ - __start_pci_fixups_header = .; \ - KEEP(*(.pci_fixup_header)) \ - __end_pci_fixups_header = .; \ - __start_pci_fixups_final = .; \ - KEEP(*(.pci_fixup_final)) \ - __end_pci_fixups_final = .; \ - __start_pci_fixups_enable = .; \ - KEEP(*(.pci_fixup_enable)) \ - __end_pci_fixups_enable = .; \ - __start_pci_fixups_resume = .; \ - KEEP(*(.pci_fixup_resume)) \ - __end_pci_fixups_resume = .; \ - __start_pci_fixups_resume_early = .; \ - KEEP(*(.pci_fixup_resume_early)) \ - __end_pci_fixups_resume_early = .; \ - __start_pci_fixups_suspend = .; \ - KEEP(*(.pci_fixup_suspend)) \ - __end_pci_fixups_suspend = .; \ - __start_pci_fixups_suspend_late = .; \ - KEEP(*(.pci_fixup_suspend_late)) \ - __end_pci_fixups_suspend_late = .; \ - } \ - \ - /* Built-in firmware blobs */ \ - .builtin_fw : AT(ADDR(.builtin_fw) - LOAD_OFFSET) { \ - __start_builtin_fw = .; \ - KEEP(*(.builtin_fw)) \ - __end_builtin_fw = .; \ + BOUNDED_SECTION_PRE_LABEL(.pci_fixup_early, _pci_fixups_early, __start, __end) \ + BOUNDED_SECTION_PRE_LABEL(.pci_fixup_header, _pci_fixups_header, __start, __end) \ + BOUNDED_SECTION_PRE_LABEL(.pci_fixup_final, _pci_fixups_final, __start, __end) \ + BOUNDED_SECTION_PRE_LABEL(.pci_fixup_enable, _pci_fixups_enable, __start, __end) \ + BOUNDED_SECTION_PRE_LABEL(.pci_fixup_resume, _pci_fixups_resume, __start, __end) \ + BOUNDED_SECTION_PRE_LABEL(.pci_fixup_suspend, _pci_fixups_suspend, __start, __end) \ + BOUNDED_SECTION_PRE_LABEL(.pci_fixup_resume_early, _pci_fixups_resume_early, __start, __end) \ + BOUNDED_SECTION_PRE_LABEL(.pci_fixup_suspend_late, _pci_fixups_suspend_late, __start, __end) \ } \ \ + FW_LOADER_BUILT_IN_DATA \ TRACEDATA \ \ + PRINTK_INDEX \ + \ /* Kernel symbol table: Normal symbols */ \ __ksymtab : AT(ADDR(__ksymtab) - LOAD_OFFSET) { \ __start___ksymtab = .; \ @@ -395,27 +522,6 @@ __stop___ksymtab_gpl = .; \ } \ \ - /* Kernel symbol table: Normal unused symbols */ \ - __ksymtab_unused : AT(ADDR(__ksymtab_unused) - LOAD_OFFSET) { \ - __start___ksymtab_unused = .; \ - KEEP(*(SORT(___ksymtab_unused+*))) \ - __stop___ksymtab_unused = .; \ - } \ - \ - /* Kernel symbol table: GPL-only unused symbols */ \ - __ksymtab_unused_gpl : AT(ADDR(__ksymtab_unused_gpl) - LOAD_OFFSET) { \ - __start___ksymtab_unused_gpl = .; \ - KEEP(*(SORT(___ksymtab_unused_gpl+*))) \ - __stop___ksymtab_unused_gpl = .; \ - } \ - \ - /* Kernel symbol table: GPL-future-only symbols */ \ - __ksymtab_gpl_future : AT(ADDR(__ksymtab_gpl_future) - LOAD_OFFSET) { \ - __start___ksymtab_gpl_future = .; \ - KEEP(*(SORT(___ksymtab_gpl_future+*))) \ - __stop___ksymtab_gpl_future = .; \ - } \ - \ /* Kernel symbol table: Normal symbols */ \ __kcrctab : AT(ADDR(__kcrctab) - LOAD_OFFSET) { \ __start___kcrctab = .; \ @@ -430,27 +536,6 @@ __stop___kcrctab_gpl = .; \ } \ \ - /* Kernel symbol table: Normal unused symbols */ \ - __kcrctab_unused : AT(ADDR(__kcrctab_unused) - LOAD_OFFSET) { \ - __start___kcrctab_unused = .; \ - KEEP(*(SORT(___kcrctab_unused+*))) \ - __stop___kcrctab_unused = .; \ - } \ - \ - /* Kernel symbol table: GPL-only unused symbols */ \ - __kcrctab_unused_gpl : AT(ADDR(__kcrctab_unused_gpl) - LOAD_OFFSET) { \ - __start___kcrctab_unused_gpl = .; \ - KEEP(*(SORT(___kcrctab_unused_gpl+*))) \ - __stop___kcrctab_unused_gpl = .; \ - } \ - \ - /* Kernel symbol table: GPL-future-only symbols */ \ - __kcrctab_gpl_future : AT(ADDR(__kcrctab_gpl_future) - LOAD_OFFSET) { \ - __start___kcrctab_gpl_future = .; \ - KEEP(*(SORT(___kcrctab_gpl_future+*))) \ - __stop___kcrctab_gpl_future = .; \ - } \ - \ /* Kernel symbol table: strings */ \ __ksymtab_strings : AT(ADDR(__ksymtab_strings) - LOAD_OFFSET) { \ *(__ksymtab_strings) \ @@ -459,48 +544,77 @@ /* __*init sections */ \ __init_rodata : AT(ADDR(__init_rodata) - LOAD_OFFSET) { \ *(.ref.rodata) \ - MEM_KEEP(init.rodata) \ - MEM_KEEP(exit.rodata) \ } \ \ /* Built-in module parameters. */ \ __param : AT(ADDR(__param) - LOAD_OFFSET) { \ - __start___param = .; \ - KEEP(*(__param)) \ - __stop___param = .; \ + BOUNDED_SECTION_BY(__param, ___param) \ } \ \ /* Built-in module versions. */ \ __modver : AT(ADDR(__modver) - LOAD_OFFSET) { \ - __start___modver = .; \ - KEEP(*(__modver)) \ - __stop___modver = .; \ - . = ALIGN((align)); \ - __end_rodata = .; \ + BOUNDED_SECTION_BY(__modver, ___modver) \ } \ - . = ALIGN((align)); + \ + KCFI_TRAPS \ + \ + RO_EXCEPTION_TABLE \ + NOTES \ + BTF \ + \ + . = ALIGN((align)); \ + __end_rodata = .; -/* RODATA & RO_DATA provided for backward compatibility. - * All archs are supposed to use RO_DATA() */ -#define RODATA RO_DATA_SECTION(4096) -#define RO_DATA(align) RO_DATA_SECTION(align) + +/* + * Non-instrumentable text section + */ +#define NOINSTR_TEXT \ + ALIGN_FUNCTION(); \ + __noinstr_text_start = .; \ + *(.noinstr.text) \ + __cpuidle_text_start = .; \ + *(.cpuidle.text) \ + __cpuidle_text_end = .; \ + __noinstr_text_end = .; + +#define TEXT_SPLIT \ + __split_text_start = .; \ + *(.text.split .text.split.[0-9a-zA-Z_]*) \ + __split_text_end = .; + +#define TEXT_UNLIKELY \ + __unlikely_text_start = .; \ + *(.text.unlikely .text.unlikely.*) \ + __unlikely_text_end = .; + +#define TEXT_HOT \ + __hot_text_start = .; \ + *(.text.hot .text.hot.*) \ + __hot_text_end = .; /* * .text section. Map to function alignment to avoid address changes * during second ld run in second ld pass when generating System.map * - * TEXT_MAIN here will match .text.fixup and .text.unlikely if dead - * code elimination is enabled, so these sections should be converted - * to use ".." first. + * TEXT_MAIN here will match symbols with a fixed pattern (for example, + * .text.hot or .text.unlikely). Match those before TEXT_MAIN to ensure + * they get grouped together. + * + * Also placing .text.hot section at the beginning of a page, this + * would help the TLB performance. */ #define TEXT_TEXT \ ALIGN_FUNCTION(); \ - *(.text.hot TEXT_MAIN .text.fixup .text.unlikely) \ - *(.text..refcount) \ - *(.ref.text) \ - MEM_KEEP(init.text*) \ - MEM_KEEP(exit.text*) \ - + *(.text.asan.* .text.tsan.*) \ + *(.text.unknown .text.unknown.*) \ + TEXT_SPLIT \ + TEXT_UNLIKELY \ + . = ALIGN(PAGE_SIZE); \ + TEXT_HOT \ + *(TEXT_MAIN .text.fixup) \ + NOINSTR_TEXT \ + *(.ref.text) /* sched.text is aling to function alignment to secure we have same * address even at second ld pass when generating System.map */ @@ -518,12 +632,6 @@ *(.spinlock.text) \ __lock_text_end = .; -#define CPUIDLE_TEXT \ - ALIGN_FUNCTION(); \ - __cpuidle_text_start = .; \ - *(.cpuidle.text) \ - __cpuidle_text_end = .; - #define KPROBES_TEXT \ ALIGN_FUNCTION(); \ __kprobes_text_start = .; \ @@ -548,6 +656,12 @@ *(.softirqentry.text) \ __softirqentry_text_end = .; +#define STATIC_CALL_TEXT \ + ALIGN_FUNCTION(); \ + __static_call_text_start = .; \ + *(.static_call.text) \ + __static_call_text_end = .; + /* Section used for early init (in .S files) */ #define HEAD_TEXT KEEP(*(.head.text)) @@ -562,12 +676,27 @@ #define EXCEPTION_TABLE(align) \ . = ALIGN(align); \ __ex_table : AT(ADDR(__ex_table) - LOAD_OFFSET) { \ - __start___ex_table = .; \ - KEEP(*(__ex_table)) \ - __stop___ex_table = .; \ + BOUNDED_SECTION_BY(__ex_table, ___ex_table) \ } /* + * .BTF + */ +#ifdef CONFIG_DEBUG_INFO_BTF +#define BTF \ + . = ALIGN(PAGE_SIZE); \ + .BTF : AT(ADDR(.BTF) - LOAD_OFFSET) { \ + BOUNDED_SECTION_BY(.BTF, _BTF) \ + } \ + . = ALIGN(PAGE_SIZE); \ + .BTF_ids : AT(ADDR(.BTF_ids) - LOAD_OFFSET) { \ + *(.BTF_ids) \ + } +#else +#define BTF +#endif + +/* * Init task */ #define INIT_TASK_DATA_SECTION(align) \ @@ -579,6 +708,7 @@ #ifdef CONFIG_CONSTRUCTORS #define KERNEL_CTORS() . = ALIGN(8); \ __ctors_start = .; \ + KEEP(*(SORT(.ctors.*))) \ KEEP(*(.ctors)) \ KEEP(*(SORT(.init_array.*))) \ KEEP(*(.init_array)) \ @@ -590,8 +720,7 @@ /* init and exit section handling */ #define INIT_DATA \ KEEP(*(SORT(___kentry+*))) \ - *(.init.data init.data.*) \ - MEM_DISCARD(init.data*) \ + *(.init.data .init.data.*) \ KERNEL_CTORS() \ MCOUNT_REC() \ *(.init.rodata .init.rodata.*) \ @@ -599,7 +728,6 @@ TRACE_SYSCALLS() \ KPROBE_BLACKLIST() \ ERROR_INJECT_WHITELIST() \ - MEM_DISCARD(init.rodata) \ CLK_OF_TABLES() \ RESERVEDMEM_OF_TABLES() \ TIMER_OF_TABLES() \ @@ -609,25 +737,24 @@ IRQCHIP_OF_MATCH_TABLE() \ ACPI_PROBE_TABLE(irqchip) \ ACPI_PROBE_TABLE(timer) \ + THERMAL_TABLE(governor) \ EARLYCON_TABLE() \ - LSM_TABLE() + LSM_TABLE() \ + EARLY_LSM_TABLE() \ + KUNIT_INIT_TABLE() #define INIT_TEXT \ *(.init.text .init.text.*) \ - *(.text.startup) \ - MEM_DISCARD(init.text*) + *(.text.startup .text.startup.*) #define EXIT_DATA \ *(.exit.data .exit.data.*) \ *(.fini_array .fini_array.*) \ - *(.dtors .dtors.*) \ - MEM_DISCARD(exit.data*) \ - MEM_DISCARD(exit.rodata*) + *(.dtors .dtors.*) #define EXIT_TEXT \ *(.exit.text) \ - *(.text.exit) \ - MEM_DISCARD(exit.text) + *(.text.exit .text.exit.*) #define EXIT_CALL \ *(.exitcall.exit) @@ -656,7 +783,9 @@ . = ALIGN(bss_align); \ .bss : AT(ADDR(.bss) - LOAD_OFFSET) { \ BSS_FIRST_SECTIONS \ + . = ALIGN(PAGE_SIZE); \ *(.bss..page_aligned) \ + . = ALIGN(PAGE_SIZE); \ *(.dynbss) \ *(BSS_MAIN) \ *(COMMON) \ @@ -700,26 +829,36 @@ /* DWARF 4 */ \ .debug_types 0 : { *(.debug_types) } \ /* DWARF 5 */ \ + .debug_addr 0 : { *(.debug_addr) } \ + .debug_line_str 0 : { *(.debug_line_str) } \ + .debug_loclists 0 : { *(.debug_loclists) } \ .debug_macro 0 : { *(.debug_macro) } \ - .debug_addr 0 : { *(.debug_addr) } + .debug_names 0 : { *(.debug_names) } \ + .debug_rnglists 0 : { *(.debug_rnglists) } \ + .debug_str_offsets 0 : { *(.debug_str_offsets) } - /* Stabs debugging sections. */ +/* Stabs debugging sections. */ #define STABS_DEBUG \ .stab 0 : { *(.stab) } \ .stabstr 0 : { *(.stabstr) } \ .stab.excl 0 : { *(.stab.excl) } \ .stab.exclstr 0 : { *(.stab.exclstr) } \ .stab.index 0 : { *(.stab.index) } \ - .stab.indexstr 0 : { *(.stab.indexstr) } \ - .comment 0 : { *(.comment) } + .stab.indexstr 0 : { *(.stab.indexstr) } + +/* Required sections not related to debugging. */ +#define ELF_DETAILS \ + .modinfo : { *(.modinfo) . = ALIGN(8); } \ + .comment 0 : { *(.comment) } \ + .symtab 0 : { *(.symtab) } \ + .strtab 0 : { *(.strtab) } \ + .shstrtab 0 : { *(.shstrtab) } #ifdef CONFIG_GENERIC_BUG #define BUG_TABLE \ . = ALIGN(8); \ __bug_table : AT(ADDR(__bug_table) - LOAD_OFFSET) { \ - __start___bug_table = .; \ - KEEP(*(__bug_table)) \ - __stop___bug_table = .; \ + BOUNDED_SECTION_BY(__bug_table, ___bug_table) \ } #else #define BUG_TABLE @@ -727,22 +866,22 @@ #ifdef CONFIG_UNWINDER_ORC #define ORC_UNWIND_TABLE \ + .orc_header : AT(ADDR(.orc_header) - LOAD_OFFSET) { \ + BOUNDED_SECTION_BY(.orc_header, _orc_header) \ + } \ . = ALIGN(4); \ .orc_unwind_ip : AT(ADDR(.orc_unwind_ip) - LOAD_OFFSET) { \ - __start_orc_unwind_ip = .; \ - KEEP(*(.orc_unwind_ip)) \ - __stop_orc_unwind_ip = .; \ + BOUNDED_SECTION_BY(.orc_unwind_ip, _orc_unwind_ip) \ } \ - . = ALIGN(6); \ + . = ALIGN(2); \ .orc_unwind : AT(ADDR(.orc_unwind) - LOAD_OFFSET) { \ - __start_orc_unwind = .; \ - KEEP(*(.orc_unwind)) \ - __stop_orc_unwind = .; \ + BOUNDED_SECTION_BY(.orc_unwind, _orc_unwind) \ } \ + text_size = _etext - _stext; \ . = ALIGN(4); \ .orc_lookup : AT(ADDR(.orc_lookup) - LOAD_OFFSET) { \ orc_lookup = .; \ - . += (((SIZEOF(.text) + LOOKUP_BLOCK_SIZE - 1) / \ + . += (((text_size + LOOKUP_BLOCK_SIZE - 1) / \ LOOKUP_BLOCK_SIZE) + 1) * 4; \ orc_lookup_end = .; \ } @@ -750,30 +889,56 @@ #define ORC_UNWIND_TABLE #endif +/* Built-in firmware blobs */ +#ifdef CONFIG_FW_LOADER +#define FW_LOADER_BUILT_IN_DATA \ + .builtin_fw : AT(ADDR(.builtin_fw) - LOAD_OFFSET) ALIGN(8) { \ + BOUNDED_SECTION_PRE_LABEL(.builtin_fw, _builtin_fw, __start, __end) \ + } +#else +#define FW_LOADER_BUILT_IN_DATA +#endif + #ifdef CONFIG_PM_TRACE #define TRACEDATA \ . = ALIGN(4); \ .tracedata : AT(ADDR(.tracedata) - LOAD_OFFSET) { \ - __tracedata_start = .; \ - KEEP(*(.tracedata)) \ - __tracedata_end = .; \ + BOUNDED_SECTION_POST_LABEL(.tracedata, __tracedata, _start, _end) \ } #else #define TRACEDATA #endif +#ifdef CONFIG_PRINTK_INDEX +#define PRINTK_INDEX \ + .printk_index : AT(ADDR(.printk_index) - LOAD_OFFSET) { \ + BOUNDED_SECTION_BY(.printk_index, _printk_index) \ + } +#else +#define PRINTK_INDEX +#endif + +/* + * Discard .note.GNU-stack, which is emitted as PROGBITS by the compiler. + * Otherwise, the type of .notes section would become PROGBITS instead of NOTES. + * + * Also, discard .note.gnu.property, otherwise it forces the notes section to + * be 8-byte aligned which causes alignment mismatches with the kernel's custom + * 4-byte aligned notes. + */ #define NOTES \ + /DISCARD/ : { \ + *(.note.GNU-stack) \ + *(.note.gnu.property) \ + } \ .notes : AT(ADDR(.notes) - LOAD_OFFSET) { \ - __start_notes = .; \ - KEEP(*(.note.*)) \ - __stop_notes = .; \ - } + BOUNDED_SECTION_BY(.note.*, _notes) \ + } NOTES_HEADERS \ + NOTES_HEADERS_RESTORE #define INIT_SETUP(initsetup_align) \ . = ALIGN(initsetup_align); \ - __setup_start = .; \ - KEEP(*(.init.setup)) \ - __setup_end = .; + BOUNDED_SECTION_POST_LABEL(.init.setup, __setup, _start, _end) #define INIT_CALLS_LEVEL(level) \ __initcall##level##_start = .; \ @@ -795,9 +960,30 @@ __initcall_end = .; #define CON_INITCALL \ - __con_initcall_start = .; \ - KEEP(*(.con_initcall.init)) \ - __con_initcall_end = .; + BOUNDED_SECTION_POST_LABEL(.con_initcall.init, __con_initcall, _start, _end) + +#define NAMED_SECTION(name) \ + . = ALIGN(8); \ + name : AT(ADDR(name) - LOAD_OFFSET) \ + { BOUNDED_SECTION_PRE_LABEL(name, name, __start_, __stop_) } + +#define RUNTIME_CONST(t,x) NAMED_SECTION(runtime_##t##_##x) + +#define RUNTIME_CONST_VARIABLES \ + RUNTIME_CONST(shift, d_hash_shift) \ + RUNTIME_CONST(ptr, dentry_hashtable) \ + RUNTIME_CONST(ptr, __dentry_cache) + +/* Alignment must be consistent with (kunit_suite *) in include/kunit/test.h */ +#define KUNIT_TABLE() \ + . = ALIGN(8); \ + BOUNDED_SECTION_POST_LABEL(.kunit_test_suites, __kunit_suites, _start, _end) + +/* Alignment must be consistent with (kunit_suite *) in include/kunit/test.h */ +#define KUNIT_INIT_TABLE() \ + . = ALIGN(8); \ + BOUNDED_SECTION_POST_LABEL(.kunit_init_test_suites, \ + __kunit_init_suites, _start, _end) #ifdef CONFIG_BLK_DEV_INITRD #define INIT_RAM_FS \ @@ -837,13 +1023,55 @@ * section definitions so that such archs put those in earlier section * definitions. */ -#define DISCARDS \ - /DISCARD/ : { \ +#ifdef RUNTIME_DISCARD_EXIT +#define EXIT_DISCARDS +#else +#define EXIT_DISCARDS \ EXIT_TEXT \ - EXIT_DATA \ - EXIT_CALL \ + EXIT_DATA +#endif + +/* + * Clang's -fprofile-arcs, -fsanitize=kernel-address, and + * -fsanitize=thread produce unwanted sections (.eh_frame + * and .init_array.*), but CONFIG_CONSTRUCTORS wants to + * keep any .init_array.* sections. + * https://llvm.org/pr46478 + */ +#ifdef CONFIG_UNWIND_TABLES +#define DISCARD_EH_FRAME +#else +#define DISCARD_EH_FRAME *(.eh_frame) +#endif +#if defined(CONFIG_GCOV_KERNEL) || defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KCSAN) +# ifdef CONFIG_CONSTRUCTORS +# define SANITIZER_DISCARDS \ + DISCARD_EH_FRAME +# else +# define SANITIZER_DISCARDS \ + *(.init_array) *(.init_array.*) \ + DISCARD_EH_FRAME +# endif +#else +# define SANITIZER_DISCARDS +#endif + +#define COMMON_DISCARDS \ + SANITIZER_DISCARDS \ + PATCHABLE_DISCARDS \ *(.discard) \ *(.discard.*) \ + *(.export_symbol) \ + *(.no_trim_symbol) \ + /* ld.bfd warns about .gnu.version* even when not emitted */ \ + *(.gnu.version*) \ + *(__tracepoint_check) \ + +#define DISCARDS \ + /DISCARD/ : { \ + EXIT_DISCARDS \ + EXIT_CALL \ + COMMON_DISCARDS \ } /** @@ -858,10 +1086,13 @@ */ #define PERCPU_INPUT(cacheline) \ __per_cpu_start = .; \ - *(.data..percpu..first) \ . = ALIGN(PAGE_SIZE); \ *(.data..percpu..page_aligned) \ . = ALIGN(cacheline); \ + __per_cpu_hot_start = .; \ + *(SORT_BY_ALIGNMENT(.data..percpu..hot.*)) \ + __per_cpu_hot_end = .; \ + . = ALIGN(cacheline); \ *(.data..percpu..read_mostly) \ . = ALIGN(cacheline); \ *(.data..percpu) \ @@ -870,52 +1101,17 @@ __per_cpu_end = .; /** - * PERCPU_VADDR - define output section for percpu area + * PERCPU_SECTION - define output section for percpu area * @cacheline: cacheline size - * @vaddr: explicit base address (optional) - * @phdr: destination PHDR (optional) * * Macro which expands to output section for percpu area. * * @cacheline is used to align subsections to avoid false cacheline * sharing between subsections for different purposes. - * - * If @vaddr is not blank, it specifies explicit base address and all - * percpu symbols will be offset from the given address. If blank, - * @vaddr always equals @laddr + LOAD_OFFSET. - * - * @phdr defines the output PHDR to use if not blank. Be warned that - * output PHDR is sticky. If @phdr is specified, the next output - * section in the linker script will go there too. @phdr should have - * a leading colon. - * - * Note that this macros defines __per_cpu_load as an absolute symbol. - * If there is no need to put the percpu section at a predetermined - * address, use PERCPU_SECTION. - */ -#define PERCPU_VADDR(cacheline, vaddr, phdr) \ - __per_cpu_load = .; \ - .data..percpu vaddr : AT(__per_cpu_load - LOAD_OFFSET) { \ - PERCPU_INPUT(cacheline) \ - } phdr \ - . = __per_cpu_load + SIZEOF(.data..percpu); - -/** - * PERCPU_SECTION - define output section for percpu area, simple version - * @cacheline: cacheline size - * - * Align to PAGE_SIZE and outputs output section for percpu area. This - * macro doesn't manipulate @vaddr or @phdr and __per_cpu_load and - * __per_cpu_start will be identical. - * - * This macro is equivalent to ALIGN(PAGE_SIZE); PERCPU_VADDR(@cacheline,,) - * except that __per_cpu_load is defined as a relative symbol against - * .data..percpu which is required for relocatable x86_32 configuration. */ #define PERCPU_SECTION(cacheline) \ . = ALIGN(PAGE_SIZE); \ .data..percpu : AT(ADDR(.data..percpu) - LOAD_OFFSET) { \ - __per_cpu_load = .; \ PERCPU_INPUT(cacheline) \ } @@ -938,12 +1134,13 @@ * matches the requirement of PAGE_ALIGNED_DATA. * * use 0 as page_align if page_aligned data is not used */ -#define RW_DATA_SECTION(cacheline, pagealigned, inittask) \ +#define RW_DATA(cacheline, pagealigned, inittask) \ . = ALIGN(PAGE_SIZE); \ .data : AT(ADDR(.data) - LOAD_OFFSET) { \ INIT_TASK_DATA(inittask) \ NOSAVE_DATA \ PAGE_ALIGNED_DATA(pagealigned) \ + CACHE_HOT_DATA(cacheline) \ CACHELINE_ALIGNED_DATA(cacheline) \ READ_MOSTLY_DATA(cacheline) \ DATA_DATA \ diff --git a/include/asm-generic/vtime.h b/include/asm-generic/vtime.h deleted file mode 100644 index b1a49677fe25..000000000000 --- a/include/asm-generic/vtime.h +++ /dev/null @@ -1 +0,0 @@ -/* no content, but patch(1) dislikes empty files */ diff --git a/include/asm-generic/word-at-a-time.h b/include/asm-generic/word-at-a-time.h index 20c93f08c993..ef3f841c6625 100644 --- a/include/asm-generic/word-at-a-time.h +++ b/include/asm-generic/word-at-a-time.h @@ -2,7 +2,8 @@ #ifndef _ASM_WORD_AT_A_TIME_H #define _ASM_WORD_AT_A_TIME_H -#include <linux/kernel.h> +#include <linux/bitops.h> +#include <linux/wordpart.h> #include <asm/byteorder.h> #ifdef __BIG_ENDIAN @@ -38,7 +39,7 @@ static inline long find_zero(unsigned long mask) return (mask >> 8) ? byte : byte + 1; } -static inline bool has_zero(unsigned long val, unsigned long *data, const struct word_at_a_time *c) +static inline unsigned long has_zero(unsigned long val, unsigned long *data, const struct word_at_a_time *c) { unsigned long rhs = val | c->low_bits; *data = rhs; diff --git a/include/asm-generic/xor.h b/include/asm-generic/xor.h index b4d843225afd..44509d48fca2 100644 --- a/include/asm-generic/xor.h +++ b/include/asm-generic/xor.h @@ -1,22 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * include/asm-generic/xor.h * * Generic optimized RAID-5 checksumming functions. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2, or (at your option) - * any later version. - * - * You should have received a copy of the GNU General Public License - * (for example /usr/src/linux/COPYING); if not, write to the Free - * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ #include <linux/prefetch.h> static void -xor_8regs_2(unsigned long bytes, unsigned long *p1, unsigned long *p2) +xor_8regs_2(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2) { long lines = bytes / (sizeof (long)) / 8; @@ -35,8 +28,9 @@ xor_8regs_2(unsigned long bytes, unsigned long *p1, unsigned long *p2) } static void -xor_8regs_3(unsigned long bytes, unsigned long *p1, unsigned long *p2, - unsigned long *p3) +xor_8regs_3(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3) { long lines = bytes / (sizeof (long)) / 8; @@ -56,8 +50,10 @@ xor_8regs_3(unsigned long bytes, unsigned long *p1, unsigned long *p2, } static void -xor_8regs_4(unsigned long bytes, unsigned long *p1, unsigned long *p2, - unsigned long *p3, unsigned long *p4) +xor_8regs_4(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4) { long lines = bytes / (sizeof (long)) / 8; @@ -78,8 +74,11 @@ xor_8regs_4(unsigned long bytes, unsigned long *p1, unsigned long *p2, } static void -xor_8regs_5(unsigned long bytes, unsigned long *p1, unsigned long *p2, - unsigned long *p3, unsigned long *p4, unsigned long *p5) +xor_8regs_5(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4, + const unsigned long * __restrict p5) { long lines = bytes / (sizeof (long)) / 8; @@ -101,7 +100,8 @@ xor_8regs_5(unsigned long bytes, unsigned long *p1, unsigned long *p2, } static void -xor_32regs_2(unsigned long bytes, unsigned long *p1, unsigned long *p2) +xor_32regs_2(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2) { long lines = bytes / (sizeof (long)) / 8; @@ -137,8 +137,9 @@ xor_32regs_2(unsigned long bytes, unsigned long *p1, unsigned long *p2) } static void -xor_32regs_3(unsigned long bytes, unsigned long *p1, unsigned long *p2, - unsigned long *p3) +xor_32regs_3(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3) { long lines = bytes / (sizeof (long)) / 8; @@ -183,8 +184,10 @@ xor_32regs_3(unsigned long bytes, unsigned long *p1, unsigned long *p2, } static void -xor_32regs_4(unsigned long bytes, unsigned long *p1, unsigned long *p2, - unsigned long *p3, unsigned long *p4) +xor_32regs_4(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4) { long lines = bytes / (sizeof (long)) / 8; @@ -238,8 +241,11 @@ xor_32regs_4(unsigned long bytes, unsigned long *p1, unsigned long *p2, } static void -xor_32regs_5(unsigned long bytes, unsigned long *p1, unsigned long *p2, - unsigned long *p3, unsigned long *p4, unsigned long *p5) +xor_32regs_5(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4, + const unsigned long * __restrict p5) { long lines = bytes / (sizeof (long)) / 8; @@ -302,7 +308,8 @@ xor_32regs_5(unsigned long bytes, unsigned long *p1, unsigned long *p2, } static void -xor_8regs_p_2(unsigned long bytes, unsigned long *p1, unsigned long *p2) +xor_8regs_p_2(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2) { long lines = bytes / (sizeof (long)) / 8 - 1; prefetchw(p1); @@ -328,8 +335,9 @@ xor_8regs_p_2(unsigned long bytes, unsigned long *p1, unsigned long *p2) } static void -xor_8regs_p_3(unsigned long bytes, unsigned long *p1, unsigned long *p2, - unsigned long *p3) +xor_8regs_p_3(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3) { long lines = bytes / (sizeof (long)) / 8 - 1; prefetchw(p1); @@ -358,8 +366,10 @@ xor_8regs_p_3(unsigned long bytes, unsigned long *p1, unsigned long *p2, } static void -xor_8regs_p_4(unsigned long bytes, unsigned long *p1, unsigned long *p2, - unsigned long *p3, unsigned long *p4) +xor_8regs_p_4(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4) { long lines = bytes / (sizeof (long)) / 8 - 1; @@ -392,8 +402,11 @@ xor_8regs_p_4(unsigned long bytes, unsigned long *p1, unsigned long *p2, } static void -xor_8regs_p_5(unsigned long bytes, unsigned long *p1, unsigned long *p2, - unsigned long *p3, unsigned long *p4, unsigned long *p5) +xor_8regs_p_5(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4, + const unsigned long * __restrict p5) { long lines = bytes / (sizeof (long)) / 8 - 1; @@ -429,7 +442,8 @@ xor_8regs_p_5(unsigned long bytes, unsigned long *p1, unsigned long *p2, } static void -xor_32regs_p_2(unsigned long bytes, unsigned long *p1, unsigned long *p2) +xor_32regs_p_2(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2) { long lines = bytes / (sizeof (long)) / 8 - 1; @@ -474,8 +488,9 @@ xor_32regs_p_2(unsigned long bytes, unsigned long *p1, unsigned long *p2) } static void -xor_32regs_p_3(unsigned long bytes, unsigned long *p1, unsigned long *p2, - unsigned long *p3) +xor_32regs_p_3(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3) { long lines = bytes / (sizeof (long)) / 8 - 1; @@ -531,8 +546,10 @@ xor_32regs_p_3(unsigned long bytes, unsigned long *p1, unsigned long *p2, } static void -xor_32regs_p_4(unsigned long bytes, unsigned long *p1, unsigned long *p2, - unsigned long *p3, unsigned long *p4) +xor_32regs_p_4(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4) { long lines = bytes / (sizeof (long)) / 8 - 1; @@ -599,8 +616,11 @@ xor_32regs_p_4(unsigned long bytes, unsigned long *p1, unsigned long *p2, } static void -xor_32regs_p_5(unsigned long bytes, unsigned long *p1, unsigned long *p2, - unsigned long *p3, unsigned long *p4, unsigned long *p5) +xor_32regs_p_5(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4, + const unsigned long * __restrict p5) { long lines = bytes / (sizeof (long)) / 8 - 1; |