diff options
Diffstat (limited to 'arch/csky')
159 files changed, 5314 insertions, 2982 deletions
diff --git a/arch/csky/Kbuild b/arch/csky/Kbuild new file mode 100644 index 000000000000..0621eaea4196 --- /dev/null +++ b/arch/csky/Kbuild @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0-only + +obj-y += kernel/ mm/ + +# for cleaning +subdir- += boot diff --git a/arch/csky/Kconfig b/arch/csky/Kconfig index da09c884cc30..d3ac36751ad1 100644 --- a/arch/csky/Kconfig +++ b/arch/csky/Kconfig @@ -2,20 +2,50 @@ config CSKY def_bool y select ARCH_32BIT_OFF_T + select ARCH_HAS_CPU_CACHE_ALIASING select ARCH_HAS_DMA_PREP_COHERENT + select ARCH_HAS_GCOV_PROFILE_ALL select ARCH_HAS_SYNC_DMA_FOR_CPU select ARCH_HAS_SYNC_DMA_FOR_DEVICE select ARCH_USE_BUILTIN_BSWAP - select ARCH_USE_QUEUED_RWLOCKS if NR_CPUS>2 + select ARCH_USE_QUEUED_RWLOCKS + select ARCH_USE_QUEUED_SPINLOCKS + select ARCH_HAS_CURRENT_STACK_POINTER + select ARCH_INLINE_READ_LOCK if !PREEMPTION + select ARCH_INLINE_READ_LOCK_BH if !PREEMPTION + select ARCH_INLINE_READ_LOCK_IRQ if !PREEMPTION + select ARCH_INLINE_READ_LOCK_IRQSAVE if !PREEMPTION + select ARCH_INLINE_READ_UNLOCK if !PREEMPTION + select ARCH_INLINE_READ_UNLOCK_BH if !PREEMPTION + select ARCH_INLINE_READ_UNLOCK_IRQ if !PREEMPTION + select ARCH_INLINE_READ_UNLOCK_IRQRESTORE if !PREEMPTION + select ARCH_INLINE_WRITE_LOCK if !PREEMPTION + select ARCH_INLINE_WRITE_LOCK_BH if !PREEMPTION + select ARCH_INLINE_WRITE_LOCK_IRQ if !PREEMPTION + select ARCH_INLINE_WRITE_LOCK_IRQSAVE if !PREEMPTION + select ARCH_INLINE_WRITE_UNLOCK if !PREEMPTION + select ARCH_INLINE_WRITE_UNLOCK_BH if !PREEMPTION + select ARCH_INLINE_WRITE_UNLOCK_IRQ if !PREEMPTION + select ARCH_INLINE_WRITE_UNLOCK_IRQRESTORE if !PREEMPTION + select ARCH_INLINE_SPIN_TRYLOCK if !PREEMPTION + select ARCH_INLINE_SPIN_TRYLOCK_BH if !PREEMPTION + select ARCH_INLINE_SPIN_LOCK if !PREEMPTION + select ARCH_INLINE_SPIN_LOCK_BH if !PREEMPTION + select ARCH_INLINE_SPIN_LOCK_IRQ if !PREEMPTION + select ARCH_INLINE_SPIN_LOCK_IRQSAVE if !PREEMPTION + select ARCH_INLINE_SPIN_UNLOCK if !PREEMPTION + select ARCH_INLINE_SPIN_UNLOCK_BH if !PREEMPTION + select ARCH_INLINE_SPIN_UNLOCK_IRQ if !PREEMPTION + select ARCH_INLINE_SPIN_UNLOCK_IRQRESTORE if !PREEMPTION + select ARCH_WANT_FRAME_POINTERS if !CPU_CK610 && $(cc-option,-mbacktrace) + select ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT select COMMON_CLK select CLKSRC_MMIO - select CLKSRC_OF select CSKY_MPINTC if CPU_CK860 select CSKY_MP_TIMER if CPU_CK860 select CSKY_APB_INTC select DMA_DIRECT_REMAP select IRQ_DOMAIN - select HANDLE_DOMAIN_IRQ select DW_APB_TIMER_OF select GENERIC_IOREMAP select GENERIC_LIB_ASHLDI3 @@ -26,7 +56,6 @@ config CSKY select GENERIC_LIB_UCMPDI2 select GENERIC_ALLOCATOR select GENERIC_ATOMIC64 - select GENERIC_CLOCKEVENTS select GENERIC_CPU_DEVICES select GENERIC_IRQ_CHIP select GENERIC_IRQ_PROBE @@ -34,22 +63,43 @@ config CSKY select GENERIC_IRQ_MULTI_HANDLER select GENERIC_SCHED_CLOCK select GENERIC_SMP_IDLE_THREAD + select GENERIC_TIME_VSYSCALL + select GENERIC_VDSO_32 + select GENERIC_GETTIMEOFDAY select GX6605S_TIMER if CPU_CK610 select HAVE_ARCH_TRACEHOOK select HAVE_ARCH_AUDITSYSCALL + select HAVE_ARCH_JUMP_LABEL if !CPU_CK610 + select HAVE_ARCH_JUMP_LABEL_RELATIVE + select HAVE_ARCH_MMAP_RND_BITS + select HAVE_ARCH_SECCOMP_FILTER + select HAVE_CONTEXT_TRACKING_USER + select HAVE_VIRT_CPU_ACCOUNTING_GEN + select HAVE_DEBUG_BUGVERBOSE + select HAVE_DEBUG_KMEMLEAK select HAVE_DYNAMIC_FTRACE + select HAVE_DYNAMIC_FTRACE_WITH_REGS + select HAVE_GENERIC_VDSO select HAVE_FUNCTION_TRACER select HAVE_FUNCTION_GRAPH_TRACER + select HAVE_FUNCTION_ERROR_INJECTION select HAVE_FTRACE_MCOUNT_RECORD select HAVE_KERNEL_GZIP select HAVE_KERNEL_LZO select HAVE_KERNEL_LZMA + select HAVE_KPROBES if !CPU_CK610 + select HAVE_KPROBES_ON_FTRACE if !CPU_CK610 + select HAVE_KRETPROBES if !CPU_CK610 + select HAVE_PAGE_SIZE_4KB select HAVE_PERF_EVENTS select HAVE_PERF_REGS select HAVE_PERF_USER_STACK_DUMP - select HAVE_DMA_API_DEBUG select HAVE_DMA_CONTIGUOUS + select HAVE_REGS_AND_STACK_ACCESS_API + select HAVE_STACKPROTECTOR select HAVE_SYSCALL_TRACEPOINTS + select HOTPLUG_CORE_SYNC_DEAD if HOTPLUG_CPU + select LOCK_MM_AND_FIND_VMA select MAY_HAVE_SPARSE_IRQ select MODULES_USE_ELF_RELA if MODULES select OF @@ -57,8 +107,18 @@ config CSKY select PERF_USE_VMALLOC if CPU_CK610 select RTC_LIB select TIMER_OF - select USB_ARCH_HAS_EHCI - select USB_ARCH_HAS_OHCI + select GENERIC_PCI_IOMAP + select HAVE_PCI + select PCI_DOMAINS_GENERIC if PCI + select PCI_SYSCALL if PCI + select PCI_MSI if PCI + select TRACE_IRQFLAGS_SUPPORT + +config LOCKDEP_SUPPORT + def_bool y + +config ARCH_SUPPORTS_UPROBES + def_bool y if !CPU_CK610 config CPU_HAS_CACHEV2 bool @@ -75,7 +135,7 @@ config CPU_HAS_TLBI config CPU_HAS_LDSTEX bool help - For SMP, CPU needs "ldex&stex" instrcutions to atomic operations. + For SMP, CPU needs "ldex&stex" instructions for atomic operations. config CPU_NEED_TLBSYNC bool @@ -110,14 +170,6 @@ config STACKTRACE_SUPPORT config TIME_LOW_RES def_bool y -config TRACE_IRQFLAGS_SUPPORT - def_bool y - -config CPU_TLB_SIZE - int - default "128" if (CPU_CK610 || CPU_CK807 || CPU_CK810) - default "1024" if (CPU_CK860) - config CPU_ASID_BITS int default "8" if (CPU_CK610 || CPU_CK807 || CPU_CK810) @@ -129,6 +181,14 @@ config L1_CACHE_SHIFT default "5" if (CPU_CK807 || CPU_CK810) default "6" if (CPU_CK860) +config ARCH_MMAP_RND_BITS_MIN + default 8 + +# max bits determined by the following formula: +# VA_BITS - PAGE_SHIFT - 3 +config ARCH_MMAP_RND_BITS_MAX + default 17 + menu "Processor type and features" choice @@ -159,6 +219,22 @@ config CPU_CK860 endchoice choice + prompt "PAGE OFFSET" + default PAGE_OFFSET_80000000 + +config PAGE_OFFSET_80000000 + bool "PAGE OFFSET 2G (user:kernel = 2:2)" + +config PAGE_OFFSET_A0000000 + bool "PAGE OFFSET 2.5G (user:kernel = 2.5:1.5)" +endchoice + +config PAGE_OFFSET + hex + default 0x80000000 if PAGE_OFFSET_80000000 + default 0xa0000000 if PAGE_OFFSET_A0000000 +choice + prompt "C-SKY PMU type" depends on PERF_EVENTS depends on CPU_CK807 || CPU_CK810 || CPU_CK860 @@ -188,6 +264,40 @@ config CPU_PM_STOP bool "stop" endchoice +menuconfig HAVE_TCM + bool "Tightly-Coupled/Sram Memory" + depends on !COMPILE_TEST + help + The implementation are not only used by TCM (Tightly-Coupled Memory) + but also used by sram on SOC bus. It follow existed linux tcm + software interface, so that old tcm application codes could be + re-used directly. + +if HAVE_TCM +config ITCM_RAM_BASE + hex "ITCM ram base" + default 0xffffffff + +config ITCM_NR_PAGES + int "Page count of ITCM size: NR*4KB" + range 1 256 + default 32 + +config HAVE_DTCM + bool "DTCM Support" + +config DTCM_RAM_BASE + hex "DTCM ram base" + depends on HAVE_DTCM + default 0xffffffff + +config DTCM_NR_PAGES + int "Page count of DTCM size: NR*4KB" + depends on HAVE_DTCM + range 1 256 + default 32 +endif + config CPU_HAS_VDSP bool "CPU has VDSP coprocessor" depends on CPU_HAS_FPU && CPU_HAS_FPUV2 @@ -196,6 +306,10 @@ config CPU_HAS_FPU bool "CPU has FPU coprocessor" depends on CPU_CK807 || CPU_CK810 || CPU_CK860 +config CPU_HAS_ICACHE_INS + bool "CPU has Icache invalidate instructions" + depends on CPU_HAS_CACHEV2 + config CPU_HAS_TEE bool "CPU has Trusted Execution Environment" depends on CPU_CK810 @@ -209,18 +323,15 @@ config NR_CPUS int "Maximum number of CPUs (2-32)" range 2 32 depends on SMP - default "2" + default "4" config HIGHMEM bool "High Memory Support" depends on !CPU_CK610 + select KMAP_LOCAL default y -config FORCE_MAX_ZONEORDER - int "Maximum zone order" - default "11" - -config RAM_BASE +config DRAM_BASE hex "DRAM start addr (the same with memory-section in dts)" default 0x0 @@ -233,6 +344,16 @@ config HOTPLUG_CPU controlled through /sys/devices/system/cpu/cpu1/hotplug/target. Say N if you want to disable CPU hotplug. + +config HAVE_EFFICIENT_UNALIGNED_STRING_OPS + bool "Enable EFFICIENT_UNALIGNED_STRING_OPS for abiv2" + depends on CPU_CK807 || CPU_CK810 || CPU_CK860 + help + Say Y here to enable EFFICIENT_UNALIGNED_STRING_OPS. Some CPU models could + deal with unaligned access by hardware. + endmenu +source "arch/csky/Kconfig.platforms" + source "kernel/Kconfig.hz" diff --git a/arch/csky/Kconfig.platforms b/arch/csky/Kconfig.platforms new file mode 100644 index 000000000000..639e17f4eacb --- /dev/null +++ b/arch/csky/Kconfig.platforms @@ -0,0 +1,9 @@ +menu "Platform drivers selection" + +config ARCH_CSKY_DW_APB_ICTL + bool "Select dw-apb interrupt controller" + select DW_APB_ICTL + default y + help + This enables support for snps dw-apb-ictl +endmenu diff --git a/arch/csky/Makefile b/arch/csky/Makefile index fb1bbbd91954..0e4237e55758 100644 --- a/arch/csky/Makefile +++ b/arch/csky/Makefile @@ -47,7 +47,7 @@ ifeq ($(CSKYABI),abiv2) KBUILD_CFLAGS += -mno-stack-size endif -ifdef CONFIG_STACKTRACE +ifdef CONFIG_FRAME_POINTER KBUILD_CFLAGS += -mbacktrace endif @@ -59,26 +59,18 @@ LDFLAGS += -EL KBUILD_AFLAGS += $(KBUILD_CFLAGS) -head-y := arch/csky/kernel/head.o - -core-y += arch/csky/kernel/ -core-y += arch/csky/mm/ core-y += arch/csky/$(CSKYABI)/ libs-y += arch/csky/lib/ \ $(shell $(CC) $(KBUILD_CFLAGS) $(KCFLAGS) -print-libgcc-file-name) boot := arch/csky/boot -core-y += $(boot)/dts/ all: zImage zImage Image uImage: vmlinux $(Q)$(MAKE) $(build)=$(boot) $(boot)/$@ -archclean: - $(Q)$(MAKE) $(clean)=$(boot) - define archhelp echo '* zImage - Compressed kernel image (arch/$(ARCH)/boot/zImage)' echo ' Image - Uncompressed kernel image (arch/$(ARCH)/boot/Image)' diff --git a/arch/csky/abiv1/Makefile b/arch/csky/abiv1/Makefile index 601ce3b2fb85..a4b2ade0fc67 100644 --- a/arch/csky/abiv1/Makefile +++ b/arch/csky/abiv1/Makefile @@ -4,5 +4,3 @@ obj-y += bswapdi.o obj-y += bswapsi.o obj-y += cacheflush.o obj-y += mmap.o -obj-y += memcpy.o -obj-y += strksyms.o diff --git a/arch/csky/abiv1/alignment.c b/arch/csky/abiv1/alignment.c index cb2a0d94a144..e5b8b4b2109a 100644 --- a/arch/csky/abiv1/alignment.c +++ b/arch/csky/abiv1/alignment.c @@ -294,7 +294,7 @@ bad_area: __func__, opcode, rz, rx, imm, addr); show_regs(regs); bust_spinlocks(0); - do_exit(SIGKILL); + make_task_dead(SIGKILL); } force_sig_fault(SIGBUS, BUS_ADRALN, (void __user *)addr); @@ -329,25 +329,11 @@ static struct ctl_table alignment_tbl[5] = { .mode = 0666, .proc_handler = &proc_dointvec }, - {} -}; - -static struct ctl_table sysctl_table[2] = { - { - .procname = "csky_alignment", - .mode = 0555, - .child = alignment_tbl}, - {} -}; - -static struct ctl_path sysctl_path[2] = { - {.procname = "csky"}, - {} }; static int __init csky_alignment_init(void) { - register_sysctl_paths(sysctl_path, sysctl_table); + register_sysctl_init("csky/csky_alignment", alignment_tbl); return 0; } diff --git a/arch/csky/abiv1/cacheflush.c b/arch/csky/abiv1/cacheflush.c index 9f1fe80cc847..171e8fb32285 100644 --- a/arch/csky/abiv1/cacheflush.c +++ b/arch/csky/abiv1/cacheflush.c @@ -4,68 +4,67 @@ #include <linux/kernel.h> #include <linux/mm.h> #include <linux/fs.h> +#include <linux/pagemap.h> #include <linux/syscalls.h> #include <linux/spinlock.h> #include <asm/page.h> #include <asm/cache.h> #include <asm/cacheflush.h> #include <asm/cachectl.h> +#include <asm/tlbflush.h> #define PG_dcache_clean PG_arch_1 -void flush_dcache_page(struct page *page) +void flush_dcache_folio(struct folio *folio) { struct address_space *mapping; - if (page == ZERO_PAGE(0)) + if (is_zero_pfn(folio_pfn(folio))) return; - mapping = page_mapping_file(page); + mapping = folio_flush_mapping(folio); - if (mapping && !page_mapcount(page)) - clear_bit(PG_dcache_clean, &page->flags); + if (mapping && !folio_mapped(folio)) + clear_bit(PG_dcache_clean, &folio->flags); else { dcache_wbinv_all(); if (mapping) icache_inv_all(); - set_bit(PG_dcache_clean, &page->flags); + set_bit(PG_dcache_clean, &folio->flags); } } +EXPORT_SYMBOL(flush_dcache_folio); + +void flush_dcache_page(struct page *page) +{ + flush_dcache_folio(page_folio(page)); +} EXPORT_SYMBOL(flush_dcache_page); -void update_mmu_cache(struct vm_area_struct *vma, unsigned long addr, - pte_t *ptep) +void update_mmu_cache_range(struct vm_fault *vmf, struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep, unsigned int nr) { unsigned long pfn = pte_pfn(*ptep); - struct page *page; + struct folio *folio; + + flush_tlb_page(vma, addr); if (!pfn_valid(pfn)) return; - page = pfn_to_page(pfn); - if (page == ZERO_PAGE(0)) + if (is_zero_pfn(pfn)) return; - if (!test_and_set_bit(PG_dcache_clean, &page->flags)) + folio = page_folio(pfn_to_page(pfn)); + if (!test_and_set_bit(PG_dcache_clean, &folio->flags)) dcache_wbinv_all(); - if (page_mapping_file(page)) { + if (folio_flush_mapping(folio)) { if (vma->vm_flags & VM_EXEC) icache_inv_all(); } } -void flush_kernel_dcache_page(struct page *page) -{ - struct address_space *mapping; - - mapping = page_mapping_file(page); - - if (!mapping || mapping_mapped(mapping)) - dcache_wbinv_all(); -} -EXPORT_SYMBOL(flush_kernel_dcache_page); - void flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) { diff --git a/arch/csky/abiv1/inc/abi/cacheflush.h b/arch/csky/abiv1/inc/abi/cacheflush.h index 79ef9e8c1afd..d011a81575d2 100644 --- a/arch/csky/abiv1/inc/abi/cacheflush.h +++ b/arch/csky/abiv1/inc/abi/cacheflush.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ABI_CSKY_CACHEFLUSH_H #define __ABI_CSKY_CACHEFLUSH_H @@ -10,17 +9,17 @@ #define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1 extern void flush_dcache_page(struct page *); +void flush_dcache_folio(struct folio *); +#define flush_dcache_folio flush_dcache_folio #define flush_cache_mm(mm) dcache_wbinv_all() #define flush_cache_page(vma, page, pfn) cache_wbinv_all() #define flush_cache_dup_mm(mm) cache_wbinv_all() -#define ARCH_HAS_FLUSH_KERNEL_DCACHE_PAGE -extern void flush_kernel_dcache_page(struct page *); - #define flush_dcache_mmap_lock(mapping) xa_lock_irq(&mapping->i_pages) #define flush_dcache_mmap_unlock(mapping) xa_unlock_irq(&mapping->i_pages) +#define ARCH_IMPLEMENTS_FLUSH_KERNEL_VMAP_RANGE 1 static inline void flush_kernel_vmap_range(void *addr, int size) { dcache_wbinv_all(); @@ -44,13 +43,12 @@ static inline void flush_anon_page(struct vm_area_struct *vma, */ extern void flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end); #define flush_cache_vmap(start, end) cache_wbinv_all() +#define flush_cache_vmap_early(start, end) do { } while (0) #define flush_cache_vunmap(start, end) cache_wbinv_all() -#define flush_icache_page(vma, page) do {} while (0); #define flush_icache_range(start, end) cache_wbinv_range(start, end) - -#define flush_icache_user_range(vma,page,addr,len) \ - flush_dcache_page(page) +#define flush_icache_mm_range(mm, start, end) cache_wbinv_range(start, end) +#define flush_icache_deferred(mm) do {} while (0); #define copy_from_user_page(vma, page, vaddr, dst, src, len) \ do { \ diff --git a/arch/csky/abiv1/inc/abi/ckmmu.h b/arch/csky/abiv1/inc/abi/ckmmu.h index ba8eb5870835..416b30c57983 100644 --- a/arch/csky/abiv1/inc/abi/ckmmu.h +++ b/arch/csky/abiv1/inc/abi/ckmmu.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_CKMMUV1_H #define __ASM_CSKY_CKMMUV1_H @@ -89,13 +88,14 @@ static inline void tlb_invalid_indexed(void) cpwcr("cpcr8", 0x02000000); } -static inline void setup_pgd(unsigned long pgd, bool kernel) +static inline void setup_pgd(pgd_t *pgd, int asid) { - cpwcr("cpcr29", pgd | BIT(0)); + cpwcr("cpcr29", __pa(pgd) | BIT(0)); + write_mmu_entryhi(asid); } -static inline unsigned long get_pgd(void) +static inline pgd_t *get_pgd(void) { - return cprcr("cpcr29") & ~BIT(0); + return __va(cprcr("cpcr29") & ~BIT(0)); } #endif /* __ASM_CSKY_CKMMUV1_H */ diff --git a/arch/csky/abiv1/inc/abi/entry.h b/arch/csky/abiv1/inc/abi/entry.h index 7ab78bd0f3b1..b6a2109b895e 100644 --- a/arch/csky/abiv1/inc/abi/entry.h +++ b/arch/csky/abiv1/inc/abi/entry.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_ENTRY_H #define __ASM_CSKY_ENTRY_H @@ -16,14 +15,16 @@ #define LSAVE_A4 40 #define LSAVE_A5 44 +#define usp ss1 + .macro USPTOKSP - mtcr sp, ss1 + mtcr sp, usp mfcr sp, ss0 .endm .macro KSPTOUSP mtcr sp, ss0 - mfcr sp, ss1 + mfcr sp, usp .endm .macro SAVE_ALL epc_inc @@ -45,7 +46,13 @@ add lr, r13 stw lr, (sp, 8) + mov lr, sp + addi lr, 32 + addi lr, 32 + addi lr, 16 + bt 2f mfcr lr, ss1 +2: stw lr, (sp, 16) stw a0, (sp, 20) @@ -72,16 +79,16 @@ .endm .macro RESTORE_ALL - psrclr ie ldw lr, (sp, 4) ldw a0, (sp, 8) mtcr a0, epc ldw a0, (sp, 12) mtcr a0, epsr btsti a0, 31 + bt 1f ldw a0, (sp, 16) mtcr a0, ss1 - +1: ldw a0, (sp, 24) ldw a1, (sp, 28) ldw a2, (sp, 32) @@ -102,9 +109,9 @@ addi sp, 32 addi sp, 8 - bt 1f + bt 2f KSPTOUSP -1: +2: rte .endm @@ -158,20 +165,12 @@ * BA Reserved C D V */ cprcr r6, cpcr30 - lsri r6, 28 - lsli r6, 28 + lsri r6, 29 + lsli r6, 29 addi r6, 0xe cpwcr r6, cpcr30 - lsri r6, 28 - addi r6, 2 - lsli r6, 28 - addi r6, 0xe + movi r6, 0 cpwcr r6, cpcr31 .endm - -.macro ANDI_R3 rx, imm - lsri \rx, 3 - andi \rx, (\imm >> 3) -.endm #endif /* __ASM_CSKY_ENTRY_H */ diff --git a/arch/csky/abiv1/inc/abi/page.h b/arch/csky/abiv1/inc/abi/page.h index c864519117c7..2d2159933b76 100644 --- a/arch/csky/abiv1/inc/abi/page.h +++ b/arch/csky/abiv1/inc/abi/page.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #include <asm/shmparam.h> diff --git a/arch/csky/abiv1/inc/abi/pgtable-bits.h b/arch/csky/abiv1/inc/abi/pgtable-bits.h index d605445aad9a..ae7a2f76dd42 100644 --- a/arch/csky/abiv1/inc/abi/pgtable-bits.h +++ b/arch/csky/abiv1/inc/abi/pgtable-bits.h @@ -1,37 +1,54 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_PGTABLE_BITS_H #define __ASM_CSKY_PGTABLE_BITS_H /* implemented in software */ -#define _PAGE_ACCESSED (1<<3) -#define PAGE_ACCESSED_BIT (3) - +#define _PAGE_PRESENT (1<<0) #define _PAGE_READ (1<<1) #define _PAGE_WRITE (1<<2) -#define _PAGE_PRESENT (1<<0) - +#define _PAGE_ACCESSED (1<<3) #define _PAGE_MODIFIED (1<<4) -#define PAGE_MODIFIED_BIT (4) + +/* We borrow bit 9 to store the exclusive marker in swap PTEs. */ +#define _PAGE_SWP_EXCLUSIVE (1<<9) /* implemented in hardware */ #define _PAGE_GLOBAL (1<<6) - #define _PAGE_VALID (1<<7) -#define PAGE_VALID_BIT (7) - #define _PAGE_DIRTY (1<<8) -#define PAGE_DIRTY_BIT (8) #define _PAGE_CACHE (3<<9) #define _PAGE_UNCACHE (2<<9) #define _PAGE_SO _PAGE_UNCACHE - #define _CACHE_MASK (7<<9) -#define _CACHE_CACHED (_PAGE_VALID | _PAGE_CACHE) -#define _CACHE_UNCACHED (_PAGE_VALID | _PAGE_UNCACHE) +#define _CACHE_CACHED _PAGE_CACHE +#define _CACHE_UNCACHED _PAGE_UNCACHE + +#define _PAGE_PROT_NONE _PAGE_READ + +/* + * Encode/decode swap entries and swap PTEs. Swap PTEs are all PTEs that + * are !pte_none() && !pte_present(). + * + * Format of swap PTE: + * bit 0: _PAGE_PRESENT (zero) + * bit 1: _PAGE_READ (zero) + * bit 2 - 5: swap type[0 - 3] + * bit 6: _PAGE_GLOBAL (zero) + * bit 7: _PAGE_VALID (zero) + * bit 8: swap type[4] + * bit 9: exclusive marker + * bit 10 - 31: swap offset + */ +#define __swp_type(x) ((((x).val >> 2) & 0xf) | \ + (((x).val >> 4) & 0x10)) +#define __swp_offset(x) ((x).val >> 10) +#define __swp_entry(type, offset) ((swp_entry_t) { \ + ((type & 0xf) << 2) | \ + ((type & 0x10) << 4) | \ + ((offset) << 10)}) #define HAVE_ARCH_UNMAPPED_AREA diff --git a/arch/csky/abiv1/inc/abi/reg_ops.h b/arch/csky/abiv1/inc/abi/reg_ops.h index a153bd3918f7..abd01a243388 100644 --- a/arch/csky/abiv1/inc/abi/reg_ops.h +++ b/arch/csky/abiv1/inc/abi/reg_ops.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ABI_REG_OPS_H #define __ABI_REG_OPS_H diff --git a/arch/csky/abiv1/inc/abi/regdef.h b/arch/csky/abiv1/inc/abi/regdef.h index 104707fbdcc1..7b386fd67070 100644 --- a/arch/csky/abiv1/inc/abi/regdef.h +++ b/arch/csky/abiv1/inc/abi/regdef.h @@ -1,10 +1,14 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_REGDEF_H #define __ASM_CSKY_REGDEF_H +#ifdef __ASSEMBLY__ #define syscallid r1 +#else +#define syscallid "r1" +#endif + #define regs_syscallid(regs) regs->regs[9] #define regs_fp(regs) regs->regs[2] diff --git a/arch/csky/abiv1/inc/abi/string.h b/arch/csky/abiv1/inc/abi/string.h index 0cd43384f8d2..de50117b904d 100644 --- a/arch/csky/abiv1/inc/abi/string.h +++ b/arch/csky/abiv1/inc/abi/string.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ABI_CSKY_STRING_H #define __ABI_CSKY_STRING_H @@ -7,4 +6,10 @@ #define __HAVE_ARCH_MEMCPY extern void *memcpy(void *, const void *, __kernel_size_t); +#define __HAVE_ARCH_MEMMOVE +extern void *memmove(void *, const void *, __kernel_size_t); + +#define __HAVE_ARCH_MEMSET +extern void *memset(void *, int, __kernel_size_t); + #endif /* __ABI_CSKY_STRING_H */ diff --git a/arch/csky/abiv1/inc/abi/switch_context.h b/arch/csky/abiv1/inc/abi/switch_context.h index 17c82686498e..ec73fd7c9f87 100644 --- a/arch/csky/abiv1/inc/abi/switch_context.h +++ b/arch/csky/abiv1/inc/abi/switch_context.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ABI_CSKY_PTRACE_H #define __ABI_CSKY_PTRACE_H diff --git a/arch/csky/abiv1/inc/abi/vdso.h b/arch/csky/abiv1/inc/abi/vdso.h index 14352f524f1d..9e6d0a2fdd2b 100644 --- a/arch/csky/abiv1/inc/abi/vdso.h +++ b/arch/csky/abiv1/inc/abi/vdso.h @@ -1,17 +1,9 @@ /* SPDX-License-Identifier: GPL-2.0 */ -#include <linux/uaccess.h> +#ifndef __ABI_CSKY_VDSO_H +#define __ABI_CSKY_VDSO_H -static inline int setup_vdso_page(unsigned short *ptr) -{ - int err = 0; +/* movi r1, 127; addi r1, (139 - 127) */ +#define SET_SYSCALL_ID .long 0x20b167f1 - /* movi r1, 127 */ - err |= __put_user(0x67f1, ptr + 0); - /* addi r1, (139 - 127) */ - err |= __put_user(0x20b1, ptr + 1); - /* trap 0 */ - err |= __put_user(0x0008, ptr + 2); - - return err; -} +#endif /* __ABI_CSKY_VDSO_H */ diff --git a/arch/csky/abiv1/memcpy.S b/arch/csky/abiv1/memcpy.S deleted file mode 100644 index 5078eb5169fa..000000000000 --- a/arch/csky/abiv1/memcpy.S +++ /dev/null @@ -1,347 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. - -#include <linux/linkage.h> - -.macro GET_FRONT_BITS rx y -#ifdef __cskyLE__ - lsri \rx, \y -#else - lsli \rx, \y -#endif -.endm - -.macro GET_AFTER_BITS rx y -#ifdef __cskyLE__ - lsli \rx, \y -#else - lsri \rx, \y -#endif -.endm - -/* void *memcpy(void *dest, const void *src, size_t n); */ -ENTRY(memcpy) - mov r7, r2 - cmplti r4, 4 - bt .L_copy_by_byte - mov r6, r2 - andi r6, 3 - cmpnei r6, 0 - jbt .L_dest_not_aligned - mov r6, r3 - andi r6, 3 - cmpnei r6, 0 - jbt .L_dest_aligned_but_src_not_aligned -.L0: - cmplti r4, 16 - jbt .L_aligned_and_len_less_16bytes - subi sp, 8 - stw r8, (sp, 0) -.L_aligned_and_len_larger_16bytes: - ldw r1, (r3, 0) - ldw r5, (r3, 4) - ldw r8, (r3, 8) - stw r1, (r7, 0) - ldw r1, (r3, 12) - stw r5, (r7, 4) - stw r8, (r7, 8) - stw r1, (r7, 12) - subi r4, 16 - addi r3, 16 - addi r7, 16 - cmplti r4, 16 - jbf .L_aligned_and_len_larger_16bytes - ldw r8, (sp, 0) - addi sp, 8 - cmpnei r4, 0 - jbf .L_return - -.L_aligned_and_len_less_16bytes: - cmplti r4, 4 - bt .L_copy_by_byte -.L1: - ldw r1, (r3, 0) - stw r1, (r7, 0) - subi r4, 4 - addi r3, 4 - addi r7, 4 - cmplti r4, 4 - jbf .L1 - br .L_copy_by_byte - -.L_return: - rts - -.L_copy_by_byte: /* len less than 4 bytes */ - cmpnei r4, 0 - jbf .L_return -.L4: - ldb r1, (r3, 0) - stb r1, (r7, 0) - addi r3, 1 - addi r7, 1 - decne r4 - jbt .L4 - rts - -/* - * If dest is not aligned, just copying some bytes makes the dest align. - * Afther that, we judge whether the src is aligned. - */ -.L_dest_not_aligned: - mov r5, r3 - rsub r5, r5, r7 - abs r5, r5 - cmplt r5, r4 - bt .L_copy_by_byte - mov r5, r7 - sub r5, r3 - cmphs r5, r4 - bf .L_copy_by_byte - mov r5, r6 -.L5: - ldb r1, (r3, 0) /* makes the dest align. */ - stb r1, (r7, 0) - addi r5, 1 - subi r4, 1 - addi r3, 1 - addi r7, 1 - cmpnei r5, 4 - jbt .L5 - cmplti r4, 4 - jbt .L_copy_by_byte - mov r6, r3 /* judge whether the src is aligned. */ - andi r6, 3 - cmpnei r6, 0 - jbf .L0 - -/* Judge the number of misaligned, 1, 2, 3? */ -.L_dest_aligned_but_src_not_aligned: - mov r5, r3 - rsub r5, r5, r7 - abs r5, r5 - cmplt r5, r4 - bt .L_copy_by_byte - bclri r3, 0 - bclri r3, 1 - ldw r1, (r3, 0) - addi r3, 4 - cmpnei r6, 2 - bf .L_dest_aligned_but_src_not_aligned_2bytes - cmpnei r6, 3 - bf .L_dest_aligned_but_src_not_aligned_3bytes - -.L_dest_aligned_but_src_not_aligned_1byte: - mov r5, r7 - sub r5, r3 - cmphs r5, r4 - bf .L_copy_by_byte - cmplti r4, 16 - bf .L11 -.L10: /* If the len is less than 16 bytes */ - GET_FRONT_BITS r1 8 - mov r5, r1 - ldw r6, (r3, 0) - mov r1, r6 - GET_AFTER_BITS r6 24 - or r5, r6 - stw r5, (r7, 0) - subi r4, 4 - addi r3, 4 - addi r7, 4 - cmplti r4, 4 - bf .L10 - subi r3, 3 - br .L_copy_by_byte -.L11: - subi sp, 16 - stw r8, (sp, 0) - stw r9, (sp, 4) - stw r10, (sp, 8) - stw r11, (sp, 12) -.L12: - ldw r5, (r3, 0) - ldw r11, (r3, 4) - ldw r8, (r3, 8) - ldw r9, (r3, 12) - - GET_FRONT_BITS r1 8 /* little or big endian? */ - mov r10, r5 - GET_AFTER_BITS r5 24 - or r5, r1 - - GET_FRONT_BITS r10 8 - mov r1, r11 - GET_AFTER_BITS r11 24 - or r11, r10 - - GET_FRONT_BITS r1 8 - mov r10, r8 - GET_AFTER_BITS r8 24 - or r8, r1 - - GET_FRONT_BITS r10 8 - mov r1, r9 - GET_AFTER_BITS r9 24 - or r9, r10 - - stw r5, (r7, 0) - stw r11, (r7, 4) - stw r8, (r7, 8) - stw r9, (r7, 12) - subi r4, 16 - addi r3, 16 - addi r7, 16 - cmplti r4, 16 - jbf .L12 - ldw r8, (sp, 0) - ldw r9, (sp, 4) - ldw r10, (sp, 8) - ldw r11, (sp, 12) - addi sp , 16 - cmplti r4, 4 - bf .L10 - subi r3, 3 - br .L_copy_by_byte - -.L_dest_aligned_but_src_not_aligned_2bytes: - cmplti r4, 16 - bf .L21 -.L20: - GET_FRONT_BITS r1 16 - mov r5, r1 - ldw r6, (r3, 0) - mov r1, r6 - GET_AFTER_BITS r6 16 - or r5, r6 - stw r5, (r7, 0) - subi r4, 4 - addi r3, 4 - addi r7, 4 - cmplti r4, 4 - bf .L20 - subi r3, 2 - br .L_copy_by_byte - rts - -.L21: /* n > 16 */ - subi sp, 16 - stw r8, (sp, 0) - stw r9, (sp, 4) - stw r10, (sp, 8) - stw r11, (sp, 12) - -.L22: - ldw r5, (r3, 0) - ldw r11, (r3, 4) - ldw r8, (r3, 8) - ldw r9, (r3, 12) - - GET_FRONT_BITS r1 16 - mov r10, r5 - GET_AFTER_BITS r5 16 - or r5, r1 - - GET_FRONT_BITS r10 16 - mov r1, r11 - GET_AFTER_BITS r11 16 - or r11, r10 - - GET_FRONT_BITS r1 16 - mov r10, r8 - GET_AFTER_BITS r8 16 - or r8, r1 - - GET_FRONT_BITS r10 16 - mov r1, r9 - GET_AFTER_BITS r9 16 - or r9, r10 - - stw r5, (r7, 0) - stw r11, (r7, 4) - stw r8, (r7, 8) - stw r9, (r7, 12) - subi r4, 16 - addi r3, 16 - addi r7, 16 - cmplti r4, 16 - jbf .L22 - ldw r8, (sp, 0) - ldw r9, (sp, 4) - ldw r10, (sp, 8) - ldw r11, (sp, 12) - addi sp, 16 - cmplti r4, 4 - bf .L20 - subi r3, 2 - br .L_copy_by_byte - - -.L_dest_aligned_but_src_not_aligned_3bytes: - cmplti r4, 16 - bf .L31 -.L30: - GET_FRONT_BITS r1 24 - mov r5, r1 - ldw r6, (r3, 0) - mov r1, r6 - GET_AFTER_BITS r6 8 - or r5, r6 - stw r5, (r7, 0) - subi r4, 4 - addi r3, 4 - addi r7, 4 - cmplti r4, 4 - bf .L30 - subi r3, 1 - br .L_copy_by_byte -.L31: - subi sp, 16 - stw r8, (sp, 0) - stw r9, (sp, 4) - stw r10, (sp, 8) - stw r11, (sp, 12) -.L32: - ldw r5, (r3, 0) - ldw r11, (r3, 4) - ldw r8, (r3, 8) - ldw r9, (r3, 12) - - GET_FRONT_BITS r1 24 - mov r10, r5 - GET_AFTER_BITS r5 8 - or r5, r1 - - GET_FRONT_BITS r10 24 - mov r1, r11 - GET_AFTER_BITS r11 8 - or r11, r10 - - GET_FRONT_BITS r1 24 - mov r10, r8 - GET_AFTER_BITS r8 8 - or r8, r1 - - GET_FRONT_BITS r10 24 - mov r1, r9 - GET_AFTER_BITS r9 8 - or r9, r10 - - stw r5, (r7, 0) - stw r11, (r7, 4) - stw r8, (r7, 8) - stw r9, (r7, 12) - subi r4, 16 - addi r3, 16 - addi r7, 16 - cmplti r4, 16 - jbf .L32 - ldw r8, (sp, 0) - ldw r9, (sp, 4) - ldw r10, (sp, 8) - ldw r11, (sp, 12) - addi sp, 16 - cmplti r4, 4 - bf .L30 - subi r3, 1 - br .L_copy_by_byte diff --git a/arch/csky/abiv1/strksyms.c b/arch/csky/abiv1/strksyms.c deleted file mode 100644 index c7ccbb27e8d7..000000000000 --- a/arch/csky/abiv1/strksyms.c +++ /dev/null @@ -1,6 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. - -#include <linux/module.h> - -EXPORT_SYMBOL(memcpy); diff --git a/arch/csky/abiv2/Makefile b/arch/csky/abiv2/Makefile index c561efa5533c..ea8005fe01a8 100644 --- a/arch/csky/abiv2/Makefile +++ b/arch/csky/abiv2/Makefile @@ -2,9 +2,11 @@ obj-y += cacheflush.o obj-$(CONFIG_CPU_HAS_FPU) += fpu.o obj-y += memcmp.o +ifeq ($(CONFIG_HAVE_EFFICIENT_UNALIGNED_STRING_OPS), y) obj-y += memcpy.o obj-y += memmove.o obj-y += memset.o +endif obj-y += strcmp.o obj-y += strcpy.o obj-y += strlen.o diff --git a/arch/csky/abiv2/cacheflush.c b/arch/csky/abiv2/cacheflush.c index 5bb887b275e1..876028b1083f 100644 --- a/arch/csky/abiv2/cacheflush.c +++ b/arch/csky/abiv2/cacheflush.c @@ -5,47 +5,88 @@ #include <linux/highmem.h> #include <linux/mm.h> #include <asm/cache.h> +#include <asm/tlbflush.h> -void flush_icache_page(struct vm_area_struct *vma, struct page *page) +void update_mmu_cache_range(struct vm_fault *vmf, struct vm_area_struct *vma, + unsigned long address, pte_t *pte, unsigned int nr) { - unsigned long start; + unsigned long pfn = pte_pfn(*pte); + struct folio *folio; + unsigned int i; - start = (unsigned long) kmap_atomic(page); + flush_tlb_page(vma, address); - cache_wbinv_range(start, start + PAGE_SIZE); + if (!pfn_valid(pfn)) + return; - kunmap_atomic((void *)start); -} + folio = page_folio(pfn_to_page(pfn)); -void flush_icache_user_range(struct vm_area_struct *vma, struct page *page, - unsigned long vaddr, int len) -{ - unsigned long kaddr; + if (test_and_set_bit(PG_dcache_clean, &folio->flags)) + return; + + icache_inv_range(address, address + nr*PAGE_SIZE); + for (i = 0; i < folio_nr_pages(folio); i++) { + unsigned long addr = (unsigned long) kmap_local_folio(folio, + i * PAGE_SIZE); - kaddr = (unsigned long) kmap_atomic(page) + (vaddr & ~PAGE_MASK); + dcache_wb_range(addr, addr + PAGE_SIZE); + if (vma->vm_flags & VM_EXEC) + icache_inv_range(addr, addr + PAGE_SIZE); + kunmap_local((void *) addr); + } +} - cache_wbinv_range(kaddr, kaddr + len); +void flush_icache_deferred(struct mm_struct *mm) +{ + unsigned int cpu = smp_processor_id(); + cpumask_t *mask = &mm->context.icache_stale_mask; - kunmap_atomic((void *)kaddr); + if (cpumask_test_cpu(cpu, mask)) { + cpumask_clear_cpu(cpu, mask); + /* + * Ensure the remote hart's writes are visible to this hart. + * This pairs with a barrier in flush_icache_mm. + */ + smp_mb(); + local_icache_inv_all(NULL); + } } -void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, - pte_t *pte) +void flush_icache_mm_range(struct mm_struct *mm, + unsigned long start, unsigned long end) { - unsigned long addr, pfn; - struct page *page; + unsigned int cpu; + cpumask_t others, *mask; - pfn = pte_pfn(*pte); - if (unlikely(!pfn_valid(pfn))) - return; + preempt_disable(); - page = pfn_to_page(pfn); - if (page == ZERO_PAGE(0)) +#ifdef CONFIG_CPU_HAS_ICACHE_INS + if (mm == current->mm) { + icache_inv_range(start, end); + preempt_enable(); return; + } +#endif + + /* Mark every hart's icache as needing a flush for this MM. */ + mask = &mm->context.icache_stale_mask; + cpumask_setall(mask); + + /* Flush this hart's I$ now, and mark it as flushed. */ + cpu = smp_processor_id(); + cpumask_clear_cpu(cpu, mask); + local_icache_inv_all(NULL); - addr = (unsigned long) kmap_atomic(page); + /* + * Flush the I$ of other harts concurrently executing, and mark them as + * flushed. + */ + cpumask_andnot(&others, mm_cpumask(mm), cpumask_of(cpu)); - cache_wbinv_range(addr, addr + PAGE_SIZE); + if (mm != current->active_mm || !cpumask_empty(&others)) { + on_each_cpu_mask(&others, local_icache_inv_all, NULL, 1); + cpumask_clear(mask); + } - kunmap_atomic((void *) addr); + preempt_enable(); } diff --git a/arch/csky/abiv2/fpu.c b/arch/csky/abiv2/fpu.c index 86d187d4e5af..5acc5c2e544e 100644 --- a/arch/csky/abiv2/fpu.c +++ b/arch/csky/abiv2/fpu.c @@ -10,11 +10,6 @@ #define MTCR_DIST 0xC0006420 #define MFCR_DIST 0xC0006020 -void __init init_fpu(void) -{ - mtcr("cr<1, 2>", 0); -} - /* * fpu_libc_helper() is to help libc to excute: * - mfcr %a, cr<1, 2> diff --git a/arch/csky/abiv2/inc/abi/cacheflush.h b/arch/csky/abiv2/inc/abi/cacheflush.h index b8db5e0b2fe3..6513ac5d2578 100644 --- a/arch/csky/abiv2/inc/abi/cacheflush.h +++ b/arch/csky/abiv2/inc/abi/cacheflush.h @@ -13,32 +13,47 @@ #define flush_cache_all() do { } while (0) #define flush_cache_mm(mm) do { } while (0) #define flush_cache_dup_mm(mm) do { } while (0) +#define flush_cache_range(vma, start, end) do { } while (0) +#define flush_cache_page(vma, vmaddr, pfn) do { } while (0) -#define flush_cache_range(vma, start, end) \ - do { \ - if (vma->vm_flags & VM_EXEC) \ - icache_inv_all(); \ - } while (0) +#define PG_dcache_clean PG_arch_1 + +static inline void flush_dcache_folio(struct folio *folio) +{ + if (test_bit(PG_dcache_clean, &folio->flags)) + clear_bit(PG_dcache_clean, &folio->flags); +} +#define flush_dcache_folio flush_dcache_folio + +#define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1 +static inline void flush_dcache_page(struct page *page) +{ + flush_dcache_folio(page_folio(page)); +} -#define flush_cache_page(vma, vmaddr, pfn) do { } while (0) -#define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 0 -#define flush_dcache_page(page) do { } while (0) #define flush_dcache_mmap_lock(mapping) do { } while (0) #define flush_dcache_mmap_unlock(mapping) do { } while (0) #define flush_icache_range(start, end) cache_wbinv_range(start, end) -void flush_icache_page(struct vm_area_struct *vma, struct page *page); -void flush_icache_user_range(struct vm_area_struct *vma, struct page *page, - unsigned long vaddr, int len); +void flush_icache_mm_range(struct mm_struct *mm, + unsigned long start, unsigned long end); +void flush_icache_deferred(struct mm_struct *mm); #define flush_cache_vmap(start, end) do { } while (0) +#define flush_cache_vmap_early(start, end) do { } while (0) #define flush_cache_vunmap(start, end) do { } while (0) #define copy_to_user_page(vma, page, vaddr, dst, src, len) \ do { \ memcpy(dst, src, len); \ - cache_wbinv_range((unsigned long)dst, (unsigned long)dst + len); \ + if (vma->vm_flags & VM_EXEC) { \ + dcache_wb_range((unsigned long)dst, \ + (unsigned long)dst + len); \ + flush_icache_mm_range(current->mm, \ + (unsigned long)dst, \ + (unsigned long)dst + len); \ + } \ } while (0) #define copy_from_user_page(vma, page, vaddr, dst, src, len) \ memcpy(dst, src, len) diff --git a/arch/csky/abiv2/inc/abi/ckmmu.h b/arch/csky/abiv2/inc/abi/ckmmu.h index 73ded7c72482..64215f2380f1 100644 --- a/arch/csky/abiv2/inc/abi/ckmmu.h +++ b/arch/csky/abiv2/inc/abi/ckmmu.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_CKMMUV2_H #define __ASM_CSKY_CKMMUV2_H @@ -78,8 +77,13 @@ static inline void tlb_read(void) static inline void tlb_invalid_all(void) { #ifdef CONFIG_CPU_HAS_TLBI - asm volatile("tlbi.alls\n":::"memory"); sync_is(); + asm volatile( + "tlbi.alls \n" + "sync.i \n" + : + : + : "memory"); #else mtcr("cr<8, 15>", 0x04000000); #endif @@ -88,8 +92,13 @@ static inline void tlb_invalid_all(void) static inline void local_tlb_invalid_all(void) { #ifdef CONFIG_CPU_HAS_TLBI - asm volatile("tlbi.all\n":::"memory"); sync_is(); + asm volatile( + "tlbi.all \n" + "sync.i \n" + : + : + : "memory"); #else tlb_invalid_all(); #endif @@ -100,16 +109,31 @@ static inline void tlb_invalid_indexed(void) mtcr("cr<8, 15>", 0x02000000); } -static inline void setup_pgd(unsigned long pgd, bool kernel) +#define NOP32 ".long 0x4820c400\n" + +static inline void setup_pgd(pgd_t *pgd, int asid) { - if (kernel) - mtcr("cr<28, 15>", pgd | BIT(0)); - else - mtcr("cr<29, 15>", pgd | BIT(0)); +#ifdef CONFIG_CPU_HAS_TLBI + sync_is(); +#else + mb(); +#endif + asm volatile( +#ifdef CONFIG_CPU_HAS_TLBI + "mtcr %1, cr<28, 15> \n" +#endif + "mtcr %1, cr<29, 15> \n" + "mtcr %0, cr< 4, 15> \n" + ".rept 64 \n" + NOP32 + ".endr \n" + : + :"r"(asid), "r"(__pa(pgd) | BIT(0)) + :"memory"); } -static inline unsigned long get_pgd(void) +static inline pgd_t *get_pgd(void) { - return mfcr("cr<29, 15>") & ~BIT(0); + return __va(mfcr("cr<29, 15>") & ~BIT(0)); } #endif /* __ASM_CSKY_CKMMUV2_H */ diff --git a/arch/csky/abiv2/inc/abi/entry.h b/arch/csky/abiv2/inc/abi/entry.h index 9897a16b45e5..cca63e699b58 100644 --- a/arch/csky/abiv2/inc/abi/entry.h +++ b/arch/csky/abiv2/inc/abi/entry.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_ENTRY_H #define __ASM_CSKY_ENTRY_H @@ -13,6 +12,8 @@ #define LSAVE_A1 28 #define LSAVE_A2 32 #define LSAVE_A3 36 +#define LSAVE_A4 40 +#define LSAVE_A5 44 #define KSPTOUSP #define USPTOKSP @@ -24,6 +25,9 @@ stw tls, (sp, 0) stw lr, (sp, 4) + RD_MEH lr + WR_MEH lr + mfcr lr, epc movi tls, \epc_inc add lr, tls @@ -31,7 +35,13 @@ mfcr lr, epsr stw lr, (sp, 12) + btsti lr, 31 + bf 1f + addi lr, sp, 152 + br 2f +1: mfcr lr, usp +2: stw lr, (sp, 16) stw a0, (sp, 20) @@ -57,15 +67,16 @@ .endm .macro RESTORE_ALL - psrclr ie ldw tls, (sp, 0) ldw lr, (sp, 4) ldw a0, (sp, 8) mtcr a0, epc ldw a0, (sp, 12) mtcr a0, epsr + btsti a0, 31 ldw a0, (sp, 16) mtcr a0, usp + mtcr a0, ss0 #ifdef CONFIG_CPU_HAS_HILO ldw a0, (sp, 140) @@ -86,9 +97,69 @@ addi sp, 40 ldm r16-r30, (sp) addi sp, 72 + bf 1f + mfcr sp, ss0 +1: rte .endm +.macro SAVE_REGS_FTRACE + subi sp, 152 + stw tls, (sp, 0) + stw lr, (sp, 4) + + mfcr lr, psr + stw lr, (sp, 12) + + addi lr, sp, 152 + stw lr, (sp, 16) + + stw a0, (sp, 20) + stw a0, (sp, 24) + stw a1, (sp, 28) + stw a2, (sp, 32) + stw a3, (sp, 36) + + addi sp, 40 + stm r4-r13, (sp) + + addi sp, 40 + stm r16-r30, (sp) +#ifdef CONFIG_CPU_HAS_HILO + mfhi lr + stw lr, (sp, 60) + mflo lr + stw lr, (sp, 64) + mfcr lr, cr14 + stw lr, (sp, 68) +#endif + subi sp, 80 +.endm + +.macro RESTORE_REGS_FTRACE + ldw tls, (sp, 0) + +#ifdef CONFIG_CPU_HAS_HILO + ldw a0, (sp, 140) + mthi a0 + ldw a0, (sp, 144) + mtlo a0 + ldw a0, (sp, 148) + mtcr a0, cr14 +#endif + + ldw a0, (sp, 24) + ldw a1, (sp, 28) + ldw a2, (sp, 32) + ldw a3, (sp, 36) + + addi sp, 40 + ldm r4-r13, (sp) + addi sp, 40 + ldm r16-r30, (sp) + addi sp, 72 +.endm + .macro SAVE_SWITCH_STACK subi sp, 64 stm r4-r11, (sp) @@ -162,6 +233,16 @@ mtcr \rx, cr<8, 15> .endm +#ifdef CONFIG_PAGE_OFFSET_80000000 +#define MSA_SET cr<30, 15> +#define MSA_CLR cr<31, 15> +#endif + +#ifdef CONFIG_PAGE_OFFSET_A0000000 +#define MSA_SET cr<31, 15> +#define MSA_CLR cr<30, 15> +#endif + .macro SETUP_MMU /* Init psr and enable ee */ lrw r6, DEFAULT_PSR_VALUE @@ -212,18 +293,15 @@ * 31 - 29 | 28 - 9 | 8 | 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0 * BA Reserved SH WA B SO SEC C D V */ - mfcr r6, cr<30, 15> /* Get MSA0 */ + mfcr r6, MSA_SET /* Get MSA */ 2: - lsri r6, 28 - lsli r6, 28 + lsri r6, 29 + lsli r6, 29 addi r6, 0x1ce - mtcr r6, cr<30, 15> /* Set MSA0 */ + mtcr r6, MSA_SET /* Set MSA */ - lsri r6, 28 - addi r6, 2 - lsli r6, 28 - addi r6, 0x1ce - mtcr r6, cr<31, 15> /* Set MSA1 */ + movi r6, 0 + mtcr r6, MSA_CLR /* Clr MSA */ /* enable MMU */ mfcr r6, cr18 @@ -233,9 +311,4 @@ jmpi 3f /* jump to va */ 3: .endm - -.macro ANDI_R3 rx, imm - lsri \rx, 3 - andi \rx, (\imm >> 3) -.endm #endif /* __ASM_CSKY_ENTRY_H */ diff --git a/arch/csky/abiv2/inc/abi/fpu.h b/arch/csky/abiv2/inc/abi/fpu.h index 22ca3cf2794a..aabb79355013 100644 --- a/arch/csky/abiv2/inc/abi/fpu.h +++ b/arch/csky/abiv2/inc/abi/fpu.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_FPU_H #define __ASM_CSKY_FPU_H @@ -9,7 +8,8 @@ int fpu_libc_helper(struct pt_regs *regs); void fpu_fpe(struct pt_regs *regs); -void __init init_fpu(void); + +static inline void init_fpu(void) { mtcr("cr<1, 2>", 0); } void save_to_user_fp(struct user_fp *user_fp); void restore_from_user_fp(struct user_fp *user_fp); diff --git a/arch/csky/abiv2/inc/abi/page.h b/arch/csky/abiv2/inc/abi/page.h index 0a70cb553dca..cf005f13cd15 100644 --- a/arch/csky/abiv2/inc/abi/page.h +++ b/arch/csky/abiv2/inc/abi/page.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. static inline void clear_user_page(void *addr, unsigned long vaddr, struct page *page) diff --git a/arch/csky/abiv2/inc/abi/pgtable-bits.h b/arch/csky/abiv2/inc/abi/pgtable-bits.h index 137f7932c83b..526152bd2156 100644 --- a/arch/csky/abiv2/inc/abi/pgtable-bits.h +++ b/arch/csky/abiv2/inc/abi/pgtable-bits.h @@ -1,37 +1,53 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_PGTABLE_BITS_H #define __ASM_CSKY_PGTABLE_BITS_H /* implemented in software */ #define _PAGE_ACCESSED (1<<7) -#define PAGE_ACCESSED_BIT (7) - #define _PAGE_READ (1<<8) #define _PAGE_WRITE (1<<9) #define _PAGE_PRESENT (1<<10) - #define _PAGE_MODIFIED (1<<11) -#define PAGE_MODIFIED_BIT (11) + +/* We borrow bit 7 to store the exclusive marker in swap PTEs. */ +#define _PAGE_SWP_EXCLUSIVE (1<<7) /* implemented in hardware */ #define _PAGE_GLOBAL (1<<0) - #define _PAGE_VALID (1<<1) -#define PAGE_VALID_BIT (1) - #define _PAGE_DIRTY (1<<2) -#define PAGE_DIRTY_BIT (2) #define _PAGE_SO (1<<5) #define _PAGE_BUF (1<<6) - #define _PAGE_CACHE (1<<3) - #define _CACHE_MASK _PAGE_CACHE -#define _CACHE_CACHED (_PAGE_VALID | _PAGE_CACHE | _PAGE_BUF) -#define _CACHE_UNCACHED (_PAGE_VALID) +#define _CACHE_CACHED (_PAGE_CACHE | _PAGE_BUF) +#define _CACHE_UNCACHED (0) + +#define _PAGE_PROT_NONE _PAGE_WRITE + +/* + * Encode/decode swap entries and swap PTEs. Swap PTEs are all PTEs that + * are !pte_none() && !pte_present(). + * + * Format of swap PTE: + * bit 0: _PAGE_GLOBAL (zero) + * bit 1: _PAGE_VALID (zero) + * bit 2 - 6: swap type + * bit 7: exclusive marker + * bit 8: swap offset[0] + * bit 9: _PAGE_WRITE (zero) + * bit 10: _PAGE_PRESENT (zero) + * bit 11 - 31: swap offset[1 - 21] + */ +#define __swp_type(x) (((x).val >> 2) & 0x1f) +#define __swp_offset(x) ((((x).val >> 8) & 0x1) | \ + (((x).val >> 10) & 0x3ffffe)) +#define __swp_entry(type, offset) ((swp_entry_t) { \ + ((type & 0x1f) << 2) | \ + ((offset & 0x1) << 8) | \ + ((offset & 0x3ffffe) << 10)}) #endif /* __ASM_CSKY_PGTABLE_BITS_H */ diff --git a/arch/csky/abiv2/inc/abi/reg_ops.h b/arch/csky/abiv2/inc/abi/reg_ops.h index ae82c3f26a6b..49ba18a64751 100644 --- a/arch/csky/abiv2/inc/abi/reg_ops.h +++ b/arch/csky/abiv2/inc/abi/reg_ops.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ABI_REG_OPS_H #define __ABI_REG_OPS_H diff --git a/arch/csky/abiv2/inc/abi/regdef.h b/arch/csky/abiv2/inc/abi/regdef.h index d7328bbc1ce7..0933addbc27b 100644 --- a/arch/csky/abiv2/inc/abi/regdef.h +++ b/arch/csky/abiv2/inc/abi/regdef.h @@ -1,10 +1,14 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_REGDEF_H #define __ASM_CSKY_REGDEF_H +#ifdef __ASSEMBLY__ #define syscallid r7 +#else +#define syscallid "r7" +#endif + #define regs_syscallid(regs) regs->regs[3] #define regs_fp(regs) regs->regs[4] diff --git a/arch/csky/abiv2/inc/abi/switch_context.h b/arch/csky/abiv2/inc/abi/switch_context.h index 73a81245a3b3..5dd5c3f4ee7e 100644 --- a/arch/csky/abiv2/inc/abi/switch_context.h +++ b/arch/csky/abiv2/inc/abi/switch_context.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ABI_CSKY_PTRACE_H #define __ABI_CSKY_PTRACE_H diff --git a/arch/csky/abiv2/inc/abi/vdso.h b/arch/csky/abiv2/inc/abi/vdso.h index b60d4a070326..40fd10d893ff 100644 --- a/arch/csky/abiv2/inc/abi/vdso.h +++ b/arch/csky/abiv2/inc/abi/vdso.h @@ -3,21 +3,7 @@ #ifndef __ABI_CSKY_VDSO_H #define __ABI_CSKY_VDSO_H -#include <linux/uaccess.h> +/* movi r7, 173 */ +#define SET_SYSCALL_ID .long 0x008bea07 -static inline int setup_vdso_page(unsigned short *ptr) -{ - int err = 0; - - /* movi r7, 173 */ - err |= __put_user(0xea07, ptr); - err |= __put_user(0x008b, ptr+1); - - /* trap 0 */ - err |= __put_user(0xc000, ptr+2); - err |= __put_user(0x2020, ptr+3); - - return err; -} - -#endif /* __ABI_CSKY_STRING_H */ +#endif /* __ABI_CSKY_VDSO_H */ diff --git a/arch/csky/abiv2/mcount.S b/arch/csky/abiv2/mcount.S index 326402e65f9e..d745e10c10b0 100644 --- a/arch/csky/abiv2/mcount.S +++ b/arch/csky/abiv2/mcount.S @@ -3,6 +3,8 @@ #include <linux/linkage.h> #include <asm/ftrace.h> +#include <abi/entry.h> +#include <asm/asm-offsets.h> /* * csky-gcc with -pg will put the following asm after prologue: @@ -44,6 +46,24 @@ jmp t1 .endm +.macro mcount_enter_regs + subi sp, 8 + stw lr, (sp, 0) + stw r8, (sp, 4) + SAVE_REGS_FTRACE +.endm + +.macro mcount_exit_regs + RESTORE_REGS_FTRACE + subi sp, 152 + ldw t1, (sp, 4) + addi sp, 152 + ldw r8, (sp, 4) + ldw lr, (sp, 8) + addi sp, 12 + jmp t1 +.endm + .macro save_return_regs subi sp, 16 stw a0, (sp, 0) @@ -85,6 +105,8 @@ ENTRY(_mcount) mov a0, lr subi a0, 4 ldw a1, (sp, 24) + lrw a2, function_trace_op + ldw a2, (a2, 0) jsr r26 @@ -122,6 +144,8 @@ ENTRY(ftrace_caller) ldw a0, (sp, 16) subi a0, 4 ldw a1, (sp, 24) + lrw a2, function_trace_op + ldw a2, (a2, 0) nop GLOBAL(ftrace_call) @@ -157,3 +181,31 @@ ENTRY(return_to_handler) jmp lr END(return_to_handler) #endif + +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS +ENTRY(ftrace_regs_caller) + mcount_enter_regs + + lrw t1, PT_FRAME_SIZE + add t1, sp + + ldw a0, (t1, 0) + subi a0, 4 + ldw a1, (t1, 8) + lrw a2, function_trace_op + ldw a2, (a2, 0) + mov a3, sp + + nop +GLOBAL(ftrace_regs_call) + nop32_stub + +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + nop +GLOBAL(ftrace_graph_regs_call) + nop32_stub +#endif + + mcount_exit_regs +ENDPROC(ftrace_regs_caller) +#endif /* CONFIG_DYNAMIC_FTRACE */ diff --git a/arch/csky/abiv2/strksyms.c b/arch/csky/abiv2/strksyms.c index 06da723d8202..8d1fd28c6cf9 100644 --- a/arch/csky/abiv2/strksyms.c +++ b/arch/csky/abiv2/strksyms.c @@ -3,10 +3,12 @@ #include <linux/module.h> +#ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_STRING_OPS EXPORT_SYMBOL(memcpy); EXPORT_SYMBOL(memset); -EXPORT_SYMBOL(memcmp); EXPORT_SYMBOL(memmove); +#endif +EXPORT_SYMBOL(memcmp); EXPORT_SYMBOL(strcmp); EXPORT_SYMBOL(strcpy); EXPORT_SYMBOL(strlen); diff --git a/arch/csky/abiv2/sysdep.h b/arch/csky/abiv2/sysdep.h index bbbedfd34777..61abe9201c50 100644 --- a/arch/csky/abiv2/sysdep.h +++ b/arch/csky/abiv2/sysdep.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __SYSDEP_H #define __SYSDEP_H diff --git a/arch/csky/boot/Makefile b/arch/csky/boot/Makefile index dbc9b1bd72f0..c3cfde28f8e6 100644 --- a/arch/csky/boot/Makefile +++ b/arch/csky/boot/Makefile @@ -1,6 +1,5 @@ # SPDX-License-Identifier: GPL-2.0-only targets := Image zImage uImage -targets += $(dtb-y) $(obj)/Image: vmlinux FORCE $(call if_changed,objcopy) diff --git a/arch/csky/configs/defconfig b/arch/csky/configs/defconfig index 7ef42895dfb0..ff559e5162aa 100644 --- a/arch/csky/configs/defconfig +++ b/arch/csky/configs/defconfig @@ -10,9 +10,6 @@ CONFIG_BSD_PROCESS_ACCT=y CONFIG_BSD_PROCESS_ACCT_V3=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y -CONFIG_DEFAULT_DEADLINE=y -CONFIG_CPU_CK807=y -CONFIG_CPU_HAS_FPU=y CONFIG_NET=y CONFIG_PACKET=y CONFIG_UNIX=y @@ -27,10 +24,7 @@ CONFIG_SERIAL_NONSTANDARD=y CONFIG_SERIAL_8250=y CONFIG_SERIAL_8250_CONSOLE=y CONFIG_SERIAL_OF_PLATFORM=y -CONFIG_TTY_PRINTK=y # CONFIG_VGA_CONSOLE is not set -CONFIG_CSKY_MPTIMER=y -CONFIG_GX6605S_TIMER=y CONFIG_PM_DEVFREQ=y CONFIG_DEVFREQ_GOV_SIMPLE_ONDEMAND=y CONFIG_DEVFREQ_GOV_PERFORMANCE=y @@ -40,7 +34,8 @@ CONFIG_GENERIC_PHY=y CONFIG_EXT4_FS=y CONFIG_FANOTIFY=y CONFIG_QUOTA=y -CONFIG_FSCACHE=m +CONFIG_NETFS_SUPPORT=m +CONFIG_FSCACHE=y CONFIG_FSCACHE_STATS=y CONFIG_CACHEFILES=m CONFIG_MSDOS_FS=y @@ -56,6 +51,4 @@ CONFIG_CRAMFS=y CONFIG_ROMFS_FS=y CONFIG_NFS_FS=y CONFIG_PRINTK_TIME=y -CONFIG_DEBUG_INFO=y -CONFIG_DEBUG_FS=y CONFIG_MAGIC_SYSRQ=y diff --git a/arch/csky/include/asm/Kbuild b/arch/csky/include/asm/Kbuild index 4d4754e6bf89..1117c28cb7e8 100644 --- a/arch/csky/include/asm/Kbuild +++ b/arch/csky/include/asm/Kbuild @@ -1,46 +1,12 @@ # SPDX-License-Identifier: GPL-2.0 generic-y += asm-offsets.h -generic-y += bugs.h -generic-y += compat.h -generic-y += current.h -generic-y += delay.h -generic-y += device.h -generic-y += div64.h -generic-y += dma.h -generic-y += dma-contiguous.h -generic-y += dma-mapping.h -generic-y += emergency-restart.h -generic-y += exec.h -generic-y += fb.h -generic-y += futex.h +generic-y += extable.h generic-y += gpio.h -generic-y += hardirq.h -generic-y += hw_irq.h -generic-y += irq.h -generic-y += irq_regs.h -generic-y += irq_work.h -generic-y += kdebug.h -generic-y += kmap_types.h -generic-y += kprobes.h generic-y += kvm_para.h -generic-y += linkage.h -generic-y += local.h -generic-y += local64.h -generic-y += mm-arch-hooks.h -generic-y += mmiowb.h -generic-y += module.h -generic-y += pci.h -generic-y += percpu.h -generic-y += preempt.h +generic-y += mcs_spinlock.h generic-y += qrwlock.h -generic-y += sections.h -generic-y += serial.h -generic-y += timex.h -generic-y += topology.h -generic-y += trace_clock.h -generic-y += unaligned.h +generic-y += qrwlock_types.h +generic-y += qspinlock.h +generic-y += parport.h generic-y += user.h -generic-y += vga.h generic-y += vmlinux.lds.h -generic-y += word-at-a-time.h -generic-y += xor.h diff --git a/arch/csky/include/asm/addrspace.h b/arch/csky/include/asm/addrspace.h index d1c2ede692ed..6fc05d44536c 100644 --- a/arch/csky/include/asm/addrspace.h +++ b/arch/csky/include/asm/addrspace.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_ADDRSPACE_H #define __ASM_CSKY_ADDRSPACE_H diff --git a/arch/csky/include/asm/asid.h b/arch/csky/include/asm/asid.h index ac08b0ffbe1f..6ff205a97a14 100644 --- a/arch/csky/include/asm/asid.h +++ b/arch/csky/include/asm/asid.h @@ -37,7 +37,7 @@ void asid_new_context(struct asid_info *info, atomic64_t *pasid, * Check the ASID is still valid for the context. If not generate a new ASID. * * @pasid: Pointer to the current ASID batch - * @cpu: current CPU ID. Must have been acquired throught get_cpu() + * @cpu: current CPU ID. Must have been acquired through get_cpu() */ static inline void asid_check_context(struct asid_info *info, atomic64_t *pasid, unsigned int cpu, diff --git a/arch/csky/include/asm/atomic.h b/arch/csky/include/asm/atomic.h index e369d73b13e3..4dab44f6143a 100644 --- a/arch/csky/include/asm/atomic.h +++ b/arch/csky/include/asm/atomic.h @@ -3,210 +3,200 @@ #ifndef __ASM_CSKY_ATOMIC_H #define __ASM_CSKY_ATOMIC_H -#include <linux/version.h> +#ifdef CONFIG_SMP +#include <asm-generic/atomic64.h> + #include <asm/cmpxchg.h> #include <asm/barrier.h> -#ifdef CONFIG_CPU_HAS_LDSTEX - -#define __atomic_add_unless __atomic_add_unless -static inline int __atomic_add_unless(atomic_t *v, int a, int u) -{ - unsigned long tmp, ret; - - smp_mb(); - - asm volatile ( - "1: ldex.w %0, (%3) \n" - " mov %1, %0 \n" - " cmpne %0, %4 \n" - " bf 2f \n" - " add %0, %2 \n" - " stex.w %0, (%3) \n" - " bez %0, 1b \n" - "2: \n" - : "=&r" (tmp), "=&r" (ret) - : "r" (a), "r"(&v->counter), "r"(u) - : "memory"); +#define __atomic_acquire_fence() __bar_brarw() - if (ret != u) - smp_mb(); +#define __atomic_release_fence() __bar_brwaw() - return ret; +static __always_inline int arch_atomic_read(const atomic_t *v) +{ + return READ_ONCE(v->counter); +} +static __always_inline void arch_atomic_set(atomic_t *v, int i) +{ + WRITE_ONCE(v->counter, i); } -#define ATOMIC_OP(op, c_op) \ -static inline void atomic_##op(int i, atomic_t *v) \ +#define ATOMIC_OP(op) \ +static __always_inline \ +void arch_atomic_##op(int i, atomic_t *v) \ { \ unsigned long tmp; \ - \ - asm volatile ( \ - "1: ldex.w %0, (%2) \n" \ - " " #op " %0, %1 \n" \ - " stex.w %0, (%2) \n" \ - " bez %0, 1b \n" \ - : "=&r" (tmp) \ - : "r" (i), "r"(&v->counter) \ - : "memory"); \ + __asm__ __volatile__ ( \ + "1: ldex.w %0, (%2) \n" \ + " " #op " %0, %1 \n" \ + " stex.w %0, (%2) \n" \ + " bez %0, 1b \n" \ + : "=&r" (tmp) \ + : "r" (i), "r" (&v->counter) \ + : "memory"); \ } -#define ATOMIC_OP_RETURN(op, c_op) \ -static inline int atomic_##op##_return(int i, atomic_t *v) \ +ATOMIC_OP(add) +ATOMIC_OP(sub) +ATOMIC_OP(and) +ATOMIC_OP( or) +ATOMIC_OP(xor) + +#undef ATOMIC_OP + +#define ATOMIC_FETCH_OP(op) \ +static __always_inline \ +int arch_atomic_fetch_##op##_relaxed(int i, atomic_t *v) \ { \ - unsigned long tmp, ret; \ - \ - smp_mb(); \ - asm volatile ( \ + register int ret, tmp; \ + __asm__ __volatile__ ( \ "1: ldex.w %0, (%3) \n" \ - " " #op " %0, %2 \n" \ " mov %1, %0 \n" \ + " " #op " %0, %2 \n" \ " stex.w %0, (%3) \n" \ " bez %0, 1b \n" \ : "=&r" (tmp), "=&r" (ret) \ - : "r" (i), "r"(&v->counter) \ + : "r" (i), "r"(&v->counter) \ : "memory"); \ - smp_mb(); \ - \ return ret; \ } -#define ATOMIC_FETCH_OP(op, c_op) \ -static inline int atomic_fetch_##op(int i, atomic_t *v) \ +#define ATOMIC_OP_RETURN(op, c_op) \ +static __always_inline \ +int arch_atomic_##op##_return_relaxed(int i, atomic_t *v) \ { \ - unsigned long tmp, ret; \ - \ - smp_mb(); \ - asm volatile ( \ - "1: ldex.w %0, (%3) \n" \ - " mov %1, %0 \n" \ - " " #op " %0, %2 \n" \ - " stex.w %0, (%3) \n" \ - " bez %0, 1b \n" \ - : "=&r" (tmp), "=&r" (ret) \ - : "r" (i), "r"(&v->counter) \ - : "memory"); \ - smp_mb(); \ - \ - return ret; \ + return arch_atomic_fetch_##op##_relaxed(i, v) c_op i; \ } -#else /* CONFIG_CPU_HAS_LDSTEX */ +#define ATOMIC_OPS(op, c_op) \ + ATOMIC_FETCH_OP(op) \ + ATOMIC_OP_RETURN(op, c_op) + +ATOMIC_OPS(add, +) +ATOMIC_OPS(sub, -) + +#define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add_relaxed +#define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub_relaxed + +#define arch_atomic_add_return_relaxed arch_atomic_add_return_relaxed +#define arch_atomic_sub_return_relaxed arch_atomic_sub_return_relaxed + +#undef ATOMIC_OPS +#undef ATOMIC_OP_RETURN + +#define ATOMIC_OPS(op) \ + ATOMIC_FETCH_OP(op) + +ATOMIC_OPS(and) +ATOMIC_OPS( or) +ATOMIC_OPS(xor) + +#define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and_relaxed +#define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or_relaxed +#define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor_relaxed -#include <linux/irqflags.h> +#undef ATOMIC_OPS -#define __atomic_add_unless __atomic_add_unless -static inline int __atomic_add_unless(atomic_t *v, int a, int u) +#undef ATOMIC_FETCH_OP + +static __always_inline int +arch_atomic_fetch_add_unless(atomic_t *v, int a, int u) { - unsigned long tmp, ret, flags; - - raw_local_irq_save(flags); - - asm volatile ( - " ldw %0, (%3) \n" - " mov %1, %0 \n" - " cmpne %0, %4 \n" - " bf 2f \n" - " add %0, %2 \n" - " stw %0, (%3) \n" - "2: \n" - : "=&r" (tmp), "=&r" (ret) - : "r" (a), "r"(&v->counter), "r"(u) + int prev, tmp; + + __asm__ __volatile__ ( + RELEASE_FENCE + "1: ldex.w %0, (%3) \n" + " cmpne %0, %4 \n" + " bf 2f \n" + " mov %1, %0 \n" + " add %1, %2 \n" + " stex.w %1, (%3) \n" + " bez %1, 1b \n" + FULL_FENCE + "2:\n" + : "=&r" (prev), "=&r" (tmp) + : "r" (a), "r" (&v->counter), "r" (u) : "memory"); - raw_local_irq_restore(flags); - - return ret; + return prev; } +#define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless -#define ATOMIC_OP(op, c_op) \ -static inline void atomic_##op(int i, atomic_t *v) \ -{ \ - unsigned long tmp, flags; \ - \ - raw_local_irq_save(flags); \ - \ - asm volatile ( \ - " ldw %0, (%2) \n" \ - " " #op " %0, %1 \n" \ - " stw %0, (%2) \n" \ - : "=&r" (tmp) \ - : "r" (i), "r"(&v->counter) \ - : "memory"); \ - \ - raw_local_irq_restore(flags); \ -} +static __always_inline bool +arch_atomic_inc_unless_negative(atomic_t *v) +{ + int rc, tmp; + + __asm__ __volatile__ ( + RELEASE_FENCE + "1: ldex.w %0, (%2) \n" + " movi %1, 0 \n" + " blz %0, 2f \n" + " movi %1, 1 \n" + " addi %0, 1 \n" + " stex.w %0, (%2) \n" + " bez %0, 1b \n" + FULL_FENCE + "2:\n" + : "=&r" (tmp), "=&r" (rc) + : "r" (&v->counter) + : "memory"); + + return tmp ? true : false; -#define ATOMIC_OP_RETURN(op, c_op) \ -static inline int atomic_##op##_return(int i, atomic_t *v) \ -{ \ - unsigned long tmp, ret, flags; \ - \ - raw_local_irq_save(flags); \ - \ - asm volatile ( \ - " ldw %0, (%3) \n" \ - " " #op " %0, %2 \n" \ - " stw %0, (%3) \n" \ - " mov %1, %0 \n" \ - : "=&r" (tmp), "=&r" (ret) \ - : "r" (i), "r"(&v->counter) \ - : "memory"); \ - \ - raw_local_irq_restore(flags); \ - \ - return ret; \ } +#define arch_atomic_inc_unless_negative arch_atomic_inc_unless_negative -#define ATOMIC_FETCH_OP(op, c_op) \ -static inline int atomic_fetch_##op(int i, atomic_t *v) \ -{ \ - unsigned long tmp, ret, flags; \ - \ - raw_local_irq_save(flags); \ - \ - asm volatile ( \ - " ldw %0, (%3) \n" \ - " mov %1, %0 \n" \ - " " #op " %0, %2 \n" \ - " stw %0, (%3) \n" \ - : "=&r" (tmp), "=&r" (ret) \ - : "r" (i), "r"(&v->counter) \ - : "memory"); \ - \ - raw_local_irq_restore(flags); \ - \ - return ret; \ +static __always_inline bool +arch_atomic_dec_unless_positive(atomic_t *v) +{ + int rc, tmp; + + __asm__ __volatile__ ( + RELEASE_FENCE + "1: ldex.w %0, (%2) \n" + " movi %1, 0 \n" + " bhz %0, 2f \n" + " movi %1, 1 \n" + " subi %0, 1 \n" + " stex.w %0, (%2) \n" + " bez %0, 1b \n" + FULL_FENCE + "2:\n" + : "=&r" (tmp), "=&r" (rc) + : "r" (&v->counter) + : "memory"); + + return tmp ? true : false; } +#define arch_atomic_dec_unless_positive arch_atomic_dec_unless_positive -#endif /* CONFIG_CPU_HAS_LDSTEX */ - -#define atomic_add_return atomic_add_return -ATOMIC_OP_RETURN(add, +) -#define atomic_sub_return atomic_sub_return -ATOMIC_OP_RETURN(sub, -) - -#define atomic_fetch_add atomic_fetch_add -ATOMIC_FETCH_OP(add, +) -#define atomic_fetch_sub atomic_fetch_sub -ATOMIC_FETCH_OP(sub, -) -#define atomic_fetch_and atomic_fetch_and -ATOMIC_FETCH_OP(and, &) -#define atomic_fetch_or atomic_fetch_or -ATOMIC_FETCH_OP(or, |) -#define atomic_fetch_xor atomic_fetch_xor -ATOMIC_FETCH_OP(xor, ^) - -#define atomic_and atomic_and -ATOMIC_OP(and, &) -#define atomic_or atomic_or -ATOMIC_OP(or, |) -#define atomic_xor atomic_xor -ATOMIC_OP(xor, ^) +static __always_inline int +arch_atomic_dec_if_positive(atomic_t *v) +{ + int dec, tmp; + + __asm__ __volatile__ ( + RELEASE_FENCE + "1: ldex.w %0, (%2) \n" + " subi %1, %0, 1 \n" + " blz %1, 2f \n" + " stex.w %1, (%2) \n" + " bez %1, 1b \n" + FULL_FENCE + "2:\n" + : "=&r" (dec), "=&r" (tmp) + : "r" (&v->counter) + : "memory"); -#undef ATOMIC_FETCH_OP -#undef ATOMIC_OP_RETURN -#undef ATOMIC_OP + return dec - 1; +} +#define arch_atomic_dec_if_positive arch_atomic_dec_if_positive +#else #include <asm-generic/atomic.h> +#endif #endif /* __ASM_CSKY_ATOMIC_H */ diff --git a/arch/csky/include/asm/barrier.h b/arch/csky/include/asm/barrier.h index a430e7fddf35..15de58b10aec 100644 --- a/arch/csky/include/asm/barrier.h +++ b/arch/csky/include/asm/barrier.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_BARRIER_H #define __ASM_CSKY_BARRIER_H @@ -8,38 +7,79 @@ #define nop() asm volatile ("nop\n":::"memory") +#ifdef CONFIG_SMP + +/* + * bar.brwarws: ordering barrier for all load/store instructions + * before/after + * + * |31|30 26|25 21|20 16|15 10|9 5|4 0| + * 1 10000 00000 00000 100001 00001 0 bw br aw ar + * + * b: before + * a: after + * r: read + * w: write + * + * Here are all combinations: + * + * bar.brw + * bar.br + * bar.bw + * bar.arw + * bar.ar + * bar.aw + * bar.brwarw + * bar.brarw + * bar.bwarw + * bar.brwar + * bar.brwaw + * bar.brar + * bar.bwaw + */ +#define FULL_FENCE ".long 0x842fc000\n" +#define ACQUIRE_FENCE ".long 0x8427c000\n" +#define RELEASE_FENCE ".long 0x842ec000\n" + +#define __bar_brw() asm volatile (".long 0x842cc000\n":::"memory") +#define __bar_br() asm volatile (".long 0x8424c000\n":::"memory") +#define __bar_bw() asm volatile (".long 0x8428c000\n":::"memory") +#define __bar_arw() asm volatile (".long 0x8423c000\n":::"memory") +#define __bar_ar() asm volatile (".long 0x8421c000\n":::"memory") +#define __bar_aw() asm volatile (".long 0x8422c000\n":::"memory") +#define __bar_brwarw() asm volatile (FULL_FENCE:::"memory") +#define __bar_brarw() asm volatile (ACQUIRE_FENCE:::"memory") +#define __bar_bwarw() asm volatile (".long 0x842bc000\n":::"memory") +#define __bar_brwar() asm volatile (".long 0x842dc000\n":::"memory") +#define __bar_brwaw() asm volatile (RELEASE_FENCE:::"memory") +#define __bar_brar() asm volatile (".long 0x8425c000\n":::"memory") +#define __bar_brar() asm volatile (".long 0x8425c000\n":::"memory") +#define __bar_bwaw() asm volatile (".long 0x842ac000\n":::"memory") + +#define __smp_mb() __bar_brwarw() +#define __smp_rmb() __bar_brar() +#define __smp_wmb() __bar_bwaw() + +#define __smp_acquire_fence() __bar_brarw() +#define __smp_release_fence() __bar_brwaw() + +#endif /* CONFIG_SMP */ + /* * sync: completion barrier, all sync.xx instructions - * guarantee the last response recieved by bus transaction + * guarantee the last response received by bus transaction * made by ld/st instructions before sync.s * sync.s: inherit from sync, but also shareable to other cores * sync.i: inherit from sync, but also flush cpu pipeline * sync.is: the same with sync.i + sync.s - * - * bar.brwarw: ordering barrier for all load/store instructions before it - * bar.brwarws: ordering barrier for all load/store instructions before it - * and shareable to other cores - * bar.brar: ordering barrier for all load instructions before it - * bar.brars: ordering barrier for all load instructions before it - * and shareable to other cores - * bar.bwaw: ordering barrier for all store instructions before it - * bar.bwaws: ordering barrier for all store instructions before it - * and shareable to other cores */ +#define mb() asm volatile ("sync\n":::"memory") #ifdef CONFIG_CPU_HAS_CACHEV2 -#define mb() asm volatile ("sync.s\n":::"memory") - -#ifdef CONFIG_SMP -#define __smp_mb() asm volatile ("bar.brwarws\n":::"memory") -#define __smp_rmb() asm volatile ("bar.brars\n":::"memory") -#define __smp_wmb() asm volatile ("bar.bwaws\n":::"memory") -#endif /* CONFIG_SMP */ - -#define sync_is() asm volatile ("sync.is\n":::"memory") - -#else /* !CONFIG_CPU_HAS_CACHEV2 */ -#define mb() asm volatile ("sync\n":::"memory") +/* + * Using three sync.is to prevent speculative PTW + */ +#define sync_is() asm volatile ("sync.is\nsync.is\nsync.is\n":::"memory") #endif #include <asm-generic/barrier.h> diff --git a/arch/csky/include/asm/bitops.h b/arch/csky/include/asm/bitops.h index 43b9838bff63..72e1b2aa29a0 100644 --- a/arch/csky/include/asm/bitops.h +++ b/arch/csky/include/asm/bitops.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_BITOPS_H #define __ASM_CSKY_BITOPS_H @@ -60,7 +59,6 @@ static __always_inline unsigned long __fls(unsigned long x) #include <asm-generic/bitops/ffz.h> #include <asm-generic/bitops/fls64.h> -#include <asm-generic/bitops/find.h> #ifndef _LINUX_BITOPS_H #error only <linux/bitops.h> can be included directly @@ -75,7 +73,6 @@ static __always_inline unsigned long __fls(unsigned long x) * bug fix, why only could use atomic!!!! */ #include <asm-generic/bitops/non-atomic.h> -#define __clear_bit(nr, vaddr) clear_bit(nr, vaddr) #include <asm-generic/bitops/le.h> #include <asm-generic/bitops/ext2-atomic.h> diff --git a/arch/csky/include/asm/bug.h b/arch/csky/include/asm/bug.h index bd7b3235bb84..03f1a5f9184a 100644 --- a/arch/csky/include/asm/bug.h +++ b/arch/csky/include/asm/bug.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_BUG_H #define __ASM_CSKY_BUG_H @@ -20,7 +19,10 @@ do { \ struct pt_regs; -void die_if_kernel(char *str, struct pt_regs *regs, int nr); +void die(struct pt_regs *regs, const char *str); +void do_trap(struct pt_regs *regs, int signo, int code, unsigned long addr); + void show_regs(struct pt_regs *regs); +void show_code(struct pt_regs *regs); #endif /* __ASM_CSKY_BUG_H */ diff --git a/arch/csky/include/asm/cache.h b/arch/csky/include/asm/cache.h index 1d5fc2f78fd7..4b5c09bf1d25 100644 --- a/arch/csky/include/asm/cache.h +++ b/arch/csky/include/asm/cache.h @@ -16,6 +16,7 @@ void dcache_wb_line(unsigned long start); void icache_inv_range(unsigned long start, unsigned long end); void icache_inv_all(void); +void local_icache_inv_all(void *priv); void dcache_wb_range(unsigned long start, unsigned long end); void dcache_wbinv_all(void); diff --git a/arch/csky/include/asm/cacheflush.h b/arch/csky/include/asm/cacheflush.h index a96da67261ae..d0f9eafe8988 100644 --- a/arch/csky/include/asm/cacheflush.h +++ b/arch/csky/include/asm/cacheflush.h @@ -1,9 +1,9 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_CACHEFLUSH_H #define __ASM_CSKY_CACHEFLUSH_H +#include <linux/mm.h> #include <abi/cacheflush.h> #endif /* __ASM_CSKY_CACHEFLUSH_H */ diff --git a/arch/csky/include/asm/cachetype.h b/arch/csky/include/asm/cachetype.h new file mode 100644 index 000000000000..98cbe3af662f --- /dev/null +++ b/arch/csky/include/asm/cachetype.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_CSKY_CACHETYPE_H +#define __ASM_CSKY_CACHETYPE_H + +#include <linux/types.h> + +#define cpu_dcache_is_aliasing() true + +#endif diff --git a/arch/csky/include/asm/checksum.h b/arch/csky/include/asm/checksum.h index 7685824291b1..aa12ef4b9080 100644 --- a/arch/csky/include/asm/checksum.h +++ b/arch/csky/include/asm/checksum.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_CHECKSUM_H #define __ASM_CSKY_CHECKSUM_H diff --git a/arch/csky/include/asm/clocksource.h b/arch/csky/include/asm/clocksource.h new file mode 100644 index 000000000000..54da0e49efa1 --- /dev/null +++ b/arch/csky/include/asm/clocksource.h @@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __ASM_VDSO_CSKY_CLOCKSOURCE_H +#define __ASM_VDSO_CSKY_CLOCKSOURCE_H + +#include <asm/vdso/clocksource.h> + +#endif diff --git a/arch/csky/include/asm/cmpxchg.h b/arch/csky/include/asm/cmpxchg.h index 89224530a0ee..916043b845f1 100644 --- a/arch/csky/include/asm/cmpxchg.h +++ b/arch/csky/include/asm/cmpxchg.h @@ -3,20 +3,38 @@ #ifndef __ASM_CSKY_CMPXCHG_H #define __ASM_CSKY_CMPXCHG_H -#ifdef CONFIG_CPU_HAS_LDSTEX +#ifdef CONFIG_SMP +#include <linux/bug.h> #include <asm/barrier.h> -extern void __bad_xchg(void); - -#define __xchg(new, ptr, size) \ +#define __xchg_relaxed(new, ptr, size) \ ({ \ __typeof__(ptr) __ptr = (ptr); \ __typeof__(new) __new = (new); \ __typeof__(*(ptr)) __ret; \ unsigned long tmp; \ switch (size) { \ + case 2: { \ + u32 ret; \ + u32 shif = ((ulong)__ptr & 2) ? 16 : 0; \ + u32 mask = 0xffff << shif; \ + __ptr = (__typeof__(ptr))((ulong)__ptr & ~2); \ + __asm__ __volatile__ ( \ + "1: ldex.w %0, (%4)\n" \ + " and %1, %0, %2\n" \ + " or %1, %1, %3\n" \ + " stex.w %1, (%4)\n" \ + " bez %1, 1b\n" \ + : "=&r" (ret), "=&r" (tmp) \ + : "r" (~mask), \ + "r" ((u32)__new << shif), \ + "r" (__ptr) \ + : "memory"); \ + __ret = (__typeof__(*(ptr))) \ + ((ret & mask) >> shif); \ + break; \ + } \ case 4: \ - smp_mb(); \ asm volatile ( \ "1: ldex.w %0, (%3) \n" \ " mov %1, %2 \n" \ @@ -25,15 +43,76 @@ extern void __bad_xchg(void); : "=&r" (__ret), "=&r" (tmp) \ : "r" (__new), "r"(__ptr) \ :); \ - smp_mb(); \ break; \ default: \ - __bad_xchg(); \ + BUILD_BUG(); \ + } \ + __ret; \ +}) + +#define arch_xchg_relaxed(ptr, x) \ + (__xchg_relaxed((x), (ptr), sizeof(*(ptr)))) + +#define __cmpxchg_relaxed(ptr, old, new, size) \ +({ \ + __typeof__(ptr) __ptr = (ptr); \ + __typeof__(new) __new = (new); \ + __typeof__(new) __tmp; \ + __typeof__(old) __old = (old); \ + __typeof__(*(ptr)) __ret; \ + switch (size) { \ + case 4: \ + asm volatile ( \ + "1: ldex.w %0, (%3) \n" \ + " cmpne %0, %4 \n" \ + " bt 2f \n" \ + " mov %1, %2 \n" \ + " stex.w %1, (%3) \n" \ + " bez %1, 1b \n" \ + "2: \n" \ + : "=&r" (__ret), "=&r" (__tmp) \ + : "r" (__new), "r"(__ptr), "r"(__old) \ + :); \ + break; \ + default: \ + BUILD_BUG(); \ + } \ + __ret; \ +}) + +#define arch_cmpxchg_relaxed(ptr, o, n) \ + (__cmpxchg_relaxed((ptr), (o), (n), sizeof(*(ptr)))) + +#define __cmpxchg_acquire(ptr, old, new, size) \ +({ \ + __typeof__(ptr) __ptr = (ptr); \ + __typeof__(new) __new = (new); \ + __typeof__(new) __tmp; \ + __typeof__(old) __old = (old); \ + __typeof__(*(ptr)) __ret; \ + switch (size) { \ + case 4: \ + asm volatile ( \ + "1: ldex.w %0, (%3) \n" \ + " cmpne %0, %4 \n" \ + " bt 2f \n" \ + " mov %1, %2 \n" \ + " stex.w %1, (%3) \n" \ + " bez %1, 1b \n" \ + ACQUIRE_FENCE \ + "2: \n" \ + : "=&r" (__ret), "=&r" (__tmp) \ + : "r" (__new), "r"(__ptr), "r"(__old) \ + :); \ + break; \ + default: \ + BUILD_BUG(); \ } \ __ret; \ }) -#define xchg(ptr, x) (__xchg((x), (ptr), sizeof(*(ptr)))) +#define arch_cmpxchg_acquire(ptr, o, n) \ + (__cmpxchg_acquire((ptr), (o), (n), sizeof(*(ptr)))) #define __cmpxchg(ptr, old, new, size) \ ({ \ @@ -44,28 +123,31 @@ extern void __bad_xchg(void); __typeof__(*(ptr)) __ret; \ switch (size) { \ case 4: \ - smp_mb(); \ asm volatile ( \ + RELEASE_FENCE \ "1: ldex.w %0, (%3) \n" \ " cmpne %0, %4 \n" \ " bt 2f \n" \ " mov %1, %2 \n" \ " stex.w %1, (%3) \n" \ " bez %1, 1b \n" \ + FULL_FENCE \ "2: \n" \ : "=&r" (__ret), "=&r" (__tmp) \ : "r" (__new), "r"(__ptr), "r"(__old) \ :); \ - smp_mb(); \ break; \ default: \ - __bad_xchg(); \ + BUILD_BUG(); \ } \ __ret; \ }) -#define cmpxchg(ptr, o, n) \ +#define arch_cmpxchg(ptr, o, n) \ (__cmpxchg((ptr), (o), (n), sizeof(*(ptr)))) + +#define arch_cmpxchg_local(ptr, o, n) \ + (__cmpxchg_relaxed((ptr), (o), (n), sizeof(*(ptr)))) #else #include <asm-generic/cmpxchg.h> #endif diff --git a/arch/csky/include/asm/elf.h b/arch/csky/include/asm/elf.h index e1ec558278bc..48b83e283ed4 100644 --- a/arch/csky/include/asm/elf.h +++ b/arch/csky/include/asm/elf.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_ELF_H #define __ASM_CSKY_ELF_H @@ -50,7 +49,6 @@ typedef elf_greg_t elf_gregset_t[ELF_NGREG]; /* * These are used to set parameters in the core dumps. */ -#define USE_ELF_CORE_DUMP #define ELF_EXEC_PAGESIZE 4096 #define ELF_CLASS ELFCLASS32 #define ELF_PLAT_INIT(_r, load_addr) { _r->a0 = 0; } diff --git a/arch/csky/include/asm/fixmap.h b/arch/csky/include/asm/fixmap.h index 380ff0a307df..49a77cbbe2a9 100644 --- a/arch/csky/include/asm/fixmap.h +++ b/arch/csky/include/asm/fixmap.h @@ -1,27 +1,33 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_FIXMAP_H #define __ASM_CSKY_FIXMAP_H #include <asm/page.h> +#include <asm/memory.h> #ifdef CONFIG_HIGHMEM #include <linux/threads.h> -#include <asm/kmap_types.h> +#include <asm/kmap_size.h> #endif enum fixed_addresses { +#ifdef CONFIG_HAVE_TCM + FIX_TCM = TCM_NR_PAGES, +#endif #ifdef CONFIG_HIGHMEM FIX_KMAP_BEGIN, - FIX_KMAP_END = FIX_KMAP_BEGIN + (KM_TYPE_NR * NR_CPUS) - 1, + FIX_KMAP_END = FIX_KMAP_BEGIN + (KM_MAX_IDX * NR_CPUS) - 1, #endif __end_of_fixed_addresses }; -#define FIXADDR_TOP 0xffffc000 #define FIXADDR_SIZE (__end_of_fixed_addresses << PAGE_SHIFT) #define FIXADDR_START (FIXADDR_TOP - FIXADDR_SIZE) #include <asm-generic/fixmap.h> +extern void fixrange_init(unsigned long start, unsigned long end, + pgd_t *pgd_base); +extern void __init fixaddr_init(void); + #endif /* __ASM_CSKY_FIXMAP_H */ diff --git a/arch/csky/include/asm/ftrace.h b/arch/csky/include/asm/ftrace.h index ba35d93ecda2..fd215c38ef27 100644 --- a/arch/csky/include/asm/ftrace.h +++ b/arch/csky/include/asm/ftrace.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_FTRACE_H #define __ASM_CSKY_FTRACE_H @@ -10,6 +9,8 @@ #define HAVE_FUNCTION_GRAPH_RET_ADDR_PTR +#define ARCH_SUPPORTS_FTRACE_OPS 1 + #define MCOUNT_ADDR ((unsigned long)_mcount) #ifndef __ASSEMBLY__ @@ -25,5 +26,9 @@ static inline unsigned long ftrace_call_adjust(unsigned long addr) struct dyn_arch_ftrace { }; + +void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr, + unsigned long frame_pointer); + #endif /* !__ASSEMBLY__ */ #endif /* __ASM_CSKY_FTRACE_H */ diff --git a/arch/csky/include/asm/futex.h b/arch/csky/include/asm/futex.h new file mode 100644 index 000000000000..6cfd312723fa --- /dev/null +++ b/arch/csky/include/asm/futex.h @@ -0,0 +1,121 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __ASM_CSKY_FUTEX_H +#define __ASM_CSKY_FUTEX_H + +#ifndef CONFIG_SMP +#include <asm-generic/futex.h> +#else +#include <linux/atomic.h> +#include <linux/futex.h> +#include <linux/uaccess.h> +#include <linux/errno.h> + +#define __futex_atomic_op(insn, ret, oldval, uaddr, oparg) \ +{ \ + u32 tmp; \ + \ + __atomic_pre_full_fence(); \ + \ + __asm__ __volatile__ ( \ + "1: ldex.w %[ov], %[u] \n" \ + " "insn" \n" \ + "2: stex.w %[t], %[u] \n" \ + " bez %[t], 1b \n" \ + " br 4f \n" \ + "3: mov %[r], %[e] \n" \ + "4: \n" \ + " .section __ex_table,\"a\" \n" \ + " .balign 4 \n" \ + " .long 1b, 3b \n" \ + " .long 2b, 3b \n" \ + " .previous \n" \ + : [r] "+r" (ret), [ov] "=&r" (oldval), \ + [u] "+m" (*uaddr), [t] "=&r" (tmp) \ + : [op] "Jr" (oparg), [e] "jr" (-EFAULT) \ + : "memory"); \ + \ + __atomic_post_full_fence(); \ +} + +static inline int +arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr) +{ + int oldval = 0, ret = 0; + + if (!access_ok(uaddr, sizeof(u32))) + return -EFAULT; + + switch (op) { + case FUTEX_OP_SET: + __futex_atomic_op("mov %[t], %[ov]", + ret, oldval, uaddr, oparg); + break; + case FUTEX_OP_ADD: + __futex_atomic_op("add %[t], %[ov], %[op]", + ret, oldval, uaddr, oparg); + break; + case FUTEX_OP_OR: + __futex_atomic_op("or %[t], %[ov], %[op]", + ret, oldval, uaddr, oparg); + break; + case FUTEX_OP_ANDN: + __futex_atomic_op("and %[t], %[ov], %[op]", + ret, oldval, uaddr, ~oparg); + break; + case FUTEX_OP_XOR: + __futex_atomic_op("xor %[t], %[ov], %[op]", + ret, oldval, uaddr, oparg); + break; + default: + ret = -ENOSYS; + } + + if (!ret) + *oval = oldval; + + return ret; +} + + + +static inline int +futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, + u32 oldval, u32 newval) +{ + int ret = 0; + u32 val, tmp; + + if (!access_ok(uaddr, sizeof(u32))) + return -EFAULT; + + __atomic_pre_full_fence(); + + __asm__ __volatile__ ( + "1: ldex.w %[v], %[u] \n" + " cmpne %[v], %[ov] \n" + " bt 4f \n" + " mov %[t], %[nv] \n" + "2: stex.w %[t], %[u] \n" + " bez %[t], 1b \n" + " br 4f \n" + "3: mov %[r], %[e] \n" + "4: \n" + " .section __ex_table,\"a\" \n" + " .balign 4 \n" + " .long 1b, 3b \n" + " .long 2b, 3b \n" + " .previous \n" + : [r] "+r" (ret), [v] "=&r" (val), [u] "+m" (*uaddr), + [t] "=&r" (tmp) + : [ov] "Jr" (oldval), [nv] "Jr" (newval), [e] "Jr" (-EFAULT) + : "memory"); + + __atomic_post_full_fence(); + + *uval = val; + return ret; +} + +#endif /* CONFIG_SMP */ +#endif /* __ASM_CSKY_FUTEX_H */ diff --git a/arch/csky/include/asm/highmem.h b/arch/csky/include/asm/highmem.h index a345a2f2c22e..1ed810effb3d 100644 --- a/arch/csky/include/asm/highmem.h +++ b/arch/csky/include/asm/highmem.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_HIGHMEM_H #define __ASM_CSKY_HIGHMEM_H @@ -9,7 +8,7 @@ #include <linux/init.h> #include <linux/interrupt.h> #include <linux/uaccess.h> -#include <asm/kmap_types.h> +#include <asm/kmap_size.h> #include <asm/cache.h> /* undef for production */ @@ -30,21 +29,15 @@ extern pte_t *pkmap_page_table; #define PKMAP_NR(virt) ((virt-PKMAP_BASE) >> PAGE_SHIFT) #define PKMAP_ADDR(nr) (PKMAP_BASE + ((nr) << PAGE_SHIFT)) -extern void *kmap_high(struct page *page); -extern void kunmap_high(struct page *page); - -extern void *kmap(struct page *page); -extern void kunmap(struct page *page); -extern void *kmap_atomic(struct page *page); -extern void __kunmap_atomic(void *kvaddr); -extern void *kmap_atomic_pfn(unsigned long pfn); -extern struct page *kmap_atomic_to_page(void *ptr); +#define ARCH_HAS_KMAP_FLUSH_TLB +extern void kmap_flush_tlb(unsigned long addr); #define flush_cache_kmaps() do {} while (0) -extern void kmap_init(void); +#define arch_kmap_local_post_map(vaddr, pteval) kmap_flush_tlb(vaddr) +#define arch_kmap_local_post_unmap(vaddr) kmap_flush_tlb(vaddr) -#define kmap_prot PAGE_KERNEL +extern void kmap_init(void); #endif /* __KERNEL__ */ diff --git a/arch/csky/include/asm/io.h b/arch/csky/include/asm/io.h index 332f51bc68fb..4725bb977b0f 100644 --- a/arch/csky/include/asm/io.h +++ b/arch/csky/include/asm/io.h @@ -1,12 +1,10 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_IO_H #define __ASM_CSKY_IO_H -#include <asm/pgtable.h> +#include <linux/pgtable.h> #include <linux/types.h> -#include <linux/version.h> /* * I/O memory access primitives. Reads are ordered relative to any @@ -34,6 +32,17 @@ #endif /* + * String version of I/O memory access operations. + */ +extern void __memcpy_fromio(void *, const volatile void __iomem *, size_t); +extern void __memcpy_toio(volatile void __iomem *, const void *, size_t); +extern void __memset_io(volatile void __iomem *, int, size_t); + +#define memset_io(c,v,l) __memset_io((c),(v),(l)) +#define memcpy_fromio(a,c,l) __memcpy_fromio((a),(c),(l)) +#define memcpy_toio(c,a,l) __memcpy_toio((c),(a),(l)) + +/* * I/O memory mapping functions. */ #define ioremap_wc(addr, size) \ diff --git a/arch/csky/include/asm/irq_work.h b/arch/csky/include/asm/irq_work.h new file mode 100644 index 000000000000..d39fcc1f5395 --- /dev/null +++ b/arch/csky/include/asm/irq_work.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __ASM_CSKY_IRQ_WORK_H +#define __ASM_CSKY_IRQ_WORK_H + +static inline bool arch_irq_work_has_interrupt(void) +{ + return true; +} + +#endif /* __ASM_CSKY_IRQ_WORK_H */ diff --git a/arch/csky/include/asm/jump_label.h b/arch/csky/include/asm/jump_label.h new file mode 100644 index 000000000000..ef2e37a10a0f --- /dev/null +++ b/arch/csky/include/asm/jump_label.h @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef __ASM_CSKY_JUMP_LABEL_H +#define __ASM_CSKY_JUMP_LABEL_H + +#ifndef __ASSEMBLY__ + +#include <linux/types.h> + +#define JUMP_LABEL_NOP_SIZE 4 + +static __always_inline bool arch_static_branch(struct static_key *key, + bool branch) +{ + asm goto( + "1: nop32 \n" + " .pushsection __jump_table, \"aw\" \n" + " .align 2 \n" + " .long 1b - ., %l[label] - . \n" + " .long %0 - . \n" + " .popsection \n" + : : "i"(&((char *)key)[branch]) : : label); + + return false; +label: + return true; +} + +static __always_inline bool arch_static_branch_jump(struct static_key *key, + bool branch) +{ + asm goto( + "1: bsr32 %l[label] \n" + " .pushsection __jump_table, \"aw\" \n" + " .align 2 \n" + " .long 1b - ., %l[label] - . \n" + " .long %0 - . \n" + " .popsection \n" + : : "i"(&((char *)key)[branch]) : : label); + + return false; +label: + return true; +} + +enum jump_label_type; +void arch_jump_label_transform_static(struct jump_entry *entry, + enum jump_label_type type); +#define arch_jump_label_transform_static arch_jump_label_transform_static + +#endif /* __ASSEMBLY__ */ +#endif /* __ASM_CSKY_JUMP_LABEL_H */ diff --git a/arch/csky/include/asm/kprobes.h b/arch/csky/include/asm/kprobes.h new file mode 100644 index 000000000000..55267cbf5204 --- /dev/null +++ b/arch/csky/include/asm/kprobes.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef __ASM_CSKY_KPROBES_H +#define __ASM_CSKY_KPROBES_H + +#include <asm-generic/kprobes.h> + +#ifdef CONFIG_KPROBES +#include <linux/types.h> +#include <linux/ptrace.h> +#include <linux/percpu.h> + +#define __ARCH_WANT_KPROBES_INSN_SLOT +#define MAX_INSN_SIZE 1 + +#define flush_insn_slot(p) do { } while (0) +#define kretprobe_blacklist_size 0 + +#include <asm/probes.h> + +struct prev_kprobe { + struct kprobe *kp; + unsigned int status; +}; + +/* Single step context for kprobe */ +struct kprobe_step_ctx { + unsigned long ss_pending; + unsigned long match_addr; +}; + +/* per-cpu kprobe control block */ +struct kprobe_ctlblk { + unsigned int kprobe_status; + unsigned long saved_sr; + struct prev_kprobe prev_kprobe; + struct kprobe_step_ctx ss_ctx; +}; + +void arch_remove_kprobe(struct kprobe *p); +int kprobe_fault_handler(struct pt_regs *regs, unsigned int trapnr); +int kprobe_breakpoint_handler(struct pt_regs *regs); +int kprobe_single_step_handler(struct pt_regs *regs); +void __kretprobe_trampoline(void); +void __kprobes *trampoline_probe_handler(struct pt_regs *regs); + +#endif /* CONFIG_KPROBES */ +#endif /* __ASM_CSKY_KPROBES_H */ diff --git a/arch/csky/include/asm/memory.h b/arch/csky/include/asm/memory.h new file mode 100644 index 000000000000..d12179801ae3 --- /dev/null +++ b/arch/csky/include/asm/memory.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __ASM_CSKY_MEMORY_H +#define __ASM_CSKY_MEMORY_H + +#include <linux/compiler.h> +#include <linux/const.h> +#include <linux/types.h> +#include <linux/sizes.h> + +#define FIXADDR_TOP _AC(0xffffc000, UL) +#define PKMAP_BASE _AC(0xff800000, UL) +#define VMALLOC_START (PAGE_OFFSET + LOWMEM_LIMIT + (PAGE_SIZE * 8)) +#define VMALLOC_END (PKMAP_BASE - (PAGE_SIZE * 2)) + +#ifdef CONFIG_HAVE_TCM +#ifdef CONFIG_HAVE_DTCM +#define TCM_NR_PAGES (CONFIG_ITCM_NR_PAGES + CONFIG_DTCM_NR_PAGES) +#else +#define TCM_NR_PAGES (CONFIG_ITCM_NR_PAGES) +#endif +#define FIXADDR_TCM _AC(FIXADDR_TOP - (TCM_NR_PAGES * PAGE_SIZE), UL) +#endif + +#endif diff --git a/arch/csky/include/asm/mmu.h b/arch/csky/include/asm/mmu.h index b382a14ea4ec..d78321901d06 100644 --- a/arch/csky/include/asm/mmu.h +++ b/arch/csky/include/asm/mmu.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_MMU_H #define __ASM_CSKY_MMU_H @@ -7,6 +6,7 @@ typedef struct { atomic64_t asid; void *vdso; + cpumask_t icache_stale_mask; } mm_context_t; #endif /* __ASM_CSKY_MMU_H */ diff --git a/arch/csky/include/asm/mmu_context.h b/arch/csky/include/asm/mmu_context.h index 0285b0ad18b6..95d99b30792c 100644 --- a/arch/csky/include/asm/mmu_context.h +++ b/arch/csky/include/asm/mmu_context.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_MMU_CONTEXT_H #define __ASM_CSKY_MMU_CONTEXT_H @@ -14,21 +13,10 @@ #include <linux/sched.h> #include <abi/ckmmu.h> -#define TLBMISS_HANDLER_SETUP_PGD(pgd) \ - setup_pgd(__pa(pgd), false) - -#define TLBMISS_HANDLER_SETUP_PGD_KERNEL(pgd) \ - setup_pgd(__pa(pgd), true) - #define ASID_MASK ((1 << CONFIG_CPU_ASID_BITS) - 1) #define cpu_asid(mm) (atomic64_read(&mm->context.asid) & ASID_MASK) #define init_new_context(tsk,mm) ({ atomic64_set(&(mm)->context.asid, 0); 0; }) -#define activate_mm(prev,next) switch_mm(prev, next, current) - -#define destroy_context(mm) do {} while (0) -#define enter_lazy_tlb(mm, tsk) do {} while (0) -#define deactivate_mm(tsk, mm) do {} while (0) void check_and_switch_context(struct mm_struct *mm, unsigned int cpu); @@ -41,7 +29,11 @@ switch_mm(struct mm_struct *prev, struct mm_struct *next, if (prev != next) check_and_switch_context(next, cpu); - TLBMISS_HANDLER_SETUP_PGD(next->pgd); - write_mmu_entryhi(next->context.asid.counter); + setup_pgd(next->pgd, next->context.asid.counter); + + flush_icache_deferred(next); } + +#include <asm-generic/mmu_context.h> + #endif /* __ASM_CSKY_MMU_CONTEXT_H */ diff --git a/arch/csky/include/asm/page.h b/arch/csky/include/asm/page.h index 9738eacefdc7..0ca6c408c07f 100644 --- a/arch/csky/include/asm/page.h +++ b/arch/csky/include/asm/page.h @@ -10,7 +10,7 @@ /* * PAGE_SHIFT determines the page size: 4KB */ -#define PAGE_SHIFT 12 +#define PAGE_SHIFT CONFIG_PAGE_SHIFT #define PAGE_SIZE (_AC(1, UL) << PAGE_SHIFT) #define PAGE_MASK (~(PAGE_SIZE - 1)) #define THREAD_SIZE (PAGE_SIZE * 2) @@ -24,22 +24,18 @@ * address region. We use them mapping kernel 1GB direct-map address area and * for more than 1GB of memory we use highmem. */ -#define PAGE_OFFSET 0x80000000 +#define PAGE_OFFSET CONFIG_PAGE_OFFSET #define SSEG_SIZE 0x20000000 #define LOWMEM_LIMIT (SSEG_SIZE * 2) -#define PHYS_OFFSET_OFFSET (CONFIG_RAM_BASE & (SSEG_SIZE - 1)) +#define PHYS_OFFSET_OFFSET (CONFIG_DRAM_BASE & (SSEG_SIZE - 1)) #ifndef __ASSEMBLY__ #include <linux/pfn.h> -#define virt_to_pfn(kaddr) (__pa(kaddr) >> PAGE_SHIFT) -#define pfn_to_virt(pfn) __va((pfn) << PAGE_SHIFT) - #define virt_addr_valid(kaddr) ((void *)(kaddr) >= (void *)PAGE_OFFSET && \ (void *)(kaddr) < high_memory) -#define pfn_valid(pfn) ((pfn) >= ARCH_PFN_OFFSET && ((pfn) - ARCH_PFN_OFFSET) < max_mapnr) extern void *memset(void *dest, int c, size_t l); extern void *memcpy(void *to, const void *from, size_t l); @@ -81,13 +77,15 @@ extern unsigned long va_pa_offset; #define __pa_symbol(x) __pa(RELOC_HIDE((unsigned long)(x), 0)) +static inline unsigned long virt_to_pfn(const void *kaddr) +{ + return __pa(kaddr) >> PAGE_SHIFT; +} + #define MAP_NR(x) PFN_DOWN((unsigned long)(x) - PAGE_OFFSET - \ PHYS_OFFSET_OFFSET) #define virt_to_page(x) (mem_map + MAP_NR(x)) -#define VM_DATA_DEFAULT_FLAGS (VM_READ | VM_WRITE | VM_EXEC | \ - VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC) - #define pfn_to_kaddr(x) __va(PFN_PHYS(x)) #include <asm-generic/memory_model.h> diff --git a/arch/csky/include/asm/pci.h b/arch/csky/include/asm/pci.h new file mode 100644 index 000000000000..42724c630d30 --- /dev/null +++ b/arch/csky/include/asm/pci.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef __ASM_CSKY_PCI_H +#define __ASM_CSKY_PCI_H + +#include <linux/types.h> +#include <linux/slab.h> +#include <linux/dma-mapping.h> + +#include <asm/io.h> + +/* Generic PCI */ +#include <asm-generic/pci.h> + +#endif /* __ASM_CSKY_PCI_H */ diff --git a/arch/csky/include/asm/perf_event.h b/arch/csky/include/asm/perf_event.h index 572093e11001..249905d8a4e8 100644 --- a/arch/csky/include/asm/perf_event.h +++ b/arch/csky/include/asm/perf_event.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_PERF_EVENT_H #define __ASM_CSKY_PERF_EVENT_H diff --git a/arch/csky/include/asm/pgalloc.h b/arch/csky/include/asm/pgalloc.h index c7c1ed27e348..9c84c9012e53 100644 --- a/arch/csky/include/asm/pgalloc.h +++ b/arch/csky/include/asm/pgalloc.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_PGALLOC_H #define __ASM_CSKY_PGALLOC_H @@ -9,7 +8,7 @@ #include <linux/sched.h> #define __HAVE_ARCH_PTE_ALLOC_ONE_KERNEL -#include <asm-generic/pgalloc.h> /* for pte_{alloc,free}_one */ +#include <asm-generic/pgalloc.h> static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd, pte_t *pte) @@ -23,8 +22,6 @@ static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, set_pmd(pmd, __pmd(__pa(page_address(pte)))); } -#define pmd_pgtable(pmd) pmd_page(pmd) - extern void pgd_init(unsigned long *p); static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm) @@ -42,17 +39,12 @@ static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm) return pte; } -static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd) -{ - free_pages((unsigned long)pgd, PGD_ORDER); -} - static inline pgd_t *pgd_alloc(struct mm_struct *mm) { pgd_t *ret; pgd_t *init; - ret = (pgd_t *) __get_free_pages(GFP_KERNEL, PGD_ORDER); + ret = (pgd_t *) __get_free_page(GFP_KERNEL); if (ret) { init = pgd_offset(&init_mm, 0UL); pgd_init((unsigned long *)ret); @@ -71,12 +63,12 @@ static inline pgd_t *pgd_alloc(struct mm_struct *mm) #define __pte_free_tlb(tlb, pte, address) \ do { \ - pgtable_pte_page_dtor(pte); \ - tlb_remove_page(tlb, pte); \ + pagetable_pte_dtor(page_ptdesc(pte)); \ + tlb_remove_page_ptdesc(tlb, page_ptdesc(pte)); \ } while (0) extern void pagetable_init(void); -extern void pre_mmu_init(void); +extern void mmu_init(unsigned long min_pfn, unsigned long max_pfn); extern void pre_trap_init(void); #endif /* __ASM_CSKY_PGALLOC_H */ diff --git a/arch/csky/include/asm/pgtable.h b/arch/csky/include/asm/pgtable.h index 4b2a41e15f2e..a397e1718ab6 100644 --- a/arch/csky/include/asm/pgtable.h +++ b/arch/csky/include/asm/pgtable.h @@ -1,10 +1,10 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_PGTABLE_H #define __ASM_CSKY_PGTABLE_H #include <asm/fixmap.h> +#include <asm/memory.h> #include <asm/addrspace.h> #include <abi/pgtable-bits.h> #include <asm-generic/pgtable-nopmd.h> @@ -13,57 +13,32 @@ #define PGDIR_SIZE (1UL << PGDIR_SHIFT) #define PGDIR_MASK (~(PGDIR_SIZE-1)) -#define USER_PTRS_PER_PGD (0x80000000UL/PGDIR_SIZE) -#define FIRST_USER_ADDRESS 0UL - -#define PKMAP_BASE (0xff800000) - -#define VMALLOC_START (0xc0008000) -#define VMALLOC_END (PKMAP_BASE - 2*PAGE_SIZE) +#define USER_PTRS_PER_PGD (PAGE_OFFSET/PGDIR_SIZE) /* * C-SKY is two-level paging structure: */ -#define PGD_ORDER 0 -#define PTE_ORDER 0 -#define PTRS_PER_PGD ((PAGE_SIZE << PGD_ORDER) / sizeof(pgd_t)) +#define PTRS_PER_PGD (PAGE_SIZE / sizeof(pgd_t)) #define PTRS_PER_PMD 1 -#define PTRS_PER_PTE ((PAGE_SIZE << PTE_ORDER) / sizeof(pte_t)) +#define PTRS_PER_PTE (PAGE_SIZE / sizeof(pte_t)) #define pte_ERROR(e) \ pr_err("%s:%d: bad pte %08lx.\n", __FILE__, __LINE__, (e).pte_low) #define pgd_ERROR(e) \ pr_err("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e)) -/* Find an entry in the third-level page table.. */ -#define __pte_offset_t(address) \ - (((address) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) -#define pte_offset_kernel(dir, address) \ - (pmd_page_vaddr(*(dir)) + __pte_offset_t(address)) -#define pte_offset_map(dir, address) \ - ((pte_t *)page_address(pmd_page(*(dir))) + __pte_offset_t(address)) +#define PFN_PTE_SHIFT PAGE_SHIFT +#define pmd_pfn(pmd) (pmd_phys(pmd) >> PAGE_SHIFT) #define pmd_page(pmd) (pfn_to_page(pmd_phys(pmd) >> PAGE_SHIFT)) #define pte_clear(mm, addr, ptep) set_pte((ptep), \ - (((unsigned int) addr & PAGE_OFFSET) ? __pte(_PAGE_GLOBAL) : __pte(0))) + (((unsigned int) addr >= PAGE_OFFSET) ? __pte(_PAGE_GLOBAL) : __pte(0))) #define pte_none(pte) (!(pte_val(pte) & ~_PAGE_GLOBAL)) #define pte_present(pte) (pte_val(pte) & _PAGE_PRESENT) #define pte_pfn(x) ((unsigned long)((x).pte_low >> PAGE_SHIFT)) #define pfn_pte(pfn, prot) __pte(((unsigned long long)(pfn) << PAGE_SHIFT) \ | pgprot_val(prot)) -#define __READABLE (_PAGE_READ | _PAGE_VALID | _PAGE_ACCESSED) -#define __WRITEABLE (_PAGE_WRITE | _PAGE_DIRTY | _PAGE_MODIFIED) - -#define _PAGE_CHG_MASK (PAGE_MASK | _PAGE_ACCESSED | _PAGE_MODIFIED | \ - _CACHE_MASK) - -#define pte_unmap(pte) ((void)(pte)) - -#define __swp_type(x) (((x).val >> 4) & 0xff) -#define __swp_offset(x) ((x).val >> 12) -#define __swp_entry(type, offset) ((swp_entry_t) {((type) << 4) | \ - ((offset) << 12) }) #define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) }) #define __swp_entry_to_pte(x) ((pte_t) { (x).val }) @@ -72,41 +47,34 @@ pgprot_val(pgprot)) /* - * CSKY can't do page protection for execute, and considers that the same like - * read. Also, write permissions imply read permissions. This is the closest - * we can get by reasonable means.. + * C-SKY only has VALID and DIRTY bit in hardware. So we need to use the + * two bits emulate PRESENT, READ, WRITE, EXEC, MODIFIED, ACCESSED. */ -#define PAGE_NONE __pgprot(_PAGE_PRESENT | _CACHE_CACHED) -#define PAGE_SHARED __pgprot(_PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE | \ +#define _PAGE_BASE (_PAGE_PRESENT | _PAGE_ACCESSED) + +#define PAGE_NONE __pgprot(_PAGE_PROT_NONE) +#define PAGE_READ __pgprot(_PAGE_BASE | _PAGE_READ | \ + _CACHE_CACHED) +#define PAGE_WRITE __pgprot(_PAGE_BASE | _PAGE_READ | _PAGE_WRITE | \ _CACHE_CACHED) -#define PAGE_COPY __pgprot(_PAGE_PRESENT | _PAGE_READ | _CACHE_CACHED) -#define PAGE_READONLY __pgprot(_PAGE_PRESENT | _PAGE_READ | _CACHE_CACHED) -#define PAGE_KERNEL __pgprot(_PAGE_PRESENT | __READABLE | __WRITEABLE | \ - _PAGE_GLOBAL | _CACHE_CACHED) -#define PAGE_USERIO __pgprot(_PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE | \ +#define PAGE_SHARED PAGE_WRITE + +#define PAGE_KERNEL __pgprot(_PAGE_BASE | _PAGE_READ | _PAGE_VALID | \ + _PAGE_WRITE | _PAGE_DIRTY | _PAGE_MODIFIED | \ + _PAGE_GLOBAL | \ _CACHE_CACHED) -#define _PAGE_IOREMAP \ - (_PAGE_PRESENT | __READABLE | __WRITEABLE | _PAGE_GLOBAL | \ - _CACHE_UNCACHED | _PAGE_SO) - -#define __P000 PAGE_NONE -#define __P001 PAGE_READONLY -#define __P010 PAGE_COPY -#define __P011 PAGE_COPY -#define __P100 PAGE_READONLY -#define __P101 PAGE_READONLY -#define __P110 PAGE_COPY -#define __P111 PAGE_COPY - -#define __S000 PAGE_NONE -#define __S001 PAGE_READONLY -#define __S010 PAGE_SHARED -#define __S011 PAGE_SHARED -#define __S100 PAGE_READONLY -#define __S101 PAGE_READONLY -#define __S110 PAGE_SHARED -#define __S111 PAGE_SHARED +#define _PAGE_IOREMAP (_PAGE_BASE | _PAGE_READ | _PAGE_VALID | \ + _PAGE_WRITE | _PAGE_DIRTY | _PAGE_MODIFIED | \ + _PAGE_GLOBAL | \ + _CACHE_UNCACHED | _PAGE_SO) + +#define _PAGE_CHG_MASK (~(unsigned long) \ + (_PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE | \ + _CACHE_MASK | _PAGE_GLOBAL)) + +#define MAX_SWAPFILES_CHECK() \ + BUILD_BUG_ON(MAX_SWAPFILES_SHIFT != 5) extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]; #define ZERO_PAGE(vaddr) (virt_to_page(empty_zero_page)) @@ -114,9 +82,6 @@ extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]; extern void load_pgd(unsigned long pg_dir); extern pte_t invalid_pte_table[PTRS_PER_PTE]; -static inline int pte_special(pte_t pte) { return 0; } -static inline pte_t pte_mkspecial(pte_t pte) { return pte; } - static inline void set_pte(pte_t *p, pte_t pte) { *p = pte; @@ -126,7 +91,6 @@ static inline void set_pte(pte_t *p, pte_t pte) /* prevent out of order excution */ smp_mb(); } -#define set_pte_at(mm, addr, ptep, pteval) set_pte(ptep, pteval) static inline pte_t *pmd_page_vaddr(pmd_t pmd) { @@ -212,7 +176,7 @@ static inline pte_t pte_mkold(pte_t pte) return pte; } -static inline pte_t pte_mkwrite(pte_t pte) +static inline pte_t pte_mkwrite_novma(pte_t pte) { pte_val(pte) |= _PAGE_WRITE; if (pte_val(pte) & _PAGE_MODIFIED) @@ -236,14 +200,22 @@ static inline pte_t pte_mkyoung(pte_t pte) return pte; } -#define __pgd_offset(address) pgd_index(address) -#define __pud_offset(address) (((address) >> PUD_SHIFT) & (PTRS_PER_PUD-1)) -#define __pmd_offset(address) (((address) >> PMD_SHIFT) & (PTRS_PER_PMD-1)) +static inline int pte_swp_exclusive(pte_t pte) +{ + return pte_val(pte) & _PAGE_SWP_EXCLUSIVE; +} -/* to find an entry in a kernel page-table-directory */ -#define pgd_offset_k(address) pgd_offset(&init_mm, address) +static inline pte_t pte_swp_mkexclusive(pte_t pte) +{ + pte_val(pte) |= _PAGE_SWP_EXCLUSIVE; + return pte; +} -#define pgd_index(address) ((address) >> PGDIR_SHIFT) +static inline pte_t pte_swp_clear_exclusive(pte_t pte) +{ + pte_val(pte) &= ~_PAGE_SWP_EXCLUSIVE; + return pte; +} #define __HAVE_PHYS_MEM_ACCESS_PROT struct file; @@ -288,31 +260,15 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) (pgprot_val(newprot))); } -/* to find an entry in a page-table-directory */ -static inline pgd_t *pgd_offset(struct mm_struct *mm, unsigned long address) -{ - return mm->pgd + pgd_index(address); -} - -/* Find an entry in the third-level page table.. */ -static inline pte_t *pte_offset(pmd_t *dir, unsigned long address) -{ - return (pte_t *) (pmd_page_vaddr(*dir)) + - ((address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)); -} - extern pgd_t swapper_pg_dir[PTRS_PER_PGD]; extern void paging_init(void); -void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, - pte_t *pte); - -/* Needs to be defined here and not in linux/mm.h, as it is arch dependent */ -#define kern_addr_valid(addr) (1) +void update_mmu_cache_range(struct vm_fault *vmf, struct vm_area_struct *vma, + unsigned long address, pte_t *pte, unsigned int nr); +#define update_mmu_cache(vma, addr, ptep) \ + update_mmu_cache_range(NULL, vma, addr, ptep, 1) #define io_remap_pfn_range(vma, vaddr, pfn, size, prot) \ remap_pfn_range(vma, vaddr, pfn, size, prot) -#include <asm-generic/pgtable.h> - #endif /* __ASM_CSKY_PGTABLE_H */ diff --git a/arch/csky/include/asm/probes.h b/arch/csky/include/asm/probes.h new file mode 100644 index 000000000000..5e526334e6f9 --- /dev/null +++ b/arch/csky/include/asm/probes.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __ASM_CSKY_PROBES_H +#define __ASM_CSKY_PROBES_H + +typedef u32 probe_opcode_t; +typedef void (probes_handler_t) (u32 opcode, long addr, struct pt_regs *); + +/* architecture specific copy of original instruction */ +struct arch_probe_insn { + probe_opcode_t *insn; + probes_handler_t *handler; + /* restore address after simulation */ + unsigned long restore; +}; + +#ifdef CONFIG_KPROBES +typedef u32 kprobe_opcode_t; +struct arch_specific_insn { + struct arch_probe_insn api; +}; +#endif + +#endif /* __ASM_CSKY_PROBES_H */ diff --git a/arch/csky/include/asm/processor.h b/arch/csky/include/asm/processor.h index 21e0bd5293dd..e487a46d1c37 100644 --- a/arch/csky/include/asm/processor.h +++ b/arch/csky/include/asm/processor.h @@ -1,14 +1,12 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_PROCESSOR_H #define __ASM_CSKY_PROCESSOR_H #include <linux/bitops.h> -#include <asm/segment.h> +#include <linux/cache.h> #include <asm/ptrace.h> #include <asm/current.h> -#include <asm/cache.h> #include <abi/reg_ops.h> #include <abi/regdef.h> #include <abi/switch_context.h> @@ -28,7 +26,7 @@ extern struct cpuinfo_csky cpu_data[]; * for a 64 bit kernel expandable to 8192EB, of which the current CSKY * implementations will "only" be able to use 1TB ... */ -#define TASK_SIZE 0x7fff8000UL +#define TASK_SIZE (PAGE_OFFSET - (PAGE_SIZE * 8)) #ifdef __KERNEL__ #define STACK_TOP TASK_SIZE @@ -41,16 +39,15 @@ extern struct cpuinfo_csky cpu_data[]; #define TASK_UNMAPPED_BASE (TASK_SIZE / 3) struct thread_struct { - unsigned long ksp; /* kernel stack pointer */ - unsigned long sr; /* saved status register */ + unsigned long sp; /* kernel stack pointer */ + unsigned long trap_no; /* saved status register */ /* FPU regs */ struct user_fp __aligned(16) user_fp; }; #define INIT_THREAD { \ - .ksp = sizeof(init_stack) + (unsigned long) &init_stack, \ - .sr = DEFAULT_PSR_VALUE, \ + .sp = sizeof(init_stack) + (unsigned long) &init_stack, \ } /* @@ -61,7 +58,6 @@ struct thread_struct { */ #define start_thread(_regs, _pc, _usp) \ do { \ - set_fs(USER_DS); /* reads from user space */ \ (_regs)->pc = (_pc); \ (_regs)->regs[1] = 0; /* ABIV1 is R7, uClibc_main rtdl arg */ \ (_regs)->regs[2] = 0; \ @@ -73,23 +69,10 @@ do { \ /* Forward declaration, a strange C thing */ struct task_struct; -/* Free all resources held by a thread. */ -static inline void release_thread(struct task_struct *dead_task) -{ -} - /* Prepare to copy thread state - unlazy all lazy status */ #define prepare_to_copy(tsk) do { } while (0) -extern int kernel_thread(int (*fn)(void *), void *arg, unsigned long flags); - -#define copy_segments(tsk, mm) do { } while (0) -#define release_segments(mm) do { } while (0) -#define forget_segments() do { } while (0) - -extern unsigned long thread_saved_pc(struct task_struct *tsk); - -unsigned long get_wchan(struct task_struct *p); +unsigned long __get_wchan(struct task_struct *p); #define KSTK_EIP(tsk) (task_pt_regs(tsk)->pc) #define KSTK_ESP(tsk) (task_pt_regs(tsk)->usp) @@ -99,4 +82,6 @@ unsigned long get_wchan(struct task_struct *p); #define cpu_relax() barrier() +register unsigned long current_stack_pointer __asm__("sp"); + #endif /* __ASM_CSKY_PROCESSOR_H */ diff --git a/arch/csky/include/asm/ptrace.h b/arch/csky/include/asm/ptrace.h index d0aba7b32417..0634b7895d81 100644 --- a/arch/csky/include/asm/ptrace.h +++ b/arch/csky/include/asm/ptrace.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_PTRACE_H #define __ASM_CSKY_PTRACE_H @@ -7,11 +6,14 @@ #include <uapi/asm/ptrace.h> #include <asm/traps.h> #include <linux/types.h> +#include <linux/compiler.h> #ifndef __ASSEMBLY__ #define PS_S 0x80000000 /* Supervisor Mode */ +#define USR_BKPT 0x1464 + #define arch_has_single_step() (1) #define current_pt_regs() \ ({ (struct pt_regs *)((char *)current_thread_info() + THREAD_SIZE) - 1; }) @@ -21,6 +23,19 @@ #define user_mode(regs) (!((regs)->sr & PS_S)) #define instruction_pointer(regs) ((regs)->pc) #define profile_pc(regs) instruction_pointer(regs) +#define trap_no(regs) ((regs->sr >> 16) & 0xff) + +static inline void instruction_pointer_set(struct pt_regs *regs, + unsigned long val) +{ + regs->pc = val; +} + +#if defined(__CSKYABIV2__) +#define MAX_REG_OFFSET offsetof(struct pt_regs, dcsr) +#else +#define MAX_REG_OFFSET offsetof(struct pt_regs, regs[9]) +#endif static inline bool in_syscall(struct pt_regs const *regs) { @@ -37,5 +52,51 @@ static inline unsigned long regs_return_value(struct pt_regs *regs) return regs->a0; } +static inline void regs_set_return_value(struct pt_regs *regs, + unsigned long val) +{ + regs->a0 = val; +} + +/* Valid only for Kernel mode traps. */ +static inline unsigned long kernel_stack_pointer(struct pt_regs *regs) +{ + return regs->usp; +} + +static inline unsigned long frame_pointer(struct pt_regs *regs) +{ + return regs->regs[4]; +} +static inline void frame_pointer_set(struct pt_regs *regs, + unsigned long val) +{ + regs->regs[4] = val; +} + +extern int regs_query_register_offset(const char *name); +extern unsigned long regs_get_kernel_stack_nth(struct pt_regs *regs, + unsigned int n); + +/* + * regs_get_register() - get register value from its offset + * @regs: pt_regs from which register value is gotten + * @offset: offset of the register. + * + * regs_get_register returns the value of a register whose offset from @regs. + * The @offset is the offset of the register in struct pt_regs. + * If @offset is bigger than MAX_REG_OFFSET, this returns 0. + */ +static inline unsigned long regs_get_register(struct pt_regs *regs, + unsigned int offset) +{ + if (unlikely(offset > MAX_REG_OFFSET)) + return 0; + + return *(unsigned long *)((unsigned long)regs + offset); +} + +asmlinkage int syscall_trace_enter(struct pt_regs *regs); +asmlinkage void syscall_trace_exit(struct pt_regs *regs); #endif /* __ASSEMBLY__ */ #endif /* __ASM_CSKY_PTRACE_H */ diff --git a/arch/csky/include/asm/seccomp.h b/arch/csky/include/asm/seccomp.h new file mode 100644 index 000000000000..d33e758126fb --- /dev/null +++ b/arch/csky/include/asm/seccomp.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _ASM_SECCOMP_H +#define _ASM_SECCOMP_H + +#include <asm-generic/seccomp.h> + +#define SECCOMP_ARCH_NATIVE AUDIT_ARCH_CSKY +#define SECCOMP_ARCH_NATIVE_NR NR_syscalls +#define SECCOMP_ARCH_NATIVE_NAME "csky" + +#endif /* _ASM_SECCOMP_H */ diff --git a/arch/csky/include/asm/sections.h b/arch/csky/include/asm/sections.h new file mode 100644 index 000000000000..83e82b7c0f6c --- /dev/null +++ b/arch/csky/include/asm/sections.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __ASM_SECTIONS_H +#define __ASM_SECTIONS_H + +#include <asm-generic/sections.h> + +extern char _start[]; + +asmlinkage void csky_start(unsigned int unused, void *dtb_start); + +#endif /* __ASM_SECTIONS_H */ diff --git a/arch/csky/include/asm/segment.h b/arch/csky/include/asm/segment.h deleted file mode 100644 index db2640d5f575..000000000000 --- a/arch/csky/include/asm/segment.h +++ /dev/null @@ -1,18 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. - -#ifndef __ASM_CSKY_SEGMENT_H -#define __ASM_CSKY_SEGMENT_H - -typedef struct { - unsigned long seg; -} mm_segment_t; - -#define KERNEL_DS ((mm_segment_t) { 0xFFFFFFFF }) - -#define USER_DS ((mm_segment_t) { 0x80000000UL }) -#define get_fs() (current_thread_info()->addr_limit) -#define set_fs(x) (current_thread_info()->addr_limit = (x)) -#define segment_eq(a, b) ((a).seg == (b).seg) - -#endif /* __ASM_CSKY_SEGMENT_H */ diff --git a/arch/csky/include/asm/shmparam.h b/arch/csky/include/asm/shmparam.h index efafe4c79fed..2fe6cea0dae9 100644 --- a/arch/csky/include/asm/shmparam.h +++ b/arch/csky/include/asm/shmparam.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_SHMPARAM_H #define __ASM_CSKY_SHMPARAM_H diff --git a/arch/csky/include/asm/smp.h b/arch/csky/include/asm/smp.h index 668b79ce29ea..d3db334f3196 100644 --- a/arch/csky/include/asm/smp.h +++ b/arch/csky/include/asm/smp.h @@ -23,7 +23,7 @@ void __init set_send_ipi(void (*func)(const struct cpumask *mask), int irq); int __cpu_disable(void); -void __cpu_die(unsigned int cpu); +static inline void __cpu_die(unsigned int cpu) { } #endif /* CONFIG_SMP */ diff --git a/arch/csky/include/asm/spinlock.h b/arch/csky/include/asm/spinlock.h index 7cf3f2b34cea..83a2005341f5 100644 --- a/arch/csky/include/asm/spinlock.h +++ b/arch/csky/include/asm/spinlock.h @@ -3,254 +3,10 @@ #ifndef __ASM_CSKY_SPINLOCK_H #define __ASM_CSKY_SPINLOCK_H -#include <linux/spinlock_types.h> -#include <asm/barrier.h> - -#ifdef CONFIG_QUEUED_RWLOCKS - -/* - * Ticket-based spin-locking. - */ -static inline void arch_spin_lock(arch_spinlock_t *lock) -{ - arch_spinlock_t lockval; - u32 ticket_next = 1 << TICKET_NEXT; - u32 *p = &lock->lock; - u32 tmp; - - asm volatile ( - "1: ldex.w %0, (%2) \n" - " mov %1, %0 \n" - " add %0, %3 \n" - " stex.w %0, (%2) \n" - " bez %0, 1b \n" - : "=&r" (tmp), "=&r" (lockval) - : "r"(p), "r"(ticket_next) - : "cc"); - - while (lockval.tickets.next != lockval.tickets.owner) - lockval.tickets.owner = READ_ONCE(lock->tickets.owner); - - smp_mb(); -} - -static inline int arch_spin_trylock(arch_spinlock_t *lock) -{ - u32 tmp, contended, res; - u32 ticket_next = 1 << TICKET_NEXT; - u32 *p = &lock->lock; - - do { - asm volatile ( - " ldex.w %0, (%3) \n" - " movi %2, 1 \n" - " rotli %1, %0, 16 \n" - " cmpne %1, %0 \n" - " bt 1f \n" - " movi %2, 0 \n" - " add %0, %0, %4 \n" - " stex.w %0, (%3) \n" - "1: \n" - : "=&r" (res), "=&r" (tmp), "=&r" (contended) - : "r"(p), "r"(ticket_next) - : "cc"); - } while (!res); - - if (!contended) - smp_mb(); - - return !contended; -} - -static inline void arch_spin_unlock(arch_spinlock_t *lock) -{ - smp_mb(); - WRITE_ONCE(lock->tickets.owner, lock->tickets.owner + 1); -} - -static inline int arch_spin_value_unlocked(arch_spinlock_t lock) -{ - return lock.tickets.owner == lock.tickets.next; -} - -static inline int arch_spin_is_locked(arch_spinlock_t *lock) -{ - return !arch_spin_value_unlocked(READ_ONCE(*lock)); -} - -static inline int arch_spin_is_contended(arch_spinlock_t *lock) -{ - struct __raw_tickets tickets = READ_ONCE(lock->tickets); - - return (tickets.next - tickets.owner) > 1; -} -#define arch_spin_is_contended arch_spin_is_contended - +#include <asm/qspinlock.h> #include <asm/qrwlock.h> /* See include/linux/spinlock.h */ #define smp_mb__after_spinlock() smp_mb() -#else /* CONFIG_QUEUED_RWLOCKS */ - -/* - * Test-and-set spin-locking. - */ -static inline void arch_spin_lock(arch_spinlock_t *lock) -{ - u32 *p = &lock->lock; - u32 tmp; - - asm volatile ( - "1: ldex.w %0, (%1) \n" - " bnez %0, 1b \n" - " movi %0, 1 \n" - " stex.w %0, (%1) \n" - " bez %0, 1b \n" - : "=&r" (tmp) - : "r"(p) - : "cc"); - smp_mb(); -} - -static inline void arch_spin_unlock(arch_spinlock_t *lock) -{ - smp_mb(); - WRITE_ONCE(lock->lock, 0); -} - -static inline int arch_spin_trylock(arch_spinlock_t *lock) -{ - u32 *p = &lock->lock; - u32 tmp; - - asm volatile ( - "1: ldex.w %0, (%1) \n" - " bnez %0, 2f \n" - " movi %0, 1 \n" - " stex.w %0, (%1) \n" - " bez %0, 1b \n" - " movi %0, 0 \n" - "2: \n" - : "=&r" (tmp) - : "r"(p) - : "cc"); - - if (!tmp) - smp_mb(); - - return !tmp; -} - -#define arch_spin_is_locked(x) (READ_ONCE((x)->lock) != 0) - -/* - * read lock/unlock/trylock - */ -static inline void arch_read_lock(arch_rwlock_t *lock) -{ - u32 *p = &lock->lock; - u32 tmp; - - asm volatile ( - "1: ldex.w %0, (%1) \n" - " blz %0, 1b \n" - " addi %0, 1 \n" - " stex.w %0, (%1) \n" - " bez %0, 1b \n" - : "=&r" (tmp) - : "r"(p) - : "cc"); - smp_mb(); -} - -static inline void arch_read_unlock(arch_rwlock_t *lock) -{ - u32 *p = &lock->lock; - u32 tmp; - - smp_mb(); - asm volatile ( - "1: ldex.w %0, (%1) \n" - " subi %0, 1 \n" - " stex.w %0, (%1) \n" - " bez %0, 1b \n" - : "=&r" (tmp) - : "r"(p) - : "cc"); -} - -static inline int arch_read_trylock(arch_rwlock_t *lock) -{ - u32 *p = &lock->lock; - u32 tmp; - - asm volatile ( - "1: ldex.w %0, (%1) \n" - " blz %0, 2f \n" - " addi %0, 1 \n" - " stex.w %0, (%1) \n" - " bez %0, 1b \n" - " movi %0, 0 \n" - "2: \n" - : "=&r" (tmp) - : "r"(p) - : "cc"); - - if (!tmp) - smp_mb(); - - return !tmp; -} - -/* - * write lock/unlock/trylock - */ -static inline void arch_write_lock(arch_rwlock_t *lock) -{ - u32 *p = &lock->lock; - u32 tmp; - - asm volatile ( - "1: ldex.w %0, (%1) \n" - " bnez %0, 1b \n" - " subi %0, 1 \n" - " stex.w %0, (%1) \n" - " bez %0, 1b \n" - : "=&r" (tmp) - : "r"(p) - : "cc"); - smp_mb(); -} - -static inline void arch_write_unlock(arch_rwlock_t *lock) -{ - smp_mb(); - WRITE_ONCE(lock->lock, 0); -} - -static inline int arch_write_trylock(arch_rwlock_t *lock) -{ - u32 *p = &lock->lock; - u32 tmp; - - asm volatile ( - "1: ldex.w %0, (%1) \n" - " bnez %0, 2f \n" - " subi %0, 1 \n" - " stex.w %0, (%1) \n" - " bez %0, 1b \n" - " movi %0, 0 \n" - "2: \n" - : "=&r" (tmp) - : "r"(p) - : "cc"); - - if (!tmp) - smp_mb(); - - return !tmp; -} - -#endif /* CONFIG_QUEUED_RWLOCKS */ #endif /* __ASM_CSKY_SPINLOCK_H */ diff --git a/arch/csky/include/asm/spinlock_types.h b/arch/csky/include/asm/spinlock_types.h index 88b82438b182..75bdf3af80ba 100644 --- a/arch/csky/include/asm/spinlock_types.h +++ b/arch/csky/include/asm/spinlock_types.h @@ -3,35 +3,7 @@ #ifndef __ASM_CSKY_SPINLOCK_TYPES_H #define __ASM_CSKY_SPINLOCK_TYPES_H -#ifndef __LINUX_SPINLOCK_TYPES_H -# error "please don't include this file directly" -#endif - -#define TICKET_NEXT 16 - -typedef struct { - union { - u32 lock; - struct __raw_tickets { - /* little endian */ - u16 owner; - u16 next; - } tickets; - }; -} arch_spinlock_t; - -#define __ARCH_SPIN_LOCK_UNLOCKED { { 0 } } - -#ifdef CONFIG_QUEUED_RWLOCKS +#include <asm-generic/qspinlock_types.h> #include <asm-generic/qrwlock_types.h> -#else /* CONFIG_NR_CPUS > 2 */ - -typedef struct { - u32 lock; -} arch_rwlock_t; - -#define __ARCH_RW_LOCK_UNLOCKED { 0 } - -#endif /* CONFIG_QUEUED_RWLOCKS */ #endif /* __ASM_CSKY_SPINLOCK_TYPES_H */ diff --git a/arch/csky/include/asm/stackprotector.h b/arch/csky/include/asm/stackprotector.h new file mode 100644 index 000000000000..d23747447166 --- /dev/null +++ b/arch/csky/include/asm/stackprotector.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_STACKPROTECTOR_H +#define _ASM_STACKPROTECTOR_H 1 + +extern unsigned long __stack_chk_guard; + +/* + * Initialize the stackprotector canary value. + * + * NOTE: this must only be called from functions that never return, + * and it must always be inlined. + */ +static __always_inline void boot_init_stack_canary(void) +{ + unsigned long canary = get_random_canary(); + + current->stack_canary = canary; + __stack_chk_guard = current->stack_canary; +} + +#endif /* __ASM_SH_STACKPROTECTOR_H */ diff --git a/arch/csky/include/asm/string.h b/arch/csky/include/asm/string.h index 73142de18355..a0d81e9d6b8f 100644 --- a/arch/csky/include/asm/string.h +++ b/arch/csky/include/asm/string.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef _CSKY_STRING_MM_H_ #define _CSKY_STRING_MM_H_ diff --git a/arch/csky/include/asm/switch_to.h b/arch/csky/include/asm/switch_to.h index 35a39e88933d..731e466415e2 100644 --- a/arch/csky/include/asm/switch_to.h +++ b/arch/csky/include/asm/switch_to.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_SWITCH_TO_H #define __ASM_CSKY_SWITCH_TO_H diff --git a/arch/csky/include/asm/syscall.h b/arch/csky/include/asm/syscall.h index f624fa3bbc22..0de5734950bf 100644 --- a/arch/csky/include/asm/syscall.h +++ b/arch/csky/include/asm/syscall.h @@ -59,15 +59,6 @@ syscall_get_arguments(struct task_struct *task, struct pt_regs *regs, memcpy(args, ®s->a1, 5 * sizeof(args[0])); } -static inline void -syscall_set_arguments(struct task_struct *task, struct pt_regs *regs, - const unsigned long *args) -{ - regs->orig_a0 = args[0]; - args++; - memcpy(®s->a1, args, 5 * sizeof(regs->a1)); -} - static inline int syscall_get_arch(struct task_struct *task) { diff --git a/arch/csky/include/asm/syscalls.h b/arch/csky/include/asm/syscalls.h index 5d48e5e0082e..ea9ce6138b9b 100644 --- a/arch/csky/include/asm/syscalls.h +++ b/arch/csky/include/asm/syscalls.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_SYSCALLS_H #define __ASM_CSKY_SYSCALLS_H diff --git a/arch/csky/include/asm/tcm.h b/arch/csky/include/asm/tcm.h new file mode 100644 index 000000000000..bd1e662ecdfa --- /dev/null +++ b/arch/csky/include/asm/tcm.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __ASM_CSKY_TCM_H +#define __ASM_CSKY_TCM_H + +#ifndef CONFIG_HAVE_TCM +#error "You should not be including tcm.h unless you have a TCM!" +#endif + +#include <linux/compiler.h> + +/* Tag variables with this */ +#define __tcmdata __section(".tcm.data") +/* Tag constants with this */ +#define __tcmconst __section(".tcm.rodata") +/* Tag functions inside TCM called from outside TCM with this */ +#define __tcmfunc __section(".tcm.text") noinline +/* Tag function inside TCM called from inside TCM with this */ +#define __tcmlocalfunc __section(".tcm.text") + +void *tcm_alloc(size_t len); +void tcm_free(void *addr, size_t len); + +#endif diff --git a/arch/csky/include/asm/thread_info.h b/arch/csky/include/asm/thread_info.h index 0b546a55a8bf..b5ed788f0c68 100644 --- a/arch/csky/include/asm/thread_info.h +++ b/arch/csky/include/asm/thread_info.h @@ -1,12 +1,10 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef _ASM_CSKY_THREAD_INFO_H #define _ASM_CSKY_THREAD_INFO_H #ifndef __ASSEMBLY__ -#include <linux/version.h> #include <asm/types.h> #include <asm/page.h> #include <asm/processor.h> @@ -18,7 +16,6 @@ struct thread_info { unsigned long flags; int preempt_count; unsigned long tp_value; - mm_segment_t addr_limit; struct restart_block restart_block; struct pt_regs *regs; unsigned int cpu; @@ -28,7 +25,6 @@ struct thread_info { { \ .task = &tsk, \ .preempt_count = INIT_PREEMPT_COUNT, \ - .addr_limit = KERNEL_DS, \ .cpu = 0, \ .restart_block = { \ .fn = do_no_restart_syscall, \ @@ -38,7 +34,13 @@ struct thread_info { #define THREAD_SIZE_ORDER (THREAD_SHIFT - PAGE_SHIFT) #define thread_saved_fp(tsk) \ - ((unsigned long)(((struct switch_stack *)(tsk->thread.ksp))->r8)) + ((unsigned long)(((struct switch_stack *)(tsk->thread.sp))->r8)) + +#define thread_saved_sp(tsk) \ + ((unsigned long)(tsk->thread.sp)) + +#define thread_saved_lr(tsk) \ + ((unsigned long)(((struct switch_stack *)(tsk->thread.sp))->r15)) static inline struct thread_info *current_thread_info(void) { @@ -54,9 +56,11 @@ static inline struct thread_info *current_thread_info(void) #define TIF_SIGPENDING 0 /* signal pending */ #define TIF_NOTIFY_RESUME 1 /* callback before returning to user */ #define TIF_NEED_RESCHED 2 /* rescheduling necessary */ -#define TIF_SYSCALL_TRACE 3 /* syscall trace active */ -#define TIF_SYSCALL_TRACEPOINT 4 /* syscall tracepoint instrumentation */ -#define TIF_SYSCALL_AUDIT 5 /* syscall auditing */ +#define TIF_UPROBE 3 /* uprobe breakpoint or singlestep */ +#define TIF_SYSCALL_TRACE 4 /* syscall trace active */ +#define TIF_SYSCALL_TRACEPOINT 5 /* syscall tracepoint instrumentation */ +#define TIF_SYSCALL_AUDIT 6 /* syscall auditing */ +#define TIF_NOTIFY_SIGNAL 7 /* signal notifications exist */ #define TIF_POLLING_NRFLAG 16 /* poll_idle() is TIF_NEED_RESCHED */ #define TIF_MEMDIE 18 /* is terminating due to OOM killer */ #define TIF_RESTORE_SIGMASK 20 /* restore signal mask in do_signal() */ @@ -68,9 +72,18 @@ static inline struct thread_info *current_thread_info(void) #define _TIF_SYSCALL_TRACE (1 << TIF_SYSCALL_TRACE) #define _TIF_SYSCALL_TRACEPOINT (1 << TIF_SYSCALL_TRACEPOINT) #define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT) +#define _TIF_NOTIFY_SIGNAL (1 << TIF_NOTIFY_SIGNAL) +#define _TIF_UPROBE (1 << TIF_UPROBE) #define _TIF_POLLING_NRFLAG (1 << TIF_POLLING_NRFLAG) #define _TIF_MEMDIE (1 << TIF_MEMDIE) #define _TIF_RESTORE_SIGMASK (1 << TIF_RESTORE_SIGMASK) #define _TIF_SECCOMP (1 << TIF_SECCOMP) +#define _TIF_WORK_MASK (_TIF_NEED_RESCHED | _TIF_SIGPENDING | \ + _TIF_NOTIFY_RESUME | _TIF_UPROBE | \ + _TIF_NOTIFY_SIGNAL) + +#define _TIF_SYSCALL_WORK (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | \ + _TIF_SYSCALL_TRACEPOINT | _TIF_SECCOMP) + #endif /* _ASM_CSKY_THREAD_INFO_H */ diff --git a/arch/csky/include/asm/tlb.h b/arch/csky/include/asm/tlb.h index fdff9b8d70c8..702861c68874 100644 --- a/arch/csky/include/asm/tlb.h +++ b/arch/csky/include/asm/tlb.h @@ -1,25 +1,9 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_TLB_H #define __ASM_CSKY_TLB_H #include <asm/cacheflush.h> - -#define tlb_start_vma(tlb, vma) \ - do { \ - if (!(tlb)->fullmm) \ - flush_cache_range(vma, (vma)->vm_start, (vma)->vm_end); \ - } while (0) - -#define tlb_end_vma(tlb, vma) \ - do { \ - if (!(tlb)->fullmm) \ - flush_tlb_range(vma, (vma)->vm_start, (vma)->vm_end); \ - } while (0) - -#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm) - #include <asm-generic/tlb.h> #endif /* __ASM_CSKY_TLB_H */ diff --git a/arch/csky/include/asm/tlbflush.h b/arch/csky/include/asm/tlbflush.h index 6845b0667703..407160b4fde7 100644 --- a/arch/csky/include/asm/tlbflush.h +++ b/arch/csky/include/asm/tlbflush.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_TLBFLUSH_H #define __ASM_TLBFLUSH_H diff --git a/arch/csky/include/asm/traps.h b/arch/csky/include/asm/traps.h index 1c081805b962..6bbbbe43165f 100644 --- a/arch/csky/include/asm/traps.h +++ b/arch/csky/include/asm/traps.h @@ -1,9 +1,10 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_TRAPS_H #define __ASM_CSKY_TRAPS_H +#include <linux/linkage.h> + #define VEC_RESET 0 #define VEC_ALIGN 1 #define VEC_ACCESS 2 @@ -41,4 +42,19 @@ do { \ void csky_alignment(struct pt_regs *regs); +asmlinkage void do_trap_unknown(struct pt_regs *regs); +asmlinkage void do_trap_zdiv(struct pt_regs *regs); +asmlinkage void do_trap_buserr(struct pt_regs *regs); +asmlinkage void do_trap_misaligned(struct pt_regs *regs); +asmlinkage void do_trap_bkpt(struct pt_regs *regs); +asmlinkage void do_trap_illinsn(struct pt_regs *regs); +asmlinkage void do_trap_fpe(struct pt_regs *regs); +asmlinkage void do_trap_priv(struct pt_regs *regs); +asmlinkage void trap_c(struct pt_regs *regs); + +asmlinkage void do_notify_resume(struct pt_regs *regs, + unsigned long thread_info_flags); + +asmlinkage void do_page_fault(struct pt_regs *regs); + #endif /* __ASM_CSKY_TRAPS_H */ diff --git a/arch/csky/include/asm/uaccess.h b/arch/csky/include/asm/uaccess.h index eaa1c3403a42..2e927c21d8a1 100644 --- a/arch/csky/include/asm/uaccess.h +++ b/arch/csky/include/asm/uaccess.h @@ -1,126 +1,17 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_UACCESS_H #define __ASM_CSKY_UACCESS_H /* - * User space memory access functions + * __put_user_fn */ -#include <linux/compiler.h> -#include <linux/errno.h> -#include <linux/types.h> -#include <linux/sched.h> -#include <linux/mm.h> -#include <linux/string.h> -#include <linux/version.h> -#include <asm/segment.h> - -static inline int access_ok(const void *addr, unsigned long size) -{ - unsigned long limit = current_thread_info()->addr_limit.seg; - - return (((unsigned long)addr < limit) && - ((unsigned long)(addr + size) < limit)); -} - -#define __addr_ok(addr) (access_ok(addr, 0)) - extern int __put_user_bad(void); -/* - * Tell gcc we read from memory instead of writing: this is because - * we do not write to any memory gcc knows about, so there are no - * aliasing issues. - */ - -/* - * These are the main single-value transfer routines. They automatically - * use the right size if we just have the right pointer type. - * - * This gets kind of ugly. We want to return _two_ values in "get_user()" - * and yet we don't want to do any pointers, because that is too much - * of a performance impact. Thus we have a few rather ugly macros here, - * and hide all the ugliness from the user. - * - * The "__xxx" versions of the user access functions are versions that - * do not verify the address space, that must have been done previously - * with a separate "access_ok()" call (this is used when we do multiple - * accesses to the same area of user memory). - * - * As we use the same address space for kernel and user data on - * Ckcore, we can just do these as direct assignments. (Of course, the - * exception handling means that it's no longer "just"...) - */ - -#define put_user(x, ptr) \ - __put_user_check((x), (ptr), sizeof(*(ptr))) - -#define __put_user(x, ptr) \ - __put_user_nocheck((x), (ptr), sizeof(*(ptr))) - -#define __ptr(x) ((unsigned long *)(x)) - -#define get_user(x, ptr) \ - __get_user_check((x), (ptr), sizeof(*(ptr))) - -#define __get_user(x, ptr) \ - __get_user_nocheck((x), (ptr), sizeof(*(ptr))) - -#define __put_user_nocheck(x, ptr, size) \ -({ \ - long __pu_err = 0; \ - typeof(*(ptr)) *__pu_addr = (ptr); \ - typeof(*(ptr)) __pu_val = (typeof(*(ptr)))(x); \ - if (__pu_addr) \ - __put_user_size(__pu_val, (__pu_addr), (size), \ - __pu_err); \ - __pu_err; \ -}) - -#define __put_user_check(x, ptr, size) \ -({ \ - long __pu_err = -EFAULT; \ - typeof(*(ptr)) *__pu_addr = (ptr); \ - typeof(*(ptr)) __pu_val = (typeof(*(ptr)))(x); \ - if (access_ok(__pu_addr, size) && __pu_addr) \ - __put_user_size(__pu_val, __pu_addr, (size), __pu_err); \ - __pu_err; \ -}) - -#define __put_user_size(x, ptr, size, retval) \ -do { \ - retval = 0; \ - switch (size) { \ - case 1: \ - __put_user_asm_b(x, ptr, retval); \ - break; \ - case 2: \ - __put_user_asm_h(x, ptr, retval); \ - break; \ - case 4: \ - __put_user_asm_w(x, ptr, retval); \ - break; \ - case 8: \ - __put_user_asm_64(x, ptr, retval); \ - break; \ - default: \ - __put_user_bad(); \ - } \ -} while (0) - -/* - * We don't tell gcc that we are accessing memory, but this is OK - * because we do not write to any memory gcc knows about, so there - * are no aliasing issues. - * - * Note that PC at a fault is the address *after* the faulting - * instruction. - */ #define __put_user_asm_b(x, ptr, err) \ do { \ int errcode; \ - asm volatile( \ + __asm__ __volatile__( \ "1: stb %1, (%2,0) \n" \ " br 3f \n" \ "2: mov %0, %3 \n" \ @@ -138,7 +29,7 @@ do { \ #define __put_user_asm_h(x, ptr, err) \ do { \ int errcode; \ - asm volatile( \ + __asm__ __volatile__( \ "1: sth %1, (%2,0) \n" \ " br 3f \n" \ "2: mov %0, %3 \n" \ @@ -156,7 +47,7 @@ do { \ #define __put_user_asm_w(x, ptr, err) \ do { \ int errcode; \ - asm volatile( \ + __asm__ __volatile__( \ "1: stw %1, (%2,0) \n" \ " br 3f \n" \ "2: mov %0, %3 \n" \ @@ -171,238 +62,142 @@ do { \ : "memory"); \ } while (0) -#define __put_user_asm_64(x, ptr, err) \ -do { \ - int tmp; \ - int errcode; \ - typeof(*(ptr))src = (typeof(*(ptr)))x; \ - typeof(*(ptr))*psrc = &src; \ - \ - asm volatile( \ - " ldw %3, (%1, 0) \n" \ - "1: stw %3, (%2, 0) \n" \ - " ldw %3, (%1, 4) \n" \ - "2: stw %3, (%2, 4) \n" \ - " br 4f \n" \ - "3: mov %0, %4 \n" \ - " br 4f \n" \ - ".section __ex_table, \"a\" \n" \ - ".align 2 \n" \ - ".long 1b, 3b \n" \ - ".long 2b, 3b \n" \ - ".previous \n" \ - "4: \n" \ - : "=r"(err), "=r"(psrc), "=r"(ptr), \ - "=r"(tmp), "=r"(errcode) \ - : "0"(err), "1"(psrc), "2"(ptr), "3"(0), "4"(-EFAULT) \ - : "memory"); \ -} while (0) - -#define __get_user_nocheck(x, ptr, size) \ -({ \ - long __gu_err; \ - __get_user_size(x, (ptr), (size), __gu_err); \ - __gu_err; \ -}) - -#define __get_user_check(x, ptr, size) \ -({ \ - int __gu_err = -EFAULT; \ - const __typeof__(*(ptr)) __user *__gu_ptr = (ptr); \ - if (access_ok(__gu_ptr, size) && __gu_ptr) \ - __get_user_size(x, __gu_ptr, size, __gu_err); \ - __gu_err; \ -}) - -#define __get_user_size(x, ptr, size, retval) \ -do { \ - switch (size) { \ - case 1: \ - __get_user_asm_common((x), ptr, "ldb", retval); \ - break; \ - case 2: \ - __get_user_asm_common((x), ptr, "ldh", retval); \ - break; \ - case 4: \ - __get_user_asm_common((x), ptr, "ldw", retval); \ - break; \ - default: \ - x = 0; \ - (retval) = __get_user_bad(); \ - } \ +#define __put_user_asm_64(x, ptr, err) \ +do { \ + int tmp; \ + int errcode; \ + \ + __asm__ __volatile__( \ + " ldw %3, (%1, 0) \n" \ + "1: stw %3, (%2, 0) \n" \ + " ldw %3, (%1, 4) \n" \ + "2: stw %3, (%2, 4) \n" \ + " br 4f \n" \ + "3: mov %0, %4 \n" \ + " br 4f \n" \ + ".section __ex_table, \"a\" \n" \ + ".align 2 \n" \ + ".long 1b, 3b \n" \ + ".long 2b, 3b \n" \ + ".previous \n" \ + "4: \n" \ + : "=r"(err), "=r"(x), "=r"(ptr), \ + "=r"(tmp), "=r"(errcode) \ + : "0"(err), "1"(x), "2"(ptr), "3"(0), \ + "4"(-EFAULT) \ + : "memory"); \ } while (0) -#define __get_user_asm_common(x, ptr, ins, err) \ -do { \ - int errcode; \ - asm volatile( \ - "1: " ins " %1, (%4,0) \n" \ - " br 3f \n" \ - /* Fix up codes */ \ - "2: mov %0, %2 \n" \ - " movi %1, 0 \n" \ - " br 3f \n" \ - ".section __ex_table,\"a\" \n" \ - ".align 2 \n" \ - ".long 1b, 2b \n" \ - ".previous \n" \ - "3: \n" \ - : "=r"(err), "=r"(x), "=r"(errcode) \ - : "0"(0), "r"(ptr), "2"(-EFAULT) \ - : "memory"); \ -} while (0) +static inline int __put_user_fn(size_t size, void __user *ptr, void *x) +{ + int retval = 0; + u32 tmp; + + switch (size) { + case 1: + tmp = *(u8 *)x; + __put_user_asm_b(tmp, ptr, retval); + break; + case 2: + tmp = *(u16 *)x; + __put_user_asm_h(tmp, ptr, retval); + break; + case 4: + tmp = *(u32 *)x; + __put_user_asm_w(tmp, ptr, retval); + break; + case 8: + __put_user_asm_64(x, (u64 *)ptr, retval); + break; + } + + return retval; +} +#define __put_user_fn __put_user_fn +/* + * __get_user_fn + */ extern int __get_user_bad(void); -#define __copy_user(to, from, n) \ +#define __get_user_asm_common(x, ptr, ins, err) \ do { \ - int w0, w1, w2, w3; \ - asm volatile( \ - "0: cmpnei %1, 0 \n" \ - " bf 8f \n" \ - " mov %3, %1 \n" \ - " or %3, %2 \n" \ - " andi %3, 3 \n" \ - " cmpnei %3, 0 \n" \ - " bf 1f \n" \ - " br 5f \n" \ - "1: cmplti %0, 16 \n" /* 4W */ \ - " bt 3f \n" \ - " ldw %3, (%2, 0) \n" \ - " ldw %4, (%2, 4) \n" \ - " ldw %5, (%2, 8) \n" \ - " ldw %6, (%2, 12) \n" \ - "2: stw %3, (%1, 0) \n" \ - "9: stw %4, (%1, 4) \n" \ - "10: stw %5, (%1, 8) \n" \ - "11: stw %6, (%1, 12) \n" \ - " addi %2, 16 \n" \ - " addi %1, 16 \n" \ - " subi %0, 16 \n" \ - " br 1b \n" \ - "3: cmplti %0, 4 \n" /* 1W */ \ - " bt 5f \n" \ - " ldw %3, (%2, 0) \n" \ - "4: stw %3, (%1, 0) \n" \ - " addi %2, 4 \n" \ - " addi %1, 4 \n" \ - " subi %0, 4 \n" \ - " br 3b \n" \ - "5: cmpnei %0, 0 \n" /* 1B */ \ - " bf 8f \n" \ - " ldb %3, (%2, 0) \n" \ - "6: stb %3, (%1, 0) \n" \ - " addi %2, 1 \n" \ - " addi %1, 1 \n" \ - " subi %0, 1 \n" \ - " br 5b \n" \ - "7: br 8f \n" \ - ".section __ex_table, \"a\" \n" \ - ".align 2 \n" \ - ".long 2b, 7b \n" \ - ".long 9b, 7b \n" \ - ".long 10b, 7b \n" \ - ".long 11b, 7b \n" \ - ".long 4b, 7b \n" \ - ".long 6b, 7b \n" \ - ".previous \n" \ - "8: \n" \ - : "=r"(n), "=r"(to), "=r"(from), "=r"(w0), \ - "=r"(w1), "=r"(w2), "=r"(w3) \ - : "0"(n), "1"(to), "2"(from) \ + int errcode; \ + __asm__ __volatile__( \ + "1: " ins " %1, (%4, 0) \n" \ + " br 3f \n" \ + "2: mov %0, %2 \n" \ + " movi %1, 0 \n" \ + " br 3f \n" \ + ".section __ex_table,\"a\" \n" \ + ".align 2 \n" \ + ".long 1b, 2b \n" \ + ".previous \n" \ + "3: \n" \ + : "=r"(err), "=r"(x), "=r"(errcode) \ + : "0"(0), "r"(ptr), "2"(-EFAULT) \ : "memory"); \ } while (0) -#define __copy_user_zeroing(to, from, n) \ +#define __get_user_asm_64(x, ptr, err) \ do { \ int tmp; \ - int nsave; \ - asm volatile( \ - "0: cmpnei %1, 0 \n" \ - " bf 7f \n" \ - " mov %3, %1 \n" \ - " or %3, %2 \n" \ - " andi %3, 3 \n" \ - " cmpnei %3, 0 \n" \ - " bf 1f \n" \ - " br 5f \n" \ - "1: cmplti %0, 16 \n" \ - " bt 3f \n" \ - "2: ldw %3, (%2, 0) \n" \ - "10: ldw %4, (%2, 4) \n" \ - " stw %3, (%1, 0) \n" \ - " stw %4, (%1, 4) \n" \ - "11: ldw %3, (%2, 8) \n" \ - "12: ldw %4, (%2, 12) \n" \ - " stw %3, (%1, 8) \n" \ - " stw %4, (%1, 12) \n" \ - " addi %2, 16 \n" \ - " addi %1, 16 \n" \ - " subi %0, 16 \n" \ - " br 1b \n" \ - "3: cmplti %0, 4 \n" \ - " bt 5f \n" \ - "4: ldw %3, (%2, 0) \n" \ - " stw %3, (%1, 0) \n" \ - " addi %2, 4 \n" \ - " addi %1, 4 \n" \ - " subi %0, 4 \n" \ - " br 3b \n" \ - "5: cmpnei %0, 0 \n" \ - " bf 7f \n" \ - "6: ldb %3, (%2, 0) \n" \ - " stb %3, (%1, 0) \n" \ - " addi %2, 1 \n" \ - " addi %1, 1 \n" \ - " subi %0, 1 \n" \ - " br 5b \n" \ - "8: mov %3, %0 \n" \ - " movi %4, 0 \n" \ - "9: stb %4, (%1, 0) \n" \ - " addi %1, 1 \n" \ - " subi %3, 1 \n" \ - " cmpnei %3, 0 \n" \ - " bt 9b \n" \ - " br 7f \n" \ - ".section __ex_table, \"a\" \n" \ - ".align 2 \n" \ - ".long 2b, 8b \n" \ - ".long 10b, 8b \n" \ - ".long 11b, 8b \n" \ - ".long 12b, 8b \n" \ - ".long 4b, 8b \n" \ - ".long 6b, 8b \n" \ - ".previous \n" \ - "7: \n" \ - : "=r"(n), "=r"(to), "=r"(from), "=r"(nsave), \ - "=r"(tmp) \ - : "0"(n), "1"(to), "2"(from) \ + int errcode; \ + \ + __asm__ __volatile__( \ + "1: ldw %3, (%2, 0) \n" \ + " stw %3, (%1, 0) \n" \ + "2: ldw %3, (%2, 4) \n" \ + " stw %3, (%1, 4) \n" \ + " br 4f \n" \ + "3: mov %0, %4 \n" \ + " br 4f \n" \ + ".section __ex_table, \"a\" \n" \ + ".align 2 \n" \ + ".long 1b, 3b \n" \ + ".long 2b, 3b \n" \ + ".previous \n" \ + "4: \n" \ + : "=r"(err), "=r"(x), "=r"(ptr), \ + "=r"(tmp), "=r"(errcode) \ + : "0"(err), "1"(x), "2"(ptr), "3"(0), \ + "4"(-EFAULT) \ : "memory"); \ } while (0) +static inline int __get_user_fn(size_t size, const void __user *ptr, void *x) +{ + int retval; + u32 tmp; + + switch (size) { + case 1: + __get_user_asm_common(tmp, ptr, "ldb", retval); + *(u8 *)x = (u8)tmp; + break; + case 2: + __get_user_asm_common(tmp, ptr, "ldh", retval); + *(u16 *)x = (u16)tmp; + break; + case 4: + __get_user_asm_common(tmp, ptr, "ldw", retval); + *(u32 *)x = (u32)tmp; + break; + case 8: + __get_user_asm_64(x, ptr, retval); + break; + } + + return retval; +} +#define __get_user_fn __get_user_fn + unsigned long raw_copy_from_user(void *to, const void *from, unsigned long n); unsigned long raw_copy_to_user(void *to, const void *from, unsigned long n); -unsigned long clear_user(void *to, unsigned long n); unsigned long __clear_user(void __user *to, unsigned long n); +#define __clear_user __clear_user -long strncpy_from_user(char *dst, const char *src, long count); -long __strncpy_from_user(char *dst, const char *src, long count); - -/* - * Return the size of a string (including the ending 0) - * - * Return 0 on exception, a value greater than N if too long - */ -long strnlen_user(const char *src, long n); - -#define strlen_user(str) strnlen_user(str, 32767) - -struct exception_table_entry { - unsigned long insn; - unsigned long nextinsn; -}; - -extern int fixup_exception(struct pt_regs *regs); +#include <asm-generic/uaccess.h> #endif /* __ASM_CSKY_UACCESS_H */ diff --git a/arch/csky/include/asm/unistd.h b/arch/csky/include/asm/unistd.h index da7a18295615..9cf97de9a26d 100644 --- a/arch/csky/include/asm/unistd.h +++ b/arch/csky/include/asm/unistd.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #include <uapi/asm/unistd.h> diff --git a/arch/csky/include/asm/uprobes.h b/arch/csky/include/asm/uprobes.h new file mode 100644 index 000000000000..600388eb93c6 --- /dev/null +++ b/arch/csky/include/asm/uprobes.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef __ASM_CSKY_UPROBES_H +#define __ASM_CSKY_UPROBES_H + +#include <asm/probes.h> + +#define MAX_UINSN_BYTES 4 + +#define UPROBE_SWBP_INSN USR_BKPT +#define UPROBE_SWBP_INSN_SIZE 2 +#define UPROBE_XOL_SLOT_BYTES MAX_UINSN_BYTES + +typedef u32 uprobe_opcode_t; + +struct arch_uprobe_task { + unsigned long saved_trap_no; +}; + +struct arch_uprobe { + union { + u8 insn[MAX_UINSN_BYTES]; + u8 ixol[MAX_UINSN_BYTES]; + }; + struct arch_probe_insn api; + unsigned long insn_size; + bool simulate; +}; + +int uprobe_breakpoint_handler(struct pt_regs *regs); +int uprobe_single_step_handler(struct pt_regs *regs); + +#endif /* __ASM_CSKY_UPROBES_H */ diff --git a/arch/csky/include/asm/vdso.h b/arch/csky/include/asm/vdso.h index d963d691f3a1..181a15edafe8 100644 --- a/arch/csky/include/asm/vdso.h +++ b/arch/csky/include/asm/vdso.h @@ -3,10 +3,20 @@ #ifndef __ASM_CSKY_VDSO_H #define __ASM_CSKY_VDSO_H -#include <abi/vdso.h> +#include <linux/types.h> -struct csky_vdso { - unsigned short rt_signal_retcode[4]; -}; +/* + * The VDSO symbols are mapped into Linux so we can just use regular symbol + * addressing to get their offsets in userspace. The symbols are mapped at an + * offset of 0, but since the linker must support setting weak undefined + * symbols to the absolute address 0 it also happens to support other low + * addresses even when the code model suggests those low addresses would not + * otherwise be available. + */ +#define VDSO_SYMBOL(base, name) \ +({ \ + extern const char __vdso_##name[]; \ + (void __user *)((unsigned long)(base) + __vdso_##name); \ +}) #endif /* __ASM_CSKY_VDSO_H */ diff --git a/arch/csky/include/asm/vdso/clocksource.h b/arch/csky/include/asm/vdso/clocksource.h new file mode 100644 index 000000000000..dfca7b4724b7 --- /dev/null +++ b/arch/csky/include/asm/vdso/clocksource.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __ASM_VDSO_CSKY_CLOCKSOURCE_H +#define __ASM_VDSO_CSKY_CLOCKSOURCE_H + +#define VDSO_ARCH_CLOCKMODES \ + VDSO_CLOCKMODE_ARCHTIMER + +#endif /* __ASM_VDSO_CSKY_CLOCKSOURCE_H */ diff --git a/arch/csky/include/asm/vdso/gettimeofday.h b/arch/csky/include/asm/vdso/gettimeofday.h new file mode 100644 index 000000000000..6c4f1446944f --- /dev/null +++ b/arch/csky/include/asm/vdso/gettimeofday.h @@ -0,0 +1,114 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __ASM_VDSO_CSKY_GETTIMEOFDAY_H +#define __ASM_VDSO_CSKY_GETTIMEOFDAY_H + +#ifndef __ASSEMBLY__ + +#include <asm/barrier.h> +#include <asm/unistd.h> +#include <abi/regdef.h> +#include <uapi/linux/time.h> + +#define VDSO_HAS_CLOCK_GETRES 1 + +static __always_inline +int gettimeofday_fallback(struct __kernel_old_timeval *_tv, + struct timezone *_tz) +{ + register struct __kernel_old_timeval *tv asm("a0") = _tv; + register struct timezone *tz asm("a1") = _tz; + register long ret asm("a0"); + register long nr asm(syscallid) = __NR_gettimeofday; + + asm volatile ("trap 0\n" + : "=r" (ret) + : "r"(tv), "r"(tz), "r"(nr) + : "memory"); + + return ret; +} + +static __always_inline +long clock_gettime_fallback(clockid_t _clkid, struct __kernel_timespec *_ts) +{ + register clockid_t clkid asm("a0") = _clkid; + register struct __kernel_timespec *ts asm("a1") = _ts; + register long ret asm("a0"); + register long nr asm(syscallid) = __NR_clock_gettime64; + + asm volatile ("trap 0\n" + : "=r" (ret) + : "r"(clkid), "r"(ts), "r"(nr) + : "memory"); + + return ret; +} + +static __always_inline +long clock_gettime32_fallback(clockid_t _clkid, struct old_timespec32 *_ts) +{ + register clockid_t clkid asm("a0") = _clkid; + register struct old_timespec32 *ts asm("a1") = _ts; + register long ret asm("a0"); + register long nr asm(syscallid) = __NR_clock_gettime; + + asm volatile ("trap 0\n" + : "=r" (ret) + : "r"(clkid), "r"(ts), "r"(nr) + : "memory"); + + return ret; +} + +static __always_inline +int clock_getres_fallback(clockid_t _clkid, struct __kernel_timespec *_ts) +{ + register clockid_t clkid asm("a0") = _clkid; + register struct __kernel_timespec *ts asm("a1") = _ts; + register long ret asm("a0"); + register long nr asm(syscallid) = __NR_clock_getres_time64; + + asm volatile ("trap 0\n" + : "=r" (ret) + : "r"(clkid), "r"(ts), "r"(nr) + : "memory"); + + return ret; +} + +static __always_inline +int clock_getres32_fallback(clockid_t _clkid, struct old_timespec32 *_ts) +{ + register clockid_t clkid asm("a0") = _clkid; + register struct old_timespec32 *ts asm("a1") = _ts; + register long ret asm("a0"); + register long nr asm(syscallid) = __NR_clock_getres; + + asm volatile ("trap 0\n" + : "=r" (ret) + : "r"(clkid), "r"(ts), "r"(nr) + : "memory"); + + return ret; +} + +uint64_t csky_pmu_read_cc(void); +static __always_inline u64 __arch_get_hw_counter(s32 clock_mode, + const struct vdso_data *vd) +{ +#ifdef CONFIG_CSKY_PMU_V1 + return csky_pmu_read_cc(); +#else + return 0; +#endif +} + +static __always_inline const struct vdso_data *__arch_get_vdso_data(void) +{ + return _vdso_data; +} + +#endif /* !__ASSEMBLY__ */ + +#endif /* __ASM_VDSO_CSKY_GETTIMEOFDAY_H */ diff --git a/arch/csky/include/asm/vdso/processor.h b/arch/csky/include/asm/vdso/processor.h new file mode 100644 index 000000000000..39a6b561d0cc --- /dev/null +++ b/arch/csky/include/asm/vdso/processor.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef __ASM_VDSO_CSKY_PROCESSOR_H +#define __ASM_VDSO_CSKY_PROCESSOR_H + +#ifndef __ASSEMBLY__ + +#define cpu_relax() barrier() + +#endif /* __ASSEMBLY__ */ + +#endif /* __ASM_VDSO_CSKY_PROCESSOR_H */ diff --git a/arch/csky/include/asm/vdso/vsyscall.h b/arch/csky/include/asm/vdso/vsyscall.h new file mode 100644 index 000000000000..c276211a7c4d --- /dev/null +++ b/arch/csky/include/asm/vdso/vsyscall.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __ASM_VDSO_CSKY_VSYSCALL_H +#define __ASM_VDSO_CSKY_VSYSCALL_H + +#ifndef __ASSEMBLY__ + +#include <vdso/datapage.h> + +extern struct vdso_data *vdso_data; + +static __always_inline struct vdso_data *__csky_get_k_vdso_data(void) +{ + return vdso_data; +} +#define __arch_get_k_vdso_data __csky_get_k_vdso_data + +#include <asm-generic/vdso/vsyscall.h> + +#endif /* !__ASSEMBLY__ */ + +#endif /* __ASM_VDSO_CSKY_VSYSCALL_H */ diff --git a/arch/csky/include/asm/vmalloc.h b/arch/csky/include/asm/vmalloc.h new file mode 100644 index 000000000000..43dca6336b4c --- /dev/null +++ b/arch/csky/include/asm/vmalloc.h @@ -0,0 +1,4 @@ +#ifndef _ASM_CSKY_VMALLOC_H +#define _ASM_CSKY_VMALLOC_H + +#endif /* _ASM_CSKY_VMALLOC_H */ diff --git a/arch/csky/include/uapi/asm/byteorder.h b/arch/csky/include/uapi/asm/byteorder.h index d150cd664873..1aedd513b65a 100644 --- a/arch/csky/include/uapi/asm/byteorder.h +++ b/arch/csky/include/uapi/asm/byteorder.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_BYTEORDER_H #define __ASM_CSKY_BYTEORDER_H diff --git a/arch/csky/include/uapi/asm/perf_regs.h b/arch/csky/include/uapi/asm/perf_regs.h index 49d4e147a559..d0a8ac6a1b77 100644 --- a/arch/csky/include/uapi/asm/perf_regs.h +++ b/arch/csky/include/uapi/asm/perf_regs.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -// Copyright (C) 2019 Hangzhou C-SKY Microsystems co.,ltd. #ifndef _ASM_CSKY_PERF_REGS_H #define _ASM_CSKY_PERF_REGS_H diff --git a/arch/csky/include/uapi/asm/ptrace.h b/arch/csky/include/uapi/asm/ptrace.h index 66b2268e324e..3be9c14334a6 100644 --- a/arch/csky/include/uapi/asm/ptrace.h +++ b/arch/csky/include/uapi/asm/ptrace.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef _CSKY_PTRACE_H #define _CSKY_PTRACE_H diff --git a/arch/csky/include/uapi/asm/sigcontext.h b/arch/csky/include/uapi/asm/sigcontext.h index 670c020f2cb8..859afb602477 100644 --- a/arch/csky/include/uapi/asm/sigcontext.h +++ b/arch/csky/include/uapi/asm/sigcontext.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_SIGCONTEXT_H #define __ASM_CSKY_SIGCONTEXT_H diff --git a/arch/csky/include/uapi/asm/unistd.h b/arch/csky/include/uapi/asm/unistd.h index 211c983c7282..7ff6a2466af1 100644 --- a/arch/csky/include/uapi/asm/unistd.h +++ b/arch/csky/include/uapi/asm/unistd.h @@ -1,7 +1,9 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. +#define __ARCH_WANT_STAT64 +#define __ARCH_WANT_NEW_STAT #define __ARCH_WANT_SYS_CLONE +#define __ARCH_WANT_SYS_CLONE3 #define __ARCH_WANT_SET_GET_RLIMIT #define __ARCH_WANT_TIME32_SYSCALLS #include <asm-generic/unistd.h> diff --git a/arch/csky/kernel/Makefile b/arch/csky/kernel/Makefile index 071d659f37b7..8a868316b912 100644 --- a/arch/csky/kernel/Makefile +++ b/arch/csky/kernel/Makefile @@ -1,9 +1,10 @@ # SPDX-License-Identifier: GPL-2.0-only -extra-y := head.o vmlinux.lds +extra-y := vmlinux.lds -obj-y += entry.o atomic.o signal.o traps.o irq.o time.o vdso.o -obj-y += power.o syscall.o syscall_table.o setup.o -obj-y += process.o cpu-probe.o ptrace.o dumpstack.o +obj-y += head.o entry.o atomic.o signal.o traps.o irq.o time.o vdso.o vdso/ +obj-y += power.o syscall.o syscall_table.o setup.o io.o +obj-y += process.o cpu-probe.o ptrace.o stacktrace.o +obj-y += probes/ obj-$(CONFIG_MODULES) += module.o obj-$(CONFIG_SMP) += smp.o @@ -12,6 +13,7 @@ obj-$(CONFIG_STACKTRACE) += stacktrace.o obj-$(CONFIG_CSKY_PMU_V1) += perf_event.o obj-$(CONFIG_PERF_EVENTS) += perf_callchain.o obj-$(CONFIG_HAVE_PERF_REGS) += perf_regs.o +obj-$(CONFIG_JUMP_LABEL) += jump_label.o ifdef CONFIG_FUNCTION_TRACER CFLAGS_REMOVE_ftrace.o = $(CC_FLAGS_FTRACE) diff --git a/arch/csky/kernel/asm-offsets.c b/arch/csky/kernel/asm-offsets.c index 9b48b1b1a61b..d1e903579473 100644 --- a/arch/csky/kernel/asm-offsets.c +++ b/arch/csky/kernel/asm-offsets.c @@ -9,7 +9,6 @@ int main(void) { /* offsets into the task struct */ - DEFINE(TASK_STATE, offsetof(struct task_struct, state)); DEFINE(TASK_THREAD_INFO, offsetof(struct task_struct, stack)); DEFINE(TASK_FLAGS, offsetof(struct task_struct, flags)); DEFINE(TASK_PTRACE, offsetof(struct task_struct, ptrace)); @@ -18,8 +17,7 @@ int main(void) DEFINE(TASK_ACTIVE_MM, offsetof(struct task_struct, active_mm)); /* offsets into the thread struct */ - DEFINE(THREAD_KSP, offsetof(struct thread_struct, ksp)); - DEFINE(THREAD_SR, offsetof(struct thread_struct, sr)); + DEFINE(THREAD_KSP, offsetof(struct thread_struct, sp)); DEFINE(THREAD_FESR, offsetof(struct thread_struct, user_fp.fesr)); DEFINE(THREAD_FCR, offsetof(struct thread_struct, user_fp.fcr)); DEFINE(THREAD_FPREG, offsetof(struct thread_struct, user_fp.vr)); @@ -27,7 +25,6 @@ int main(void) /* offsets into the thread_info struct */ DEFINE(TINFO_FLAGS, offsetof(struct thread_info, flags)); DEFINE(TINFO_PREEMPT, offsetof(struct thread_info, preempt_count)); - DEFINE(TINFO_ADDR_LIMIT, offsetof(struct thread_info, addr_limit)); DEFINE(TINFO_TP_VALUE, offsetof(struct thread_info, tp_value)); DEFINE(TINFO_TASK, offsetof(struct thread_info, task)); @@ -72,6 +69,7 @@ int main(void) DEFINE(PT_RLO, offsetof(struct pt_regs, rlo)); #endif DEFINE(PT_USP, offsetof(struct pt_regs, usp)); + DEFINE(PT_FRAME_SIZE, sizeof(struct pt_regs)); /* offsets into the irq_cpustat_t struct */ DEFINE(CPUSTAT_SOFTIRQ_PENDING, offsetof(irq_cpustat_t, diff --git a/arch/csky/kernel/atomic.S b/arch/csky/kernel/atomic.S index 5b84f11485ae..e73e548f7855 100644 --- a/arch/csky/kernel/atomic.S +++ b/arch/csky/kernel/atomic.S @@ -14,13 +14,19 @@ */ ENTRY(csky_cmpxchg) USPTOKSP + + RD_MEH a3 + WR_MEH a3 + mfcr a3, epc addi a3, TRAP0_SIZE - subi sp, 8 + subi sp, 16 stw a3, (sp, 0) mfcr a3, epsr stw a3, (sp, 4) + mfcr a3, usp + stw a3, (sp, 8) psrset ee #ifdef CONFIG_CPU_HAS_LDSTEX @@ -34,11 +40,11 @@ ENTRY(csky_cmpxchg) 2: sync.is #else -1: +GLOBAL(csky_cmpxchg_ldw) ldw a3, (a2) cmpne a0, a3 bt16 3f -2: +GLOBAL(csky_cmpxchg_stw) stw a1, (a2) 3: #endif @@ -47,23 +53,9 @@ ENTRY(csky_cmpxchg) mtcr a3, epc ldw a3, (sp, 4) mtcr a3, epsr - addi sp, 8 + ldw a3, (sp, 8) + mtcr a3, usp + addi sp, 16 KSPTOUSP rte END(csky_cmpxchg) - -#ifndef CONFIG_CPU_HAS_LDSTEX -/* - * Called from tlbmodified exception - */ -ENTRY(csky_cmpxchg_fixup) - mfcr a0, epc - lrw a1, 2b - cmpne a1, a0 - bt 1f - subi a1, (2b - 1b) - stw a1, (sp, LSAVE_PC) -1: - rts -END(csky_cmpxchg_fixup) -#endif diff --git a/arch/csky/kernel/dumpstack.c b/arch/csky/kernel/dumpstack.c deleted file mode 100644 index d67f9777cfd9..000000000000 --- a/arch/csky/kernel/dumpstack.c +++ /dev/null @@ -1,49 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. - -#include <linux/ptrace.h> - -int kstack_depth_to_print = 48; - -void show_trace(unsigned long *stack) -{ - unsigned long *stack_end; - unsigned long *stack_start; - unsigned long *fp; - unsigned long addr; - - addr = (unsigned long) stack & THREAD_MASK; - stack_start = (unsigned long *) addr; - stack_end = (unsigned long *) (addr + THREAD_SIZE); - - fp = stack; - pr_info("\nCall Trace:"); - - while (fp > stack_start && fp < stack_end) { -#ifdef CONFIG_STACKTRACE - addr = fp[1]; - fp = (unsigned long *) fp[0]; -#else - addr = *fp++; -#endif - if (__kernel_text_address(addr)) - pr_cont("\n[<%08lx>] %pS", addr, (void *)addr); - } - pr_cont("\n"); -} - -void show_stack(struct task_struct *task, unsigned long *stack) -{ - if (!stack) { - if (task) - stack = (unsigned long *)thread_saved_fp(task); - else -#ifdef CONFIG_STACKTRACE - asm volatile("mov %0, r8\n":"=r"(stack)::"memory"); -#else - stack = (unsigned long *)&stack; -#endif - } - - show_trace(stack); -} diff --git a/arch/csky/kernel/entry.S b/arch/csky/kernel/entry.S index a7a5b67df898..c68cdcc76d60 100644 --- a/arch/csky/kernel/entry.S +++ b/arch/csky/kernel/entry.S @@ -9,153 +9,75 @@ #include <asm/unistd.h> #include <asm/asm-offsets.h> #include <linux/threads.h> -#include <asm/setup.h> #include <asm/page.h> #include <asm/thread_info.h> -#define PTE_INDX_MSK 0xffc -#define PTE_INDX_SHIFT 10 -#define _PGDIR_SHIFT 22 - .macro zero_fp #ifdef CONFIG_STACKTRACE movi r8, 0 #endif .endm -.macro tlbop_begin name, val0, val1, val2 -ENTRY(csky_\name) - mtcr a3, ss2 - mtcr r6, ss3 - mtcr a2, ss4 - - RD_PGDR r6 - RD_MEH a3 -#ifdef CONFIG_CPU_HAS_TLBI - tlbi.vaas a3 - sync.is - - btsti a3, 31 - bf 1f - RD_PGDR_K r6 -1: -#else - bgeni a2, 31 - WR_MCIR a2 - bgeni a2, 25 - WR_MCIR a2 +.macro context_tracking +#ifdef CONFIG_CONTEXT_TRACKING_USER + mfcr a0, epsr + btsti a0, 31 + bt 1f + jbsr user_exit_callable + ldw a0, (sp, LSAVE_A0) + ldw a1, (sp, LSAVE_A1) + ldw a2, (sp, LSAVE_A2) + ldw a3, (sp, LSAVE_A3) +#if defined(__CSKYABIV1__) + ldw r6, (sp, LSAVE_A4) + ldw r7, (sp, LSAVE_A5) #endif - bclri r6, 0 - lrw a2, va_pa_offset - ld.w a2, (a2, 0) - subu r6, a2 - bseti r6, 31 - - mov a2, a3 - lsri a2, _PGDIR_SHIFT - lsli a2, 2 - addu r6, a2 - ldw r6, (r6) - - lrw a2, va_pa_offset - ld.w a2, (a2, 0) - subu r6, a2 - bseti r6, 31 - - lsri a3, PTE_INDX_SHIFT - lrw a2, PTE_INDX_MSK - and a3, a2 - addu r6, a3 - ldw a3, (r6) - - movi a2, (_PAGE_PRESENT | \val0) - and a3, a2 - cmpne a3, a2 - bt \name - - /* First read/write the page, just update the flags */ - ldw a3, (r6) - bgeni a2, PAGE_VALID_BIT - bseti a2, PAGE_ACCESSED_BIT - bseti a2, \val1 - bseti a2, \val2 - or a3, a2 - stw a3, (r6) - - /* Some cpu tlb-hardrefill bypass the cache */ -#ifdef CONFIG_CPU_NEED_TLBSYNC - movi a2, 0x22 - bseti a2, 6 - mtcr r6, cr22 - mtcr a2, cr17 - sync +1: #endif +.endm - mfcr a3, ss2 - mfcr r6, ss3 - mfcr a2, ss4 - rte -\name: - mfcr a3, ss2 - mfcr r6, ss3 - mfcr a2, ss4 +.text +ENTRY(csky_pagefault) SAVE_ALL 0 -.endm -.macro tlbop_end is_write zero_fp - RD_MEH a2 - psrset ee, ie + context_tracking + psrset ee mov a0, sp - movi a1, \is_write jbsr do_page_fault jmpi ret_from_exception -.endm - -.text - -tlbop_begin tlbinvalidl, _PAGE_READ, PAGE_VALID_BIT, PAGE_ACCESSED_BIT -tlbop_end 0 - -tlbop_begin tlbinvalids, _PAGE_WRITE, PAGE_DIRTY_BIT, PAGE_MODIFIED_BIT -tlbop_end 1 - -tlbop_begin tlbmodified, _PAGE_WRITE, PAGE_DIRTY_BIT, PAGE_MODIFIED_BIT -#ifndef CONFIG_CPU_HAS_LDSTEX -jbsr csky_cmpxchg_fixup -#endif -tlbop_end 1 ENTRY(csky_systemcall) SAVE_ALL TRAP0_SIZE zero_fp - + context_tracking psrset ee, ie - lrw r11, __NR_syscalls - cmphs syscallid, r11 /* Check nr of syscall */ + lrw r9, __NR_syscalls + cmphs syscallid, r9 /* Check nr of syscall */ bt ret_from_exception - lrw r13, sys_call_table - ixw r13, syscallid - ldw r11, (r13) - cmpnei r11, 0 + lrw r9, sys_call_table + ixw r9, syscallid + ldw syscallid, (r9) + cmpnei syscallid, 0 bf ret_from_exception mov r9, sp bmaski r10, THREAD_SHIFT andn r9, r10 - ldw r12, (r9, TINFO_FLAGS) - ANDI_R3 r12, (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_TRACEPOINT | _TIF_SYSCALL_AUDIT) - cmpnei r12, 0 + ldw r10, (r9, TINFO_FLAGS) + lrw r9, _TIF_SYSCALL_WORK + and r10, r9 + cmpnei r10, 0 bt csky_syscall_trace #if defined(__CSKYABIV2__) subi sp, 8 stw r5, (sp, 0x4) stw r4, (sp, 0x0) - jsr r11 /* Do system call */ + jsr syscallid /* Do system call */ addi sp, 8 #else - jsr r11 + jsr syscallid #endif stw a0, (sp, LSAVE_A0) /* Save return value */ jmpi ret_from_exception @@ -163,6 +85,8 @@ ENTRY(csky_systemcall) csky_syscall_trace: mov a0, sp /* sp = pt_regs pointer */ jbsr syscall_trace_enter + cmpnei a0, 0 + bt 1f /* Prepare args before do system call */ ldw a0, (sp, LSAVE_A0) ldw a1, (sp, LSAVE_A1) @@ -170,18 +94,20 @@ csky_syscall_trace: ldw a3, (sp, LSAVE_A3) #if defined(__CSKYABIV2__) subi sp, 8 - stw r5, (sp, 0x4) - stw r4, (sp, 0x0) + ldw r9, (sp, LSAVE_A4) + stw r9, (sp, 0x0) + ldw r9, (sp, LSAVE_A5) + stw r9, (sp, 0x4) + jsr syscallid /* Do system call */ + addi sp, 8 #else ldw r6, (sp, LSAVE_A4) ldw r7, (sp, LSAVE_A5) -#endif - jsr r11 /* Do system call */ -#if defined(__CSKYABIV2__) - addi sp, 8 + jsr syscallid /* Do system call */ #endif stw a0, (sp, LSAVE_A0) /* Save return value */ +1: mov a0, sp /* right now, sp --> pt_regs */ jbsr syscall_trace_exit br ret_from_exception @@ -197,18 +123,20 @@ ENTRY(ret_from_fork) mov r9, sp bmaski r10, THREAD_SHIFT andn r9, r10 - ldw r12, (r9, TINFO_FLAGS) - ANDI_R3 r12, (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_TRACEPOINT | _TIF_SYSCALL_AUDIT) - cmpnei r12, 0 + ldw r10, (r9, TINFO_FLAGS) + lrw r9, _TIF_SYSCALL_WORK + and r10, r9 + cmpnei r10, 0 bf ret_from_exception mov a0, sp /* sp = pt_regs pointer */ jbsr syscall_trace_exit ret_from_exception: - ld syscallid, (sp, LSAVE_PSR) - btsti syscallid, 31 - bt 1f + psrclr ie + ld r9, (sp, LSAVE_PSR) + btsti r9, 31 + bt 1f /* * Load address of current->thread_info, Then get address of task_struct * Get task_needreshed in task_struct @@ -217,22 +145,46 @@ ret_from_exception: bmaski r10, THREAD_SHIFT andn r9, r10 - ldw r12, (r9, TINFO_FLAGS) - andi r12, (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED) - cmpnei r12, 0 + ldw r10, (r9, TINFO_FLAGS) + lrw r9, _TIF_WORK_MASK + and r10, r9 + cmpnei r10, 0 bt exit_work +#ifdef CONFIG_CONTEXT_TRACKING_USER + jbsr user_enter_callable +#endif 1: +#ifdef CONFIG_PREEMPTION + mov r9, sp + bmaski r10, THREAD_SHIFT + andn r9, r10 + + ldw r10, (r9, TINFO_PREEMPT) + cmpnei r10, 0 + bt 2f + jbsr preempt_schedule_irq /* irq en/disable is done inside */ +2: +#endif + +#ifdef CONFIG_TRACE_IRQFLAGS + ld r10, (sp, LSAVE_PSR) + btsti r10, 6 + bf 2f + jbsr trace_hardirqs_on +2: +#endif RESTORE_ALL exit_work: - lrw syscallid, ret_from_exception - mov lr, syscallid + lrw r9, ret_from_exception + mov lr, r9 - btsti r12, TIF_NEED_RESCHED + btsti r10, TIF_NEED_RESCHED bt work_resched + psrset ie mov a0, sp - mov a1, r12 + mov a1, r10 jmpi do_notify_resume work_resched: @@ -241,6 +193,7 @@ work_resched: ENTRY(csky_trap) SAVE_ALL 0 zero_fp + context_tracking psrset ee mov a0, sp /* Push Stack pointer arg */ jbsr trap_c /* Call C-level trap handler */ @@ -254,6 +207,9 @@ ENTRY(csky_trap) ENTRY(csky_get_tls) USPTOKSP + RD_MEH a0 + WR_MEH a0 + /* increase epc for continue */ mfcr a0, epc addi a0, TRAP0_SIZE @@ -275,36 +231,17 @@ ENTRY(csky_get_tls) ENTRY(csky_irq) SAVE_ALL 0 zero_fp + context_tracking psrset ee -#ifdef CONFIG_PREEMPT - mov r9, sp /* Get current stack pointer */ - bmaski r10, THREAD_SHIFT - andn r9, r10 /* Get thread_info */ - - /* - * Get task_struct->stack.preempt_count for current, - * and increase 1. - */ - ldw r12, (r9, TINFO_PREEMPT) - addi r12, 1 - stw r12, (r9, TINFO_PREEMPT) +#ifdef CONFIG_TRACE_IRQFLAGS + jbsr trace_hardirqs_off #endif + mov a0, sp - jbsr csky_do_IRQ + jbsr generic_handle_arch_irq -#ifdef CONFIG_PREEMPT - subi r12, 1 - stw r12, (r9, TINFO_PREEMPT) - cmpnei r12, 0 - bt 2f - ldw r12, (r9, TINFO_FLAGS) - btsti r12, TIF_NEED_RESCHED - bf 2f - jbsr preempt_schedule_irq /* irq en/disable is done inside */ -#endif -2: jmpi ret_from_exception /* @@ -316,11 +253,6 @@ ENTRY(__switch_to) lrw a3, TASK_THREAD addu a3, a0 - mfcr a2, psr /* Save PSR value */ - stw a2, (a3, THREAD_SR) /* Save PSR in task struct */ - bclri a2, 6 /* Disable interrupts */ - mtcr a2, psr - SAVE_SWITCH_STACK stw sp, (a3, THREAD_KSP) @@ -331,12 +263,9 @@ ENTRY(__switch_to) ldw sp, (a3, THREAD_KSP) /* Set next kernel sp */ - ldw a2, (a3, THREAD_SR) /* Set next PSR */ - mtcr a2, psr - #if defined(__CSKYABIV2__) - addi r7, a1, TASK_THREAD_INFO - ldw tls, (r7, TINFO_TP_VALUE) + addi a3, a1, TASK_THREAD_INFO + ldw tls, (a3, TINFO_TP_VALUE) #endif RESTORE_SWITCH_STACK diff --git a/arch/csky/kernel/ftrace.c b/arch/csky/kernel/ftrace.c index 44f4880179b7..50bfcf129078 100644 --- a/arch/csky/kernel/ftrace.c +++ b/arch/csky/kernel/ftrace.c @@ -3,6 +3,7 @@ #include <linux/ftrace.h> #include <linux/uaccess.h> +#include <linux/stop_machine.h> #include <asm/cacheflush.h> #ifdef CONFIG_DYNAMIC_FTRACE @@ -71,7 +72,8 @@ static int ftrace_check_current_nop(unsigned long hook) uint16_t olds[7]; unsigned long hook_pos = hook - 2; - if (probe_kernel_read((void *)olds, (void *)hook_pos, sizeof(nops))) + if (copy_from_kernel_nofault((void *)olds, (void *)hook_pos, + sizeof(nops))) return -EFAULT; if (memcmp((void *)nops, (void *)olds, sizeof(nops))) { @@ -96,7 +98,7 @@ static int ftrace_modify_code(unsigned long hook, unsigned long target, make_jbsr(target, hook, call, nolr); - ret = probe_kernel_write((void *)hook_pos, enable ? call : nops, + ret = copy_to_kernel_nofault((void *)hook_pos, enable ? call : nops, sizeof(nops)); if (ret) return -EPERM; @@ -126,14 +128,20 @@ int ftrace_update_ftrace_func(ftrace_func_t func) { int ret = ftrace_modify_code((unsigned long)&ftrace_call, (unsigned long)func, true, true); + if (!ret) + ret = ftrace_modify_code((unsigned long)&ftrace_regs_call, + (unsigned long)func, true, true); return ret; } +#endif /* CONFIG_DYNAMIC_FTRACE */ -int __init ftrace_dyn_arch_init(void) +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS +int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, + unsigned long addr) { - return 0; + return ftrace_modify_code(rec->ip, addr, true, true); } -#endif /* CONFIG_DYNAMIC_FTRACE */ +#endif #ifdef CONFIG_FUNCTION_GRAPH_TRACER void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr, @@ -190,5 +198,37 @@ int ftrace_disable_ftrace_graph_caller(void) #endif /* CONFIG_DYNAMIC_FTRACE */ #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ +#ifdef CONFIG_DYNAMIC_FTRACE +#ifndef CONFIG_CPU_HAS_ICACHE_INS +struct ftrace_modify_param { + int command; + atomic_t cpu_count; +}; + +static int __ftrace_modify_code(void *data) +{ + struct ftrace_modify_param *param = data; + + if (atomic_inc_return(¶m->cpu_count) == 1) { + ftrace_modify_all_code(param->command); + atomic_inc(¶m->cpu_count); + } else { + while (atomic_read(¶m->cpu_count) <= num_online_cpus()) + cpu_relax(); + local_icache_inv_all(NULL); + } + + return 0; +} + +void arch_ftrace_update_code(int command) +{ + struct ftrace_modify_param param = { command, ATOMIC_INIT(0) }; + + stop_machine(__ftrace_modify_code, ¶m, cpu_online_mask); +} +#endif +#endif /* CONFIG_DYNAMIC_FTRACE */ + /* _mcount is defined in abi's mcount.S */ EXPORT_SYMBOL(_mcount); diff --git a/arch/csky/kernel/head.S b/arch/csky/kernel/head.S index 61989f9241c0..7e3e4f15b052 100644 --- a/arch/csky/kernel/head.S +++ b/arch/csky/kernel/head.S @@ -21,6 +21,17 @@ END(_start) ENTRY(_start_smp_secondary) SETUP_MMU +#ifdef CONFIG_PAGE_OFFSET_80000000 + lrw r6, secondary_msa1 + ld.w r6, (r6, 0) + mtcr r6, cr<31, 15> +#endif + + lrw r6, secondary_pgd + ld.w r6, (r6, 0) + mtcr r6, cr<28, 15> + mtcr r6, cr<29, 15> + /* set stack point */ lrw r6, secondary_stack ld.w r6, (r6, 0) diff --git a/arch/csky/kernel/io.c b/arch/csky/kernel/io.c new file mode 100644 index 000000000000..5883f13fa2b1 --- /dev/null +++ b/arch/csky/kernel/io.c @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/export.h> +#include <linux/types.h> +#include <linux/io.h> + +/* + * Copy data from IO memory space to "real" memory space. + */ +void __memcpy_fromio(void *to, const volatile void __iomem *from, size_t count) +{ + while (count && !IS_ALIGNED((unsigned long)from, 4)) { + *(u8 *)to = __raw_readb(from); + from++; + to++; + count--; + } + + while (count >= 4) { + *(u32 *)to = __raw_readl(from); + from += 4; + to += 4; + count -= 4; + } + + while (count) { + *(u8 *)to = __raw_readb(from); + from++; + to++; + count--; + } +} +EXPORT_SYMBOL(__memcpy_fromio); + +/* + * Copy data from "real" memory space to IO memory space. + */ +void __memcpy_toio(volatile void __iomem *to, const void *from, size_t count) +{ + while (count && !IS_ALIGNED((unsigned long)to, 4)) { + __raw_writeb(*(u8 *)from, to); + from++; + to++; + count--; + } + + while (count >= 4) { + __raw_writel(*(u32 *)from, to); + from += 4; + to += 4; + count -= 4; + } + + while (count) { + __raw_writeb(*(u8 *)from, to); + from++; + to++; + count--; + } +} +EXPORT_SYMBOL(__memcpy_toio); + +/* + * "memset" on IO memory space. + */ +void __memset_io(volatile void __iomem *dst, int c, size_t count) +{ + u32 qc = (u8)c; + + qc |= qc << 8; + qc |= qc << 16; + + while (count && !IS_ALIGNED((unsigned long)dst, 4)) { + __raw_writeb(c, dst); + dst++; + count--; + } + + while (count >= 4) { + __raw_writel(qc, dst); + dst += 4; + count -= 4; + } + + while (count) { + __raw_writeb(c, dst); + dst++; + count--; + } +} +EXPORT_SYMBOL(__memset_io); diff --git a/arch/csky/kernel/irq.c b/arch/csky/kernel/irq.c index 03a1930f1cbb..fcdaf3156286 100644 --- a/arch/csky/kernel/irq.c +++ b/arch/csky/kernel/irq.c @@ -15,8 +15,3 @@ void __init init_IRQ(void) setup_smp_ipi(); #endif } - -asmlinkage void __irq_entry csky_do_IRQ(struct pt_regs *regs) -{ - handle_arch_irq(regs); -} diff --git a/arch/csky/kernel/jump_label.c b/arch/csky/kernel/jump_label.c new file mode 100644 index 000000000000..d0e8b21447e1 --- /dev/null +++ b/arch/csky/kernel/jump_label.c @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include <linux/jump_label.h> +#include <linux/kernel.h> +#include <linux/memory.h> +#include <linux/mutex.h> +#include <linux/uaccess.h> +#include <asm/cacheflush.h> + +#define NOP32_HI 0xc400 +#define NOP32_LO 0x4820 +#define BSR_LINK 0xe000 + +void arch_jump_label_transform(struct jump_entry *entry, + enum jump_label_type type) +{ + unsigned long addr = jump_entry_code(entry); + u16 insn[2]; + int ret = 0; + + if (type == JUMP_LABEL_JMP) { + long offset = jump_entry_target(entry) - jump_entry_code(entry); + + if (WARN_ON(offset & 1 || offset < -67108864 || offset >= 67108864)) + return; + + offset = offset >> 1; + + insn[0] = BSR_LINK | + ((uint16_t)((unsigned long) offset >> 16) & 0x3ff); + insn[1] = (uint16_t)((unsigned long) offset & 0xffff); + } else { + insn[0] = NOP32_HI; + insn[1] = NOP32_LO; + } + + ret = copy_to_kernel_nofault((void *)addr, insn, 4); + WARN_ON(ret); + + flush_icache_range(addr, addr + 4); +} + +void arch_jump_label_transform_static(struct jump_entry *entry, + enum jump_label_type type) +{ + /* + * We use the same instructions in the arch_static_branch and + * arch_static_branch_jump inline functions, so there's no + * need to patch them up here. + * The core will call arch_jump_label_transform when those + * instructions need to be replaced. + */ + arch_jump_label_transform(entry, type); +} diff --git a/arch/csky/kernel/module.c b/arch/csky/kernel/module.c index b5ad7d9de18c..0b56a8cd12a3 100644 --- a/arch/csky/kernel/module.c +++ b/arch/csky/kernel/module.c @@ -10,7 +10,6 @@ #include <linux/string.h> #include <linux/kernel.h> #include <linux/spinlock.h> -#include <asm/pgtable.h> #ifdef CONFIG_CPU_CK810 #define IS_BSR32(hi16, lo16) (((hi16) & 0xFC00) == 0xE000) @@ -41,7 +40,7 @@ static void jsri_2_lrw_jsr(uint32_t *location) } } #else -static void inline jsri_2_lrw_jsr(uint32_t *location) +static inline void jsri_2_lrw_jsr(uint32_t *location) { return; } @@ -69,7 +68,7 @@ int apply_relocate_add(Elf32_Shdr *sechdrs, const char *strtab, *location = rel[i].r_addend + sym->st_value; break; case R_CSKY_PC32: - /* Add the value, subtract its postition */ + /* Add the value, subtract its position */ *location = rel[i].r_addend + sym->st_value - (uint32_t)location; break; diff --git a/arch/csky/kernel/perf_callchain.c b/arch/csky/kernel/perf_callchain.c index e68ff375c8f8..1612f4354087 100644 --- a/arch/csky/kernel/perf_callchain.c +++ b/arch/csky/kernel/perf_callchain.c @@ -12,12 +12,17 @@ struct stackframe { static int unwind_frame_kernel(struct stackframe *frame) { - if (kstack_end((void *)frame->fp)) + unsigned long low = (unsigned long)task_stack_page(current); + unsigned long high = low + THREAD_SIZE; + + if (unlikely(frame->fp < low || frame->fp > high)) return -EPERM; - if (frame->fp & 0x3 || frame->fp < TASK_SIZE) + + if (kstack_end((void *)frame->fp) || frame->fp & 0x3) return -EPERM; *frame = *(struct stackframe *)frame->fp; + if (__kernel_text_address(frame->lr)) { int graph = 0; @@ -44,7 +49,7 @@ static unsigned long user_backtrace(struct perf_callchain_entry_ctx *entry, { struct stackframe buftail; unsigned long lr = 0; - unsigned long *user_frame_tail = (unsigned long *)fp; + unsigned long __user *user_frame_tail = (unsigned long __user *)fp; /* Check accessibility of one struct frame_tail beyond */ if (!access_ok(user_frame_tail, sizeof(buftail))) @@ -83,10 +88,6 @@ void perf_callchain_user(struct perf_callchain_entry_ctx *entry, { unsigned long fp = 0; - /* C-SKY does not support virtualization. */ - if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) - return; - fp = regs->regs[4]; perf_callchain_store(entry, regs->pc); @@ -107,12 +108,6 @@ void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, { struct stackframe fr; - /* C-SKY does not support virtualization. */ - if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { - pr_warn("C-SKY does not support perf in guest mode!"); - return; - } - fr.fp = regs->regs[4]; fr.lr = regs->lr; walk_stackframe(&fr, entry); diff --git a/arch/csky/kernel/perf_event.c b/arch/csky/kernel/perf_event.c index 1a29f1157449..e5f18420ce64 100644 --- a/arch/csky/kernel/perf_event.c +++ b/arch/csky/kernel/perf_event.c @@ -87,7 +87,7 @@ static int csky_pmu_irq; }) /* cycle counter */ -static uint64_t csky_pmu_read_cc(void) +uint64_t csky_pmu_read_cc(void) { uint32_t lo, hi, tmp; uint64_t result; @@ -1319,7 +1319,7 @@ int csky_pmu_device_probe(struct platform_device *pdev, pr_notice("[perf] PMU request irq fail!\n"); } - ret = cpuhp_setup_state(CPUHP_AP_PERF_ONLINE, "AP_PERF_ONLINE", + ret = cpuhp_setup_state(CPUHP_AP_PERF_CSKY_ONLINE, "AP_PERF_ONLINE", csky_pmu_starting_cpu, csky_pmu_dying_cpu); if (ret) { diff --git a/arch/csky/kernel/perf_regs.c b/arch/csky/kernel/perf_regs.c index eb32838b8210..09b7f88a2d6a 100644 --- a/arch/csky/kernel/perf_regs.c +++ b/arch/csky/kernel/perf_regs.c @@ -32,8 +32,7 @@ u64 perf_reg_abi(struct task_struct *task) } void perf_get_regs_user(struct perf_regs *regs_user, - struct pt_regs *regs, - struct pt_regs *regs_user_copy) + struct pt_regs *regs) { regs_user->regs = task_pt_regs(current); regs_user->abi = perf_reg_abi(current); diff --git a/arch/csky/kernel/power.c b/arch/csky/kernel/power.c index 923ee4e381b8..86ee202906f8 100644 --- a/arch/csky/kernel/power.c +++ b/arch/csky/kernel/power.c @@ -9,16 +9,14 @@ EXPORT_SYMBOL(pm_power_off); void machine_power_off(void) { local_irq_disable(); - if (pm_power_off) - pm_power_off(); + do_kernel_power_off(); asm volatile ("bkpt"); } void machine_halt(void) { local_irq_disable(); - if (pm_power_off) - pm_power_off(); + do_kernel_power_off(); asm volatile ("bkpt"); } diff --git a/arch/csky/kernel/probes/Makefile b/arch/csky/kernel/probes/Makefile new file mode 100644 index 000000000000..1c7c6e6cb25b --- /dev/null +++ b/arch/csky/kernel/probes/Makefile @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_KPROBES) += kprobes.o decode-insn.o simulate-insn.o +obj-$(CONFIG_KPROBES) += kprobes_trampoline.o +obj-$(CONFIG_KPROBES_ON_FTRACE) += ftrace.o +obj-$(CONFIG_UPROBES) += uprobes.o decode-insn.o simulate-insn.o + +CFLAGS_REMOVE_simulate-insn.o = $(CC_FLAGS_FTRACE) diff --git a/arch/csky/kernel/probes/decode-insn.c b/arch/csky/kernel/probes/decode-insn.c new file mode 100644 index 000000000000..bbc4edc25dc9 --- /dev/null +++ b/arch/csky/kernel/probes/decode-insn.c @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: GPL-2.0+ + +#include <linux/kernel.h> +#include <linux/kprobes.h> +#include <linux/module.h> +#include <linux/kallsyms.h> +#include <asm/sections.h> + +#include "decode-insn.h" +#include "simulate-insn.h" + +/* Return: + * INSN_REJECTED If instruction is one not allowed to kprobe, + * INSN_GOOD_NO_SLOT If instruction is supported but doesn't use its slot. + */ +enum probe_insn __kprobes +csky_probe_decode_insn(probe_opcode_t *addr, struct arch_probe_insn *api) +{ + probe_opcode_t insn = le32_to_cpu(*addr); + + CSKY_INSN_SET_SIMULATE(br16, insn); + CSKY_INSN_SET_SIMULATE(bt16, insn); + CSKY_INSN_SET_SIMULATE(bf16, insn); + CSKY_INSN_SET_SIMULATE(jmp16, insn); + CSKY_INSN_SET_SIMULATE(jsr16, insn); + CSKY_INSN_SET_SIMULATE(lrw16, insn); + CSKY_INSN_SET_SIMULATE(pop16, insn); + + CSKY_INSN_SET_SIMULATE(br32, insn); + CSKY_INSN_SET_SIMULATE(bt32, insn); + CSKY_INSN_SET_SIMULATE(bf32, insn); + CSKY_INSN_SET_SIMULATE(jmp32, insn); + CSKY_INSN_SET_SIMULATE(jsr32, insn); + CSKY_INSN_SET_SIMULATE(lrw32, insn); + CSKY_INSN_SET_SIMULATE(pop32, insn); + + CSKY_INSN_SET_SIMULATE(bez32, insn); + CSKY_INSN_SET_SIMULATE(bnez32, insn); + CSKY_INSN_SET_SIMULATE(bnezad32, insn); + CSKY_INSN_SET_SIMULATE(bhsz32, insn); + CSKY_INSN_SET_SIMULATE(bhz32, insn); + CSKY_INSN_SET_SIMULATE(blsz32, insn); + CSKY_INSN_SET_SIMULATE(blz32, insn); + CSKY_INSN_SET_SIMULATE(bsr32, insn); + CSKY_INSN_SET_SIMULATE(jmpi32, insn); + CSKY_INSN_SET_SIMULATE(jsri32, insn); + + return INSN_GOOD; +} diff --git a/arch/csky/kernel/probes/decode-insn.h b/arch/csky/kernel/probes/decode-insn.h new file mode 100644 index 000000000000..9c4ad48fee0d --- /dev/null +++ b/arch/csky/kernel/probes/decode-insn.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ + +#ifndef __CSKY_KERNEL_KPROBES_DECODE_INSN_H +#define __CSKY_KERNEL_KPROBES_DECODE_INSN_H + +#include <asm/sections.h> +#include <asm/kprobes.h> + +enum probe_insn { + INSN_REJECTED, + INSN_GOOD_NO_SLOT, + INSN_GOOD, +}; + +#define is_insn32(insn) ((insn & 0xc000) == 0xc000) + +enum probe_insn __kprobes +csky_probe_decode_insn(probe_opcode_t *addr, struct arch_probe_insn *asi); + +#endif /* __CSKY_KERNEL_KPROBES_DECODE_INSN_H */ diff --git a/arch/csky/kernel/probes/ftrace.c b/arch/csky/kernel/probes/ftrace.c new file mode 100644 index 000000000000..834cffcfbce3 --- /dev/null +++ b/arch/csky/kernel/probes/ftrace.c @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/kprobes.h> + +/* Ftrace callback handler for kprobes -- called under preepmt disabled */ +void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, + struct ftrace_ops *ops, struct ftrace_regs *fregs) +{ + int bit; + bool lr_saver = false; + struct kprobe *p; + struct kprobe_ctlblk *kcb; + struct pt_regs *regs; + + bit = ftrace_test_recursion_trylock(ip, parent_ip); + if (bit < 0) + return; + + regs = ftrace_get_regs(fregs); + p = get_kprobe((kprobe_opcode_t *)ip); + if (!p) { + p = get_kprobe((kprobe_opcode_t *)(ip - MCOUNT_INSN_SIZE)); + if (unlikely(!p) || kprobe_disabled(p)) + goto out; + lr_saver = true; + } + + kcb = get_kprobe_ctlblk(); + if (kprobe_running()) { + kprobes_inc_nmissed_count(p); + } else { + unsigned long orig_ip = instruction_pointer(regs); + + if (lr_saver) + ip -= MCOUNT_INSN_SIZE; + instruction_pointer_set(regs, ip); + __this_cpu_write(current_kprobe, p); + kcb->kprobe_status = KPROBE_HIT_ACTIVE; + if (!p->pre_handler || !p->pre_handler(p, regs)) { + /* + * Emulate singlestep (and also recover regs->pc) + * as if there is a nop + */ + instruction_pointer_set(regs, + (unsigned long)p->addr + MCOUNT_INSN_SIZE); + if (unlikely(p->post_handler)) { + kcb->kprobe_status = KPROBE_HIT_SSDONE; + p->post_handler(p, regs, 0); + } + instruction_pointer_set(regs, orig_ip); + } + /* + * If pre_handler returns !0, it changes regs->pc. We have to + * skip emulating post_handler. + */ + __this_cpu_write(current_kprobe, NULL); + } +out: + ftrace_test_recursion_unlock(bit); +} +NOKPROBE_SYMBOL(kprobe_ftrace_handler); + +int arch_prepare_kprobe_ftrace(struct kprobe *p) +{ + p->ainsn.api.insn = NULL; + return 0; +} diff --git a/arch/csky/kernel/probes/kprobes.c b/arch/csky/kernel/probes/kprobes.c new file mode 100644 index 000000000000..3c6e5c725d81 --- /dev/null +++ b/arch/csky/kernel/probes/kprobes.c @@ -0,0 +1,412 @@ +// SPDX-License-Identifier: GPL-2.0+ + +#define pr_fmt(fmt) "kprobes: " fmt + +#include <linux/kprobes.h> +#include <linux/extable.h> +#include <linux/slab.h> +#include <linux/stop_machine.h> +#include <asm/ptrace.h> +#include <linux/uaccess.h> +#include <asm/sections.h> +#include <asm/cacheflush.h> + +#include "decode-insn.h" + +DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL; +DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk); + +static void __kprobes +post_kprobe_handler(struct kprobe_ctlblk *, struct pt_regs *); + +struct csky_insn_patch { + kprobe_opcode_t *addr; + u32 opcode; + atomic_t cpu_count; +}; + +static int __kprobes patch_text_cb(void *priv) +{ + struct csky_insn_patch *param = priv; + unsigned int addr = (unsigned int)param->addr; + + if (atomic_inc_return(¶m->cpu_count) == num_online_cpus()) { + *(u16 *) addr = cpu_to_le16(param->opcode); + dcache_wb_range(addr, addr + 2); + atomic_inc(¶m->cpu_count); + } else { + while (atomic_read(¶m->cpu_count) <= num_online_cpus()) + cpu_relax(); + } + + icache_inv_range(addr, addr + 2); + + return 0; +} + +static int __kprobes patch_text(kprobe_opcode_t *addr, u32 opcode) +{ + struct csky_insn_patch param = { addr, opcode, ATOMIC_INIT(0) }; + + return stop_machine_cpuslocked(patch_text_cb, ¶m, cpu_online_mask); +} + +static void __kprobes arch_prepare_ss_slot(struct kprobe *p) +{ + unsigned long offset = is_insn32(p->opcode) ? 4 : 2; + + p->ainsn.api.restore = (unsigned long)p->addr + offset; + + patch_text(p->ainsn.api.insn, p->opcode); +} + +static void __kprobes arch_prepare_simulate(struct kprobe *p) +{ + p->ainsn.api.restore = 0; +} + +static void __kprobes arch_simulate_insn(struct kprobe *p, struct pt_regs *regs) +{ + struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); + + if (p->ainsn.api.handler) + p->ainsn.api.handler((u32)p->opcode, (long)p->addr, regs); + + post_kprobe_handler(kcb, regs); +} + +int __kprobes arch_prepare_kprobe(struct kprobe *p) +{ + unsigned long probe_addr = (unsigned long)p->addr; + + if (probe_addr & 0x1) + return -EILSEQ; + + /* copy instruction */ + p->opcode = le32_to_cpu(*p->addr); + + /* decode instruction */ + switch (csky_probe_decode_insn(p->addr, &p->ainsn.api)) { + case INSN_REJECTED: /* insn not supported */ + return -EINVAL; + + case INSN_GOOD_NO_SLOT: /* insn need simulation */ + p->ainsn.api.insn = NULL; + break; + + case INSN_GOOD: /* instruction uses slot */ + p->ainsn.api.insn = get_insn_slot(); + if (!p->ainsn.api.insn) + return -ENOMEM; + break; + } + + /* prepare the instruction */ + if (p->ainsn.api.insn) + arch_prepare_ss_slot(p); + else + arch_prepare_simulate(p); + + return 0; +} + +/* install breakpoint in text */ +void __kprobes arch_arm_kprobe(struct kprobe *p) +{ + patch_text(p->addr, USR_BKPT); +} + +/* remove breakpoint from text */ +void __kprobes arch_disarm_kprobe(struct kprobe *p) +{ + patch_text(p->addr, p->opcode); +} + +void __kprobes arch_remove_kprobe(struct kprobe *p) +{ + if (p->ainsn.api.insn) { + free_insn_slot(p->ainsn.api.insn, 0); + p->ainsn.api.insn = NULL; + } +} + +static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb) +{ + kcb->prev_kprobe.kp = kprobe_running(); + kcb->prev_kprobe.status = kcb->kprobe_status; +} + +static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb) +{ + __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp); + kcb->kprobe_status = kcb->prev_kprobe.status; +} + +static void __kprobes set_current_kprobe(struct kprobe *p) +{ + __this_cpu_write(current_kprobe, p); +} + +/* + * Interrupts need to be disabled before single-step mode is set, and not + * reenabled until after single-step mode ends. + * Without disabling interrupt on local CPU, there is a chance of + * interrupt occurrence in the period of exception return and start of + * out-of-line single-step, that result in wrongly single stepping + * into the interrupt handler. + */ +static void __kprobes kprobes_save_local_irqflag(struct kprobe_ctlblk *kcb, + struct pt_regs *regs) +{ + kcb->saved_sr = regs->sr; + regs->sr &= ~BIT(6); +} + +static void __kprobes kprobes_restore_local_irqflag(struct kprobe_ctlblk *kcb, + struct pt_regs *regs) +{ + regs->sr = kcb->saved_sr; +} + +static void __kprobes +set_ss_context(struct kprobe_ctlblk *kcb, unsigned long addr, struct kprobe *p) +{ + unsigned long offset = is_insn32(p->opcode) ? 4 : 2; + + kcb->ss_ctx.ss_pending = true; + kcb->ss_ctx.match_addr = addr + offset; +} + +static void __kprobes clear_ss_context(struct kprobe_ctlblk *kcb) +{ + kcb->ss_ctx.ss_pending = false; + kcb->ss_ctx.match_addr = 0; +} + +#define TRACE_MODE_SI BIT(14) +#define TRACE_MODE_MASK ~(0x3 << 14) +#define TRACE_MODE_RUN 0 + +static void __kprobes setup_singlestep(struct kprobe *p, + struct pt_regs *regs, + struct kprobe_ctlblk *kcb, int reenter) +{ + unsigned long slot; + + if (reenter) { + save_previous_kprobe(kcb); + set_current_kprobe(p); + kcb->kprobe_status = KPROBE_REENTER; + } else { + kcb->kprobe_status = KPROBE_HIT_SS; + } + + if (p->ainsn.api.insn) { + /* prepare for single stepping */ + slot = (unsigned long)p->ainsn.api.insn; + + set_ss_context(kcb, slot, p); /* mark pending ss */ + + /* IRQs and single stepping do not mix well. */ + kprobes_save_local_irqflag(kcb, regs); + regs->sr = (regs->sr & TRACE_MODE_MASK) | TRACE_MODE_SI; + instruction_pointer_set(regs, slot); + } else { + /* insn simulation */ + arch_simulate_insn(p, regs); + } +} + +static int __kprobes reenter_kprobe(struct kprobe *p, + struct pt_regs *regs, + struct kprobe_ctlblk *kcb) +{ + switch (kcb->kprobe_status) { + case KPROBE_HIT_SSDONE: + case KPROBE_HIT_ACTIVE: + kprobes_inc_nmissed_count(p); + setup_singlestep(p, regs, kcb, 1); + break; + case KPROBE_HIT_SS: + case KPROBE_REENTER: + pr_warn("Failed to recover from reentered kprobes.\n"); + dump_kprobe(p); + BUG(); + break; + default: + WARN_ON(1); + return 0; + } + + return 1; +} + +static void __kprobes +post_kprobe_handler(struct kprobe_ctlblk *kcb, struct pt_regs *regs) +{ + struct kprobe *cur = kprobe_running(); + + if (!cur) + return; + + /* return addr restore if non-branching insn */ + if (cur->ainsn.api.restore != 0) + regs->pc = cur->ainsn.api.restore; + + /* restore back original saved kprobe variables and continue */ + if (kcb->kprobe_status == KPROBE_REENTER) { + restore_previous_kprobe(kcb); + return; + } + + /* call post handler */ + kcb->kprobe_status = KPROBE_HIT_SSDONE; + if (cur->post_handler) { + /* post_handler can hit breakpoint and single step + * again, so we enable D-flag for recursive exception. + */ + cur->post_handler(cur, regs, 0); + } + + reset_current_kprobe(); +} + +int __kprobes kprobe_fault_handler(struct pt_regs *regs, unsigned int trapnr) +{ + struct kprobe *cur = kprobe_running(); + struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); + + switch (kcb->kprobe_status) { + case KPROBE_HIT_SS: + case KPROBE_REENTER: + /* + * We are here because the instruction being single + * stepped caused a page fault. We reset the current + * kprobe and the ip points back to the probe address + * and allow the page fault handler to continue as a + * normal page fault. + */ + regs->pc = (unsigned long) cur->addr; + BUG_ON(!instruction_pointer(regs)); + + if (kcb->kprobe_status == KPROBE_REENTER) + restore_previous_kprobe(kcb); + else + reset_current_kprobe(); + + break; + case KPROBE_HIT_ACTIVE: + case KPROBE_HIT_SSDONE: + /* + * In case the user-specified fault handler returned + * zero, try to fix up. + */ + if (fixup_exception(regs)) + return 1; + } + return 0; +} + +int __kprobes +kprobe_breakpoint_handler(struct pt_regs *regs) +{ + struct kprobe *p, *cur_kprobe; + struct kprobe_ctlblk *kcb; + unsigned long addr = instruction_pointer(regs); + + kcb = get_kprobe_ctlblk(); + cur_kprobe = kprobe_running(); + + p = get_kprobe((kprobe_opcode_t *) addr); + + if (p) { + if (cur_kprobe) { + if (reenter_kprobe(p, regs, kcb)) + return 1; + } else { + /* Probe hit */ + set_current_kprobe(p); + kcb->kprobe_status = KPROBE_HIT_ACTIVE; + + /* + * If we have no pre-handler or it returned 0, we + * continue with normal processing. If we have a + * pre-handler and it returned non-zero, it will + * modify the execution path and no need to single + * stepping. Let's just reset current kprobe and exit. + * + * pre_handler can hit a breakpoint and can step thru + * before return. + */ + if (!p->pre_handler || !p->pre_handler(p, regs)) + setup_singlestep(p, regs, kcb, 0); + else + reset_current_kprobe(); + } + return 1; + } + + /* + * The breakpoint instruction was removed right + * after we hit it. Another cpu has removed + * either a probepoint or a debugger breakpoint + * at this address. In either case, no further + * handling of this interrupt is appropriate. + * Return back to original instruction, and continue. + */ + return 0; +} + +int __kprobes +kprobe_single_step_handler(struct pt_regs *regs) +{ + struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); + + if ((kcb->ss_ctx.ss_pending) + && (kcb->ss_ctx.match_addr == instruction_pointer(regs))) { + clear_ss_context(kcb); /* clear pending ss */ + + kprobes_restore_local_irqflag(kcb, regs); + regs->sr = (regs->sr & TRACE_MODE_MASK) | TRACE_MODE_RUN; + + post_kprobe_handler(kcb, regs); + return 1; + } + return 0; +} + +/* + * Provide a blacklist of symbols identifying ranges which cannot be kprobed. + * This blacklist is exposed to userspace via debugfs (kprobes/blacklist). + */ +int __init arch_populate_kprobe_blacklist(void) +{ + int ret; + + ret = kprobe_add_area_blacklist((unsigned long)__irqentry_text_start, + (unsigned long)__irqentry_text_end); + return ret; +} + +void __kprobes __used *trampoline_probe_handler(struct pt_regs *regs) +{ + return (void *)kretprobe_trampoline_handler(regs, NULL); +} + +void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri, + struct pt_regs *regs) +{ + ri->ret_addr = (kprobe_opcode_t *)regs->lr; + ri->fp = NULL; + regs->lr = (unsigned long) &__kretprobe_trampoline; +} + +int __kprobes arch_trampoline_kprobe(struct kprobe *p) +{ + return 0; +} + +int __init arch_init_kprobes(void) +{ + return 0; +} diff --git a/arch/csky/kernel/probes/kprobes_trampoline.S b/arch/csky/kernel/probes/kprobes_trampoline.S new file mode 100644 index 000000000000..ba48ad04a847 --- /dev/null +++ b/arch/csky/kernel/probes/kprobes_trampoline.S @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ + +#include <linux/linkage.h> + +#include <abi/entry.h> + +ENTRY(__kretprobe_trampoline) + SAVE_REGS_FTRACE + + mov a0, sp /* pt_regs */ + + jbsr trampoline_probe_handler + + /* use the result as the return-address */ + mov lr, a0 + + RESTORE_REGS_FTRACE + rts +ENDPROC(__kretprobe_trampoline) diff --git a/arch/csky/kernel/probes/simulate-insn.c b/arch/csky/kernel/probes/simulate-insn.c new file mode 100644 index 000000000000..d6e8d092c9b7 --- /dev/null +++ b/arch/csky/kernel/probes/simulate-insn.c @@ -0,0 +1,390 @@ +// SPDX-License-Identifier: GPL-2.0+ + +#include <linux/bitops.h> +#include <linux/kernel.h> +#include <linux/kprobes.h> + +#include "decode-insn.h" +#include "simulate-insn.h" + +static inline bool csky_insn_reg_get_val(struct pt_regs *regs, + unsigned long index, + unsigned long *ptr) +{ + if (index < 14) + *ptr = *(®s->a0 + index); + + if (index > 15 && index < 31) + *ptr = *(®s->exregs[0] + index - 16); + + switch (index) { + case 14: + *ptr = regs->usp; + break; + case 15: + *ptr = regs->lr; + break; + case 31: + *ptr = regs->tls; + break; + default: + goto fail; + } + + return true; +fail: + return false; +} + +static inline bool csky_insn_reg_set_val(struct pt_regs *regs, + unsigned long index, + unsigned long val) +{ + if (index < 14) + *(®s->a0 + index) = val; + + if (index > 15 && index < 31) + *(®s->exregs[0] + index - 16) = val; + + switch (index) { + case 14: + regs->usp = val; + break; + case 15: + regs->lr = val; + break; + case 31: + regs->tls = val; + break; + default: + goto fail; + } + + return true; +fail: + return false; +} + +void __kprobes +simulate_br16(u32 opcode, long addr, struct pt_regs *regs) +{ + instruction_pointer_set(regs, + addr + sign_extend32((opcode & 0x3ff) << 1, 9)); +} + +void __kprobes +simulate_br32(u32 opcode, long addr, struct pt_regs *regs) +{ + instruction_pointer_set(regs, + addr + sign_extend32((opcode & 0xffff0000) >> 15, 15)); +} + +void __kprobes +simulate_bt16(u32 opcode, long addr, struct pt_regs *regs) +{ + if (regs->sr & 1) + instruction_pointer_set(regs, + addr + sign_extend32((opcode & 0x3ff) << 1, 9)); + else + instruction_pointer_set(regs, addr + 2); +} + +void __kprobes +simulate_bt32(u32 opcode, long addr, struct pt_regs *regs) +{ + if (regs->sr & 1) + instruction_pointer_set(regs, + addr + sign_extend32((opcode & 0xffff0000) >> 15, 15)); + else + instruction_pointer_set(regs, addr + 4); +} + +void __kprobes +simulate_bf16(u32 opcode, long addr, struct pt_regs *regs) +{ + if (!(regs->sr & 1)) + instruction_pointer_set(regs, + addr + sign_extend32((opcode & 0x3ff) << 1, 9)); + else + instruction_pointer_set(regs, addr + 2); +} + +void __kprobes +simulate_bf32(u32 opcode, long addr, struct pt_regs *regs) +{ + if (!(regs->sr & 1)) + instruction_pointer_set(regs, + addr + sign_extend32((opcode & 0xffff0000) >> 15, 15)); + else + instruction_pointer_set(regs, addr + 4); +} + +void __kprobes +simulate_jmp16(u32 opcode, long addr, struct pt_regs *regs) +{ + unsigned long tmp = (opcode >> 2) & 0xf; + + csky_insn_reg_get_val(regs, tmp, &tmp); + + instruction_pointer_set(regs, tmp & 0xfffffffe); +} + +void __kprobes +simulate_jmp32(u32 opcode, long addr, struct pt_regs *regs) +{ + unsigned long tmp = opcode & 0x1f; + + csky_insn_reg_get_val(regs, tmp, &tmp); + + instruction_pointer_set(regs, tmp & 0xfffffffe); +} + +void __kprobes +simulate_jsr16(u32 opcode, long addr, struct pt_regs *regs) +{ + unsigned long tmp = (opcode >> 2) & 0xf; + + csky_insn_reg_get_val(regs, tmp, &tmp); + + regs->lr = addr + 2; + + instruction_pointer_set(regs, tmp & 0xfffffffe); +} + +void __kprobes +simulate_jsr32(u32 opcode, long addr, struct pt_regs *regs) +{ + unsigned long tmp = opcode & 0x1f; + + csky_insn_reg_get_val(regs, tmp, &tmp); + + regs->lr = addr + 4; + + instruction_pointer_set(regs, tmp & 0xfffffffe); +} + +void __kprobes +simulate_lrw16(u32 opcode, long addr, struct pt_regs *regs) +{ + unsigned long val; + unsigned long tmp = (opcode & 0x300) >> 3; + unsigned long offset = ((opcode & 0x1f) | tmp) << 2; + + tmp = (opcode & 0xe0) >> 5; + + val = *(unsigned int *)(instruction_pointer(regs) + offset); + + csky_insn_reg_set_val(regs, tmp, val); +} + +void __kprobes +simulate_lrw32(u32 opcode, long addr, struct pt_regs *regs) +{ + unsigned long val; + unsigned long offset = (opcode & 0xffff0000) >> 14; + unsigned long tmp = opcode & 0x0000001f; + + val = *(unsigned int *) + ((instruction_pointer(regs) + offset) & 0xfffffffc); + + csky_insn_reg_set_val(regs, tmp, val); +} + +void __kprobes +simulate_pop16(u32 opcode, long addr, struct pt_regs *regs) +{ + unsigned long *tmp = (unsigned long *)regs->usp; + int i; + + for (i = 0; i < (opcode & 0xf); i++) { + csky_insn_reg_set_val(regs, i + 4, *tmp); + tmp += 1; + } + + if (opcode & 0x10) { + csky_insn_reg_set_val(regs, 15, *tmp); + tmp += 1; + } + + regs->usp = (unsigned long)tmp; + + instruction_pointer_set(regs, regs->lr); +} + +void __kprobes +simulate_pop32(u32 opcode, long addr, struct pt_regs *regs) +{ + unsigned long *tmp = (unsigned long *)regs->usp; + int i; + + for (i = 0; i < ((opcode & 0xf0000) >> 16); i++) { + csky_insn_reg_set_val(regs, i + 4, *tmp); + tmp += 1; + } + + if (opcode & 0x100000) { + csky_insn_reg_set_val(regs, 15, *tmp); + tmp += 1; + } + + for (i = 0; i < ((opcode & 0xe00000) >> 21); i++) { + csky_insn_reg_set_val(regs, i + 16, *tmp); + tmp += 1; + } + + if (opcode & 0x1000000) { + csky_insn_reg_set_val(regs, 29, *tmp); + tmp += 1; + } + + regs->usp = (unsigned long)tmp; + + instruction_pointer_set(regs, regs->lr); +} + +void __kprobes +simulate_bez32(u32 opcode, long addr, struct pt_regs *regs) +{ + unsigned long tmp = opcode & 0x1f; + + csky_insn_reg_get_val(regs, tmp, &tmp); + + if (tmp == 0) { + instruction_pointer_set(regs, + addr + sign_extend32((opcode & 0xffff0000) >> 15, 15)); + } else + instruction_pointer_set(regs, addr + 4); +} + +void __kprobes +simulate_bnez32(u32 opcode, long addr, struct pt_regs *regs) +{ + unsigned long tmp = opcode & 0x1f; + + csky_insn_reg_get_val(regs, tmp, &tmp); + + if (tmp != 0) { + instruction_pointer_set(regs, + addr + sign_extend32((opcode & 0xffff0000) >> 15, 15)); + } else + instruction_pointer_set(regs, addr + 4); +} + +void __kprobes +simulate_bnezad32(u32 opcode, long addr, struct pt_regs *regs) +{ + unsigned long tmp = opcode & 0x1f; + long val; + + csky_insn_reg_get_val(regs, tmp, (unsigned long *)&val); + + val -= 1; + + if (val > 0) { + instruction_pointer_set(regs, + addr + sign_extend32((opcode & 0xffff0000) >> 15, 15)); + } else + instruction_pointer_set(regs, addr + 4); + + csky_insn_reg_set_val(regs, tmp, (unsigned long)val); +} + +void __kprobes +simulate_bhsz32(u32 opcode, long addr, struct pt_regs *regs) +{ + unsigned long tmp = opcode & 0x1f; + unsigned long val; + + csky_insn_reg_get_val(regs, tmp, &val); + + if ((long) val >= 0) { + instruction_pointer_set(regs, + addr + sign_extend32((opcode & 0xffff0000) >> 15, 15)); + } else + instruction_pointer_set(regs, addr + 4); +} + +void __kprobes +simulate_bhz32(u32 opcode, long addr, struct pt_regs *regs) +{ + unsigned long tmp = opcode & 0x1f; + unsigned long val; + + csky_insn_reg_get_val(regs, tmp, &val); + + if ((long) val > 0) { + instruction_pointer_set(regs, + addr + sign_extend32((opcode & 0xffff0000) >> 15, 15)); + } else + instruction_pointer_set(regs, addr + 4); +} + +void __kprobes +simulate_blsz32(u32 opcode, long addr, struct pt_regs *regs) +{ + unsigned long tmp = opcode & 0x1f; + unsigned long val; + + csky_insn_reg_get_val(regs, tmp, &val); + + if ((long) val <= 0) { + instruction_pointer_set(regs, + addr + sign_extend32((opcode & 0xffff0000) >> 15, 15)); + } else + instruction_pointer_set(regs, addr + 4); +} + +void __kprobes +simulate_blz32(u32 opcode, long addr, struct pt_regs *regs) +{ + unsigned long tmp = opcode & 0x1f; + unsigned long val; + + csky_insn_reg_get_val(regs, tmp, &val); + + if ((long) val < 0) { + instruction_pointer_set(regs, + addr + sign_extend32((opcode & 0xffff0000) >> 15, 15)); + } else + instruction_pointer_set(regs, addr + 4); +} + +void __kprobes +simulate_bsr32(u32 opcode, long addr, struct pt_regs *regs) +{ + unsigned long tmp; + + tmp = (opcode & 0xffff) << 16; + tmp |= (opcode & 0xffff0000) >> 16; + + instruction_pointer_set(regs, + addr + sign_extend32((tmp & 0x3ffffff) << 1, 15)); + + regs->lr = addr + 4; +} + +void __kprobes +simulate_jmpi32(u32 opcode, long addr, struct pt_regs *regs) +{ + unsigned long val; + unsigned long offset = ((opcode & 0xffff0000) >> 14); + + val = *(unsigned int *) + ((instruction_pointer(regs) + offset) & 0xfffffffc); + + instruction_pointer_set(regs, val); +} + +void __kprobes +simulate_jsri32(u32 opcode, long addr, struct pt_regs *regs) +{ + unsigned long val; + unsigned long offset = ((opcode & 0xffff0000) >> 14); + + val = *(unsigned int *) + ((instruction_pointer(regs) + offset) & 0xfffffffc); + + regs->lr = addr + 4; + + instruction_pointer_set(regs, val); +} diff --git a/arch/csky/kernel/probes/simulate-insn.h b/arch/csky/kernel/probes/simulate-insn.h new file mode 100644 index 000000000000..ba4cb7ef062e --- /dev/null +++ b/arch/csky/kernel/probes/simulate-insn.h @@ -0,0 +1,49 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ + +#ifndef __CSKY_KERNEL_PROBES_SIMULATE_INSN_H +#define __CSKY_KERNEL_PROBES_SIMULATE_INSN_H + +#define __CSKY_INSN_FUNCS(name, mask, val) \ +static __always_inline bool csky_insn_is_##name(probe_opcode_t code) \ +{ \ + BUILD_BUG_ON(~(mask) & (val)); \ + return (code & (mask)) == (val); \ +} \ +void simulate_##name(u32 opcode, long addr, struct pt_regs *regs); + +#define CSKY_INSN_SET_SIMULATE(name, code) \ + do { \ + if (csky_insn_is_##name(code)) { \ + api->handler = simulate_##name; \ + return INSN_GOOD_NO_SLOT; \ + } \ + } while (0) + +__CSKY_INSN_FUNCS(br16, 0xfc00, 0x0400) +__CSKY_INSN_FUNCS(bt16, 0xfc00, 0x0800) +__CSKY_INSN_FUNCS(bf16, 0xfc00, 0x0c00) +__CSKY_INSN_FUNCS(jmp16, 0xffc3, 0x7800) +__CSKY_INSN_FUNCS(jsr16, 0xffc3, 0x7801) +__CSKY_INSN_FUNCS(lrw16, 0xfc00, 0x1000) +__CSKY_INSN_FUNCS(pop16, 0xffe0, 0x1480) + +__CSKY_INSN_FUNCS(br32, 0x0000ffff, 0x0000e800) +__CSKY_INSN_FUNCS(bt32, 0x0000ffff, 0x0000e860) +__CSKY_INSN_FUNCS(bf32, 0x0000ffff, 0x0000e840) +__CSKY_INSN_FUNCS(jmp32, 0xffffffe0, 0x0000e8c0) +__CSKY_INSN_FUNCS(jsr32, 0xffffffe0, 0x0000e8e0) +__CSKY_INSN_FUNCS(lrw32, 0x0000ffe0, 0x0000ea80) +__CSKY_INSN_FUNCS(pop32, 0xfe00ffff, 0x0000ebc0) + +__CSKY_INSN_FUNCS(bez32, 0x0000ffe0, 0x0000e900) +__CSKY_INSN_FUNCS(bnez32, 0x0000ffe0, 0x0000e920) +__CSKY_INSN_FUNCS(bnezad32, 0x0000ffe0, 0x0000e820) +__CSKY_INSN_FUNCS(bhsz32, 0x0000ffe0, 0x0000e9a0) +__CSKY_INSN_FUNCS(bhz32, 0x0000ffe0, 0x0000e940) +__CSKY_INSN_FUNCS(blsz32, 0x0000ffe0, 0x0000e960) +__CSKY_INSN_FUNCS(blz32, 0x0000ffe0, 0x0000e980) +__CSKY_INSN_FUNCS(bsr32, 0x0000fc00, 0x0000e000) +__CSKY_INSN_FUNCS(jmpi32, 0x0000ffff, 0x0000eac0) +__CSKY_INSN_FUNCS(jsri32, 0x0000ffff, 0x0000eae0) + +#endif /* __CSKY_KERNEL_PROBES_SIMULATE_INSN_H */ diff --git a/arch/csky/kernel/probes/uprobes.c b/arch/csky/kernel/probes/uprobes.c new file mode 100644 index 000000000000..936bea6fd32d --- /dev/null +++ b/arch/csky/kernel/probes/uprobes.c @@ -0,0 +1,158 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2014-2016 Pratyush Anand <panand@redhat.com> + */ +#include <linux/highmem.h> +#include <linux/ptrace.h> +#include <linux/uprobes.h> +#include <asm/cacheflush.h> + +#include "decode-insn.h" + +#define UPROBE_TRAP_NR UINT_MAX + +bool is_swbp_insn(uprobe_opcode_t *insn) +{ + return (*insn & 0xffff) == UPROBE_SWBP_INSN; +} + +unsigned long uprobe_get_swbp_addr(struct pt_regs *regs) +{ + return instruction_pointer(regs); +} + +int arch_uprobe_analyze_insn(struct arch_uprobe *auprobe, struct mm_struct *mm, + unsigned long addr) +{ + probe_opcode_t insn; + + insn = *(probe_opcode_t *)(&auprobe->insn[0]); + + auprobe->insn_size = is_insn32(insn) ? 4 : 2; + + switch (csky_probe_decode_insn(&insn, &auprobe->api)) { + case INSN_REJECTED: + return -EINVAL; + + case INSN_GOOD_NO_SLOT: + auprobe->simulate = true; + break; + + default: + break; + } + + return 0; +} + +int arch_uprobe_pre_xol(struct arch_uprobe *auprobe, struct pt_regs *regs) +{ + struct uprobe_task *utask = current->utask; + + utask->autask.saved_trap_no = current->thread.trap_no; + current->thread.trap_no = UPROBE_TRAP_NR; + + instruction_pointer_set(regs, utask->xol_vaddr); + + user_enable_single_step(current); + + return 0; +} + +int arch_uprobe_post_xol(struct arch_uprobe *auprobe, struct pt_regs *regs) +{ + struct uprobe_task *utask = current->utask; + + WARN_ON_ONCE(current->thread.trap_no != UPROBE_TRAP_NR); + current->thread.trap_no = utask->autask.saved_trap_no; + + instruction_pointer_set(regs, utask->vaddr + auprobe->insn_size); + + user_disable_single_step(current); + + return 0; +} + +bool arch_uprobe_xol_was_trapped(struct task_struct *t) +{ + if (t->thread.trap_no != UPROBE_TRAP_NR) + return true; + + return false; +} + +bool arch_uprobe_skip_sstep(struct arch_uprobe *auprobe, struct pt_regs *regs) +{ + probe_opcode_t insn; + unsigned long addr; + + if (!auprobe->simulate) + return false; + + insn = *(probe_opcode_t *)(&auprobe->insn[0]); + addr = instruction_pointer(regs); + + if (auprobe->api.handler) + auprobe->api.handler(insn, addr, regs); + + return true; +} + +void arch_uprobe_abort_xol(struct arch_uprobe *auprobe, struct pt_regs *regs) +{ + struct uprobe_task *utask = current->utask; + + current->thread.trap_no = utask->autask.saved_trap_no; + + /* + * Task has received a fatal signal, so reset back to probed + * address. + */ + instruction_pointer_set(regs, utask->vaddr); + + user_disable_single_step(current); +} + +bool arch_uretprobe_is_alive(struct return_instance *ret, enum rp_check ctx, + struct pt_regs *regs) +{ + if (ctx == RP_CHECK_CHAIN_CALL) + return regs->usp <= ret->stack; + else + return regs->usp < ret->stack; +} + +unsigned long +arch_uretprobe_hijack_return_addr(unsigned long trampoline_vaddr, + struct pt_regs *regs) +{ + unsigned long ra; + + ra = regs->lr; + + regs->lr = trampoline_vaddr; + + return ra; +} + +int arch_uprobe_exception_notify(struct notifier_block *self, + unsigned long val, void *data) +{ + return NOTIFY_DONE; +} + +int uprobe_breakpoint_handler(struct pt_regs *regs) +{ + if (uprobe_pre_sstep_notifier(regs)) + return 1; + + return 0; +} + +int uprobe_single_step_handler(struct pt_regs *regs) +{ + if (uprobe_post_sstep_notifier(regs)) + return 1; + + return 0; +} diff --git a/arch/csky/kernel/process.c b/arch/csky/kernel/process.c index f320d9248a22..0c6e4b17fe00 100644 --- a/arch/csky/kernel/process.c +++ b/arch/csky/kernel/process.c @@ -2,7 +2,6 @@ // Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #include <linux/module.h> -#include <linux/version.h> #include <linux/sched.h> #include <linux/sched/task_stack.h> #include <linux/sched/debug.h> @@ -10,12 +9,19 @@ #include <linux/kallsyms.h> #include <linux/uaccess.h> #include <linux/ptrace.h> +#include <linux/elfcore.h> #include <asm/elf.h> #include <abi/reg_ops.h> struct cpuinfo_csky cpu_data[NR_CPUS]; +#ifdef CONFIG_STACKPROTECTOR +#include <linux/stackprotector.h> +unsigned long __stack_chk_guard __read_mostly; +EXPORT_SYMBOL(__stack_chk_guard); +#endif + asmlinkage void ret_from_fork(void); asmlinkage void ret_from_kernel_thread(void); @@ -24,21 +30,11 @@ asmlinkage void ret_from_kernel_thread(void); */ void flush_thread(void){} -/* - * Return saved PC from a blocked thread - */ -unsigned long thread_saved_pc(struct task_struct *tsk) -{ - struct switch_stack *sw = (struct switch_stack *)tsk->thread.ksp; - - return sw->r15; -} - -int copy_thread(unsigned long clone_flags, - unsigned long usp, - unsigned long kthread_arg, - struct task_struct *p) +int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) { + unsigned long clone_flags = args->flags; + unsigned long usp = args->stack; + unsigned long tls = args->tls; struct switch_stack *childstack; struct pt_regs *childregs = task_pt_regs(p); @@ -49,14 +45,14 @@ int copy_thread(unsigned long clone_flags, childstack = ((struct switch_stack *) childregs) - 1; memset(childstack, 0, sizeof(struct switch_stack)); - /* setup ksp for switch_to !!! */ - p->thread.ksp = (unsigned long)childstack; + /* setup thread.sp for switch_to !!! */ + p->thread.sp = (unsigned long)childstack; - if (unlikely(p->flags & PF_KTHREAD)) { + if (unlikely(args->fn)) { memset(childregs, 0, sizeof(struct pt_regs)); childstack->r15 = (unsigned long) ret_from_kernel_thread; - childstack->r10 = kthread_arg; - childstack->r9 = usp; + childstack->r10 = (unsigned long) args->fn_arg; + childstack->r9 = (unsigned long) args->fn; childregs->sr = mfcr("psr"); } else { *childregs = *(current_pt_regs()); @@ -64,7 +60,7 @@ int copy_thread(unsigned long clone_flags, childregs->usp = usp; if (clone_flags & CLONE_SETTLS) task_thread_info(p)->tp_value = childregs->tls - = childregs->regs[0]; + = tls; childregs->a0 = 0; childstack->r15 = (unsigned long) ret_from_fork; @@ -74,12 +70,11 @@ int copy_thread(unsigned long clone_flags, } /* Fill in the fpu structure for a core dump. */ -int dump_fpu(struct pt_regs *regs, struct user_fp *fpu) +int elf_core_copy_task_fpregs(struct task_struct *t, elf_fpregset_t *fpu) { memcpy(fpu, ¤t->thread.user_fp, sizeof(*fpu)); return 1; } -EXPORT_SYMBOL(dump_fpu); int dump_task_regs(struct task_struct *tsk, elf_gregset_t *pr_regs) { @@ -91,37 +86,6 @@ int dump_task_regs(struct task_struct *tsk, elf_gregset_t *pr_regs) return 1; } -unsigned long get_wchan(struct task_struct *p) -{ - unsigned long lr; - unsigned long *fp, *stack_start, *stack_end; - int count = 0; - - if (!p || p == current || p->state == TASK_RUNNING) - return 0; - - stack_start = (unsigned long *)end_of_stack(p); - stack_end = (unsigned long *)(task_stack_page(p) + THREAD_SIZE); - - fp = (unsigned long *) thread_saved_fp(p); - do { - if (fp < stack_start || fp > stack_end) - return 0; -#ifdef CONFIG_STACKTRACE - lr = fp[1]; - fp = (unsigned long *)fp[0]; -#else - lr = *fp++; -#endif - if (!in_sched_functions(lr) && - __kernel_text_address(lr)) - return lr; - } while (count++ < 16); - - return 0; -} -EXPORT_SYMBOL(get_wchan); - #ifndef CONFIG_CPU_PM_NONE void arch_cpu_idle(void) { @@ -136,6 +100,5 @@ void arch_cpu_idle(void) #ifdef CONFIG_CPU_PM_STOP asm volatile("stop\n"); #endif - local_irq_enable(); } #endif diff --git a/arch/csky/kernel/ptrace.c b/arch/csky/kernel/ptrace.c index 313623a19ecb..0f7e7b653c72 100644 --- a/arch/csky/kernel/ptrace.c +++ b/arch/csky/kernel/ptrace.c @@ -12,17 +12,16 @@ #include <linux/sched/task_stack.h> #include <linux/signal.h> #include <linux/smp.h> -#include <linux/tracehook.h> #include <linux/uaccess.h> #include <linux/user.h> #include <asm/thread_info.h> #include <asm/page.h> -#include <asm/pgtable.h> #include <asm/processor.h> #include <asm/asm-offsets.h> #include <abi/regdef.h> +#include <abi/ckmmu.h> #define CREATE_TRACE_POINTS #include <trace/events/syscalls.h> @@ -41,6 +40,9 @@ static void singlestep_disable(struct task_struct *tsk) regs = task_pt_regs(tsk); regs->sr = (regs->sr & TRACE_MODE_MASK) | TRACE_MODE_RUN; + + /* Enable irq */ + regs->sr |= BIT(6); } static void singlestep_enable(struct task_struct *tsk) @@ -49,6 +51,9 @@ static void singlestep_enable(struct task_struct *tsk) regs = task_pt_regs(tsk); regs->sr = (regs->sr & TRACE_MODE_MASK) | TRACE_MODE_SI; + + /* Disable irq */ + regs->sr &= ~BIT(6); } /* @@ -71,17 +76,14 @@ enum csky_regset { static int gpr_get(struct task_struct *target, const struct user_regset *regset, - unsigned int pos, unsigned int count, - void *kbuf, void __user *ubuf) + struct membuf to) { - struct pt_regs *regs; - - regs = task_pt_regs(target); + struct pt_regs *regs = task_pt_regs(target); /* Abiv1 regs->tls is fake and we need sync here. */ regs->tls = task_thread_info(target)->tp_value; - return user_regset_copyout(&pos, &count, &kbuf, &ubuf, regs, 0, -1); + return membuf_write(&to, regs, sizeof(*regs)); } static int gpr_set(struct task_struct *target, @@ -96,7 +98,8 @@ static int gpr_set(struct task_struct *target, if (ret) return ret; - regs.sr = task_pt_regs(target)->sr; + /* BIT(0) of regs.sr is Condition Code/Carry bit */ + regs.sr = (regs.sr & BIT(0)) | (task_pt_regs(target)->sr & ~BIT(0)); #ifdef CONFIG_CPU_HAS_HILO regs.dcsr = task_pt_regs(target)->dcsr; #endif @@ -109,8 +112,7 @@ static int gpr_set(struct task_struct *target, static int fpr_get(struct task_struct *target, const struct user_regset *regset, - unsigned int pos, unsigned int count, - void *kbuf, void __user *ubuf) + struct membuf to) { struct user_fp *regs = (struct user_fp *)&target->thread.user_fp; @@ -126,9 +128,9 @@ static int fpr_get(struct task_struct *target, for (i = 0; i < 32; i++) tmp.vr[64 + i] = regs->vr[32 + i]; - return user_regset_copyout(&pos, &count, &kbuf, &ubuf, &tmp, 0, -1); + return membuf_write(&to, &tmp, sizeof(tmp)); #else - return user_regset_copyout(&pos, &count, &kbuf, &ubuf, regs, 0, -1); + return membuf_write(&to, regs, sizeof(*regs)); #endif } @@ -168,16 +170,16 @@ static const struct user_regset csky_regsets[] = { .n = sizeof(struct pt_regs) / sizeof(u32), .size = sizeof(u32), .align = sizeof(u32), - .get = &gpr_get, - .set = &gpr_set, + .regset_get = gpr_get, + .set = gpr_set, }, [REGSET_FPR] = { .core_note_type = NT_PRFPREG, .n = sizeof(struct user_fp) / sizeof(u32), .size = sizeof(u32), .align = sizeof(u32), - .get = &fpr_get, - .set = &fpr_set, + .regset_get = fpr_get, + .set = fpr_set, }, }; @@ -193,6 +195,109 @@ const struct user_regset_view *task_user_regset_view(struct task_struct *task) return &user_csky_view; } +struct pt_regs_offset { + const char *name; + int offset; +}; + +#define REG_OFFSET_NAME(r) {.name = #r, .offset = offsetof(struct pt_regs, r)} +#define REG_OFFSET_END {.name = NULL, .offset = 0} + +static const struct pt_regs_offset regoffset_table[] = { + REG_OFFSET_NAME(tls), + REG_OFFSET_NAME(lr), + REG_OFFSET_NAME(pc), + REG_OFFSET_NAME(sr), + REG_OFFSET_NAME(usp), + REG_OFFSET_NAME(orig_a0), + REG_OFFSET_NAME(a0), + REG_OFFSET_NAME(a1), + REG_OFFSET_NAME(a2), + REG_OFFSET_NAME(a3), + REG_OFFSET_NAME(regs[0]), + REG_OFFSET_NAME(regs[1]), + REG_OFFSET_NAME(regs[2]), + REG_OFFSET_NAME(regs[3]), + REG_OFFSET_NAME(regs[4]), + REG_OFFSET_NAME(regs[5]), + REG_OFFSET_NAME(regs[6]), + REG_OFFSET_NAME(regs[7]), + REG_OFFSET_NAME(regs[8]), + REG_OFFSET_NAME(regs[9]), +#if defined(__CSKYABIV2__) + REG_OFFSET_NAME(exregs[0]), + REG_OFFSET_NAME(exregs[1]), + REG_OFFSET_NAME(exregs[2]), + REG_OFFSET_NAME(exregs[3]), + REG_OFFSET_NAME(exregs[4]), + REG_OFFSET_NAME(exregs[5]), + REG_OFFSET_NAME(exregs[6]), + REG_OFFSET_NAME(exregs[7]), + REG_OFFSET_NAME(exregs[8]), + REG_OFFSET_NAME(exregs[9]), + REG_OFFSET_NAME(exregs[10]), + REG_OFFSET_NAME(exregs[11]), + REG_OFFSET_NAME(exregs[12]), + REG_OFFSET_NAME(exregs[13]), + REG_OFFSET_NAME(exregs[14]), + REG_OFFSET_NAME(rhi), + REG_OFFSET_NAME(rlo), + REG_OFFSET_NAME(dcsr), +#endif + REG_OFFSET_END, +}; + +/** + * regs_query_register_offset() - query register offset from its name + * @name: the name of a register + * + * regs_query_register_offset() returns the offset of a register in struct + * pt_regs from its name. If the name is invalid, this returns -EINVAL; + */ +int regs_query_register_offset(const char *name) +{ + const struct pt_regs_offset *roff; + + for (roff = regoffset_table; roff->name != NULL; roff++) + if (!strcmp(roff->name, name)) + return roff->offset; + return -EINVAL; +} + +/** + * regs_within_kernel_stack() - check the address in the stack + * @regs: pt_regs which contains kernel stack pointer. + * @addr: address which is checked. + * + * regs_within_kernel_stack() checks @addr is within the kernel stack page(s). + * If @addr is within the kernel stack, it returns true. If not, returns false. + */ +static bool regs_within_kernel_stack(struct pt_regs *regs, unsigned long addr) +{ + return (addr & ~(THREAD_SIZE - 1)) == + (kernel_stack_pointer(regs) & ~(THREAD_SIZE - 1)); +} + +/** + * regs_get_kernel_stack_nth() - get Nth entry of the stack + * @regs: pt_regs which contains kernel stack pointer. + * @n: stack entry number. + * + * regs_get_kernel_stack_nth() returns @n th entry of the kernel stack which + * is specified by @regs. If the @n th entry is NOT in the kernel stack, + * this returns 0. + */ +unsigned long regs_get_kernel_stack_nth(struct pt_regs *regs, unsigned int n) +{ + unsigned long *addr = (unsigned long *)kernel_stack_pointer(regs); + + addr += n; + if (regs_within_kernel_stack(regs, (unsigned long)addr)) + return *addr; + else + return 0; +} + void ptrace_disable(struct task_struct *child) { singlestep_disable(child); @@ -212,16 +317,20 @@ long arch_ptrace(struct task_struct *child, long request, return ret; } -asmlinkage void syscall_trace_enter(struct pt_regs *regs) +asmlinkage int syscall_trace_enter(struct pt_regs *regs) { if (test_thread_flag(TIF_SYSCALL_TRACE)) - if (tracehook_report_syscall_entry(regs)) - syscall_set_nr(current, regs, -1); + if (ptrace_report_syscall_entry(regs)) + return -1; + + if (secure_computing() == -1) + return -1; if (test_thread_flag(TIF_SYSCALL_TRACEPOINT)) trace_sys_enter(regs, syscall_get_nr(current, regs)); audit_syscall_entry(regs_syscallid(regs), regs->a0, regs->a1, regs->a2, regs->a3); + return 0; } asmlinkage void syscall_trace_exit(struct pt_regs *regs) @@ -229,19 +338,132 @@ asmlinkage void syscall_trace_exit(struct pt_regs *regs) audit_syscall_exit(regs); if (test_thread_flag(TIF_SYSCALL_TRACE)) - tracehook_report_syscall_exit(regs, 0); + ptrace_report_syscall_exit(regs, 0); if (test_thread_flag(TIF_SYSCALL_TRACEPOINT)) trace_sys_exit(regs, syscall_get_return_value(current, regs)); } -extern void show_stack(struct task_struct *task, unsigned long *stack); -void show_regs(struct pt_regs *fp) +#ifdef CONFIG_CPU_CK860 +static void show_iutlb(void) { - unsigned long *sp; - unsigned char *tp; - int i; + int entry, i; + unsigned long flags; + unsigned long oldpid; + unsigned long entryhi[16], entrylo0[16], entrylo1[16]; + + oldpid = read_mmu_entryhi(); + + entry = 0x8000; + + local_irq_save(flags); + + for (i = 0; i < 16; i++) { + write_mmu_index(entry); + tlb_read(); + entryhi[i] = read_mmu_entryhi(); + entrylo0[i] = read_mmu_entrylo0(); + entrylo1[i] = read_mmu_entrylo1(); + + entry++; + } + + local_irq_restore(flags); + + write_mmu_entryhi(oldpid); + + printk("\n\n\n"); + for (i = 0; i < 16; i++) + printk("iutlb[%d]: entryhi - 0x%lx; entrylo0 - 0x%lx;" + " entrylo1 - 0x%lx\n", + i, entryhi[i], entrylo0[i], entrylo1[i]); + printk("\n\n\n"); +} +static void show_dutlb(void) +{ + int entry, i; + unsigned long flags; + unsigned long oldpid; + unsigned long entryhi[16], entrylo0[16], entrylo1[16]; + + oldpid = read_mmu_entryhi(); + + entry = 0x4000; + + local_irq_save(flags); + + for (i = 0; i < 16; i++) { + write_mmu_index(entry); + tlb_read(); + entryhi[i] = read_mmu_entryhi(); + entrylo0[i] = read_mmu_entrylo0(); + entrylo1[i] = read_mmu_entrylo1(); + + entry++; + } + + local_irq_restore(flags); + + write_mmu_entryhi(oldpid); + + printk("\n\n\n"); + for (i = 0; i < 16; i++) + printk("dutlb[%d]: entryhi - 0x%lx; entrylo0 - 0x%lx;" + " entrylo1 - 0x%lx\n", + i, entryhi[i], entrylo0[i], entrylo1[i]); + printk("\n\n\n"); +} + +static unsigned long entryhi[1024], entrylo0[1024], entrylo1[1024]; +static void show_jtlb(void) +{ + int entry; + unsigned long flags; + unsigned long oldpid; + + oldpid = read_mmu_entryhi(); + + entry = 0; + + local_irq_save(flags); + while (entry < 1024) { + write_mmu_index(entry); + tlb_read(); + entryhi[entry] = read_mmu_entryhi(); + entrylo0[entry] = read_mmu_entrylo0(); + entrylo1[entry] = read_mmu_entrylo1(); + + entry++; + } + local_irq_restore(flags); + + write_mmu_entryhi(oldpid); + + printk("\n\n\n"); + + for (entry = 0; entry < 1024; entry++) + printk("jtlb[%x]: entryhi - 0x%lx; entrylo0 - 0x%lx;" + " entrylo1 - 0x%lx\n", + entry, entryhi[entry], entrylo0[entry], entrylo1[entry]); + printk("\n\n\n"); +} + +static void show_tlb(void) +{ + show_iutlb(); + show_dutlb(); + show_jtlb(); +} +#else +static void show_tlb(void) +{ + return; +} +#endif + +void show_regs(struct pt_regs *fp) +{ pr_info("\nCURRENT PROCESS:\n\n"); pr_info("COMM=%s PID=%d\n", current->comm, current->pid); @@ -260,9 +482,10 @@ void show_regs(struct pt_regs *fp) pr_info("PC: 0x%08lx (%pS)\n", (long)fp->pc, (void *)fp->pc); pr_info("LR: 0x%08lx (%pS)\n", (long)fp->lr, (void *)fp->lr); - pr_info("SP: 0x%08lx\n", (long)fp); - pr_info("orig_a0: 0x%08lx\n", fp->orig_a0); + pr_info("SP: 0x%08lx\n", (long)fp->usp); pr_info("PSR: 0x%08lx\n", (long)fp->sr); + pr_info("orig_a0: 0x%08lx\n", fp->orig_a0); + pr_info("PT_REGS: 0x%08lx\n", (long)fp); pr_info(" a0: 0x%08lx a1: 0x%08lx a2: 0x%08lx a3: 0x%08lx\n", fp->a0, fp->a1, fp->a2, fp->a3); @@ -288,29 +511,11 @@ void show_regs(struct pt_regs *fp) fp->regs[0], fp->regs[1], fp->regs[2], fp->regs[3]); pr_info("r10: 0x%08lx r11: 0x%08lx r12: 0x%08lx r13: 0x%08lx\n", fp->regs[4], fp->regs[5], fp->regs[6], fp->regs[7]); - pr_info("r14: 0x%08lx r1: 0x%08lx r15: 0x%08lx\n", - fp->regs[8], fp->regs[9], fp->lr); + pr_info("r14: 0x%08lx r1: 0x%08lx\n", + fp->regs[8], fp->regs[9]); #endif - pr_info("\nCODE:"); - tp = ((unsigned char *) fp->pc) - 0x20; - tp += ((int)tp % 4) ? 2 : 0; - for (sp = (unsigned long *) tp, i = 0; (i < 0x40); i += 4) { - if ((i % 0x10) == 0) - pr_cont("\n%08x: ", (int) (tp + i)); - pr_cont("%08x ", (int) *sp++); - } - pr_cont("\n"); - - pr_info("\nKERNEL STACK:"); - tp = ((unsigned char *) fp) - 0x40; - for (sp = (unsigned long *) tp, i = 0; (i < 0xc0); i += 4) { - if ((i % 0x10) == 0) - pr_cont("\n%08x: ", (int) (tp + i)); - pr_cont("%08x ", (int) *sp++); - } - pr_cont("\n"); + show_tlb(); - show_stack(NULL, (unsigned long *)fp->regs[4]); return; } diff --git a/arch/csky/kernel/setup.c b/arch/csky/kernel/setup.c index 23ee604aafdb..51012e90780d 100644 --- a/arch/csky/kernel/setup.c +++ b/arch/csky/kernel/setup.c @@ -7,104 +7,54 @@ #include <linux/of.h> #include <linux/of_fdt.h> #include <linux/start_kernel.h> -#include <linux/dma-contiguous.h> -#include <linux/screen_info.h> +#include <linux/dma-map-ops.h> #include <asm/sections.h> #include <asm/mmu_context.h> #include <asm/pgalloc.h> -#ifdef CONFIG_DUMMY_CONSOLE -struct screen_info screen_info = { - .orig_video_lines = 30, - .orig_video_cols = 80, - .orig_video_mode = 0, - .orig_video_ega_bx = 0, - .orig_video_isVGA = 1, - .orig_video_points = 8 -}; -#endif - -phys_addr_t __init_memblock memblock_end_of_REG0(void) -{ - return (memblock.memory.regions[0].base + - memblock.memory.regions[0].size); -} - -phys_addr_t __init_memblock memblock_start_of_REG1(void) -{ - return memblock.memory.regions[1].base; -} - -size_t __init_memblock memblock_size_of_REG1(void) -{ - return memblock.memory.regions[1].size; -} - static void __init csky_memblock_init(void) { - unsigned long zone_size[MAX_NR_ZONES]; - unsigned long zhole_size[MAX_NR_ZONES]; + unsigned long lowmem_size = PFN_DOWN(LOWMEM_LIMIT - PHYS_OFFSET_OFFSET); + unsigned long sseg_size = PFN_DOWN(SSEG_SIZE - PHYS_OFFSET_OFFSET); + unsigned long max_zone_pfn[MAX_NR_ZONES] = { 0 }; signed long size; - memblock_reserve(__pa(_stext), _end - _stext); -#ifdef CONFIG_BLK_DEV_INITRD - memblock_reserve(__pa(initrd_start), initrd_end - initrd_start); -#endif + memblock_reserve(__pa(_start), _end - _start); early_init_fdt_reserve_self(); early_init_fdt_scan_reserved_mem(); memblock_dump_all(); - memset(zone_size, 0, sizeof(zone_size)); - memset(zhole_size, 0, sizeof(zhole_size)); - min_low_pfn = PFN_UP(memblock_start_of_DRAM()); - max_pfn = PFN_DOWN(memblock_end_of_DRAM()); - - max_low_pfn = PFN_UP(memblock_end_of_REG0()); - if (max_low_pfn == 0) - max_low_pfn = max_pfn; + max_low_pfn = max_pfn = PFN_DOWN(memblock_end_of_DRAM()); size = max_pfn - min_low_pfn; - if (memblock.memory.cnt > 1) { - zone_size[ZONE_NORMAL] = - PFN_DOWN(memblock_start_of_REG1()) - min_low_pfn; - zhole_size[ZONE_NORMAL] = - PFN_DOWN(memblock_start_of_REG1()) - max_low_pfn; - } else { - if (size <= PFN_DOWN(LOWMEM_LIMIT - PHYS_OFFSET_OFFSET)) - zone_size[ZONE_NORMAL] = max_pfn - min_low_pfn; - else { - zone_size[ZONE_NORMAL] = - PFN_DOWN(LOWMEM_LIMIT - PHYS_OFFSET_OFFSET); - max_low_pfn = min_low_pfn + zone_size[ZONE_NORMAL]; - } + if (size >= lowmem_size) { + max_low_pfn = min_low_pfn + lowmem_size; +#ifdef CONFIG_PAGE_OFFSET_80000000 + write_mmu_msa1(read_mmu_msa0() + SSEG_SIZE); +#endif + } else if (size > sseg_size) { + max_low_pfn = min_low_pfn + sseg_size; } -#ifdef CONFIG_HIGHMEM - size = 0; - if (memblock.memory.cnt > 1) { - size = PFN_DOWN(memblock_size_of_REG1()); - highstart_pfn = PFN_DOWN(memblock_start_of_REG1()); - } else { - size = max_pfn - min_low_pfn - - PFN_DOWN(LOWMEM_LIMIT - PHYS_OFFSET_OFFSET); - highstart_pfn = min_low_pfn + - PFN_DOWN(LOWMEM_LIMIT - PHYS_OFFSET_OFFSET); - } + max_zone_pfn[ZONE_NORMAL] = max_low_pfn; - if (size > 0) - zone_size[ZONE_HIGHMEM] = size; + mmu_init(min_low_pfn, max_low_pfn); - highend_pfn = max_pfn; +#ifdef CONFIG_HIGHMEM + max_zone_pfn[ZONE_HIGHMEM] = max_pfn; + + highstart_pfn = max_low_pfn; + highend_pfn = max_pfn; #endif memblock_set_current_limit(PFN_PHYS(max_low_pfn)); dma_contiguous_reserve(0); - free_area_init_node(0, zone_size, min_low_pfn, zhole_size); + free_area_init(max_zone_pfn); } void __init setup_arch(char **cmdline_p) @@ -116,10 +66,7 @@ void __init setup_arch(char **cmdline_p) pr_info("Phys. mem: %ldMB\n", (unsigned long) memblock_phys_mem_size()/1024/1024); - init_mm.start_code = (unsigned long) _stext; - init_mm.end_code = (unsigned long) _etext; - init_mm.end_data = (unsigned long) _edata; - init_mm.brk = (unsigned long) _end; + setup_initial_init_mm(_start, _etext, _edata, _end); parse_early_param(); @@ -133,28 +80,36 @@ void __init setup_arch(char **cmdline_p) sparse_init(); + fixaddr_init(); + #ifdef CONFIG_HIGHMEM kmap_init(); #endif - -#if defined(CONFIG_VT) && defined(CONFIG_DUMMY_CONSOLE) - conswitchp = &dummy_con; -#endif } unsigned long va_pa_offset; EXPORT_SYMBOL(va_pa_offset); +static inline unsigned long read_mmu_msa(void) +{ +#ifdef CONFIG_PAGE_OFFSET_80000000 + return read_mmu_msa0(); +#endif + +#ifdef CONFIG_PAGE_OFFSET_A0000000 + return read_mmu_msa1(); +#endif +} + asmlinkage __visible void __init csky_start(unsigned int unused, void *dtb_start) { /* Clean up bss section */ memset(__bss_start, 0, __bss_stop - __bss_start); - va_pa_offset = read_mmu_msa0() & ~(SSEG_SIZE - 1); + va_pa_offset = read_mmu_msa() & ~(SSEG_SIZE - 1); pre_trap_init(); - pre_mmu_init(); if (dtb_start == NULL) early_init_dt_scan(__dtb_start); diff --git a/arch/csky/kernel/signal.c b/arch/csky/kernel/signal.c index 9b1b7c039ddf..10da0fefd431 100644 --- a/arch/csky/kernel/signal.c +++ b/arch/csky/kernel/signal.c @@ -3,7 +3,7 @@ #include <linux/signal.h> #include <linux/uaccess.h> #include <linux/syscalls.h> -#include <linux/tracehook.h> +#include <linux/resume_user_mode.h> #include <asm/traps.h> #include <asm/ucontext.h> @@ -52,10 +52,14 @@ static long restore_sigcontext(struct pt_regs *regs, struct sigcontext __user *sc) { int err = 0; + unsigned long sr = regs->sr; /* sc_pt_regs is structured the same as the start of pt_regs */ err |= __copy_from_user(regs, &sc->sc_pt_regs, sizeof(struct pt_regs)); + /* BIT(0) of regs->sr is Condition Code/Carry bit */ + regs->sr = (sr & ~1) | (regs->sr & 1); + /* Restore the floating-point state. */ err |= restore_fpu_state(sc); @@ -132,9 +136,8 @@ static inline void __user *get_sigframe(struct ksignal *ksig, static int setup_rt_frame(struct ksignal *ksig, sigset_t *set, struct pt_regs *regs) { - struct rt_sigframe *frame; + struct rt_sigframe __user *frame; int err = 0; - struct csky_vdso *vdso = current->mm->context.vdso; frame = get_sigframe(ksig, regs, sizeof(*frame)); if (!access_ok(frame, sizeof(*frame))) @@ -152,7 +155,8 @@ setup_rt_frame(struct ksignal *ksig, sigset_t *set, struct pt_regs *regs) return -EFAULT; /* Set up to return from userspace. */ - regs->lr = (unsigned long)(vdso->rt_signal_retcode); + regs->lr = (unsigned long)VDSO_SYMBOL( + current->mm->context.vdso, rt_sigreturn); /* * Set up registers for signal handler. @@ -192,7 +196,7 @@ static void handle_signal(struct ksignal *ksig, struct pt_regs *regs) regs->a0 = -EINTR; break; } - /* fallthrough */ + fallthrough; case -ERESTARTNOINTR: regs->a0 = regs->orig_a0; regs->pc -= TRAP0_SIZE; @@ -251,12 +255,13 @@ static void do_signal(struct pt_regs *regs) asmlinkage void do_notify_resume(struct pt_regs *regs, unsigned long thread_info_flags) { + if (thread_info_flags & _TIF_UPROBE) + uprobe_notify_resume(regs); + /* Handle pending signal delivery */ - if (thread_info_flags & _TIF_SIGPENDING) + if (thread_info_flags & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) do_signal(regs); - if (thread_info_flags & _TIF_NOTIFY_RESUME) { - clear_thread_flag(TIF_NOTIFY_RESUME); - tracehook_notify_resume(regs); - } + if (thread_info_flags & _TIF_NOTIFY_RESUME) + resume_user_mode_work(regs); } diff --git a/arch/csky/kernel/smp.c b/arch/csky/kernel/smp.c index b753d382e4ce..92dbbf3e0205 100644 --- a/arch/csky/kernel/smp.c +++ b/arch/csky/kernel/smp.c @@ -12,8 +12,10 @@ #include <linux/delay.h> #include <linux/err.h> #include <linux/irq.h> +#include <linux/irq_work.h> #include <linux/irqdomain.h> #include <linux/of.h> +#include <linux/seq_file.h> #include <linux/sched/task_stack.h> #include <linux/sched/mm.h> #include <linux/sched/hotplug.h> @@ -21,22 +23,28 @@ #include <asm/traps.h> #include <asm/sections.h> #include <asm/mmu_context.h> -#include <asm/pgalloc.h> - -struct ipi_data_struct { - unsigned long bits ____cacheline_aligned; -}; -static DEFINE_PER_CPU(struct ipi_data_struct, ipi_data); +#ifdef CONFIG_CPU_HAS_FPU +#include <abi/fpu.h> +#endif enum ipi_message_type { IPI_EMPTY, IPI_RESCHEDULE, IPI_CALL_FUNC, + IPI_IRQ_WORK, IPI_MAX }; +struct ipi_data_struct { + unsigned long bits ____cacheline_aligned; + unsigned long stats[IPI_MAX] ____cacheline_aligned; +}; +static DEFINE_PER_CPU(struct ipi_data_struct, ipi_data); + static irqreturn_t handle_ipi(int irq, void *dev) { + unsigned long *stats = this_cpu_ptr(&ipi_data)->stats; + while (true) { unsigned long ops; @@ -44,11 +52,20 @@ static irqreturn_t handle_ipi(int irq, void *dev) if (ops == 0) return IRQ_HANDLED; - if (ops & (1 << IPI_RESCHEDULE)) + if (ops & (1 << IPI_RESCHEDULE)) { + stats[IPI_RESCHEDULE]++; scheduler_ipi(); + } - if (ops & (1 << IPI_CALL_FUNC)) + if (ops & (1 << IPI_CALL_FUNC)) { + stats[IPI_CALL_FUNC]++; generic_smp_call_function_interrupt(); + } + + if (ops & (1 << IPI_IRQ_WORK)) { + stats[IPI_IRQ_WORK]++; + irq_work_run(); + } BUG_ON((ops >> IPI_MAX) != 0); } @@ -80,6 +97,29 @@ send_ipi_message(const struct cpumask *to_whom, enum ipi_message_type operation) send_arch_ipi(to_whom); } +static const char * const ipi_names[] = { + [IPI_EMPTY] = "Empty interrupts", + [IPI_RESCHEDULE] = "Rescheduling interrupts", + [IPI_CALL_FUNC] = "Function call interrupts", + [IPI_IRQ_WORK] = "Irq work interrupts", +}; + +int arch_show_interrupts(struct seq_file *p, int prec) +{ + unsigned int cpu, i; + + for (i = 0; i < IPI_MAX; i++) { + seq_printf(p, "%*s%u:%s", prec - 1, "IPI", i, + prec >= 4 ? " " : ""); + for_each_online_cpu(cpu) + seq_printf(p, "%10lu ", + per_cpu_ptr(&ipi_data, cpu)->stats[i]); + seq_printf(p, " %s\n", ipi_names[i]); + } + + return 0; +} + void arch_send_call_function_ipi_mask(struct cpumask *mask) { send_ipi_message(mask, IPI_CALL_FUNC); @@ -100,14 +140,17 @@ void smp_send_stop(void) on_each_cpu(ipi_stop, NULL, 1); } -void smp_send_reschedule(int cpu) +void arch_smp_send_reschedule(int cpu) { send_ipi_message(cpumask_of(cpu), IPI_RESCHEDULE); } -void __init smp_prepare_boot_cpu(void) +#ifdef CONFIG_IRQ_WORK +void arch_irq_work_raise(void) { + send_ipi_message(cpumask_of(smp_processor_id()), IPI_IRQ_WORK); } +#endif void __init smp_prepare_cpus(unsigned int max_cpus) { @@ -120,7 +163,7 @@ void __init setup_smp_ipi(void) int rc; if (ipi_irq == 0) - panic("%s IRQ mapping failed\n", __func__); + return; rc = request_percpu_irq(ipi_irq, handle_ipi, "IPI Interrupt", &ipi_dummy_dev); @@ -133,15 +176,13 @@ void __init setup_smp_ipi(void) void __init setup_smp(void) { struct device_node *node = NULL; - int cpu; + unsigned int cpu; for_each_of_cpu_node(node) { if (!of_device_is_available(node)) continue; - if (of_property_read_u32(node, "reg", &cpu)) - continue; - + cpu = of_get_cpu_hwid(node, 0); if (cpu >= NR_CPUS) continue; @@ -153,8 +194,11 @@ void __init setup_smp(void) extern void _start_smp_secondary(void); volatile unsigned int secondary_hint; +volatile unsigned int secondary_hint2; volatile unsigned int secondary_ccr; volatile unsigned int secondary_stack; +volatile unsigned int secondary_msa1; +volatile unsigned int secondary_pgd; int __cpu_up(unsigned int cpu, struct task_struct *tidle) { @@ -163,7 +207,10 @@ int __cpu_up(unsigned int cpu, struct task_struct *tidle) secondary_stack = (unsigned int) task_stack_page(tidle) + THREAD_SIZE - 8; secondary_hint = mfcr("cr31"); + secondary_hint2 = mfcr("cr<21, 1>"); secondary_ccr = mfcr("cr18"); + secondary_msa1 = read_mmu_msa1(); + secondary_pgd = mfcr("cr<29, 15>"); /* * Because other CPUs are in reset status, we must flush data @@ -192,25 +239,19 @@ void __init smp_cpus_done(unsigned int max_cpus) { } -int setup_profiling_timer(unsigned int multiplier) -{ - return -EINVAL; -} - void csky_start_secondary(void) { struct mm_struct *mm = &init_mm; unsigned int cpu = smp_processor_id(); mtcr("cr31", secondary_hint); + mtcr("cr<21, 1>", secondary_hint2); mtcr("cr18", secondary_ccr); mtcr("vbr", vec_base); flush_tlb_all(); write_mmu_pagemask(0); - TLBMISS_HANDLER_SETUP_PGD(swapper_pg_dir); - TLBMISS_HANDLER_SETUP_PGD_KERNEL(swapper_pg_dir); #ifdef CONFIG_CPU_HAS_FPU init_fpu(); @@ -229,7 +270,6 @@ void csky_start_secondary(void) pr_info("CPU%u Online: %s...\n", cpu, __func__); local_irq_enable(); - preempt_disable(); cpu_startup_entry(CPUHP_AP_ONLINE_IDLE); } @@ -247,25 +287,21 @@ int __cpu_disable(void) return 0; } -void __cpu_die(unsigned int cpu) +void arch_cpuhp_cleanup_dead_cpu(unsigned int cpu) { - if (!cpu_wait_death(cpu, 5)) { - pr_crit("CPU%u: shutdown failed\n", cpu); - return; - } pr_notice("CPU%u: shutdown\n", cpu); } -void arch_cpu_idle_dead(void) +void __noreturn arch_cpu_idle_dead(void) { idle_task_exit(); - cpu_report_death(); + cpuhp_ap_report_dead(); while (!secondary_stack) arch_cpu_idle(); - local_irq_disable(); + raw_local_irq_disable(); asm volatile( "mov sp, %0\n" @@ -273,5 +309,7 @@ void arch_cpu_idle_dead(void) "jmpi csky_start_secondary" : : "r" (secondary_stack)); + + BUG(); } #endif diff --git a/arch/csky/kernel/stacktrace.c b/arch/csky/kernel/stacktrace.c index fec777a643f1..27ecd63e321b 100644 --- a/arch/csky/kernel/stacktrace.c +++ b/arch/csky/kernel/stacktrace.c @@ -1,57 +1,156 @@ // SPDX-License-Identifier: GPL-2.0 -/* Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. */ #include <linux/sched/debug.h> #include <linux/sched/task_stack.h> #include <linux/stacktrace.h> #include <linux/ftrace.h> +#include <linux/ptrace.h> -void save_stack_trace(struct stack_trace *trace) +#ifdef CONFIG_FRAME_POINTER + +struct stackframe { + unsigned long fp; + unsigned long ra; +}; + +void notrace walk_stackframe(struct task_struct *task, struct pt_regs *regs, + bool (*fn)(unsigned long, void *), void *arg) { - save_stack_trace_tsk(current, trace); + unsigned long fp, sp, pc; + + if (regs) { + fp = frame_pointer(regs); + sp = user_stack_pointer(regs); + pc = instruction_pointer(regs); + } else if (task == NULL || task == current) { + const register unsigned long current_fp __asm__ ("r8"); + fp = current_fp; + sp = current_stack_pointer; + pc = (unsigned long)walk_stackframe; + } else { + /* task blocked in __switch_to */ + fp = thread_saved_fp(task); + sp = thread_saved_sp(task); + pc = thread_saved_lr(task); + } + + for (;;) { + unsigned long low, high; + struct stackframe *frame; + + if (unlikely(!__kernel_text_address(pc) || fn(pc, arg))) + break; + + /* Validate frame pointer */ + low = sp; + high = ALIGN(sp, THREAD_SIZE); + if (unlikely(fp < low || fp > high || fp & 0x3)) + break; + /* Unwind stack frame */ + frame = (struct stackframe *)fp; + sp = fp; + fp = frame->fp; + pc = ftrace_graph_ret_addr(current, NULL, frame->ra, + (unsigned long *)(fp - 8)); + } } -EXPORT_SYMBOL_GPL(save_stack_trace); -void save_stack_trace_tsk(struct task_struct *tsk, struct stack_trace *trace) +#else /* !CONFIG_FRAME_POINTER */ + +static void notrace walk_stackframe(struct task_struct *task, + struct pt_regs *regs, bool (*fn)(unsigned long, void *), void *arg) { - unsigned long *fp, *stack_start, *stack_end; - unsigned long addr; - int skip = trace->skip; - int savesched; - int graph_idx = 0; + unsigned long sp, pc; + unsigned long *ksp; - if (tsk == current) { - asm volatile("mov %0, r8\n":"=r"(fp)); - savesched = 1; + if (regs) { + sp = user_stack_pointer(regs); + pc = instruction_pointer(regs); + } else if (task == NULL || task == current) { + sp = current_stack_pointer; + pc = (unsigned long)walk_stackframe; } else { - fp = (unsigned long *)thread_saved_fp(tsk); - savesched = 0; + /* task blocked in __switch_to */ + sp = thread_saved_sp(task); + pc = thread_saved_lr(task); } - addr = (unsigned long) fp & THREAD_MASK; - stack_start = (unsigned long *) addr; - stack_end = (unsigned long *) (addr + THREAD_SIZE); - - while (fp > stack_start && fp < stack_end) { - unsigned long lpp, fpp; + if (unlikely(sp & 0x3)) + return; - fpp = fp[0]; - lpp = fp[1]; - if (!__kernel_text_address(lpp)) + ksp = (unsigned long *)sp; + while (!kstack_end(ksp)) { + if (__kernel_text_address(pc) && unlikely(fn(pc, arg))) break; - else - lpp = ftrace_graph_ret_addr(tsk, &graph_idx, lpp, NULL); - - if (savesched || !in_sched_functions(lpp)) { - if (skip) { - skip--; - } else { - trace->entries[trace->nr_entries++] = lpp; - if (trace->nr_entries >= trace->max_entries) - break; - } - } - fp = (unsigned long *)fpp; + pc = (*ksp++) - 0x4; } } +#endif /* CONFIG_FRAME_POINTER */ + +static bool print_trace_address(unsigned long pc, void *arg) +{ + print_ip_sym((const char *)arg, pc); + return false; +} + +void show_stack(struct task_struct *task, unsigned long *sp, const char *loglvl) +{ + pr_cont("Call Trace:\n"); + walk_stackframe(task, NULL, print_trace_address, (void *)loglvl); +} + +static bool save_wchan(unsigned long pc, void *arg) +{ + if (!in_sched_functions(pc)) { + unsigned long *p = arg; + *p = pc; + return true; + } + return false; +} + +unsigned long __get_wchan(struct task_struct *task) +{ + unsigned long pc = 0; + + walk_stackframe(task, NULL, save_wchan, &pc); + return pc; +} + +#ifdef CONFIG_STACKTRACE +static bool __save_trace(unsigned long pc, void *arg, bool nosched) +{ + struct stack_trace *trace = arg; + + if (unlikely(nosched && in_sched_functions(pc))) + return false; + if (unlikely(trace->skip > 0)) { + trace->skip--; + return false; + } + + trace->entries[trace->nr_entries++] = pc; + return (trace->nr_entries >= trace->max_entries); +} + +static bool save_trace(unsigned long pc, void *arg) +{ + return __save_trace(pc, arg, false); +} + +/* + * Save stack-backtrace addresses into a stack_trace buffer. + */ +void save_stack_trace_tsk(struct task_struct *tsk, struct stack_trace *trace) +{ + walk_stackframe(tsk, NULL, save_trace, trace); +} EXPORT_SYMBOL_GPL(save_stack_trace_tsk); + +void save_stack_trace(struct stack_trace *trace) +{ + save_stack_trace_tsk(NULL, trace); +} +EXPORT_SYMBOL_GPL(save_stack_trace); + +#endif /* CONFIG_STACKTRACE */ diff --git a/arch/csky/kernel/time.c b/arch/csky/kernel/time.c index b5fc9447d93f..52379d866fe4 100644 --- a/arch/csky/kernel/time.c +++ b/arch/csky/kernel/time.c @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 // Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. -#include <linux/clk-provider.h> #include <linux/clocksource.h> +#include <linux/of_clk.h> void __init time_init(void) { diff --git a/arch/csky/kernel/traps.c b/arch/csky/kernel/traps.c index b057480e7463..c2246b07cc9c 100644 --- a/arch/csky/kernel/traps.c +++ b/arch/csky/kernel/traps.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 // Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. +#include <linux/cpu.h> #include <linux/sched.h> #include <linux/signal.h> #include <linux/kernel.h> @@ -14,6 +15,9 @@ #include <linux/kallsyms.h> #include <linux/rtc.h> #include <linux/uaccess.h> +#include <linux/kprobes.h> +#include <linux/kdebug.h> +#include <linux/sched/debug.h> #include <asm/setup.h> #include <asm/traps.h> @@ -26,6 +30,8 @@ #include <abi/fpu.h> #endif +int show_unhandled_signals = 1; + /* Defined in entry.S */ asmlinkage void csky_trap(void); @@ -34,9 +40,7 @@ asmlinkage void csky_cmpxchg(void); asmlinkage void csky_get_tls(void); asmlinkage void csky_irq(void); -asmlinkage void csky_tlbinvalidl(void); -asmlinkage void csky_tlbinvalids(void); -asmlinkage void csky_tlbmodified(void); +asmlinkage void csky_pagefault(void); /* Defined in head.S */ asmlinkage void _start_smp_secondary(void); @@ -61,9 +65,9 @@ void __init trap_init(void) VEC_INIT(VEC_TRAP3, csky_get_tls); /* setup MMU TLB exception */ - VEC_INIT(VEC_TLBINVALIDL, csky_tlbinvalidl); - VEC_INIT(VEC_TLBINVALIDS, csky_tlbinvalids); - VEC_INIT(VEC_TLBMODIFIED, csky_tlbmodified); + VEC_INIT(VEC_TLBINVALIDL, csky_pagefault); + VEC_INIT(VEC_TLBINVALIDS, csky_pagefault); + VEC_INIT(VEC_TLBMODIFIED, csky_pagefault); #ifdef CONFIG_CPU_HAS_FPU init_fpu(); @@ -76,93 +80,184 @@ void __init trap_init(void) #endif } -void die_if_kernel(char *str, struct pt_regs *regs, int nr) +static DEFINE_SPINLOCK(die_lock); + +void die(struct pt_regs *regs, const char *str) { - if (user_mode(regs)) - return; + static int die_counter; + int ret; + oops_enter(); + + spin_lock_irq(&die_lock); console_verbose(); - pr_err("%s: %08x\n", str, nr); + bust_spinlocks(1); + + pr_emerg("%s [#%d]\n", str, ++die_counter); + print_modules(); show_regs(regs); + show_stack(current, (unsigned long *)regs->regs[4], KERN_INFO); + + ret = notify_die(DIE_OOPS, str, regs, 0, trap_no(regs), SIGSEGV); + + bust_spinlocks(0); add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE); - do_exit(SIGSEGV); + spin_unlock_irq(&die_lock); + oops_exit(); + + if (in_interrupt()) + panic("Fatal exception in interrupt"); + if (panic_on_oops) + panic("Fatal exception"); + if (ret != NOTIFY_STOP) + make_task_dead(SIGSEGV); } -void buserr(struct pt_regs *regs) +void do_trap(struct pt_regs *regs, int signo, int code, unsigned long addr) { -#ifdef CONFIG_CPU_CK810 - static unsigned long prev_pc; + struct task_struct *tsk = current; - if ((regs->pc == prev_pc) && prev_pc != 0) { - prev_pc = 0; + if (show_unhandled_signals && unhandled_signal(tsk, signo) + && printk_ratelimit()) { + pr_info("%s[%d]: unhandled signal %d code 0x%x at 0x%08lx", + tsk->comm, task_pid_nr(tsk), signo, code, addr); + print_vma_addr(KERN_CONT " in ", instruction_pointer(regs)); + pr_cont("\n"); + show_regs(regs); + } + + force_sig_fault(signo, code, (void __user *)addr); +} + +static void do_trap_error(struct pt_regs *regs, int signo, int code, + unsigned long addr, const char *str) +{ + current->thread.trap_no = trap_no(regs); + + if (user_mode(regs)) { + do_trap(regs, signo, code, addr); } else { - prev_pc = regs->pc; - return; + if (!fixup_exception(regs)) + die(regs, str); } +} + +#define DO_ERROR_INFO(name, signo, code, str) \ +asmlinkage __visible void name(struct pt_regs *regs) \ +{ \ + do_trap_error(regs, signo, code, regs->pc, "Oops - " str); \ +} + +DO_ERROR_INFO(do_trap_unknown, + SIGILL, ILL_ILLTRP, "unknown exception"); +DO_ERROR_INFO(do_trap_zdiv, + SIGFPE, FPE_INTDIV, "error zero div exception"); +DO_ERROR_INFO(do_trap_buserr, + SIGSEGV, ILL_ILLADR, "error bus error exception"); + +asmlinkage void do_trap_misaligned(struct pt_regs *regs) +{ +#ifdef CONFIG_CPU_NEED_SOFTALIGN + csky_alignment(regs); +#else + current->thread.trap_no = trap_no(regs); + do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->pc, + "Oops - load/store address misaligned"); #endif +} - die_if_kernel("Kernel mode BUS error", regs, 0); +asmlinkage void do_trap_bkpt(struct pt_regs *regs) +{ +#ifdef CONFIG_KPROBES + if (kprobe_single_step_handler(regs)) + return; +#endif +#ifdef CONFIG_UPROBES + if (uprobe_single_step_handler(regs)) + return; +#endif + if (user_mode(regs)) { + send_sig(SIGTRAP, current, 0); + return; + } - pr_err("User mode Bus Error\n"); - show_regs(regs); + do_trap_error(regs, SIGILL, ILL_ILLTRP, regs->pc, + "Oops - illegal trap exception"); +} - force_sig_fault(SIGSEGV, 0, (void __user *)regs->pc); +asmlinkage void do_trap_illinsn(struct pt_regs *regs) +{ + current->thread.trap_no = trap_no(regs); + +#ifdef CONFIG_KPROBES + if (kprobe_breakpoint_handler(regs)) + return; +#endif +#ifdef CONFIG_UPROBES + if (uprobe_breakpoint_handler(regs)) + return; +#endif +#ifndef CONFIG_CPU_NO_USER_BKPT + if (*(uint16_t *)instruction_pointer(regs) != USR_BKPT) { + send_sig(SIGTRAP, current, 0); + return; + } +#endif + + do_trap_error(regs, SIGILL, ILL_ILLOPC, regs->pc, + "Oops - illegal instruction exception"); } -#define USR_BKPT 0x1464 -asmlinkage void trap_c(struct pt_regs *regs) +asmlinkage void do_trap_fpe(struct pt_regs *regs) { - int sig; - unsigned long vector; - siginfo_t info; +#ifdef CONFIG_CPU_HAS_FPU + return fpu_fpe(regs); +#else + do_trap_error(regs, SIGILL, ILL_ILLOPC, regs->pc, + "Oops - fpu instruction exception"); +#endif +} - vector = (mfcr("psr") >> 16) & 0xff; +asmlinkage void do_trap_priv(struct pt_regs *regs) +{ +#ifdef CONFIG_CPU_HAS_FPU + if (user_mode(regs) && fpu_libc_helper(regs)) + return; +#endif + do_trap_error(regs, SIGILL, ILL_PRVOPC, regs->pc, + "Oops - illegal privileged exception"); +} - switch (vector) { +asmlinkage void trap_c(struct pt_regs *regs) +{ + switch (trap_no(regs)) { case VEC_ZERODIV: - die_if_kernel("Kernel mode ZERO DIV", regs, vector); - sig = SIGFPE; + do_trap_zdiv(regs); break; - /* ptrace */ case VEC_TRACE: - info.si_code = TRAP_TRACE; - sig = SIGTRAP; + do_trap_bkpt(regs); break; case VEC_ILLEGAL: - die_if_kernel("Kernel mode ILLEGAL", regs, vector); -#ifndef CONFIG_CPU_NO_USER_BKPT - if (*(uint16_t *)instruction_pointer(regs) != USR_BKPT) -#endif - { - sig = SIGILL; - break; - } - /* gdbserver breakpoint */ + do_trap_illinsn(regs); + break; case VEC_TRAP1: - /* jtagserver breakpoint */ case VEC_BREAKPOINT: - die_if_kernel("Kernel mode BKPT", regs, vector); - info.si_code = TRAP_BRKPT; - sig = SIGTRAP; + do_trap_bkpt(regs); break; case VEC_ACCESS: - return buserr(regs); -#ifdef CONFIG_CPU_NEED_SOFTALIGN + do_trap_buserr(regs); + break; case VEC_ALIGN: - return csky_alignment(regs); -#endif -#ifdef CONFIG_CPU_HAS_FPU + do_trap_misaligned(regs); + break; case VEC_FPE: - die_if_kernel("Kernel mode FPE", regs, vector); - return fpu_fpe(regs); + do_trap_fpe(regs); + break; case VEC_PRIV: - die_if_kernel("Kernel mode PRIV", regs, vector); - if (fpu_libc_helper(regs)) - return; -#endif + do_trap_priv(regs); + break; default: - sig = SIGSEGV; + do_trap_unknown(regs); break; } - send_sig(sig, current, 0); } diff --git a/arch/csky/kernel/vdso.c b/arch/csky/kernel/vdso.c index 60ff7adfad1d..2ca886e4a458 100644 --- a/arch/csky/kernel/vdso.c +++ b/arch/csky/kernel/vdso.c @@ -1,86 +1,97 @@ // SPDX-License-Identifier: GPL-2.0 // Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. -#include <linux/kernel.h> -#include <linux/err.h> -#include <linux/sched.h> -#include <linux/mm.h> -#include <linux/init.h> #include <linux/binfmts.h> #include <linux/elf.h> -#include <linux/vmalloc.h> -#include <linux/unistd.h> -#include <linux/uaccess.h> - -#include <asm/vdso.h> -#include <asm/cacheflush.h> - -static struct page *vdso_page; +#include <linux/err.h> +#include <linux/mm.h> +#include <linux/slab.h> -static int __init init_vdso(void) -{ - struct csky_vdso *vdso; - int err = 0; +#include <asm/page.h> +#include <vdso/datapage.h> - vdso_page = alloc_page(GFP_KERNEL); - if (!vdso_page) - panic("Cannot allocate vdso"); +extern char vdso_start[], vdso_end[]; - vdso = vmap(&vdso_page, 1, 0, PAGE_KERNEL); - if (!vdso) - panic("Cannot map vdso"); +static unsigned int vdso_pages; +static struct page **vdso_pagelist; - clear_page(vdso); +static union vdso_data_store vdso_data_store __page_aligned_data; +struct vdso_data *vdso_data = vdso_data_store.data; - err = setup_vdso_page(vdso->rt_signal_retcode); - if (err) - panic("Cannot set signal return code, err: %x.", err); +static int __init vdso_init(void) +{ + unsigned int i; + + vdso_pages = (vdso_end - vdso_start) >> PAGE_SHIFT; + vdso_pagelist = + kcalloc(vdso_pages + 1, sizeof(struct page *), GFP_KERNEL); + if (unlikely(vdso_pagelist == NULL)) { + pr_err("vdso: pagelist allocation failed\n"); + return -ENOMEM; + } - dcache_wb_range((unsigned long)vdso, (unsigned long)vdso + 16); + for (i = 0; i < vdso_pages; i++) { + struct page *pg; - vunmap(vdso); + pg = virt_to_page(vdso_start + (i << PAGE_SHIFT)); + vdso_pagelist[i] = pg; + } + vdso_pagelist[i] = virt_to_page(vdso_data); return 0; } -subsys_initcall(init_vdso); +arch_initcall(vdso_init); -int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) +int arch_setup_additional_pages(struct linux_binprm *bprm, + int uses_interp) { - int ret; - unsigned long addr; struct mm_struct *mm = current->mm; + unsigned long vdso_base, vdso_len; + int ret; - down_write(&mm->mmap_sem); + vdso_len = (vdso_pages + 1) << PAGE_SHIFT; - addr = get_unmapped_area(NULL, STACK_TOP, PAGE_SIZE, 0, 0); - if (IS_ERR_VALUE(addr)) { - ret = addr; - goto up_fail; + mmap_write_lock(mm); + vdso_base = get_unmapped_area(NULL, 0, vdso_len, 0, 0); + if (IS_ERR_VALUE(vdso_base)) { + ret = vdso_base; + goto end; } - ret = install_special_mapping( - mm, - addr, - PAGE_SIZE, - VM_READ|VM_EXEC|VM_MAYREAD|VM_MAYWRITE|VM_MAYEXEC, - &vdso_page); - if (ret) - goto up_fail; + /* + * Put vDSO base into mm struct. We need to do this before calling + * install_special_mapping or the perf counter mmap tracking code + * will fail to recognise it as a vDSO (since arch_vma_name fails). + */ + mm->context.vdso = (void *)vdso_base; + + ret = + install_special_mapping(mm, vdso_base, vdso_pages << PAGE_SHIFT, + (VM_READ | VM_EXEC | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC), + vdso_pagelist); + + if (unlikely(ret)) { + mm->context.vdso = NULL; + goto end; + } - mm->context.vdso = (void *)addr; + vdso_base += (vdso_pages << PAGE_SHIFT); + ret = install_special_mapping(mm, vdso_base, PAGE_SIZE, + (VM_READ | VM_MAYREAD), &vdso_pagelist[vdso_pages]); -up_fail: - up_write(&mm->mmap_sem); + if (unlikely(ret)) + mm->context.vdso = NULL; +end: + mmap_write_unlock(mm); return ret; } const char *arch_vma_name(struct vm_area_struct *vma) { - if (vma->vm_mm == NULL) - return NULL; - - if (vma->vm_start == (long)vma->vm_mm->context.vdso) + if (vma->vm_mm && (vma->vm_start == (long)vma->vm_mm->context.vdso)) return "[vdso]"; - else - return NULL; + if (vma->vm_mm && (vma->vm_start == + (long)vma->vm_mm->context.vdso + PAGE_SIZE)) + return "[vdso_data]"; + return NULL; } diff --git a/arch/csky/kernel/vdso/.gitignore b/arch/csky/kernel/vdso/.gitignore new file mode 100644 index 000000000000..3a19def868ec --- /dev/null +++ b/arch/csky/kernel/vdso/.gitignore @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only +vdso.lds +*.tmp +vdso-syms.S diff --git a/arch/csky/kernel/vdso/Makefile b/arch/csky/kernel/vdso/Makefile new file mode 100644 index 000000000000..ddf784a62c11 --- /dev/null +++ b/arch/csky/kernel/vdso/Makefile @@ -0,0 +1,60 @@ +# SPDX-License-Identifier: GPL-2.0-only + +# Include the generic Makefile to check the built vdso. +include $(srctree)/lib/vdso/Makefile + +# Symbols present in the vdso +vdso-syms += rt_sigreturn +vdso-syms += vgettimeofday + +# Files to link into the vdso +obj-vdso = $(patsubst %, %.o, $(vdso-syms)) note.o + +ifneq ($(c-gettimeofday-y),) + CFLAGS_vgettimeofday.o += -include $(c-gettimeofday-y) +endif + +ccflags-y := -fno-stack-protector -DBUILD_VDSO32 + +# Build rules +targets := $(obj-vdso) vdso.so vdso.so.dbg vdso.lds vdso-dummy.o +obj-vdso := $(addprefix $(obj)/, $(obj-vdso)) + +obj-y += vdso.o vdso-syms.o +CPPFLAGS_vdso.lds += -P -C -U$(ARCH) + +# Disable gcov profiling for VDSO code +GCOV_PROFILE := n +KCOV_INSTRUMENT := n + +# Force dependency +$(obj)/vdso.o: $(obj)/vdso.so + +SYSCFLAGS_vdso.so.dbg = $(c_flags) +$(obj)/vdso.so.dbg: $(src)/vdso.lds $(obj-vdso) FORCE + $(call if_changed,vdsold) +SYSCFLAGS_vdso.so.dbg = -shared -s -Wl,-soname=linux-vdso.so.1 \ + -Wl,--build-id=sha1 -Wl,--hash-style=both + +$(obj)/vdso-syms.S: $(obj)/vdso.so FORCE + $(call if_changed,so2s) + +# strip rule for the .so file +$(obj)/%.so: OBJCOPYFLAGS := -S +$(obj)/%.so: $(obj)/%.so.dbg FORCE + $(call if_changed,objcopy) + +# actual build commands +# The DSO images are built using a special linker script +# Make sure only to export the intended __vdso_xxx symbol offsets. +quiet_cmd_vdsold = VDSOLD $@ + cmd_vdsold = $(CC) $(KBUILD_CFLAGS) $(call cc-option, -no-pie) -nostdlib -nostartfiles $(SYSCFLAGS_$(@F)) \ + -Wl,-T,$(filter-out FORCE,$^) -o $@.tmp && \ + $(CROSS_COMPILE)objcopy \ + $(patsubst %, -G __vdso_%, $(vdso-syms)) $@.tmp $@ && \ + rm $@.tmp + +# Extracts symbol offsets from the VDSO, converting them into an assembly file +# that contains the same symbols at the same offsets. +quiet_cmd_so2s = SO2S $@ + cmd_so2s = $(NM) -D $< | $(srctree)/$(src)/so2s.sh > $@ diff --git a/arch/csky/kernel/vdso/note.S b/arch/csky/kernel/vdso/note.S new file mode 100644 index 000000000000..2a956c942211 --- /dev/null +++ b/arch/csky/kernel/vdso/note.S @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * This supplies .note.* sections to go into the PT_NOTE inside the vDSO text. + * Here we can supply some information useful to userland. + */ + +#include <linux/elfnote.h> +#include <linux/version.h> + +ELFNOTE_START(Linux, 0, "a") + .long LINUX_VERSION_CODE +ELFNOTE_END diff --git a/arch/csky/kernel/vdso/rt_sigreturn.S b/arch/csky/kernel/vdso/rt_sigreturn.S new file mode 100644 index 000000000000..0a6bd1216118 --- /dev/null +++ b/arch/csky/kernel/vdso/rt_sigreturn.S @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#include <linux/linkage.h> +#include <asm/unistd.h> +#include <abi/vdso.h> + + .text +ENTRY(__vdso_rt_sigreturn) + .cfi_startproc + .cfi_signal_frame + SET_SYSCALL_ID + trap 0 + .cfi_endproc +ENDPROC(__vdso_rt_sigreturn) diff --git a/arch/csky/kernel/vdso/so2s.sh b/arch/csky/kernel/vdso/so2s.sh new file mode 100755 index 000000000000..69da3d529c6d --- /dev/null +++ b/arch/csky/kernel/vdso/so2s.sh @@ -0,0 +1,5 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0+ + +sed 's!\([0-9a-f]*\) T \([a-z0-9_]*\)\(@@LINUX_5.10\)*!.global \2\n.set \2,0x\1!' \ +| grep '^\.' diff --git a/arch/csky/kernel/vdso/vdso.S b/arch/csky/kernel/vdso/vdso.S new file mode 100644 index 000000000000..5162ca069494 --- /dev/null +++ b/arch/csky/kernel/vdso/vdso.S @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#include <linux/init.h> +#include <linux/linkage.h> +#include <asm/page.h> + + __PAGE_ALIGNED_DATA + + .globl vdso_start, vdso_end + .balign PAGE_SIZE +vdso_start: + .incbin "arch/csky/kernel/vdso/vdso.so" + .balign PAGE_SIZE +vdso_end: + + .previous diff --git a/arch/csky/kernel/vdso/vdso.lds.S b/arch/csky/kernel/vdso/vdso.lds.S new file mode 100644 index 000000000000..590a6c79fff7 --- /dev/null +++ b/arch/csky/kernel/vdso/vdso.lds.S @@ -0,0 +1,58 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#include <asm/page.h> + +OUTPUT_ARCH(csky) + +SECTIONS +{ + PROVIDE(_vdso_data = . + PAGE_SIZE); + . = SIZEOF_HEADERS; + + .hash : { *(.hash) } :text + .gnu.hash : { *(.gnu.hash) } + .dynsym : { *(.dynsym) } + .dynstr : { *(.dynstr) } + .gnu.version : { *(.gnu.version) } + .gnu.version_d : { *(.gnu.version_d) } + .gnu.version_r : { *(.gnu.version_r) } + + .note : { *(.note.*) } :text :note + .dynamic : { *(.dynamic) } :text :dynamic + + .eh_frame_hdr : { *(.eh_frame_hdr) } :text :eh_frame_hdr + .eh_frame : { KEEP (*(.eh_frame)) } :text + + .rodata : { *(.rodata .rodata.* .gnu.linkonce.r.*) } + + . = 0x800; + .text : { *(.text .text.*) } :text + + .data : { + *(.got.plt) *(.got) + *(.data .data.* .gnu.linkonce.d.*) + *(.dynbss) + *(.bss .bss.* .gnu.linkonce.b.*) + } +} + +PHDRS +{ + text PT_LOAD FLAGS(5) FILEHDR PHDRS; /* PF_R|PF_X */ + dynamic PT_DYNAMIC FLAGS(4); /* PF_R */ + note PT_NOTE FLAGS(4); /* PF_R */ + eh_frame_hdr PT_GNU_EH_FRAME; +} + +VERSION +{ + LINUX_5.10 { + global: + __vdso_rt_sigreturn; + __vdso_clock_gettime; + __vdso_clock_gettime64; + __vdso_gettimeofday; + __vdso_clock_getres; + local: *; + }; +} diff --git a/arch/csky/kernel/vdso/vgettimeofday.c b/arch/csky/kernel/vdso/vgettimeofday.c new file mode 100644 index 000000000000..55af30e83752 --- /dev/null +++ b/arch/csky/kernel/vdso/vgettimeofday.c @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include <linux/time.h> +#include <linux/types.h> +#include <vdso/gettime.h> + +extern +int __vdso_clock_gettime(clockid_t clock, + struct old_timespec32 *ts) +{ + return __cvdso_clock_gettime32(clock, ts); +} + +int __vdso_clock_gettime64(clockid_t clock, + struct __kernel_timespec *ts) +{ + return __cvdso_clock_gettime(clock, ts); +} + +int __vdso_gettimeofday(struct __kernel_old_timeval *tv, + struct timezone *tz) +{ + return __cvdso_gettimeofday(tv, tz); +} + +int __vdso_clock_getres(clockid_t clock_id, + struct old_timespec32 *res) +{ + return __cvdso_clock_getres_time32(clock_id, res); +} diff --git a/arch/csky/kernel/vmlinux.lds.S b/arch/csky/kernel/vmlinux.lds.S index 2ff37beaf2bf..d718961786d2 100644 --- a/arch/csky/kernel/vmlinux.lds.S +++ b/arch/csky/kernel/vmlinux.lds.S @@ -2,6 +2,7 @@ #include <asm/vmlinux.lds.h> #include <asm/page.h> +#include <asm/memory.h> OUTPUT_ARCH(csky) ENTRY(_start) @@ -21,22 +22,18 @@ SECTIONS { . = PAGE_OFFSET + PHYS_OFFSET_OFFSET; - _stext = .; - __init_begin = .; + _start = .; HEAD_TEXT_SECTION - INIT_TEXT_SECTION(PAGE_SIZE) - INIT_DATA_SECTION(PAGE_SIZE) - PERCPU_SECTION(L1_CACHE_BYTES) . = ALIGN(PAGE_SIZE); - __init_end = .; .text : AT(ADDR(.text) - LOAD_OFFSET) { _text = .; + _stext = .; + VBR_BASE IRQENTRY_TEXT SOFTIRQENTRY_TEXT TEXT_TEXT SCHED_TEXT - CPUIDLE_TEXT LOCK_TEXT KPROBES_TEXT *(.fixup) @@ -46,20 +43,73 @@ SECTIONS /* __init_begin __init_end must be page aligned for free_initmem */ . = ALIGN(PAGE_SIZE); - + __init_begin = .; + INIT_TEXT_SECTION(PAGE_SIZE) + INIT_DATA_SECTION(PAGE_SIZE) + PERCPU_SECTION(L1_CACHE_BYTES) + . = ALIGN(PAGE_SIZE); + __init_end = .; _sdata = .; RO_DATA(PAGE_SIZE) RW_DATA(L1_CACHE_BYTES, PAGE_SIZE, THREAD_SIZE) _edata = .; +#ifdef CONFIG_HAVE_TCM + .tcm_start : { + . = ALIGN(PAGE_SIZE); + __tcm_start = .; + } + + .text_data_tcm FIXADDR_TCM : AT(__tcm_start) + { + . = ALIGN(4); + __stcm_text_data = .; + *(.tcm.text) + *(.tcm.rodata) +#ifndef CONFIG_HAVE_DTCM + *(.tcm.data) +#endif + . = ALIGN(4); + __etcm_text_data = .; + } + + . = ADDR(.tcm_start) + SIZEOF(.tcm_start) + SIZEOF(.text_data_tcm); + +#ifdef CONFIG_HAVE_DTCM + #define ITCM_SIZE CONFIG_ITCM_NR_PAGES * PAGE_SIZE + + .dtcm_start : { + __dtcm_start = .; + } + + .data_tcm FIXADDR_TCM + ITCM_SIZE : AT(__dtcm_start) + { + . = ALIGN(4); + __stcm_data = .; + *(.tcm.data) + . = ALIGN(4); + __etcm_data = .; + } + + . = ADDR(.dtcm_start) + SIZEOF(.data_tcm); + + .tcm_end : AT(ADDR(.dtcm_start) + SIZEOF(.data_tcm)) { +#else + .tcm_end : AT(ADDR(.tcm_start) + SIZEOF(.text_data_tcm)) { +#endif + . = ALIGN(PAGE_SIZE); + __tcm_end = .; + } +#endif + EXCEPTION_TABLE(L1_CACHE_BYTES) BSS_SECTION(L1_CACHE_BYTES, PAGE_SIZE, L1_CACHE_BYTES) - VBR_BASE _end = . ; STABS_DEBUG DWARF_DEBUG + ELF_DETAILS DISCARDS } diff --git a/arch/csky/lib/Makefile b/arch/csky/lib/Makefile index 078e2d5f32e1..d0ce6e2d7ab2 100644 --- a/arch/csky/lib/Makefile +++ b/arch/csky/lib/Makefile @@ -1,2 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-only lib-y := usercopy.o delay.o +obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o +ifneq ($(CONFIG_HAVE_EFFICIENT_UNALIGNED_STRING_OPS), y) +lib-y += string.o +endif diff --git a/arch/csky/lib/delay.c b/arch/csky/lib/delay.c index 22570b0790d6..f5db317313bb 100644 --- a/arch/csky/lib/delay.c +++ b/arch/csky/lib/delay.c @@ -5,7 +5,7 @@ #include <linux/init.h> #include <linux/delay.h> -void __delay(unsigned long loops) +void __aligned(8) __delay(unsigned long loops) { asm volatile ( "mov r0, r0\n" diff --git a/arch/csky/lib/error-inject.c b/arch/csky/lib/error-inject.c new file mode 100644 index 000000000000..c15fb36fe067 --- /dev/null +++ b/arch/csky/lib/error-inject.c @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/error-injection.h> +#include <linux/kprobes.h> + +void override_function_with_return(struct pt_regs *regs) +{ + instruction_pointer_set(regs, regs->lr); +} +NOKPROBE_SYMBOL(override_function_with_return); diff --git a/arch/csky/lib/string.c b/arch/csky/lib/string.c new file mode 100644 index 000000000000..d65626fcaeac --- /dev/null +++ b/arch/csky/lib/string.c @@ -0,0 +1,134 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * String functions optimized for hardware which doesn't + * handle unaligned memory accesses efficiently. + * + * Copyright (C) 2021 Matteo Croce + */ + +#include <linux/types.h> +#include <linux/module.h> + +/* Minimum size for a word copy to be convenient */ +#define BYTES_LONG sizeof(long) +#define WORD_MASK (BYTES_LONG - 1) +#define MIN_THRESHOLD (BYTES_LONG * 2) + +/* convenience union to avoid cast between different pointer types */ +union types { + u8 *as_u8; + unsigned long *as_ulong; + uintptr_t as_uptr; +}; + +union const_types { + const u8 *as_u8; + unsigned long *as_ulong; + uintptr_t as_uptr; +}; + +void *memcpy(void *dest, const void *src, size_t count) +{ + union const_types s = { .as_u8 = src }; + union types d = { .as_u8 = dest }; + int distance = 0; + + if (count < MIN_THRESHOLD) + goto copy_remainder; + + /* Copy a byte at time until destination is aligned. */ + for (; d.as_uptr & WORD_MASK; count--) + *d.as_u8++ = *s.as_u8++; + + distance = s.as_uptr & WORD_MASK; + + if (distance) { + unsigned long last, next; + + /* + * s is distance bytes ahead of d, and d just reached + * the alignment boundary. Move s backward to word align it + * and shift data to compensate for distance, in order to do + * word-by-word copy. + */ + s.as_u8 -= distance; + + next = s.as_ulong[0]; + for (; count >= BYTES_LONG; count -= BYTES_LONG) { + last = next; + next = s.as_ulong[1]; + + d.as_ulong[0] = last >> (distance * 8) | + next << ((BYTES_LONG - distance) * 8); + + d.as_ulong++; + s.as_ulong++; + } + + /* Restore s with the original offset. */ + s.as_u8 += distance; + } else { + /* + * If the source and dest lower bits are the same, do a simple + * 32/64 bit wide copy. + */ + for (; count >= BYTES_LONG; count -= BYTES_LONG) + *d.as_ulong++ = *s.as_ulong++; + } + +copy_remainder: + while (count--) + *d.as_u8++ = *s.as_u8++; + + return dest; +} +EXPORT_SYMBOL(memcpy); + +/* + * Simply check if the buffer overlaps an call memcpy() in case, + * otherwise do a simple one byte at time backward copy. + */ +void *memmove(void *dest, const void *src, size_t count) +{ + if (dest < src || src + count <= dest) + return memcpy(dest, src, count); + + if (dest > src) { + const char *s = src + count; + char *tmp = dest + count; + + while (count--) + *--tmp = *--s; + } + return dest; +} +EXPORT_SYMBOL(memmove); + +void *memset(void *s, int c, size_t count) +{ + union types dest = { .as_u8 = s }; + + if (count >= MIN_THRESHOLD) { + unsigned long cu = (unsigned long)c; + + /* Compose an ulong with 'c' repeated 4/8 times */ + cu |= cu << 8; + cu |= cu << 16; + /* Suppress warning on 32 bit machines */ + cu |= (cu << 16) << 16; + + for (; count && dest.as_uptr & WORD_MASK; count--) + *dest.as_u8++ = c; + + /* Copy using the largest size allowed */ + for (; count >= BYTES_LONG; count -= BYTES_LONG) + *dest.as_ulong++ = cu; + } + + /* copy the remainder */ + while (count--) + *dest.as_u8++ = c; + + return s; +} +EXPORT_SYMBOL(memset); diff --git a/arch/csky/lib/usercopy.c b/arch/csky/lib/usercopy.c index 647a23986fb5..3c01c54421ca 100644 --- a/arch/csky/lib/usercopy.c +++ b/arch/csky/lib/usercopy.c @@ -7,10 +7,70 @@ unsigned long raw_copy_from_user(void *to, const void *from, unsigned long n) { - if (access_ok(from, n)) - __copy_user_zeroing(to, from, n); - else - memset(to, 0, n); + int tmp, nsave; + + __asm__ __volatile__( + "0: cmpnei %1, 0 \n" + " bf 7f \n" + " mov %3, %1 \n" + " or %3, %2 \n" + " andi %3, 3 \n" + " cmpnei %3, 0 \n" + " bf 1f \n" + " br 5f \n" + "1: cmplti %0, 16 \n" + " bt 3f \n" + "2: ldw %3, (%2, 0) \n" + "10: ldw %4, (%2, 4) \n" + " stw %3, (%1, 0) \n" + " stw %4, (%1, 4) \n" + "11: ldw %3, (%2, 8) \n" + "12: ldw %4, (%2, 12) \n" + " stw %3, (%1, 8) \n" + " stw %4, (%1, 12) \n" + " addi %2, 16 \n" + " addi %1, 16 \n" + " subi %0, 16 \n" + " br 1b \n" + "3: cmplti %0, 4 \n" + " bt 5f \n" + "4: ldw %3, (%2, 0) \n" + " stw %3, (%1, 0) \n" + " addi %2, 4 \n" + " addi %1, 4 \n" + " subi %0, 4 \n" + " br 3b \n" + "5: cmpnei %0, 0 \n" + " bf 7f \n" + "6: ldb %3, (%2, 0) \n" + " stb %3, (%1, 0) \n" + " addi %2, 1 \n" + " addi %1, 1 \n" + " subi %0, 1 \n" + " br 5b \n" + "8: stw %3, (%1, 0) \n" + " subi %0, 4 \n" + " bf 7f \n" + "9: subi %0, 8 \n" + " bf 7f \n" + "13: stw %3, (%1, 8) \n" + " subi %0, 12 \n" + " bf 7f \n" + ".section __ex_table, \"a\" \n" + ".align 2 \n" + ".long 2b, 7f \n" + ".long 4b, 7f \n" + ".long 6b, 7f \n" + ".long 10b, 8b \n" + ".long 11b, 9b \n" + ".long 12b,13b \n" + ".previous \n" + "7: \n" + : "=r"(n), "=r"(to), "=r"(from), "=r"(nsave), + "=r"(tmp) + : "0"(n), "1"(to), "2"(from) + : "memory"); + return n; } EXPORT_SYMBOL(raw_copy_from_user); @@ -18,229 +78,69 @@ EXPORT_SYMBOL(raw_copy_from_user); unsigned long raw_copy_to_user(void *to, const void *from, unsigned long n) { - if (access_ok(to, n)) - __copy_user(to, from, n); - return n; -} -EXPORT_SYMBOL(raw_copy_to_user); - - -/* - * copy a null terminated string from userspace. - */ -#define __do_strncpy_from_user(dst, src, count, res) \ -do { \ - int tmp; \ - long faultres; \ - asm volatile( \ - " cmpnei %3, 0 \n" \ - " bf 4f \n" \ - "1: cmpnei %1, 0 \n" \ - " bf 5f \n" \ - "2: ldb %4, (%3, 0) \n" \ - " stb %4, (%2, 0) \n" \ - " cmpnei %4, 0 \n" \ - " bf 3f \n" \ - " addi %3, 1 \n" \ - " addi %2, 1 \n" \ - " subi %1, 1 \n" \ - " br 1b \n" \ - "3: subu %0, %1 \n" \ - " br 5f \n" \ - "4: mov %0, %5 \n" \ - " br 5f \n" \ - ".section __ex_table, \"a\" \n" \ - ".align 2 \n" \ - ".long 2b, 4b \n" \ - ".previous \n" \ - "5: \n" \ - : "=r"(res), "=r"(count), "=r"(dst), \ - "=r"(src), "=r"(tmp), "=r"(faultres) \ - : "5"(-EFAULT), "0"(count), "1"(count), \ - "2"(dst), "3"(src) \ - : "memory", "cc"); \ -} while (0) - -/* - * __strncpy_from_user: - Copy a NUL terminated string from userspace, - * with less checking. - * @dst: Destination address, in kernel space. This buffer must be at - * least @count bytes long. - * @src: Source address, in user space. - * @count: Maximum number of bytes to copy, including the trailing NUL. - * - * Copies a NUL-terminated string from userspace to kernel space. - * Caller must check the specified block with access_ok() before calling - * this function. - * - * On success, returns the length of the string (not including the trailing - * NUL). - * - * If access to userspace fails, returns -EFAULT (some data may have been - * copied). - * - * If @count is smaller than the length of the string, copies @count bytes - * and returns @count. - */ -long __strncpy_from_user(char *dst, const char *src, long count) -{ - long res; - - __do_strncpy_from_user(dst, src, count, res); - return res; -} -EXPORT_SYMBOL(__strncpy_from_user); - -/* - * strncpy_from_user: - Copy a NUL terminated string from userspace. - * @dst: Destination address, in kernel space. This buffer must be at - * least @count bytes long. - * @src: Source address, in user space. - * @count: Maximum number of bytes to copy, including the trailing NUL. - * - * Copies a NUL-terminated string from userspace to kernel space. - * - * On success, returns the length of the string (not including the trailing - * NUL). - * - * If access to userspace fails, returns -EFAULT (some data may have been - * copied). - * - * If @count is smaller than the length of the string, copies @count bytes - * and returns @count. - */ -long strncpy_from_user(char *dst, const char *src, long count) -{ - long res = -EFAULT; - - if (access_ok(src, 1)) - __do_strncpy_from_user(dst, src, count, res); - return res; -} -EXPORT_SYMBOL(strncpy_from_user); - -/* - * strlen_user: - Get the size of a string in user space. - * @str: The string to measure. - * @n: The maximum valid length - * - * Get the size of a NUL-terminated string in user space. - * - * Returns the size of the string INCLUDING the terminating NUL. - * On exception, returns 0. - * If the string is too long, returns a value greater than @n. - */ -long strnlen_user(const char *s, long n) -{ - unsigned long res, tmp; - - if (s == NULL) - return 0; - - asm volatile( - " cmpnei %1, 0 \n" - " bf 3f \n" - "1: cmpnei %0, 0 \n" - " bf 3f \n" - "2: ldb %3, (%1, 0) \n" + int w0, w1, w2, w3; + + __asm__ __volatile__( + "0: cmpnei %1, 0 \n" + " bf 8f \n" + " mov %3, %1 \n" + " or %3, %2 \n" + " andi %3, 3 \n" " cmpnei %3, 0 \n" - " bf 3f \n" - " subi %0, 1 \n" - " addi %1, 1 \n" + " bf 1f \n" + " br 5f \n" + "1: cmplti %0, 16 \n" /* 4W */ + " bt 3f \n" + " ldw %3, (%2, 0) \n" + " ldw %4, (%2, 4) \n" + " ldw %5, (%2, 8) \n" + " ldw %6, (%2, 12) \n" + "2: stw %3, (%1, 0) \n" + "9: stw %4, (%1, 4) \n" + "10: stw %5, (%1, 8) \n" + "11: stw %6, (%1, 12) \n" + " addi %2, 16 \n" + " addi %1, 16 \n" + " subi %0, 16 \n" " br 1b \n" - "3: subu %2, %0 \n" + "3: cmplti %0, 4 \n" /* 1W */ + " bt 5f \n" + " ldw %3, (%2, 0) \n" + "4: stw %3, (%1, 0) \n" + " addi %2, 4 \n" + " addi %1, 4 \n" + " subi %0, 4 \n" + " br 3b \n" + "5: cmpnei %0, 0 \n" /* 1B */ + " bf 13f \n" + " ldb %3, (%2, 0) \n" + "6: stb %3, (%1, 0) \n" " addi %2, 1 \n" - " br 5f \n" - "4: movi %0, 0 \n" - " br 5f \n" + " addi %1, 1 \n" + " subi %0, 1 \n" + " br 5b \n" + "7: subi %0, 4 \n" + "8: subi %0, 4 \n" + "12: subi %0, 4 \n" + " br 13f \n" ".section __ex_table, \"a\" \n" ".align 2 \n" - ".long 2b, 4b \n" + ".long 2b, 13f \n" + ".long 4b, 13f \n" + ".long 6b, 13f \n" + ".long 9b, 12b \n" + ".long 10b, 8b \n" + ".long 11b, 7b \n" ".previous \n" - "5: \n" - : "=r"(n), "=r"(s), "=r"(res), "=r"(tmp) - : "0"(n), "1"(s), "2"(n) - : "memory", "cc"); + "13: \n" + : "=r"(n), "=r"(to), "=r"(from), "=r"(w0), + "=r"(w1), "=r"(w2), "=r"(w3) + : "0"(n), "1"(to), "2"(from) + : "memory"); - return res; -} -EXPORT_SYMBOL(strnlen_user); - -#define __do_clear_user(addr, size) \ -do { \ - int __d0, zvalue, tmp; \ - \ - asm volatile( \ - "0: cmpnei %1, 0 \n" \ - " bf 7f \n" \ - " mov %3, %1 \n" \ - " andi %3, 3 \n" \ - " cmpnei %3, 0 \n" \ - " bf 1f \n" \ - " br 5f \n" \ - "1: cmplti %0, 32 \n" /* 4W */ \ - " bt 3f \n" \ - "8: stw %2, (%1, 0) \n" \ - "10: stw %2, (%1, 4) \n" \ - "11: stw %2, (%1, 8) \n" \ - "12: stw %2, (%1, 12) \n" \ - "13: stw %2, (%1, 16) \n" \ - "14: stw %2, (%1, 20) \n" \ - "15: stw %2, (%1, 24) \n" \ - "16: stw %2, (%1, 28) \n" \ - " addi %1, 32 \n" \ - " subi %0, 32 \n" \ - " br 1b \n" \ - "3: cmplti %0, 4 \n" /* 1W */ \ - " bt 5f \n" \ - "4: stw %2, (%1, 0) \n" \ - " addi %1, 4 \n" \ - " subi %0, 4 \n" \ - " br 3b \n" \ - "5: cmpnei %0, 0 \n" /* 1B */ \ - "9: bf 7f \n" \ - "6: stb %2, (%1, 0) \n" \ - " addi %1, 1 \n" \ - " subi %0, 1 \n" \ - " br 5b \n" \ - ".section __ex_table,\"a\" \n" \ - ".align 2 \n" \ - ".long 8b, 9b \n" \ - ".long 10b, 9b \n" \ - ".long 11b, 9b \n" \ - ".long 12b, 9b \n" \ - ".long 13b, 9b \n" \ - ".long 14b, 9b \n" \ - ".long 15b, 9b \n" \ - ".long 16b, 9b \n" \ - ".long 4b, 9b \n" \ - ".long 6b, 9b \n" \ - ".previous \n" \ - "7: \n" \ - : "=r"(size), "=r" (__d0), \ - "=r"(zvalue), "=r"(tmp) \ - : "0"(size), "1"(addr), "2"(0) \ - : "memory", "cc"); \ -} while (0) - -/* - * clear_user: - Zero a block of memory in user space. - * @to: Destination address, in user space. - * @n: Number of bytes to zero. - * - * Zero a block of memory in user space. - * - * Returns number of bytes that could not be cleared. - * On success, this will be zero. - */ -unsigned long -clear_user(void __user *to, unsigned long n) -{ - if (access_ok(to, n)) - __do_clear_user(to, n); return n; } -EXPORT_SYMBOL(clear_user); +EXPORT_SYMBOL(raw_copy_to_user); /* * __clear_user: - Zero a block of memory in user space, with less checking. @@ -256,7 +156,59 @@ EXPORT_SYMBOL(clear_user); unsigned long __clear_user(void __user *to, unsigned long n) { - __do_clear_user(to, n); + int data, value, tmp; + + __asm__ __volatile__( + "0: cmpnei %1, 0 \n" + " bf 7f \n" + " mov %3, %1 \n" + " andi %3, 3 \n" + " cmpnei %3, 0 \n" + " bf 1f \n" + " br 5f \n" + "1: cmplti %0, 32 \n" /* 4W */ + " bt 3f \n" + "8: stw %2, (%1, 0) \n" + "10: stw %2, (%1, 4) \n" + "11: stw %2, (%1, 8) \n" + "12: stw %2, (%1, 12) \n" + "13: stw %2, (%1, 16) \n" + "14: stw %2, (%1, 20) \n" + "15: stw %2, (%1, 24) \n" + "16: stw %2, (%1, 28) \n" + " addi %1, 32 \n" + " subi %0, 32 \n" + " br 1b \n" + "3: cmplti %0, 4 \n" /* 1W */ + " bt 5f \n" + "4: stw %2, (%1, 0) \n" + " addi %1, 4 \n" + " subi %0, 4 \n" + " br 3b \n" + "5: cmpnei %0, 0 \n" /* 1B */ + "9: bf 7f \n" + "6: stb %2, (%1, 0) \n" + " addi %1, 1 \n" + " subi %0, 1 \n" + " br 5b \n" + ".section __ex_table,\"a\" \n" + ".align 2 \n" + ".long 8b, 9b \n" + ".long 10b, 9b \n" + ".long 11b, 9b \n" + ".long 12b, 9b \n" + ".long 13b, 9b \n" + ".long 14b, 9b \n" + ".long 15b, 9b \n" + ".long 16b, 9b \n" + ".long 4b, 9b \n" + ".long 6b, 9b \n" + ".previous \n" + "7: \n" + : "=r"(n), "=r" (data), "=r"(value), "=r"(tmp) + : "0"(n), "1"(to), "2"(0) + : "memory"); + return n; } EXPORT_SYMBOL(__clear_user); diff --git a/arch/csky/mm/Makefile b/arch/csky/mm/Makefile index c94ef6481098..6e7696e55f71 100644 --- a/arch/csky/mm/Makefile +++ b/arch/csky/mm/Makefile @@ -1,8 +1,10 @@ # SPDX-License-Identifier: GPL-2.0-only ifeq ($(CONFIG_CPU_HAS_CACHEV2),y) obj-y += cachev2.o +CFLAGS_REMOVE_cachev2.o = $(CC_FLAGS_FTRACE) else obj-y += cachev1.o +CFLAGS_REMOVE_cachev1.o = $(CC_FLAGS_FTRACE) endif obj-y += dma-mapping.o @@ -14,3 +16,4 @@ obj-y += syscache.o obj-y += tlb.o obj-y += asid.o obj-y += context.o +obj-$(CONFIG_HAVE_TCM) += tcm.o diff --git a/arch/csky/mm/asid.c b/arch/csky/mm/asid.c index b2e914745c1d..7fb6c417bbac 100644 --- a/arch/csky/mm/asid.c +++ b/arch/csky/mm/asid.c @@ -27,7 +27,7 @@ static void flush_context(struct asid_info *info) u64 asid; /* Update the list of reserved ASIDs and the ASID bitmap. */ - bitmap_clear(info->map, 0, NUM_CTXT_ASIDS(info)); + bitmap_zero(info->map, NUM_CTXT_ASIDS(info)); for_each_possible_cpu(i) { asid = atomic64_xchg_relaxed(&active_asid(info, i), 0); @@ -178,8 +178,7 @@ int asid_allocator_init(struct asid_info *info, */ WARN_ON(NUM_CTXT_ASIDS(info) - 1 <= num_possible_cpus()); atomic64_set(&info->generation, ASID_FIRST_VERSION(info)); - info->map = kcalloc(BITS_TO_LONGS(NUM_CTXT_ASIDS(info)), - sizeof(*info->map), GFP_KERNEL); + info->map = bitmap_zalloc(NUM_CTXT_ASIDS(info), GFP_KERNEL); if (!info->map) return -ENOMEM; diff --git a/arch/csky/mm/cachev1.c b/arch/csky/mm/cachev1.c index 494ec912abff..5a5a9804a0e3 100644 --- a/arch/csky/mm/cachev1.c +++ b/arch/csky/mm/cachev1.c @@ -94,6 +94,11 @@ void icache_inv_all(void) cache_op_all(INS_CACHE|CACHE_INV, 0); } +void local_icache_inv_all(void *priv) +{ + cache_op_all(INS_CACHE|CACHE_INV, 0); +} + void dcache_wb_range(unsigned long start, unsigned long end) { cache_op_range(start, end, DATA_CACHE|CACHE_CLR, 0); diff --git a/arch/csky/mm/cachev2.c b/arch/csky/mm/cachev2.c index b61be6518e21..7a9664adce43 100644 --- a/arch/csky/mm/cachev2.c +++ b/arch/csky/mm/cachev2.c @@ -3,15 +3,24 @@ #include <linux/spinlock.h> #include <linux/smp.h> +#include <linux/mm.h> #include <asm/cache.h> #include <asm/barrier.h> -inline void dcache_wb_line(unsigned long start) +/* for L1-cache */ +#define INS_CACHE (1 << 0) +#define DATA_CACHE (1 << 1) +#define CACHE_INV (1 << 4) +#define CACHE_CLR (1 << 5) +#define CACHE_OMS (1 << 6) + +void local_icache_inv_all(void *priv) { - asm volatile("dcache.cval1 %0\n"::"r"(start):"memory"); + mtcr("cr17", INS_CACHE|CACHE_INV); sync_is(); } +#ifdef CONFIG_CPU_HAS_ICACHE_INS void icache_inv_range(unsigned long start, unsigned long end) { unsigned long i = start & ~(L1_CACHE_BYTES - 1); @@ -20,43 +29,66 @@ void icache_inv_range(unsigned long start, unsigned long end) asm volatile("icache.iva %0\n"::"r"(i):"memory"); sync_is(); } +#else +struct cache_range { + unsigned long start; + unsigned long end; +}; + +static DEFINE_SPINLOCK(cache_lock); -void icache_inv_all(void) +static inline void cache_op_line(unsigned long i, unsigned int val) { - asm volatile("icache.ialls\n":::"memory"); - sync_is(); + mtcr("cr22", i); + mtcr("cr17", val); } -void dcache_wb_range(unsigned long start, unsigned long end) +void local_icache_inv_range(void *priv) { - unsigned long i = start & ~(L1_CACHE_BYTES - 1); + struct cache_range *param = priv; + unsigned long i = param->start & ~(L1_CACHE_BYTES - 1); + unsigned long flags; + + spin_lock_irqsave(&cache_lock, flags); + + for (; i < param->end; i += L1_CACHE_BYTES) + cache_op_line(i, INS_CACHE | CACHE_INV | CACHE_OMS); + + spin_unlock_irqrestore(&cache_lock, flags); - for (; i < end; i += L1_CACHE_BYTES) - asm volatile("dcache.cval1 %0\n"::"r"(i):"memory"); sync_is(); } -void dcache_inv_range(unsigned long start, unsigned long end) +void icache_inv_range(unsigned long start, unsigned long end) { - unsigned long i = start & ~(L1_CACHE_BYTES - 1); + struct cache_range param = { start, end }; - for (; i < end; i += L1_CACHE_BYTES) - asm volatile("dcache.civa %0\n"::"r"(i):"memory"); + if (irqs_disabled()) + local_icache_inv_range(¶m); + else + on_each_cpu(local_icache_inv_range, ¶m, 1); +} +#endif + +inline void dcache_wb_line(unsigned long start) +{ + asm volatile("dcache.cval1 %0\n"::"r"(start):"memory"); sync_is(); } -void cache_wbinv_range(unsigned long start, unsigned long end) +void dcache_wb_range(unsigned long start, unsigned long end) { unsigned long i = start & ~(L1_CACHE_BYTES - 1); for (; i < end; i += L1_CACHE_BYTES) asm volatile("dcache.cval1 %0\n"::"r"(i):"memory"); sync_is(); +} - i = start & ~(L1_CACHE_BYTES - 1); - for (; i < end; i += L1_CACHE_BYTES) - asm volatile("icache.iva %0\n"::"r"(i):"memory"); - sync_is(); +void cache_wbinv_range(unsigned long start, unsigned long end) +{ + dcache_wb_range(start, end); + icache_inv_range(start, end); } EXPORT_SYMBOL(cache_wbinv_range); diff --git a/arch/csky/mm/dma-mapping.c b/arch/csky/mm/dma-mapping.c index 8f6571ae27c8..82447029feb4 100644 --- a/arch/csky/mm/dma-mapping.c +++ b/arch/csky/mm/dma-mapping.c @@ -2,16 +2,13 @@ // Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #include <linux/cache.h> -#include <linux/dma-mapping.h> -#include <linux/dma-contiguous.h> -#include <linux/dma-noncoherent.h> +#include <linux/dma-map-ops.h> #include <linux/genalloc.h> #include <linux/highmem.h> #include <linux/io.h> #include <linux/mm.h> #include <linux/scatterlist.h> #include <linux/types.h> -#include <linux/version.h> #include <asm/cache.h> static inline void cache_op(phys_addr_t paddr, size_t size, diff --git a/arch/csky/mm/fault.c b/arch/csky/mm/fault.c index f76618b630f9..a885518ce1dd 100644 --- a/arch/csky/mm/fault.c +++ b/arch/csky/mm/fault.c @@ -1,36 +1,18 @@ // SPDX-License-Identifier: GPL-2.0 // Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. -#include <linux/signal.h> -#include <linux/module.h> -#include <linux/sched.h> -#include <linux/interrupt.h> -#include <linux/kernel.h> -#include <linux/errno.h> -#include <linux/string.h> -#include <linux/types.h> -#include <linux/ptrace.h> -#include <linux/mman.h> -#include <linux/mm.h> -#include <linux/smp.h> -#include <linux/version.h> -#include <linux/vt_kern.h> #include <linux/extable.h> -#include <linux/uaccess.h> +#include <linux/kprobes.h> +#include <linux/mmu_context.h> #include <linux/perf_event.h> -#include <asm/hardirq.h> -#include <asm/mmu_context.h> -#include <asm/traps.h> -#include <asm/page.h> - int fixup_exception(struct pt_regs *regs) { const struct exception_table_entry *fixup; fixup = search_exception_tables(instruction_pointer(regs)); if (fixup) { - regs->pc = fixup->nextinsn; + regs->pc = fixup->fixup; return 1; } @@ -38,179 +20,279 @@ int fixup_exception(struct pt_regs *regs) return 0; } -/* - * This routine handles page faults. It determines the address, - * and the problem, and then passes it off to one of the appropriate - * routines. - */ -asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long write, - unsigned long mmu_meh) +static inline bool is_write(struct pt_regs *regs) { - struct vm_area_struct *vma = NULL; - struct task_struct *tsk = current; - struct mm_struct *mm = tsk->mm; - int si_code; - int fault; - unsigned long address = mmu_meh & PAGE_MASK; + switch (trap_no(regs)) { + case VEC_TLBINVALIDS: + return true; + case VEC_TLBMODIFIED: + return true; + } - si_code = SEGV_MAPERR; + return false; +} -#ifndef CONFIG_CPU_HAS_TLBI - /* - * We fault-in kernel-space virtual memory on-demand. The - * 'reference' page table is init_mm.pgd. - * - * NOTE! We MUST NOT take any locks for this case. We may - * be in an interrupt or a critical region, and should - * only copy the information from the master page table, - * nothing more. - */ - if (unlikely(address >= VMALLOC_START) && - unlikely(address <= VMALLOC_END)) { - /* - * Synchronize this task's top level page-table - * with the 'reference' page table. - * - * Do _not_ use "tsk" here. We might be inside - * an interrupt in the middle of a task switch.. - */ - int offset = __pgd_offset(address); - pgd_t *pgd, *pgd_k; - pud_t *pud, *pud_k; - pmd_t *pmd, *pmd_k; - pte_t *pte_k; +#ifdef CONFIG_CPU_HAS_LDSTEX +static inline void csky_cmpxchg_fixup(struct pt_regs *regs) +{ + return; +} +#else +extern unsigned long csky_cmpxchg_ldw; +extern unsigned long csky_cmpxchg_stw; +static inline void csky_cmpxchg_fixup(struct pt_regs *regs) +{ + if (trap_no(regs) != VEC_TLBMODIFIED) + return; - unsigned long pgd_base; + if (instruction_pointer(regs) == csky_cmpxchg_stw) + instruction_pointer_set(regs, csky_cmpxchg_ldw); + return; +} +#endif - pgd_base = (unsigned long)__va(get_pgd()); - pgd = (pgd_t *)pgd_base + offset; - pgd_k = init_mm.pgd + offset; +static inline void no_context(struct pt_regs *regs, unsigned long addr) +{ + current->thread.trap_no = trap_no(regs); - if (!pgd_present(*pgd_k)) - goto no_context; - set_pgd(pgd, *pgd_k); + /* Are we prepared to handle this kernel fault? */ + if (fixup_exception(regs)) + return; - pud = (pud_t *)pgd; - pud_k = (pud_t *)pgd_k; - if (!pud_present(*pud_k)) - goto no_context; + /* + * Oops. The kernel tried to access some bad page. We'll have to + * terminate things with extreme prejudice. + */ + bust_spinlocks(1); + pr_alert("Unable to handle kernel paging request at virtual " + "addr 0x%08lx, pc: 0x%08lx\n", addr, regs->pc); + die(regs, "Oops"); + make_task_dead(SIGKILL); +} - pmd = pmd_offset(pud, address); - pmd_k = pmd_offset(pud_k, address); - if (!pmd_present(*pmd_k)) - goto no_context; - set_pmd(pmd, *pmd_k); +static inline void mm_fault_error(struct pt_regs *regs, unsigned long addr, vm_fault_t fault) +{ + current->thread.trap_no = trap_no(regs); - pte_k = pte_offset_kernel(pmd_k, address); - if (!pte_present(*pte_k)) - goto no_context; + if (fault & VM_FAULT_OOM) { + /* + * We ran out of memory, call the OOM killer, and return the userspace + * (which will retry the fault, or kill us if we got oom-killed). + */ + if (!user_mode(regs)) { + no_context(regs, addr); + return; + } + pagefault_out_of_memory(); + return; + } else if (fault & VM_FAULT_SIGBUS) { + /* Kernel mode? Handle exceptions or die */ + if (!user_mode(regs)) { + no_context(regs, addr); + return; + } + do_trap(regs, SIGBUS, BUS_ADRERR, addr); return; } -#endif + BUG(); +} - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); +static inline void bad_area_nosemaphore(struct pt_regs *regs, struct mm_struct *mm, int code, unsigned long addr) +{ /* - * If we're in an interrupt or have no user - * context, we must not take the fault.. + * Something tried to access memory that isn't in our memory map. + * Fix it, but check if it's kernel or user first. */ - if (in_atomic() || !mm) - goto bad_area_nosemaphore; - - down_read(&mm->mmap_sem); - vma = find_vma(mm, address); - if (!vma) - goto bad_area; - if (vma->vm_start <= address) - goto good_area; - if (!(vma->vm_flags & VM_GROWSDOWN)) - goto bad_area; - if (expand_stack(vma, address)) - goto bad_area; + /* User mode accesses just cause a SIGSEGV */ + if (user_mode(regs)) { + do_trap(regs, SIGSEGV, code, addr); + return; + } + + no_context(regs, addr); +} + +static inline void vmalloc_fault(struct pt_regs *regs, int code, unsigned long addr) +{ + pgd_t *pgd, *pgd_k; + pud_t *pud, *pud_k; + pmd_t *pmd, *pmd_k; + pte_t *pte_k; + int offset; + + /* User mode accesses just cause a SIGSEGV */ + if (user_mode(regs)) { + do_trap(regs, SIGSEGV, code, addr); + return; + } + /* - * Ok, we have a good vm_area for this memory access, so - * we can handle it.. + * Synchronize this task's top level page-table + * with the 'reference' page table. + * + * Do _not_ use "tsk" here. We might be inside + * an interrupt in the middle of a task switch.. */ -good_area: - si_code = SEGV_ACCERR; + offset = pgd_index(addr); + + pgd = get_pgd() + offset; + pgd_k = init_mm.pgd + offset; + + if (!pgd_present(*pgd_k)) { + no_context(regs, addr); + return; + } + set_pgd(pgd, *pgd_k); + + pud = (pud_t *)pgd; + pud_k = (pud_t *)pgd_k; + if (!pud_present(*pud_k)) { + no_context(regs, addr); + return; + } + + pmd = pmd_offset(pud, addr); + pmd_k = pmd_offset(pud_k, addr); + if (!pmd_present(*pmd_k)) { + no_context(regs, addr); + return; + } + set_pmd(pmd, *pmd_k); + + pte_k = pte_offset_kernel(pmd_k, addr); + if (!pte_present(*pte_k)) { + no_context(regs, addr); + return; + } + + flush_tlb_one(addr); +} - if (write) { +static inline bool access_error(struct pt_regs *regs, struct vm_area_struct *vma) +{ + if (is_write(regs)) { if (!(vma->vm_flags & VM_WRITE)) - goto bad_area; + return true; } else { - if (!(vma->vm_flags & (VM_READ | VM_WRITE | VM_EXEC))) - goto bad_area; + if (unlikely(!vma_is_accessible(vma))) + return true; } + return false; +} + +/* + * This routine handles page faults. It determines the address and the + * problem, and then passes it off to one of the appropriate routines. + */ +asmlinkage void do_page_fault(struct pt_regs *regs) +{ + struct task_struct *tsk; + struct vm_area_struct *vma; + struct mm_struct *mm; + unsigned long addr = read_mmu_entryhi() & PAGE_MASK; + unsigned int flags = FAULT_FLAG_DEFAULT; + int code = SEGV_MAPERR; + vm_fault_t fault; + + tsk = current; + mm = tsk->mm; + + csky_cmpxchg_fixup(regs); + + if (kprobe_page_fault(regs, tsk->thread.trap_no)) + return; /* - * If for any reason at all we couldn't handle the fault, - * make sure we exit gracefully rather than endlessly redo - * the fault. + * Fault-in kernel-space virtual memory on-demand. + * The 'reference' page table is init_mm.pgd. + * + * NOTE! We MUST NOT take any locks for this case. We may + * be in an interrupt or a critical region, and should + * only copy the information from the master page table, + * nothing more. */ - fault = handle_mm_fault(vma, address, write ? FAULT_FLAG_WRITE : 0); - if (unlikely(fault & VM_FAULT_ERROR)) { - if (fault & VM_FAULT_OOM) - goto out_of_memory; - else if (fault & VM_FAULT_SIGBUS) - goto do_sigbus; - else if (fault & VM_FAULT_SIGSEGV) - goto bad_area; - BUG(); - } - if (fault & VM_FAULT_MAJOR) { - tsk->maj_flt++; - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs, - address); - } else { - tsk->min_flt++; - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs, - address); + if (unlikely((addr >= VMALLOC_START) && (addr <= VMALLOC_END))) { + vmalloc_fault(regs, code, addr); + return; } - up_read(&mm->mmap_sem); - return; + /* Enable interrupts if they were enabled in the parent context. */ + if (likely(regs->sr & BIT(6))) + local_irq_enable(); /* - * Something tried to access memory that isn't in our memory map.. - * Fix it, but check if it's kernel or user first.. + * If we're in an interrupt, have no user context, or are running + * in an atomic region, then we must not take the fault. */ -bad_area: - up_read(&mm->mmap_sem); + if (unlikely(faulthandler_disabled() || !mm)) { + no_context(regs, addr); + return; + } -bad_area_nosemaphore: - /* User mode accesses just cause a SIGSEGV */ - if (user_mode(regs)) { - force_sig_fault(SIGSEGV, si_code, (void __user *)address); + if (user_mode(regs)) + flags |= FAULT_FLAG_USER; + + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr); + + if (is_write(regs)) + flags |= FAULT_FLAG_WRITE; +retry: + vma = lock_mm_and_find_vma(mm, addr, regs); + if (unlikely(!vma)) { + bad_area_nosemaphore(regs, mm, code, addr); return; } -no_context: - /* Are we prepared to handle this kernel fault? */ - if (fixup_exception(regs)) + /* + * Ok, we have a good vm_area for this memory access, so + * we can handle it. + */ + code = SEGV_ACCERR; + + if (unlikely(access_error(regs, vma))) { + mmap_read_unlock(mm); + bad_area_nosemaphore(regs, mm, code, addr); return; + } /* - * Oops. The kernel tried to access some bad page. We'll have to - * terminate things with extreme prejudice. + * If for any reason at all we could not handle the fault, + * make sure we exit gracefully rather than endlessly redo + * the fault. */ - bust_spinlocks(1); - pr_alert("Unable to handle kernel paging request at virtual " - "address 0x%08lx, pc: 0x%08lx\n", address, regs->pc); - die_if_kernel("Oops", regs, write); + fault = handle_mm_fault(vma, addr, flags, regs); -out_of_memory: /* - * We ran out of memory, call the OOM killer, and return the userspace - * (which will retry the fault, or kill us if we got oom-killed). + * If we need to retry but a fatal signal is pending, handle the + * signal first. We do not need to release the mmap_lock because it + * would already be released in __lock_page_or_retry in mm/filemap.c. */ - pagefault_out_of_memory(); - return; + if (fault_signal_pending(fault, regs)) { + if (!user_mode(regs)) + no_context(regs, addr); + return; + } -do_sigbus: - up_read(&mm->mmap_sem); + /* The fault is fully completed (including releasing mmap lock) */ + if (fault & VM_FAULT_COMPLETED) + return; - /* Kernel mode? Handle exceptions or die */ - if (!user_mode(regs)) - goto no_context; + if (unlikely((fault & VM_FAULT_RETRY) && (flags & FAULT_FLAG_ALLOW_RETRY))) { + flags |= FAULT_FLAG_TRIED; - force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)address); + /* + * No need to mmap_read_unlock(mm) as we would + * have already released it in __lock_page_or_retry + * in mm/filemap.c. + */ + goto retry; + } + + mmap_read_unlock(mm); + + if (unlikely(fault & VM_FAULT_ERROR)) { + mm_fault_error(regs, addr, fault); + return; + } + return; } diff --git a/arch/csky/mm/highmem.c b/arch/csky/mm/highmem.c index 3317b774f6dc..4161df3c6c15 100644 --- a/arch/csky/mm/highmem.c +++ b/arch/csky/mm/highmem.c @@ -9,195 +9,28 @@ #include <asm/tlbflush.h> #include <asm/cacheflush.h> -static pte_t *kmap_pte; - unsigned long highstart_pfn, highend_pfn; -void *kmap(struct page *page) -{ - void *addr; - - might_sleep(); - if (!PageHighMem(page)) - return page_address(page); - addr = kmap_high(page); - flush_tlb_one((unsigned long)addr); - - return addr; -} -EXPORT_SYMBOL(kmap); - -void kunmap(struct page *page) -{ - BUG_ON(in_interrupt()); - if (!PageHighMem(page)) - return; - kunmap_high(page); -} -EXPORT_SYMBOL(kunmap); - -void *kmap_atomic(struct page *page) -{ - unsigned long vaddr; - int idx, type; - - preempt_disable(); - pagefault_disable(); - if (!PageHighMem(page)) - return page_address(page); - - type = kmap_atomic_idx_push(); - idx = type + KM_TYPE_NR*smp_processor_id(); - vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx); -#ifdef CONFIG_DEBUG_HIGHMEM - BUG_ON(!pte_none(*(kmap_pte - idx))); -#endif - set_pte(kmap_pte-idx, mk_pte(page, PAGE_KERNEL)); - flush_tlb_one((unsigned long)vaddr); - - return (void *)vaddr; -} -EXPORT_SYMBOL(kmap_atomic); - -void __kunmap_atomic(void *kvaddr) -{ - unsigned long vaddr = (unsigned long) kvaddr & PAGE_MASK; - int idx; - - if (vaddr < FIXADDR_START) - goto out; - -#ifdef CONFIG_DEBUG_HIGHMEM - idx = KM_TYPE_NR*smp_processor_id() + kmap_atomic_idx(); - - BUG_ON(vaddr != __fix_to_virt(FIX_KMAP_BEGIN + idx)); - - pte_clear(&init_mm, vaddr, kmap_pte - idx); - flush_tlb_one(vaddr); -#else - (void) idx; /* to kill a warning */ -#endif - kmap_atomic_idx_pop(); -out: - pagefault_enable(); - preempt_enable(); -} -EXPORT_SYMBOL(__kunmap_atomic); - -/* - * This is the same as kmap_atomic() but can map memory that doesn't - * have a struct page associated with it. - */ -void *kmap_atomic_pfn(unsigned long pfn) -{ - unsigned long vaddr; - int idx, type; - - pagefault_disable(); - - type = kmap_atomic_idx_push(); - idx = type + KM_TYPE_NR*smp_processor_id(); - vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx); - set_pte(kmap_pte-idx, pfn_pte(pfn, PAGE_KERNEL)); - flush_tlb_one(vaddr); - - return (void *) vaddr; -} - -struct page *kmap_atomic_to_page(void *ptr) +void kmap_flush_tlb(unsigned long addr) { - unsigned long idx, vaddr = (unsigned long)ptr; - pte_t *pte; - - if (vaddr < FIXADDR_START) - return virt_to_page(ptr); - - idx = virt_to_fix(vaddr); - pte = kmap_pte - (idx - FIX_KMAP_BEGIN); - return pte_page(*pte); + flush_tlb_one(addr); } +EXPORT_SYMBOL(kmap_flush_tlb); -static void __init fixrange_init(unsigned long start, unsigned long end, - pgd_t *pgd_base) -{ -#ifdef CONFIG_HIGHMEM - pgd_t *pgd; - pud_t *pud; - pmd_t *pmd; - pte_t *pte; - int i, j, k; - unsigned long vaddr; - - vaddr = start; - i = __pgd_offset(vaddr); - j = __pud_offset(vaddr); - k = __pmd_offset(vaddr); - pgd = pgd_base + i; - - for ( ; (i < PTRS_PER_PGD) && (vaddr != end); pgd++, i++) { - pud = (pud_t *)pgd; - for ( ; (j < PTRS_PER_PUD) && (vaddr != end); pud++, j++) { - pmd = (pmd_t *)pud; - for (; (k < PTRS_PER_PMD) && (vaddr != end); pmd++, k++) { - if (pmd_none(*pmd)) { - pte = (pte_t *) memblock_alloc_low(PAGE_SIZE, PAGE_SIZE); - if (!pte) - panic("%s: Failed to allocate %lu bytes align=%lx\n", - __func__, PAGE_SIZE, - PAGE_SIZE); - - set_pmd(pmd, __pmd(__pa(pte))); - BUG_ON(pte != pte_offset_kernel(pmd, 0)); - } - vaddr += PMD_SIZE; - } - k = 0; - } - j = 0; - } -#endif -} - -void __init fixaddr_kmap_pages_init(void) +void __init kmap_init(void) { unsigned long vaddr; - pgd_t *pgd_base; -#ifdef CONFIG_HIGHMEM pgd_t *pgd; pmd_t *pmd; pud_t *pud; pte_t *pte; -#endif - pgd_base = swapper_pg_dir; - /* - * Fixed mappings: - */ - vaddr = __fix_to_virt(__end_of_fixed_addresses - 1) & PMD_MASK; - fixrange_init(vaddr, 0, pgd_base); - -#ifdef CONFIG_HIGHMEM - /* - * Permanent kmaps: - */ vaddr = PKMAP_BASE; - fixrange_init(vaddr, vaddr + PAGE_SIZE*LAST_PKMAP, pgd_base); + fixrange_init(vaddr, vaddr + PAGE_SIZE*LAST_PKMAP, swapper_pg_dir); - pgd = swapper_pg_dir + __pgd_offset(vaddr); + pgd = swapper_pg_dir + pgd_index(vaddr); pud = (pud_t *)pgd; pmd = pmd_offset(pud, vaddr); pte = pte_offset_kernel(pmd, vaddr); pkmap_page_table = pte; -#endif -} - -void __init kmap_init(void) -{ - unsigned long vaddr; - - fixaddr_kmap_pages_init(); - - vaddr = __fix_to_virt(FIX_KMAP_BEGIN); - - kmap_pte = pte_offset_kernel((pmd_t *)pgd_offset_k(vaddr), vaddr); } diff --git a/arch/csky/mm/init.c b/arch/csky/mm/init.c index d4c2292ea46b..bde7cabd23df 100644 --- a/arch/csky/mm/init.c +++ b/arch/csky/mm/init.c @@ -19,33 +19,83 @@ #include <linux/swap.h> #include <linux/proc_fs.h> #include <linux/pfn.h> +#include <linux/initrd.h> #include <asm/setup.h> #include <asm/cachectl.h> #include <asm/dma.h> -#include <asm/pgtable.h> #include <asm/pgalloc.h> #include <asm/mmu_context.h> #include <asm/sections.h> #include <asm/tlb.h> +#include <asm/cacheflush.h> + +#define PTRS_KERN_TABLE \ + ((PTRS_PER_PGD - USER_PTRS_PER_PGD) * PTRS_PER_PTE) pgd_t swapper_pg_dir[PTRS_PER_PGD] __page_aligned_bss; pte_t invalid_pte_table[PTRS_PER_PTE] __page_aligned_bss; +pte_t kernel_pte_tables[PTRS_KERN_TABLE] __page_aligned_bss; + +EXPORT_SYMBOL(invalid_pte_table); unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)] __page_aligned_bss; EXPORT_SYMBOL(empty_zero_page); +#ifdef CONFIG_BLK_DEV_INITRD +static void __init setup_initrd(void) +{ + unsigned long size; + + if (initrd_start >= initrd_end) { + pr_err("initrd not found or empty"); + goto disable; + } + + if (__pa(initrd_end) > PFN_PHYS(max_low_pfn)) { + pr_err("initrd extends beyond end of memory"); + goto disable; + } + + size = initrd_end - initrd_start; + + if (memblock_is_region_reserved(__pa(initrd_start), size)) { + pr_err("INITRD: 0x%08lx+0x%08lx overlaps in-use memory region", + __pa(initrd_start), size); + goto disable; + } + + memblock_reserve(__pa(initrd_start), size); + + pr_info("Initial ramdisk at: 0x%p (%lu bytes)\n", + (void *)(initrd_start), size); + + initrd_below_start_ok = 1; + + return; + +disable: + initrd_start = initrd_end = 0; + + pr_err(" - disabling initrd\n"); +} +#endif + void __init mem_init(void) { #ifdef CONFIG_HIGHMEM unsigned long tmp; - max_mapnr = highend_pfn; + set_max_mapnr(highend_pfn - ARCH_PFN_OFFSET); #else - max_mapnr = max_low_pfn; + set_max_mapnr(max_low_pfn - ARCH_PFN_OFFSET); #endif high_memory = (void *) __va(max_low_pfn << PAGE_SHIFT); +#ifdef CONFIG_BLK_DEV_INITRD + setup_initrd(); +#endif + memblock_free_all(); #ifdef CONFIG_HIGHMEM @@ -57,27 +107,11 @@ void __init mem_init(void) free_highmem_page(page); } #endif - mem_init_print_info(NULL); } -extern char __init_begin[], __init_end[]; - void free_initmem(void) { - unsigned long addr; - - addr = (unsigned long) &__init_begin; - - while (addr < (unsigned long) &__init_end) { - ClearPageReserved(virt_to_page(addr)); - init_page_count(virt_to_page(addr)); - free_page(addr); - totalram_pages_inc(); - addr += PAGE_SIZE; - } - - pr_info("Freeing unused kernel memory: %dk freed\n", - ((unsigned int)&__init_end - (unsigned int)&__init_begin) >> 10); + free_initmem_default(-1); } void pgd_init(unsigned long *p) @@ -86,18 +120,100 @@ void pgd_init(unsigned long *p) for (i = 0; i < PTRS_PER_PGD; i++) p[i] = __pa(invalid_pte_table); + + flush_tlb_all(); + local_icache_inv_all(NULL); } -void __init pre_mmu_init(void) +void __init mmu_init(unsigned long min_pfn, unsigned long max_pfn) { - /* - * Setup page-table and enable TLB-hardrefill - */ + int i; + + for (i = 0; i < USER_PTRS_PER_PGD; i++) + swapper_pg_dir[i].pgd = __pa(invalid_pte_table); + + for (i = USER_PTRS_PER_PGD; i < PTRS_PER_PGD; i++) + swapper_pg_dir[i].pgd = + __pa(kernel_pte_tables + (PTRS_PER_PTE * (i - USER_PTRS_PER_PGD))); + + for (i = 0; i < PTRS_KERN_TABLE; i++) + set_pte(&kernel_pte_tables[i], __pte(_PAGE_GLOBAL)); + + for (i = min_pfn; i < max_pfn; i++) + set_pte(&kernel_pte_tables[i - PFN_DOWN(va_pa_offset)], pfn_pte(i, PAGE_KERNEL)); + flush_tlb_all(); - pgd_init((unsigned long *)swapper_pg_dir); - TLBMISS_HANDLER_SETUP_PGD(swapper_pg_dir); - TLBMISS_HANDLER_SETUP_PGD_KERNEL(swapper_pg_dir); + local_icache_inv_all(NULL); /* Setup page mask to 4k */ write_mmu_pagemask(0); + + setup_pgd(swapper_pg_dir, 0); +} + +void __init fixrange_init(unsigned long start, unsigned long end, + pgd_t *pgd_base) +{ + pgd_t *pgd; + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + int i, j, k; + unsigned long vaddr; + + vaddr = start; + i = pgd_index(vaddr); + j = pud_index(vaddr); + k = pmd_index(vaddr); + pgd = pgd_base + i; + + for ( ; (i < PTRS_PER_PGD) && (vaddr != end); pgd++, i++) { + pud = (pud_t *)pgd; + for ( ; (j < PTRS_PER_PUD) && (vaddr != end); pud++, j++) { + pmd = (pmd_t *)pud; + for (; (k < PTRS_PER_PMD) && (vaddr != end); pmd++, k++) { + if (pmd_none(*pmd)) { + pte = (pte_t *) memblock_alloc_low(PAGE_SIZE, PAGE_SIZE); + if (!pte) + panic("%s: Failed to allocate %lu bytes align=%lx\n", + __func__, PAGE_SIZE, + PAGE_SIZE); + + set_pmd(pmd, __pmd(__pa(pte))); + BUG_ON(pte != pte_offset_kernel(pmd, 0)); + } + vaddr += PMD_SIZE; + } + k = 0; + } + j = 0; + } } + +void __init fixaddr_init(void) +{ + unsigned long vaddr; + + vaddr = __fix_to_virt(__end_of_fixed_addresses - 1) & PMD_MASK; + fixrange_init(vaddr, vaddr + PMD_SIZE, swapper_pg_dir); +} + +static const pgprot_t protection_map[16] = { + [VM_NONE] = PAGE_NONE, + [VM_READ] = PAGE_READ, + [VM_WRITE] = PAGE_READ, + [VM_WRITE | VM_READ] = PAGE_READ, + [VM_EXEC] = PAGE_READ, + [VM_EXEC | VM_READ] = PAGE_READ, + [VM_EXEC | VM_WRITE] = PAGE_READ, + [VM_EXEC | VM_WRITE | VM_READ] = PAGE_READ, + [VM_SHARED] = PAGE_NONE, + [VM_SHARED | VM_READ] = PAGE_READ, + [VM_SHARED | VM_WRITE] = PAGE_WRITE, + [VM_SHARED | VM_WRITE | VM_READ] = PAGE_WRITE, + [VM_SHARED | VM_EXEC] = PAGE_READ, + [VM_SHARED | VM_EXEC | VM_READ] = PAGE_READ, + [VM_SHARED | VM_EXEC | VM_WRITE] = PAGE_WRITE, + [VM_SHARED | VM_EXEC | VM_WRITE | VM_READ] = PAGE_WRITE +}; +DECLARE_VM_GET_PAGE_PROT diff --git a/arch/csky/mm/syscache.c b/arch/csky/mm/syscache.c index c4645e4e97f4..cd847ad62c7e 100644 --- a/arch/csky/mm/syscache.c +++ b/arch/csky/mm/syscache.c @@ -3,7 +3,7 @@ #include <linux/syscalls.h> #include <asm/page.h> -#include <asm/cache.h> +#include <asm/cacheflush.h> #include <asm/cachectl.h> SYSCALL_DEFINE3(cacheflush, @@ -12,17 +12,17 @@ SYSCALL_DEFINE3(cacheflush, int, cache) { switch (cache) { - case ICACHE: - icache_inv_range((unsigned long)addr, - (unsigned long)addr + bytes); - break; + case BCACHE: case DCACHE: dcache_wb_range((unsigned long)addr, (unsigned long)addr + bytes); - break; - case BCACHE: - cache_wbinv_range((unsigned long)addr, - (unsigned long)addr + bytes); + if (cache != BCACHE) + break; + fallthrough; + case ICACHE: + flush_icache_mm_range(current->mm, + (unsigned long)addr, + (unsigned long)addr + bytes); break; default: return -EINVAL; diff --git a/arch/csky/mm/tcm.c b/arch/csky/mm/tcm.c new file mode 100644 index 000000000000..ddeb36328819 --- /dev/null +++ b/arch/csky/mm/tcm.c @@ -0,0 +1,169 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/highmem.h> +#include <linux/genalloc.h> +#include <asm/tlbflush.h> +#include <asm/fixmap.h> + +#if (CONFIG_ITCM_RAM_BASE == 0xffffffff) +#error "You should define ITCM_RAM_BASE" +#endif + +#ifdef CONFIG_HAVE_DTCM +#if (CONFIG_DTCM_RAM_BASE == 0xffffffff) +#error "You should define DTCM_RAM_BASE" +#endif + +#if (CONFIG_DTCM_RAM_BASE == CONFIG_ITCM_RAM_BASE) +#error "You should define correct DTCM_RAM_BASE" +#endif +#endif + +extern char __tcm_start, __tcm_end, __dtcm_start; + +static struct gen_pool *tcm_pool; + +static void __init tcm_mapping_init(void) +{ + pte_t *tcm_pte; + unsigned long vaddr, paddr; + int i; + + paddr = CONFIG_ITCM_RAM_BASE; + + if (pfn_valid(PFN_DOWN(CONFIG_ITCM_RAM_BASE))) + goto panic; + +#ifndef CONFIG_HAVE_DTCM + for (i = 0; i < TCM_NR_PAGES; i++) { +#else + for (i = 0; i < CONFIG_ITCM_NR_PAGES; i++) { +#endif + vaddr = __fix_to_virt(FIX_TCM - i); + + tcm_pte = + pte_offset_kernel((pmd_t *)pgd_offset_k(vaddr), vaddr); + + set_pte(tcm_pte, pfn_pte(__phys_to_pfn(paddr), PAGE_KERNEL)); + + flush_tlb_one(vaddr); + + paddr = paddr + PAGE_SIZE; + } + +#ifdef CONFIG_HAVE_DTCM + if (pfn_valid(PFN_DOWN(CONFIG_DTCM_RAM_BASE))) + goto panic; + + paddr = CONFIG_DTCM_RAM_BASE; + + for (i = 0; i < CONFIG_DTCM_NR_PAGES; i++) { + vaddr = __fix_to_virt(FIX_TCM - CONFIG_ITCM_NR_PAGES - i); + + tcm_pte = + pte_offset_kernel((pmd_t *) pgd_offset_k(vaddr), vaddr); + + set_pte(tcm_pte, pfn_pte(__phys_to_pfn(paddr), PAGE_KERNEL)); + + flush_tlb_one(vaddr); + + paddr = paddr + PAGE_SIZE; + } +#endif + +#ifndef CONFIG_HAVE_DTCM + memcpy((void *)__fix_to_virt(FIX_TCM), + &__tcm_start, &__tcm_end - &__tcm_start); + + pr_info("%s: mapping tcm va:0x%08lx to pa:0x%08x\n", + __func__, __fix_to_virt(FIX_TCM), CONFIG_ITCM_RAM_BASE); + + pr_info("%s: __tcm_start va:0x%08lx size:%d\n", + __func__, (unsigned long)&__tcm_start, &__tcm_end - &__tcm_start); +#else + memcpy((void *)__fix_to_virt(FIX_TCM), + &__tcm_start, &__dtcm_start - &__tcm_start); + + pr_info("%s: mapping itcm va:0x%08lx to pa:0x%08x\n", + __func__, __fix_to_virt(FIX_TCM), CONFIG_ITCM_RAM_BASE); + + pr_info("%s: __itcm_start va:0x%08lx size:%d\n", + __func__, (unsigned long)&__tcm_start, &__dtcm_start - &__tcm_start); + + memcpy((void *)__fix_to_virt(FIX_TCM - CONFIG_ITCM_NR_PAGES), + &__dtcm_start, &__tcm_end - &__dtcm_start); + + pr_info("%s: mapping dtcm va:0x%08lx to pa:0x%08x\n", + __func__, __fix_to_virt(FIX_TCM - CONFIG_ITCM_NR_PAGES), + CONFIG_DTCM_RAM_BASE); + + pr_info("%s: __dtcm_start va:0x%08lx size:%d\n", + __func__, (unsigned long)&__dtcm_start, &__tcm_end - &__dtcm_start); + +#endif + return; +panic: + panic("TCM init error"); +} + +void *tcm_alloc(size_t len) +{ + unsigned long vaddr; + + if (!tcm_pool) + return NULL; + + vaddr = gen_pool_alloc(tcm_pool, len); + if (!vaddr) + return NULL; + + return (void *) vaddr; +} +EXPORT_SYMBOL(tcm_alloc); + +void tcm_free(void *addr, size_t len) +{ + gen_pool_free(tcm_pool, (unsigned long) addr, len); +} +EXPORT_SYMBOL(tcm_free); + +static int __init tcm_setup_pool(void) +{ +#ifndef CONFIG_HAVE_DTCM + u32 pool_size = (u32) (TCM_NR_PAGES * PAGE_SIZE) + - (u32) (&__tcm_end - &__tcm_start); + + u32 tcm_pool_start = __fix_to_virt(FIX_TCM) + + (u32) (&__tcm_end - &__tcm_start); +#else + u32 pool_size = (u32) (CONFIG_DTCM_NR_PAGES * PAGE_SIZE) + - (u32) (&__tcm_end - &__dtcm_start); + + u32 tcm_pool_start = __fix_to_virt(FIX_TCM - CONFIG_ITCM_NR_PAGES) + + (u32) (&__tcm_end - &__dtcm_start); +#endif + int ret; + + tcm_pool = gen_pool_create(2, -1); + + ret = gen_pool_add(tcm_pool, tcm_pool_start, pool_size, -1); + if (ret) { + pr_err("%s: gen_pool add failed!\n", __func__); + return ret; + } + + pr_info("%s: Added %d bytes @ 0x%08x to memory pool\n", + __func__, pool_size, tcm_pool_start); + + return 0; +} + +static int __init tcm_init(void) +{ + tcm_mapping_init(); + + tcm_setup_pool(); + + return 0; +} +arch_initcall(tcm_init); diff --git a/arch/csky/mm/tlb.c b/arch/csky/mm/tlb.c index eb3ba6c9c927..9234c5e5ceaf 100644 --- a/arch/csky/mm/tlb.c +++ b/arch/csky/mm/tlb.c @@ -7,7 +7,6 @@ #include <linux/sched.h> #include <asm/mmu_context.h> -#include <asm/pgtable.h> #include <asm/setup.h> /* @@ -25,7 +24,13 @@ void flush_tlb_all(void) void flush_tlb_mm(struct mm_struct *mm) { #ifdef CONFIG_CPU_HAS_TLBI - asm volatile("tlbi.asids %0"::"r"(cpu_asid(mm))); + sync_is(); + asm volatile( + "tlbi.asids %0 \n" + "sync.i \n" + : + : "r" (cpu_asid(mm)) + : "memory"); #else tlb_invalid_all(); #endif @@ -54,11 +59,17 @@ void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, end &= TLB_ENTRY_SIZE_MASK; #ifdef CONFIG_CPU_HAS_TLBI + sync_is(); while (start < end) { - asm volatile("tlbi.vas %0"::"r"(start | newpid)); + asm volatile( + "tlbi.vas %0 \n" + : + : "r" (start | newpid) + : "memory"); + start += 2*PAGE_SIZE; } - sync_is(); + asm volatile("sync.i\n"); #else { unsigned long flags, oldpid; @@ -88,11 +99,17 @@ void flush_tlb_kernel_range(unsigned long start, unsigned long end) end &= TLB_ENTRY_SIZE_MASK; #ifdef CONFIG_CPU_HAS_TLBI + sync_is(); while (start < end) { - asm volatile("tlbi.vaas %0"::"r"(start)); + asm volatile( + "tlbi.vaas %0 \n" + : + : "r" (start) + : "memory"); + start += 2*PAGE_SIZE; } - sync_is(); + asm volatile("sync.i\n"); #else { unsigned long flags, oldpid; @@ -122,8 +139,13 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr) addr &= TLB_ENTRY_SIZE_MASK; #ifdef CONFIG_CPU_HAS_TLBI - asm volatile("tlbi.vas %0"::"r"(addr | newpid)); sync_is(); + asm volatile( + "tlbi.vas %0 \n" + "sync.i \n" + : + : "r" (addr | newpid) + : "memory"); #else { int oldpid, idx; @@ -148,8 +170,13 @@ void flush_tlb_one(unsigned long addr) addr &= TLB_ENTRY_SIZE_MASK; #ifdef CONFIG_CPU_HAS_TLBI - asm volatile("tlbi.vaas %0"::"r"(addr)); sync_is(); + asm volatile( + "tlbi.vaas %0 \n" + "sync.i \n" + : + : "r" (addr) + : "memory"); #else { int oldpid, idx; |