summaryrefslogtreecommitdiff
path: root/arch/mips/mm
diff options
context:
space:
mode:
Diffstat (limited to 'arch/mips/mm')
-rw-r--r--arch/mips/mm/c-octeon.c5
-rw-r--r--arch/mips/mm/c-r3k.c5
-rw-r--r--arch/mips/mm/c-r4k.c142
-rw-r--r--arch/mips/mm/cache.c94
-rw-r--r--arch/mips/mm/cex-gen.S2
-rw-r--r--arch/mips/mm/dma-noncoherent.c3
-rw-r--r--arch/mips/mm/fault.c17
-rw-r--r--arch/mips/mm/hugetlbpage.c10
-rw-r--r--arch/mips/mm/init.c94
-rw-r--r--arch/mips/mm/ioremap.c8
-rw-r--r--arch/mips/mm/ioremap64.c4
-rw-r--r--arch/mips/mm/mmap.c8
-rw-r--r--arch/mips/mm/page-funcs.S2
-rw-r--r--arch/mips/mm/page.c202
-rw-r--r--arch/mips/mm/pgtable-32.c19
-rw-r--r--arch/mips/mm/pgtable-64.c29
-rw-r--r--arch/mips/mm/pgtable.c6
-rw-r--r--arch/mips/mm/physaddr.c2
-rw-r--r--arch/mips/mm/tlb-funcs.S2
-rw-r--r--arch/mips/mm/tlb-r3k.c6
-rw-r--r--arch/mips/mm/tlb-r4k.c116
-rw-r--r--arch/mips/mm/tlbex.c244
22 files changed, 464 insertions, 556 deletions
diff --git a/arch/mips/mm/c-octeon.c b/arch/mips/mm/c-octeon.c
index c7ed589de882..b7393b61cfa7 100644
--- a/arch/mips/mm/c-octeon.c
+++ b/arch/mips/mm/c-octeon.c
@@ -83,8 +83,13 @@ static void octeon_flush_icache_all_cores(struct vm_area_struct *vma)
else
mask = *cpu_online_mask;
cpumask_clear_cpu(cpu, &mask);
+#ifdef CONFIG_CAVIUM_OCTEON_SOC
for_each_cpu(cpu, &mask)
octeon_send_ipi_single(cpu, SMP_ICACHE_FLUSH);
+#else
+ smp_call_function_many(&mask, (smp_call_func_t)octeon_local_flush_icache,
+ NULL, 1);
+#endif
preempt_enable();
#endif
diff --git a/arch/mips/mm/c-r3k.c b/arch/mips/mm/c-r3k.c
index df6755ca1892..5869df848fab 100644
--- a/arch/mips/mm/c-r3k.c
+++ b/arch/mips/mm/c-r3k.c
@@ -261,10 +261,6 @@ static void r3k_flush_cache_page(struct vm_area_struct *vma,
r3k_flush_icache_range(kaddr, kaddr + PAGE_SIZE);
}
-static void local_r3k_flush_data_cache_page(void *addr)
-{
-}
-
static void r3k_flush_data_cache_page(unsigned long addr)
{
}
@@ -302,7 +298,6 @@ void r3k_cache_init(void)
__flush_kernel_vmap_range = r3k_flush_kernel_vmap_range;
- local_flush_data_cache_page = local_r3k_flush_data_cache_page;
flush_data_cache_page = r3k_flush_data_cache_page;
_dma_cache_wback_inv = r3k_dma_cache_wback_inv;
diff --git a/arch/mips/mm/c-r4k.c b/arch/mips/mm/c-r4k.c
index a549fa98c2f4..10413b6f6662 100644
--- a/arch/mips/mm/c-r4k.c
+++ b/arch/mips/mm/c-r4k.c
@@ -110,20 +110,6 @@ static unsigned long dcache_size __read_mostly;
static unsigned long vcache_size __read_mostly;
static unsigned long scache_size __read_mostly;
-/*
- * Dummy cache handling routines for machines without boardcaches
- */
-static void cache_noop(void) {}
-
-static struct bcache_ops no_sc_ops = {
- .bc_enable = (void *)cache_noop,
- .bc_disable = (void *)cache_noop,
- .bc_wback_inv = (void *)cache_noop,
- .bc_inv = (void *)cache_noop
-};
-
-struct bcache_ops *bcops = &no_sc_ops;
-
#define cpu_is_r4600_v1_x() ((read_c0_prid() & 0xfffffff0) == 0x00002010)
#define cpu_is_r4600_v2_x() ((read_c0_prid() & 0xfffffff0) == 0x00002020)
@@ -201,24 +187,6 @@ static void r4k_blast_dcache_user_page_setup(void)
#endif
-static void (* r4k_blast_dcache_page_indexed)(unsigned long addr);
-
-static void r4k_blast_dcache_page_indexed_setup(void)
-{
- unsigned long dc_lsize = cpu_dcache_line_size();
-
- if (dc_lsize == 0)
- r4k_blast_dcache_page_indexed = (void *)cache_noop;
- else if (dc_lsize == 16)
- r4k_blast_dcache_page_indexed = blast_dcache16_page_indexed;
- else if (dc_lsize == 32)
- r4k_blast_dcache_page_indexed = blast_dcache32_page_indexed;
- else if (dc_lsize == 64)
- r4k_blast_dcache_page_indexed = blast_dcache64_page_indexed;
- else if (dc_lsize == 128)
- r4k_blast_dcache_page_indexed = blast_dcache128_page_indexed;
-}
-
void (* r4k_blast_dcache)(void);
EXPORT_SYMBOL(r4k_blast_dcache);
@@ -280,39 +248,6 @@ static inline void tx49_blast_icache32(void)
addr | ws, 32);
}
-static inline void blast_icache32_r4600_v1_page_indexed(unsigned long page)
-{
- unsigned long flags;
-
- local_irq_save(flags);
- blast_icache32_page_indexed(page);
- local_irq_restore(flags);
-}
-
-static inline void tx49_blast_icache32_page_indexed(unsigned long page)
-{
- unsigned long indexmask = current_cpu_data.icache.waysize - 1;
- unsigned long start = INDEX_BASE + (page & indexmask);
- unsigned long end = start + PAGE_SIZE;
- unsigned long ws_inc = 1UL << current_cpu_data.icache.waybit;
- unsigned long ws_end = current_cpu_data.icache.ways <<
- current_cpu_data.icache.waybit;
- unsigned long ws, addr;
-
- CACHE32_UNROLL32_ALIGN2;
- /* I'm in even chunk. blast odd chunks */
- for (ws = 0; ws < ws_end; ws += ws_inc)
- for (addr = start + 0x400; addr < end; addr += 0x400 * 2)
- cache_unroll(32, kernel_cache, Index_Invalidate_I,
- addr | ws, 32);
- CACHE32_UNROLL32_ALIGN;
- /* I'm in odd chunk. blast even chunks */
- for (ws = 0; ws < ws_end; ws += ws_inc)
- for (addr = start; addr < end; addr += 0x400 * 2)
- cache_unroll(32, kernel_cache, Index_Invalidate_I,
- addr | ws, 32);
-}
-
static void (* r4k_blast_icache_page)(unsigned long addr);
static void r4k_blast_icache_page_setup(void)
@@ -355,34 +290,6 @@ static void r4k_blast_icache_user_page_setup(void)
#endif
-static void (* r4k_blast_icache_page_indexed)(unsigned long addr);
-
-static void r4k_blast_icache_page_indexed_setup(void)
-{
- unsigned long ic_lsize = cpu_icache_line_size();
-
- if (ic_lsize == 0)
- r4k_blast_icache_page_indexed = (void *)cache_noop;
- else if (ic_lsize == 16)
- r4k_blast_icache_page_indexed = blast_icache16_page_indexed;
- else if (ic_lsize == 32) {
- if (IS_ENABLED(CONFIG_WAR_R4600_V1_INDEX_ICACHEOP) &&
- cpu_is_r4600_v1_x())
- r4k_blast_icache_page_indexed =
- blast_icache32_r4600_v1_page_indexed;
- else if (IS_ENABLED(CONFIG_WAR_TX49XX_ICACHE_INDEX_INV))
- r4k_blast_icache_page_indexed =
- tx49_blast_icache32_page_indexed;
- else if (current_cpu_type() == CPU_LOONGSON2EF)
- r4k_blast_icache_page_indexed =
- loongson2_blast_icache32_page_indexed;
- else
- r4k_blast_icache_page_indexed =
- blast_icache32_page_indexed;
- } else if (ic_lsize == 64)
- r4k_blast_icache_page_indexed = blast_icache64_page_indexed;
-}
-
void (* r4k_blast_icache)(void);
EXPORT_SYMBOL(r4k_blast_icache);
@@ -428,24 +335,6 @@ static void r4k_blast_scache_page_setup(void)
r4k_blast_scache_page = blast_scache128_page;
}
-static void (* r4k_blast_scache_page_indexed)(unsigned long addr);
-
-static void r4k_blast_scache_page_indexed_setup(void)
-{
- unsigned long sc_lsize = cpu_scache_line_size();
-
- if (scache_size == 0)
- r4k_blast_scache_page_indexed = (void *)cache_noop;
- else if (sc_lsize == 16)
- r4k_blast_scache_page_indexed = blast_scache16_page_indexed;
- else if (sc_lsize == 32)
- r4k_blast_scache_page_indexed = blast_scache32_page_indexed;
- else if (sc_lsize == 64)
- r4k_blast_scache_page_indexed = blast_scache64_page_indexed;
- else if (sc_lsize == 128)
- r4k_blast_scache_page_indexed = blast_scache128_page_indexed;
-}
-
static void (* r4k_blast_scache)(void);
static void r4k_blast_scache_setup(void)
@@ -679,13 +568,14 @@ static inline void local_r4k_flush_cache_page(void *args)
if ((mm == current->active_mm) && (pte_val(*ptep) & _PAGE_VALID))
vaddr = NULL;
else {
+ struct folio *folio = page_folio(page);
/*
* Use kmap_coherent or kmap_atomic to do flushes for
* another ASID than the current one.
*/
map_coherent = (cpu_has_dc_aliases &&
- page_mapcount(page) &&
- !Page_dcache_dirty(page));
+ folio_mapped(folio) &&
+ !folio_test_dcache_dirty(folio));
if (map_coherent)
vaddr = kmap_coherent(page, addr);
else
@@ -1595,10 +1485,6 @@ static void loongson3_sc_init(void)
return;
}
-extern int r5k_sc_init(void);
-extern int rm7k_sc_init(void);
-extern int mips_sc_init(void);
-
static void setup_scache(void)
{
struct cpuinfo_mips *c = &current_cpu_data;
@@ -1764,7 +1650,7 @@ static void coherency_setup(void)
/*
* c0_status.cu=0 specifies that updates by the sc instruction use
- * the coherency mode specified by the TLB; 1 means cachable
+ * the coherency mode specified by the TLB; 1 means cacheable
* coherent update on write will be used. Not all processors have
* this bit and; some wire it to zero, others like Toshiba had the
* silly idea of putting something else there ...
@@ -1821,13 +1707,10 @@ void r4k_cache_init(void)
setup_scache();
r4k_blast_dcache_page_setup();
- r4k_blast_dcache_page_indexed_setup();
r4k_blast_dcache_setup();
r4k_blast_icache_page_setup();
- r4k_blast_icache_page_indexed_setup();
r4k_blast_icache_setup();
r4k_blast_scache_page_setup();
- r4k_blast_scache_page_indexed_setup();
r4k_blast_scache_setup();
r4k_blast_scache_node_setup();
#ifdef CONFIG_EVA
@@ -1859,7 +1742,6 @@ void r4k_cache_init(void)
__flush_kernel_vmap_range = r4k_flush_kernel_vmap_range;
flush_icache_all = r4k_flush_icache_all;
- local_flush_data_cache_page = local_r4k_flush_data_cache_page;
flush_data_cache_page = r4k_flush_data_cache_page;
flush_icache_range = r4k_flush_icache_range;
local_flush_icache_range = local_r4k_flush_icache_range;
@@ -1867,15 +1749,9 @@ void r4k_cache_init(void)
__local_flush_icache_user_range = local_r4k_flush_icache_user_range;
#ifdef CONFIG_DMA_NONCOHERENT
- if (dma_default_coherent) {
- _dma_cache_wback_inv = (void *)cache_noop;
- _dma_cache_wback = (void *)cache_noop;
- _dma_cache_inv = (void *)cache_noop;
- } else {
- _dma_cache_wback_inv = r4k_dma_cache_wback_inv;
- _dma_cache_wback = r4k_dma_cache_wback_inv;
- _dma_cache_inv = r4k_dma_cache_inv;
- }
+ _dma_cache_wback_inv = r4k_dma_cache_wback_inv;
+ _dma_cache_wback = r4k_dma_cache_wback_inv;
+ _dma_cache_inv = r4k_dma_cache_inv;
#endif /* CONFIG_DMA_NONCOHERENT */
build_clear_page();
@@ -1908,7 +1784,6 @@ void r4k_cache_init(void)
/* I$ fills from D$ just by emptying the write buffers */
flush_cache_page = (void *)b5k_instruction_hazard;
flush_cache_range = (void *)b5k_instruction_hazard;
- local_flush_data_cache_page = (void *)b5k_instruction_hazard;
flush_data_cache_page = (void *)b5k_instruction_hazard;
flush_icache_range = (void *)b5k_instruction_hazard;
local_flush_icache_range = (void *)b5k_instruction_hazard;
@@ -1928,7 +1803,6 @@ void r4k_cache_init(void)
flush_cache_range = (void *)cache_noop;
flush_icache_all = (void *)cache_noop;
flush_data_cache_page = (void *)cache_noop;
- local_flush_data_cache_page = (void *)cache_noop;
break;
}
}
@@ -1950,7 +1824,7 @@ static struct notifier_block r4k_cache_pm_notifier_block = {
.notifier_call = r4k_cache_pm_notifier,
};
-int __init r4k_cache_init_pm(void)
+static int __init r4k_cache_init_pm(void)
{
return cpu_pm_register_notifier(&r4k_cache_pm_notifier_block);
}
diff --git a/arch/mips/mm/cache.c b/arch/mips/mm/cache.c
index 11b3e7ddafd5..e3b4224c9a40 100644
--- a/arch/mips/mm/cache.c
+++ b/arch/mips/mm/cache.c
@@ -17,6 +17,7 @@
#include <linux/highmem.h>
#include <linux/pagemap.h>
+#include <asm/bcache.h>
#include <asm/cacheflush.h>
#include <asm/processor.h>
#include <asm/cpu.h>
@@ -48,14 +49,30 @@ void (*__flush_kernel_vmap_range)(unsigned long vaddr, int size);
EXPORT_SYMBOL_GPL(__flush_kernel_vmap_range);
/* MIPS specific cache operations */
-void (*local_flush_data_cache_page)(void * addr);
void (*flush_data_cache_page)(unsigned long addr);
void (*flush_icache_all)(void);
-EXPORT_SYMBOL_GPL(local_flush_data_cache_page);
EXPORT_SYMBOL(flush_data_cache_page);
EXPORT_SYMBOL(flush_icache_all);
+/*
+ * Dummy cache handling routine
+ */
+
+void cache_noop(void) {}
+
+#ifdef CONFIG_BOARD_SCACHE
+
+static struct bcache_ops no_sc_ops = {
+ .bc_enable = (void *)cache_noop,
+ .bc_disable = (void *)cache_noop,
+ .bc_wback_inv = (void *)cache_noop,
+ .bc_inv = (void *)cache_noop
+};
+
+struct bcache_ops *bcops = &no_sc_ops;
+#endif
+
#ifdef CONFIG_DMA_NONCOHERENT
/* DMA cache operations. */
@@ -82,40 +99,38 @@ SYSCALL_DEFINE3(cacheflush, unsigned long, addr, unsigned long, bytes,
return 0;
}
-void __flush_dcache_page(struct page *page)
+void __flush_dcache_folio_pages(struct folio *folio, struct page *page,
+ unsigned int nr)
{
- struct address_space *mapping = page_mapping_file(page);
+ struct address_space *mapping = folio_flush_mapping(folio);
unsigned long addr;
+ unsigned int i;
if (mapping && !mapping_mapped(mapping)) {
- SetPageDcacheDirty(page);
+ folio_set_dcache_dirty(folio);
return;
}
/*
- * We could delay the flush for the !page_mapping case too. But that
+ * We could delay the flush for the !folio_mapping case too. But that
* case is for exec env/arg pages and those are %99 certainly going to
* get faulted into the tlb (and thus flushed) anyways.
*/
- if (PageHighMem(page))
- addr = (unsigned long)kmap_atomic(page);
- else
- addr = (unsigned long)page_address(page);
-
- flush_data_cache_page(addr);
-
- if (PageHighMem(page))
- kunmap_atomic((void *)addr);
+ for (i = 0; i < nr; i++) {
+ addr = (unsigned long)kmap_local_page(page + i);
+ flush_data_cache_page(addr);
+ kunmap_local((void *)addr);
+ }
}
-
-EXPORT_SYMBOL(__flush_dcache_page);
+EXPORT_SYMBOL(__flush_dcache_folio_pages);
void __flush_anon_page(struct page *page, unsigned long vmaddr)
{
unsigned long addr = (unsigned long) page_address(page);
+ struct folio *folio = page_folio(page);
if (pages_do_alias(addr, vmaddr)) {
- if (page_mapcount(page) && !Page_dcache_dirty(page)) {
+ if (folio_mapped(folio) && !folio_test_dcache_dirty(folio)) {
void *kaddr;
kaddr = kmap_coherent(page, vmaddr);
@@ -130,27 +145,29 @@ EXPORT_SYMBOL(__flush_anon_page);
void __update_cache(unsigned long address, pte_t pte)
{
- struct page *page;
+ struct folio *folio;
unsigned long pfn, addr;
int exec = !pte_no_exec(pte) && !cpu_has_ic_fills_f_dc;
+ unsigned int i;
pfn = pte_pfn(pte);
if (unlikely(!pfn_valid(pfn)))
return;
- page = pfn_to_page(pfn);
- if (Page_dcache_dirty(page)) {
- if (PageHighMem(page))
- addr = (unsigned long)kmap_atomic(page);
- else
- addr = (unsigned long)page_address(page);
-
- if (exec || pages_do_alias(addr, address & PAGE_MASK))
- flush_data_cache_page(addr);
- if (PageHighMem(page))
- kunmap_atomic((void *)addr);
+ folio = page_folio(pfn_to_page(pfn));
+ address &= PAGE_MASK;
+ address -= offset_in_folio(folio, pfn << PAGE_SHIFT);
+
+ if (folio_test_dcache_dirty(folio)) {
+ for (i = 0; i < folio_nr_pages(folio); i++) {
+ addr = (unsigned long)kmap_local_folio(folio, i);
- ClearPageDcacheDirty(page);
+ if (exec || pages_do_alias(addr, address))
+ flush_data_cache_page(addr);
+ kunmap_local((void *)addr);
+ address += PAGE_SIZE;
+ }
+ folio_clear_dcache_dirty(folio);
}
}
@@ -188,22 +205,13 @@ static inline void setup_protection_map(void)
void cpu_cache_init(void)
{
- if (cpu_has_3k_cache) {
- extern void __weak r3k_cache_init(void);
-
+ if (IS_ENABLED(CONFIG_CPU_R3000) && cpu_has_3k_cache)
r3k_cache_init();
- }
- if (cpu_has_4k_cache) {
- extern void __weak r4k_cache_init(void);
-
+ if (IS_ENABLED(CONFIG_CPU_R4K_CACHE_TLB) && cpu_has_4k_cache)
r4k_cache_init();
- }
-
- if (cpu_has_octeon_cache) {
- extern void __weak octeon_cache_init(void);
+ if (IS_ENABLED(CONFIG_CPU_CAVIUM_OCTEON) && cpu_has_octeon_cache)
octeon_cache_init();
- }
setup_protection_map();
}
diff --git a/arch/mips/mm/cex-gen.S b/arch/mips/mm/cex-gen.S
index 45dff5cd4b8e..e528583d1331 100644
--- a/arch/mips/mm/cex-gen.S
+++ b/arch/mips/mm/cex-gen.S
@@ -25,7 +25,7 @@
* This is a very bad place to be. Our cache error
* detection has triggered. If we have write-back data
* in the cache, we may not be able to recover. As a
- * first-order desperate measure, turn off KSEG0 cacheing.
+ * first-order desperate measure, turn off KSEG0 caching.
*/
mfc0 k0,CP0_CONFIG
li k1,~CONF_CM_CMASK
diff --git a/arch/mips/mm/dma-noncoherent.c b/arch/mips/mm/dma-noncoherent.c
index 3c4fc97b9f39..ab4f2a75a7d0 100644
--- a/arch/mips/mm/dma-noncoherent.c
+++ b/arch/mips/mm/dma-noncoherent.c
@@ -137,8 +137,7 @@ void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size,
#endif
#ifdef CONFIG_ARCH_HAS_SETUP_DMA_OPS
-void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
- const struct iommu_ops *iommu, bool coherent)
+void arch_setup_dma_ops(struct device *dev, bool coherent)
{
dev->dma_coherent = coherent;
}
diff --git a/arch/mips/mm/fault.c b/arch/mips/mm/fault.c
index a27045f5a556..37fedeaca2e9 100644
--- a/arch/mips/mm/fault.c
+++ b/arch/mips/mm/fault.c
@@ -26,6 +26,7 @@
#include <asm/mmu_context.h>
#include <asm/ptrace.h>
#include <asm/highmem.h> /* For VMALLOC_END */
+#include <asm/traps.h>
#include <linux/kdebug.h>
int show_unhandled_signals = 1;
@@ -82,8 +83,8 @@ static void __do_page_fault(struct pt_regs *regs, unsigned long write,
if (unlikely(address >= VMALLOC_START && address <= VMALLOC_END))
goto VMALLOC_FAULT_TARGET;
-#ifdef MODULE_START
- if (unlikely(address >= MODULE_START && address < MODULE_END))
+#ifdef MODULES_VADDR
+ if (unlikely(address >= MODULES_VADDR && address < MODULES_END))
goto VMALLOC_FAULT_TARGET;
#endif
@@ -99,21 +100,13 @@ static void __do_page_fault(struct pt_regs *regs, unsigned long write,
perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
retry:
- mmap_read_lock(mm);
- vma = find_vma(mm, address);
+ vma = lock_mm_and_find_vma(mm, address, regs);
if (!vma)
- goto bad_area;
- if (vma->vm_start <= address)
- goto good_area;
- if (!(vma->vm_flags & VM_GROWSDOWN))
- goto bad_area;
- if (expand_stack(vma, address))
- goto bad_area;
+ goto bad_area_nosemaphore;
/*
* Ok, we have a good vm_area for this memory access, so
* we can handle it..
*/
-good_area:
si_code = SEGV_ACCERR;
if (write) {
diff --git a/arch/mips/mm/hugetlbpage.c b/arch/mips/mm/hugetlbpage.c
index 7eaff5b07873..0b9e15555b59 100644
--- a/arch/mips/mm/hugetlbpage.c
+++ b/arch/mips/mm/hugetlbpage.c
@@ -57,13 +57,3 @@ pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr,
}
return (pte_t *) pmd;
}
-
-int pmd_huge(pmd_t pmd)
-{
- return (pmd_val(pmd) & _PAGE_HUGE) != 0;
-}
-
-int pud_huge(pud_t pud)
-{
- return (pud_val(pud) & _PAGE_HUGE) != 0;
-}
diff --git a/arch/mips/mm/init.c b/arch/mips/mm/init.c
index 5a8002839550..a673d3d68254 100644
--- a/arch/mips/mm/init.c
+++ b/arch/mips/mm/init.c
@@ -31,6 +31,7 @@
#include <linux/gfp.h>
#include <linux/kcore.h>
#include <linux/initrd.h>
+#include <linux/execmem.h>
#include <asm/bootinfo.h>
#include <asm/cachectl.h>
@@ -38,6 +39,7 @@
#include <asm/dma.h>
#include <asm/maar.h>
#include <asm/mmu_context.h>
+#include <asm/mmzone.h>
#include <asm/sections.h>
#include <asm/pgalloc.h>
#include <asm/tlb.h>
@@ -57,24 +59,16 @@ EXPORT_SYMBOL(zero_page_mask);
/*
* Not static inline because used by IP27 special magic initialization code
*/
-void setup_zero_pages(void)
+static void __init setup_zero_pages(void)
{
- unsigned int order, i;
- struct page *page;
+ unsigned int order;
if (cpu_has_vce)
order = 3;
else
order = 0;
- empty_zero_page = __get_free_pages(GFP_KERNEL | __GFP_ZERO, order);
- if (!empty_zero_page)
- panic("Oh boy, that early out of memory?");
-
- page = virt_to_page((void *)empty_zero_page);
- split_page(page, order);
- for (i = 0; i < (1 << order); i++, page++)
- mark_page_reserved(page);
+ empty_zero_page = (unsigned long)memblock_alloc_or_panic(PAGE_SIZE << order, PAGE_SIZE);
zero_page_mask = ((PAGE_SIZE << order) - 1) & PAGE_MASK;
}
@@ -88,7 +82,7 @@ static void *__kmap_pgprot(struct page *page, unsigned long addr, pgprot_t prot)
pte_t pte;
int tlbidx;
- BUG_ON(Page_dcache_dirty(page));
+ BUG_ON(folio_test_dcache_dirty(page_folio(page)));
preempt_disable();
pagefault_disable();
@@ -169,11 +163,12 @@ void kunmap_coherent(void)
void copy_user_highpage(struct page *to, struct page *from,
unsigned long vaddr, struct vm_area_struct *vma)
{
+ struct folio *src = page_folio(from);
void *vfrom, *vto;
vto = kmap_atomic(to);
if (cpu_has_dc_aliases &&
- page_mapcount(from) && !Page_dcache_dirty(from)) {
+ folio_mapped(src) && !folio_test_dcache_dirty(src)) {
vfrom = kmap_coherent(from, vaddr);
copy_page(vto, vfrom);
kunmap_coherent();
@@ -194,15 +189,17 @@ void copy_to_user_page(struct vm_area_struct *vma,
struct page *page, unsigned long vaddr, void *dst, const void *src,
unsigned long len)
{
+ struct folio *folio = page_folio(page);
+
if (cpu_has_dc_aliases &&
- page_mapcount(page) && !Page_dcache_dirty(page)) {
+ folio_mapped(folio) && !folio_test_dcache_dirty(folio)) {
void *vto = kmap_coherent(page, vaddr) + (vaddr & ~PAGE_MASK);
memcpy(vto, src, len);
kunmap_coherent();
} else {
memcpy(dst, src, len);
if (cpu_has_dc_aliases)
- SetPageDcacheDirty(page);
+ folio_set_dcache_dirty(folio);
}
if (vma->vm_flags & VM_EXEC)
flush_cache_page(vma, vaddr, page_to_pfn(page));
@@ -212,15 +209,17 @@ void copy_from_user_page(struct vm_area_struct *vma,
struct page *page, unsigned long vaddr, void *dst, const void *src,
unsigned long len)
{
+ struct folio *folio = page_folio(page);
+
if (cpu_has_dc_aliases &&
- page_mapcount(page) && !Page_dcache_dirty(page)) {
+ folio_mapped(folio) && !folio_test_dcache_dirty(folio)) {
void *vfrom = kmap_coherent(page, vaddr) + (vaddr & ~PAGE_MASK);
memcpy(dst, vfrom, len);
kunmap_coherent();
} else {
memcpy(dst, src, len);
if (cpu_has_dc_aliases)
- SetPageDcacheDirty(page);
+ folio_set_dcache_dirty(folio);
}
}
EXPORT_SYMBOL_GPL(copy_from_user_page);
@@ -426,44 +425,16 @@ void __init paging_init(void)
static struct kcore_list kcore_kseg0;
#endif
-static inline void __init mem_init_free_highmem(void)
-{
-#ifdef CONFIG_HIGHMEM
- unsigned long tmp;
-
- if (cpu_has_dc_aliases)
- return;
-
- for (tmp = highstart_pfn; tmp < highend_pfn; tmp++) {
- struct page *page = pfn_to_page(tmp);
-
- if (!memblock_is_memory(PFN_PHYS(tmp)))
- SetPageReserved(page);
- else
- free_highmem_page(page);
- }
-#endif
-}
-
-void __init mem_init(void)
+void __init arch_mm_preinit(void)
{
/*
- * When _PFN_SHIFT is greater than PAGE_SHIFT we won't have enough PTE
+ * When PFN_PTE_SHIFT is greater than PAGE_SHIFT we won't have enough PTE
* bits to hold a full 32b physical address on MIPS32 systems.
*/
- BUILD_BUG_ON(IS_ENABLED(CONFIG_32BIT) && (_PFN_SHIFT > PAGE_SHIFT));
-
-#ifdef CONFIG_HIGHMEM
- max_mapnr = highend_pfn ? highend_pfn : max_low_pfn;
-#else
- max_mapnr = max_low_pfn;
-#endif
- high_memory = (void *) __va(max_low_pfn << PAGE_SHIFT);
+ BUILD_BUG_ON(IS_ENABLED(CONFIG_32BIT) && (PFN_PTE_SHIFT > PAGE_SHIFT));
maar_init();
- memblock_free_all();
setup_zero_pages(); /* Setup zeroed pages. */
- mem_init_free_highmem();
#ifdef CONFIG_64BIT
if ((unsigned long) &_text > (unsigned long) CKSEG0)
@@ -473,6 +444,11 @@ void __init mem_init(void)
0x80000000 - 4, KCORE_TEXT);
#endif
}
+#else /* CONFIG_NUMA */
+void __init arch_mm_preinit(void)
+{
+ setup_zero_pages(); /* This comes from node 0 */
+}
#endif /* !CONFIG_NUMA */
void free_init_pages(const char *what, unsigned long begin, unsigned long end)
@@ -568,3 +544,25 @@ EXPORT_SYMBOL_GPL(invalid_pmd_table);
#endif
pte_t invalid_pte_table[PTRS_PER_PTE] __page_aligned_bss;
EXPORT_SYMBOL(invalid_pte_table);
+
+#ifdef CONFIG_EXECMEM
+#ifdef MODULES_VADDR
+static struct execmem_info execmem_info __ro_after_init;
+
+struct execmem_info __init *execmem_arch_setup(void)
+{
+ execmem_info = (struct execmem_info){
+ .ranges = {
+ [EXECMEM_DEFAULT] = {
+ .start = MODULES_VADDR,
+ .end = MODULES_END,
+ .pgprot = PAGE_KERNEL,
+ .alignment = 1,
+ },
+ },
+ };
+
+ return &execmem_info;
+}
+#endif
+#endif /* CONFIG_EXECMEM */
diff --git a/arch/mips/mm/ioremap.c b/arch/mips/mm/ioremap.c
index b6dad2fd5575..c6c4576cd4a8 100644
--- a/arch/mips/mm/ioremap.c
+++ b/arch/mips/mm/ioremap.c
@@ -44,9 +44,9 @@ static int __ioremap_check_ram(unsigned long start_pfn, unsigned long nr_pages,
* ioremap_prot gives the caller control over cache coherency attributes (CCA)
*/
void __iomem *ioremap_prot(phys_addr_t phys_addr, unsigned long size,
- unsigned long prot_val)
+ pgprot_t prot)
{
- unsigned long flags = prot_val & _CACHE_MASK;
+ unsigned long flags = pgprot_val(prot) & _CACHE_MASK;
unsigned long offset, pfn, last_pfn;
struct vm_struct *area;
phys_addr_t last_addr;
@@ -72,6 +72,10 @@ void __iomem *ioremap_prot(phys_addr_t phys_addr, unsigned long size,
flags == _CACHE_UNCACHED)
return (void __iomem *) CKSEG1ADDR(phys_addr);
+ /* Early remaps should use the unmapped regions til' VM is available */
+ if (WARN_ON_ONCE(!slab_is_available()))
+ return NULL;
+
/*
* Don't allow anybody to remap RAM that may be allocated by the page
* allocator, since that could lead to races & data clobbering.
diff --git a/arch/mips/mm/ioremap64.c b/arch/mips/mm/ioremap64.c
index 15e7820d6a5f..acc03ba20098 100644
--- a/arch/mips/mm/ioremap64.c
+++ b/arch/mips/mm/ioremap64.c
@@ -3,9 +3,9 @@
#include <ioremap.h>
void __iomem *ioremap_prot(phys_addr_t offset, unsigned long size,
- unsigned long prot_val)
+ pgprot_t prot)
{
- unsigned long flags = prot_val & _CACHE_MASK;
+ unsigned long flags = pgprot_val(prot) & _CACHE_MASK;
u64 base = (flags == _CACHE_UNCACHED ? IO_BASE : UNCAC_BASE);
void __iomem *addr;
diff --git a/arch/mips/mm/mmap.c b/arch/mips/mm/mmap.c
index 00fe90c6db3e..5d2a1225785b 100644
--- a/arch/mips/mm/mmap.c
+++ b/arch/mips/mm/mmap.c
@@ -34,7 +34,7 @@ static unsigned long arch_get_unmapped_area_common(struct file *filp,
struct vm_area_struct *vma;
unsigned long addr = addr0;
int do_color_align;
- struct vm_unmapped_area_info info;
+ struct vm_unmapped_area_info info = {};
if (unlikely(len > TASK_SIZE))
return -ENOMEM;
@@ -92,14 +92,14 @@ static unsigned long arch_get_unmapped_area_common(struct file *filp,
*/
}
- info.flags = 0;
info.low_limit = mm->mmap_base;
info.high_limit = TASK_SIZE;
return vm_unmapped_area(&info);
}
unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr0,
- unsigned long len, unsigned long pgoff, unsigned long flags)
+ unsigned long len, unsigned long pgoff, unsigned long flags,
+ vm_flags_t vm_flags)
{
return arch_get_unmapped_area_common(filp,
addr0, len, pgoff, flags, UP);
@@ -111,7 +111,7 @@ unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr0,
*/
unsigned long arch_get_unmapped_area_topdown(struct file *filp,
unsigned long addr0, unsigned long len, unsigned long pgoff,
- unsigned long flags)
+ unsigned long flags, vm_flags_t vm_flags)
{
return arch_get_unmapped_area_common(filp,
addr0, len, pgoff, flags, DOWN);
diff --git a/arch/mips/mm/page-funcs.S b/arch/mips/mm/page-funcs.S
index 43181ac0a1af..42d0516ca18a 100644
--- a/arch/mips/mm/page-funcs.S
+++ b/arch/mips/mm/page-funcs.S
@@ -8,8 +8,8 @@
* Copyright (C) 2012 MIPS Technologies, Inc.
* Copyright (C) 2012 Ralf Baechle <ralf@linux-mips.org>
*/
+#include <linux/export.h>
#include <asm/asm.h>
-#include <asm/export.h>
#include <asm/regdef.h>
#ifdef CONFIG_SIBYTE_DMA_PAGEOPS
diff --git a/arch/mips/mm/page.c b/arch/mips/mm/page.c
index d3b4459d0fe8..1df237bd4a72 100644
--- a/arch/mips/mm/page.c
+++ b/arch/mips/mm/page.c
@@ -24,6 +24,7 @@
#include <asm/bootinfo.h>
#include <asm/mipsregs.h>
#include <asm/mmu_context.h>
+#include <asm/regdef.h>
#include <asm/cpu.h>
#ifdef CONFIG_SIBYTE_DMA_PAGEOPS
@@ -34,19 +35,6 @@
#include <asm/uasm.h>
-/* Registers used in the assembled routines. */
-#define ZERO 0
-#define AT 2
-#define A0 4
-#define A1 5
-#define A2 6
-#define T0 8
-#define T1 9
-#define T2 10
-#define T3 11
-#define T9 25
-#define RA 31
-
/* Handle labels (which must be positive integers). */
enum label_id {
label_clear_nopref = 1,
@@ -106,16 +94,16 @@ pg_addiu(u32 **buf, unsigned int reg1, unsigned int reg2, unsigned int off)
IS_ENABLED(CONFIG_CPU_DADDI_WORKAROUNDS) &&
r4k_daddiu_bug()) {
if (off > 0x7fff) {
- uasm_i_lui(buf, T9, uasm_rel_hi(off));
- uasm_i_addiu(buf, T9, T9, uasm_rel_lo(off));
+ uasm_i_lui(buf, GPR_T9, uasm_rel_hi(off));
+ uasm_i_addiu(buf, GPR_T9, GPR_T9, uasm_rel_lo(off));
} else
- uasm_i_addiu(buf, T9, ZERO, off);
- uasm_i_daddu(buf, reg1, reg2, T9);
+ uasm_i_addiu(buf, GPR_T9, GPR_ZERO, off);
+ uasm_i_daddu(buf, reg1, reg2, GPR_T9);
} else {
if (off > 0x7fff) {
- uasm_i_lui(buf, T9, uasm_rel_hi(off));
- uasm_i_addiu(buf, T9, T9, uasm_rel_lo(off));
- UASM_i_ADDU(buf, reg1, reg2, T9);
+ uasm_i_lui(buf, GPR_T9, uasm_rel_hi(off));
+ uasm_i_addiu(buf, GPR_T9, GPR_T9, uasm_rel_lo(off));
+ UASM_i_ADDU(buf, reg1, reg2, GPR_T9);
} else
UASM_i_ADDIU(buf, reg1, reg2, off);
}
@@ -233,9 +221,9 @@ static void set_prefetch_parameters(void)
static void build_clear_store(u32 **buf, int off)
{
if (cpu_has_64bit_gp_regs || cpu_has_64bit_zero_reg) {
- uasm_i_sd(buf, ZERO, off, A0);
+ uasm_i_sd(buf, GPR_ZERO, off, GPR_A0);
} else {
- uasm_i_sw(buf, ZERO, off, A0);
+ uasm_i_sw(buf, GPR_ZERO, off, GPR_A0);
}
}
@@ -246,10 +234,10 @@ static inline void build_clear_pref(u32 **buf, int off)
if (pref_bias_clear_store) {
_uasm_i_pref(buf, pref_dst_mode, pref_bias_clear_store + off,
- A0);
+ GPR_A0);
} else if (cache_line_size == (half_clear_loop_size << 1)) {
if (cpu_has_cache_cdex_s) {
- uasm_i_cache(buf, Create_Dirty_Excl_SD, off, A0);
+ uasm_i_cache(buf, Create_Dirty_Excl_SD, off, GPR_A0);
} else if (cpu_has_cache_cdex_p) {
if (IS_ENABLED(CONFIG_WAR_R4600_V1_HIT_CACHEOP) &&
cpu_is_r4600_v1_x()) {
@@ -261,9 +249,9 @@ static inline void build_clear_pref(u32 **buf, int off)
if (IS_ENABLED(CONFIG_WAR_R4600_V2_HIT_CACHEOP) &&
cpu_is_r4600_v2_x())
- uasm_i_lw(buf, ZERO, ZERO, AT);
+ uasm_i_lw(buf, GPR_ZERO, GPR_ZERO, GPR_AT);
- uasm_i_cache(buf, Create_Dirty_Excl_D, off, A0);
+ uasm_i_cache(buf, Create_Dirty_Excl_D, off, GPR_A0);
}
}
}
@@ -301,12 +289,12 @@ void build_clear_page(void)
off = PAGE_SIZE - pref_bias_clear_store;
if (off > 0xffff || !pref_bias_clear_store)
- pg_addiu(&buf, A2, A0, off);
+ pg_addiu(&buf, GPR_A2, GPR_A0, off);
else
- uasm_i_ori(&buf, A2, A0, off);
+ uasm_i_ori(&buf, GPR_A2, GPR_A0, off);
if (IS_ENABLED(CONFIG_WAR_R4600_V2_HIT_CACHEOP) && cpu_is_r4600_v2_x())
- uasm_i_lui(&buf, AT, uasm_rel_hi(0xa0000000));
+ uasm_i_lui(&buf, GPR_AT, uasm_rel_hi(0xa0000000));
off = cache_line_size ? min(8, pref_bias_clear_store / cache_line_size)
* cache_line_size : 0;
@@ -320,36 +308,36 @@ void build_clear_page(void)
build_clear_store(&buf, off);
off += clear_word_size;
} while (off < half_clear_loop_size);
- pg_addiu(&buf, A0, A0, 2 * off);
+ pg_addiu(&buf, GPR_A0, GPR_A0, 2 * off);
off = -off;
do {
build_clear_pref(&buf, off);
if (off == -clear_word_size)
- uasm_il_bne(&buf, &r, A0, A2, label_clear_pref);
+ uasm_il_bne(&buf, &r, GPR_A0, GPR_A2, label_clear_pref);
build_clear_store(&buf, off);
off += clear_word_size;
} while (off < 0);
if (pref_bias_clear_store) {
- pg_addiu(&buf, A2, A0, pref_bias_clear_store);
+ pg_addiu(&buf, GPR_A2, GPR_A0, pref_bias_clear_store);
uasm_l_clear_nopref(&l, buf);
off = 0;
do {
build_clear_store(&buf, off);
off += clear_word_size;
} while (off < half_clear_loop_size);
- pg_addiu(&buf, A0, A0, 2 * off);
+ pg_addiu(&buf, GPR_A0, GPR_A0, 2 * off);
off = -off;
do {
if (off == -clear_word_size)
- uasm_il_bne(&buf, &r, A0, A2,
+ uasm_il_bne(&buf, &r, GPR_A0, GPR_A2,
label_clear_nopref);
build_clear_store(&buf, off);
off += clear_word_size;
} while (off < 0);
}
- uasm_i_jr(&buf, RA);
+ uasm_i_jr(&buf, GPR_RA);
uasm_i_nop(&buf);
BUG_ON(buf > &__clear_page_end);
@@ -369,18 +357,18 @@ void build_clear_page(void)
static void build_copy_load(u32 **buf, int reg, int off)
{
if (cpu_has_64bit_gp_regs) {
- uasm_i_ld(buf, reg, off, A1);
+ uasm_i_ld(buf, reg, off, GPR_A1);
} else {
- uasm_i_lw(buf, reg, off, A1);
+ uasm_i_lw(buf, reg, off, GPR_A1);
}
}
static void build_copy_store(u32 **buf, int reg, int off)
{
if (cpu_has_64bit_gp_regs) {
- uasm_i_sd(buf, reg, off, A0);
+ uasm_i_sd(buf, reg, off, GPR_A0);
} else {
- uasm_i_sw(buf, reg, off, A0);
+ uasm_i_sw(buf, reg, off, GPR_A0);
}
}
@@ -390,7 +378,7 @@ static inline void build_copy_load_pref(u32 **buf, int off)
return;
if (pref_bias_copy_load)
- _uasm_i_pref(buf, pref_src_mode, pref_bias_copy_load + off, A1);
+ _uasm_i_pref(buf, pref_src_mode, pref_bias_copy_load + off, GPR_A1);
}
static inline void build_copy_store_pref(u32 **buf, int off)
@@ -400,10 +388,10 @@ static inline void build_copy_store_pref(u32 **buf, int off)
if (pref_bias_copy_store) {
_uasm_i_pref(buf, pref_dst_mode, pref_bias_copy_store + off,
- A0);
+ GPR_A0);
} else if (cache_line_size == (half_copy_loop_size << 1)) {
if (cpu_has_cache_cdex_s) {
- uasm_i_cache(buf, Create_Dirty_Excl_SD, off, A0);
+ uasm_i_cache(buf, Create_Dirty_Excl_SD, off, GPR_A0);
} else if (cpu_has_cache_cdex_p) {
if (IS_ENABLED(CONFIG_WAR_R4600_V1_HIT_CACHEOP) &&
cpu_is_r4600_v1_x()) {
@@ -415,9 +403,9 @@ static inline void build_copy_store_pref(u32 **buf, int off)
if (IS_ENABLED(CONFIG_WAR_R4600_V2_HIT_CACHEOP) &&
cpu_is_r4600_v2_x())
- uasm_i_lw(buf, ZERO, ZERO, AT);
+ uasm_i_lw(buf, GPR_ZERO, GPR_ZERO, GPR_AT);
- uasm_i_cache(buf, Create_Dirty_Excl_D, off, A0);
+ uasm_i_cache(buf, Create_Dirty_Excl_D, off, GPR_A0);
}
}
}
@@ -454,12 +442,12 @@ void build_copy_page(void)
off = PAGE_SIZE - pref_bias_copy_load;
if (off > 0xffff || !pref_bias_copy_load)
- pg_addiu(&buf, A2, A0, off);
+ pg_addiu(&buf, GPR_A2, GPR_A0, off);
else
- uasm_i_ori(&buf, A2, A0, off);
+ uasm_i_ori(&buf, GPR_A2, GPR_A0, off);
if (IS_ENABLED(CONFIG_WAR_R4600_V2_HIT_CACHEOP) && cpu_is_r4600_v2_x())
- uasm_i_lui(&buf, AT, uasm_rel_hi(0xa0000000));
+ uasm_i_lui(&buf, GPR_AT, uasm_rel_hi(0xa0000000));
off = cache_line_size ? min(8, pref_bias_copy_load / cache_line_size) *
cache_line_size : 0;
@@ -476,126 +464,126 @@ void build_copy_page(void)
uasm_l_copy_pref_both(&l, buf);
do {
build_copy_load_pref(&buf, off);
- build_copy_load(&buf, T0, off);
+ build_copy_load(&buf, GPR_T0, off);
build_copy_load_pref(&buf, off + copy_word_size);
- build_copy_load(&buf, T1, off + copy_word_size);
+ build_copy_load(&buf, GPR_T1, off + copy_word_size);
build_copy_load_pref(&buf, off + 2 * copy_word_size);
- build_copy_load(&buf, T2, off + 2 * copy_word_size);
+ build_copy_load(&buf, GPR_T2, off + 2 * copy_word_size);
build_copy_load_pref(&buf, off + 3 * copy_word_size);
- build_copy_load(&buf, T3, off + 3 * copy_word_size);
+ build_copy_load(&buf, GPR_T3, off + 3 * copy_word_size);
build_copy_store_pref(&buf, off);
- build_copy_store(&buf, T0, off);
+ build_copy_store(&buf, GPR_T0, off);
build_copy_store_pref(&buf, off + copy_word_size);
- build_copy_store(&buf, T1, off + copy_word_size);
+ build_copy_store(&buf, GPR_T1, off + copy_word_size);
build_copy_store_pref(&buf, off + 2 * copy_word_size);
- build_copy_store(&buf, T2, off + 2 * copy_word_size);
+ build_copy_store(&buf, GPR_T2, off + 2 * copy_word_size);
build_copy_store_pref(&buf, off + 3 * copy_word_size);
- build_copy_store(&buf, T3, off + 3 * copy_word_size);
+ build_copy_store(&buf, GPR_T3, off + 3 * copy_word_size);
off += 4 * copy_word_size;
} while (off < half_copy_loop_size);
- pg_addiu(&buf, A1, A1, 2 * off);
- pg_addiu(&buf, A0, A0, 2 * off);
+ pg_addiu(&buf, GPR_A1, GPR_A1, 2 * off);
+ pg_addiu(&buf, GPR_A0, GPR_A0, 2 * off);
off = -off;
do {
build_copy_load_pref(&buf, off);
- build_copy_load(&buf, T0, off);
+ build_copy_load(&buf, GPR_T0, off);
build_copy_load_pref(&buf, off + copy_word_size);
- build_copy_load(&buf, T1, off + copy_word_size);
+ build_copy_load(&buf, GPR_T1, off + copy_word_size);
build_copy_load_pref(&buf, off + 2 * copy_word_size);
- build_copy_load(&buf, T2, off + 2 * copy_word_size);
+ build_copy_load(&buf, GPR_T2, off + 2 * copy_word_size);
build_copy_load_pref(&buf, off + 3 * copy_word_size);
- build_copy_load(&buf, T3, off + 3 * copy_word_size);
+ build_copy_load(&buf, GPR_T3, off + 3 * copy_word_size);
build_copy_store_pref(&buf, off);
- build_copy_store(&buf, T0, off);
+ build_copy_store(&buf, GPR_T0, off);
build_copy_store_pref(&buf, off + copy_word_size);
- build_copy_store(&buf, T1, off + copy_word_size);
+ build_copy_store(&buf, GPR_T1, off + copy_word_size);
build_copy_store_pref(&buf, off + 2 * copy_word_size);
- build_copy_store(&buf, T2, off + 2 * copy_word_size);
+ build_copy_store(&buf, GPR_T2, off + 2 * copy_word_size);
build_copy_store_pref(&buf, off + 3 * copy_word_size);
if (off == -(4 * copy_word_size))
- uasm_il_bne(&buf, &r, A2, A0, label_copy_pref_both);
- build_copy_store(&buf, T3, off + 3 * copy_word_size);
+ uasm_il_bne(&buf, &r, GPR_A2, GPR_A0, label_copy_pref_both);
+ build_copy_store(&buf, GPR_T3, off + 3 * copy_word_size);
off += 4 * copy_word_size;
} while (off < 0);
if (pref_bias_copy_load - pref_bias_copy_store) {
- pg_addiu(&buf, A2, A0,
+ pg_addiu(&buf, GPR_A2, GPR_A0,
pref_bias_copy_load - pref_bias_copy_store);
uasm_l_copy_pref_store(&l, buf);
off = 0;
do {
- build_copy_load(&buf, T0, off);
- build_copy_load(&buf, T1, off + copy_word_size);
- build_copy_load(&buf, T2, off + 2 * copy_word_size);
- build_copy_load(&buf, T3, off + 3 * copy_word_size);
+ build_copy_load(&buf, GPR_T0, off);
+ build_copy_load(&buf, GPR_T1, off + copy_word_size);
+ build_copy_load(&buf, GPR_T2, off + 2 * copy_word_size);
+ build_copy_load(&buf, GPR_T3, off + 3 * copy_word_size);
build_copy_store_pref(&buf, off);
- build_copy_store(&buf, T0, off);
+ build_copy_store(&buf, GPR_T0, off);
build_copy_store_pref(&buf, off + copy_word_size);
- build_copy_store(&buf, T1, off + copy_word_size);
+ build_copy_store(&buf, GPR_T1, off + copy_word_size);
build_copy_store_pref(&buf, off + 2 * copy_word_size);
- build_copy_store(&buf, T2, off + 2 * copy_word_size);
+ build_copy_store(&buf, GPR_T2, off + 2 * copy_word_size);
build_copy_store_pref(&buf, off + 3 * copy_word_size);
- build_copy_store(&buf, T3, off + 3 * copy_word_size);
+ build_copy_store(&buf, GPR_T3, off + 3 * copy_word_size);
off += 4 * copy_word_size;
} while (off < half_copy_loop_size);
- pg_addiu(&buf, A1, A1, 2 * off);
- pg_addiu(&buf, A0, A0, 2 * off);
+ pg_addiu(&buf, GPR_A1, GPR_A1, 2 * off);
+ pg_addiu(&buf, GPR_A0, GPR_A0, 2 * off);
off = -off;
do {
- build_copy_load(&buf, T0, off);
- build_copy_load(&buf, T1, off + copy_word_size);
- build_copy_load(&buf, T2, off + 2 * copy_word_size);
- build_copy_load(&buf, T3, off + 3 * copy_word_size);
+ build_copy_load(&buf, GPR_T0, off);
+ build_copy_load(&buf, GPR_T1, off + copy_word_size);
+ build_copy_load(&buf, GPR_T2, off + 2 * copy_word_size);
+ build_copy_load(&buf, GPR_T3, off + 3 * copy_word_size);
build_copy_store_pref(&buf, off);
- build_copy_store(&buf, T0, off);
+ build_copy_store(&buf, GPR_T0, off);
build_copy_store_pref(&buf, off + copy_word_size);
- build_copy_store(&buf, T1, off + copy_word_size);
+ build_copy_store(&buf, GPR_T1, off + copy_word_size);
build_copy_store_pref(&buf, off + 2 * copy_word_size);
- build_copy_store(&buf, T2, off + 2 * copy_word_size);
+ build_copy_store(&buf, GPR_T2, off + 2 * copy_word_size);
build_copy_store_pref(&buf, off + 3 * copy_word_size);
if (off == -(4 * copy_word_size))
- uasm_il_bne(&buf, &r, A2, A0,
+ uasm_il_bne(&buf, &r, GPR_A2, GPR_A0,
label_copy_pref_store);
- build_copy_store(&buf, T3, off + 3 * copy_word_size);
+ build_copy_store(&buf, GPR_T3, off + 3 * copy_word_size);
off += 4 * copy_word_size;
} while (off < 0);
}
if (pref_bias_copy_store) {
- pg_addiu(&buf, A2, A0, pref_bias_copy_store);
+ pg_addiu(&buf, GPR_A2, GPR_A0, pref_bias_copy_store);
uasm_l_copy_nopref(&l, buf);
off = 0;
do {
- build_copy_load(&buf, T0, off);
- build_copy_load(&buf, T1, off + copy_word_size);
- build_copy_load(&buf, T2, off + 2 * copy_word_size);
- build_copy_load(&buf, T3, off + 3 * copy_word_size);
- build_copy_store(&buf, T0, off);
- build_copy_store(&buf, T1, off + copy_word_size);
- build_copy_store(&buf, T2, off + 2 * copy_word_size);
- build_copy_store(&buf, T3, off + 3 * copy_word_size);
+ build_copy_load(&buf, GPR_T0, off);
+ build_copy_load(&buf, GPR_T1, off + copy_word_size);
+ build_copy_load(&buf, GPR_T2, off + 2 * copy_word_size);
+ build_copy_load(&buf, GPR_T3, off + 3 * copy_word_size);
+ build_copy_store(&buf, GPR_T0, off);
+ build_copy_store(&buf, GPR_T1, off + copy_word_size);
+ build_copy_store(&buf, GPR_T2, off + 2 * copy_word_size);
+ build_copy_store(&buf, GPR_T3, off + 3 * copy_word_size);
off += 4 * copy_word_size;
} while (off < half_copy_loop_size);
- pg_addiu(&buf, A1, A1, 2 * off);
- pg_addiu(&buf, A0, A0, 2 * off);
+ pg_addiu(&buf, GPR_A1, GPR_A1, 2 * off);
+ pg_addiu(&buf, GPR_A0, GPR_A0, 2 * off);
off = -off;
do {
- build_copy_load(&buf, T0, off);
- build_copy_load(&buf, T1, off + copy_word_size);
- build_copy_load(&buf, T2, off + 2 * copy_word_size);
- build_copy_load(&buf, T3, off + 3 * copy_word_size);
- build_copy_store(&buf, T0, off);
- build_copy_store(&buf, T1, off + copy_word_size);
- build_copy_store(&buf, T2, off + 2 * copy_word_size);
+ build_copy_load(&buf, GPR_T0, off);
+ build_copy_load(&buf, GPR_T1, off + copy_word_size);
+ build_copy_load(&buf, GPR_T2, off + 2 * copy_word_size);
+ build_copy_load(&buf, GPR_T3, off + 3 * copy_word_size);
+ build_copy_store(&buf, GPR_T0, off);
+ build_copy_store(&buf, GPR_T1, off + copy_word_size);
+ build_copy_store(&buf, GPR_T2, off + 2 * copy_word_size);
if (off == -(4 * copy_word_size))
- uasm_il_bne(&buf, &r, A2, A0,
+ uasm_il_bne(&buf, &r, GPR_A2, GPR_A0,
label_copy_nopref);
- build_copy_store(&buf, T3, off + 3 * copy_word_size);
+ build_copy_store(&buf, GPR_T3, off + 3 * copy_word_size);
off += 4 * copy_word_size;
} while (off < 0);
}
- uasm_i_jr(&buf, RA);
+ uasm_i_jr(&buf, GPR_RA);
uasm_i_nop(&buf);
BUG_ON(buf > &__copy_page_end);
diff --git a/arch/mips/mm/pgtable-32.c b/arch/mips/mm/pgtable-32.c
index 61891af25019..e2cf2166d5cb 100644
--- a/arch/mips/mm/pgtable-32.c
+++ b/arch/mips/mm/pgtable-32.c
@@ -13,9 +13,9 @@
#include <asm/pgalloc.h>
#include <asm/tlbflush.h>
-void pgd_init(unsigned long page)
+void pgd_init(void *addr)
{
- unsigned long *p = (unsigned long *) page;
+ unsigned long *p = (unsigned long *)addr;
int i;
for (i = 0; i < USER_PTRS_PER_PGD; i+=8) {
@@ -31,16 +31,6 @@ void pgd_init(unsigned long page)
}
#if defined(CONFIG_TRANSPARENT_HUGEPAGE)
-pmd_t mk_pmd(struct page *page, pgprot_t prot)
-{
- pmd_t pmd;
-
- pmd_val(pmd) = (page_to_pfn(page) << _PFN_SHIFT) | pgprot_val(prot);
-
- return pmd;
-}
-
-
void set_pmd_at(struct mm_struct *mm, unsigned long addr,
pmd_t *pmdp, pmd_t pmd)
{
@@ -61,9 +51,8 @@ void __init pagetable_init(void)
#endif
/* Initialize the entire pgd. */
- pgd_init((unsigned long)swapper_pg_dir);
- pgd_init((unsigned long)swapper_pg_dir
- + sizeof(pgd_t) * USER_PTRS_PER_PGD);
+ pgd_init(swapper_pg_dir);
+ pgd_init(&swapper_pg_dir[USER_PTRS_PER_PGD]);
pgd_base = swapper_pg_dir;
diff --git a/arch/mips/mm/pgtable-64.c b/arch/mips/mm/pgtable-64.c
index 7536f7804c44..b24f865de357 100644
--- a/arch/mips/mm/pgtable-64.c
+++ b/arch/mips/mm/pgtable-64.c
@@ -13,7 +13,7 @@
#include <asm/pgalloc.h>
#include <asm/tlbflush.h>
-void pgd_init(unsigned long page)
+void pgd_init(void *addr)
{
unsigned long *p, *end;
unsigned long entry;
@@ -26,7 +26,7 @@ void pgd_init(unsigned long page)
entry = (unsigned long)invalid_pte_table;
#endif
- p = (unsigned long *) page;
+ p = (unsigned long *) addr;
end = p + PTRS_PER_PGD;
do {
@@ -43,11 +43,12 @@ void pgd_init(unsigned long page)
}
#ifndef __PAGETABLE_PMD_FOLDED
-void pmd_init(unsigned long addr, unsigned long pagetable)
+void pmd_init(void *addr)
{
unsigned long *p, *end;
+ unsigned long pagetable = (unsigned long)invalid_pte_table;
- p = (unsigned long *) addr;
+ p = (unsigned long *)addr;
end = p + PTRS_PER_PMD;
do {
@@ -66,9 +67,10 @@ EXPORT_SYMBOL_GPL(pmd_init);
#endif
#ifndef __PAGETABLE_PUD_FOLDED
-void pud_init(unsigned long addr, unsigned long pagetable)
+void pud_init(void *addr)
{
unsigned long *p, *end;
+ unsigned long pagetable = (unsigned long)invalid_pmd_table;
p = (unsigned long *)addr;
end = p + PTRS_PER_PUD;
@@ -87,20 +89,13 @@ void pud_init(unsigned long addr, unsigned long pagetable)
}
#endif
-pmd_t mk_pmd(struct page *page, pgprot_t prot)
-{
- pmd_t pmd;
-
- pmd_val(pmd) = (page_to_pfn(page) << _PFN_SHIFT) | pgprot_val(prot);
-
- return pmd;
-}
-
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
void set_pmd_at(struct mm_struct *mm, unsigned long addr,
pmd_t *pmdp, pmd_t pmd)
{
*pmdp = pmd;
}
+#endif
void __init pagetable_init(void)
{
@@ -108,12 +103,12 @@ void __init pagetable_init(void)
pgd_t *pgd_base;
/* Initialize the entire pgd. */
- pgd_init((unsigned long)swapper_pg_dir);
+ pgd_init(swapper_pg_dir);
#ifndef __PAGETABLE_PUD_FOLDED
- pud_init((unsigned long)invalid_pud_table, (unsigned long)invalid_pmd_table);
+ pud_init(invalid_pud_table);
#endif
#ifndef __PAGETABLE_PMD_FOLDED
- pmd_init((unsigned long)invalid_pmd_table, (unsigned long)invalid_pte_table);
+ pmd_init(invalid_pmd_table);
#endif
pgd_base = swapper_pg_dir;
/*
diff --git a/arch/mips/mm/pgtable.c b/arch/mips/mm/pgtable.c
index 3b7590660a04..10835414819f 100644
--- a/arch/mips/mm/pgtable.c
+++ b/arch/mips/mm/pgtable.c
@@ -10,12 +10,12 @@
pgd_t *pgd_alloc(struct mm_struct *mm)
{
- pgd_t *ret, *init;
+ pgd_t *init, *ret;
- ret = (pgd_t *) __get_free_pages(GFP_KERNEL, PGD_TABLE_ORDER);
+ ret = __pgd_alloc(mm, PGD_TABLE_ORDER);
if (ret) {
init = pgd_offset(&init_mm, 0UL);
- pgd_init((unsigned long)ret);
+ pgd_init(ret);
memcpy(ret + USER_PTRS_PER_PGD, init + USER_PTRS_PER_PGD,
(PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t));
}
diff --git a/arch/mips/mm/physaddr.c b/arch/mips/mm/physaddr.c
index f9b8c85e9843..a6b1bf82057a 100644
--- a/arch/mips/mm/physaddr.c
+++ b/arch/mips/mm/physaddr.c
@@ -30,7 +30,7 @@ static inline bool __debug_virt_addr_valid(unsigned long x)
phys_addr_t __virt_to_phys(volatile const void *x)
{
WARN(!__debug_virt_addr_valid((unsigned long)x),
- "virt_to_phys used for non-linear address: %pK (%pS)\n",
+ "virt_to_phys used for non-linear address: %p (%pS)\n",
x, x);
return __virt_to_phys_nodebug(x);
diff --git a/arch/mips/mm/tlb-funcs.S b/arch/mips/mm/tlb-funcs.S
index 00fef578c8cd..2705d7dcb33e 100644
--- a/arch/mips/mm/tlb-funcs.S
+++ b/arch/mips/mm/tlb-funcs.S
@@ -11,8 +11,8 @@
* Copyright (C) 2012 MIPS Technologies, Inc.
* Copyright (C) 2012 Ralf Baechle <ralf@linux-mips.org>
*/
+#include <linux/export.h>
#include <asm/asm.h>
-#include <asm/export.h>
#include <asm/regdef.h>
#define FASTPATH_SIZE 128
diff --git a/arch/mips/mm/tlb-r3k.c b/arch/mips/mm/tlb-r3k.c
index 53dfa2b9316b..173f7b36033b 100644
--- a/arch/mips/mm/tlb-r3k.c
+++ b/arch/mips/mm/tlb-r3k.c
@@ -23,11 +23,11 @@
#include <asm/io.h>
#include <asm/bootinfo.h>
#include <asm/cpu.h>
+#include <asm/setup.h>
+#include <asm/tlbex.h>
#undef DEBUG_TLB
-extern void build_tlb_refill_handler(void);
-
/* CP0 hazard avoidance. */
#define BARRIER \
__asm__ __volatile__( \
@@ -183,7 +183,7 @@ void __update_tlb(struct vm_area_struct *vma, unsigned long address, pte_t pte)
int idx, pid;
/*
- * Handle debugger faulting in for debugee.
+ * Handle debugger faulting in for debuggee.
*/
if (current->active_mm != vma->vm_mm)
return;
diff --git a/arch/mips/mm/tlb-r4k.c b/arch/mips/mm/tlb-r4k.c
index 1b939abbe4ca..44a662536148 100644
--- a/arch/mips/mm/tlb-r4k.c
+++ b/arch/mips/mm/tlb-r4k.c
@@ -12,9 +12,11 @@
#include <linux/init.h>
#include <linux/sched.h>
#include <linux/smp.h>
+#include <linux/memblock.h>
#include <linux/mm.h>
#include <linux/hugetlb.h>
#include <linux/export.h>
+#include <linux/sort.h>
#include <asm/cpu.h>
#include <asm/cpu-type.h>
@@ -22,9 +24,9 @@
#include <asm/hazards.h>
#include <asm/mmu_context.h>
#include <asm/tlb.h>
+#include <asm/tlbex.h>
#include <asm/tlbmisc.h>
-
-extern void build_tlb_refill_handler(void);
+#include <asm/setup.h>
/*
* LOONGSON-2 has a 4 entry itlb which is a subset of jtlb, LOONGSON-3 has
@@ -297,11 +299,11 @@ void __update_tlb(struct vm_area_struct * vma, unsigned long address, pte_t pte)
p4d_t *p4dp;
pud_t *pudp;
pmd_t *pmdp;
- pte_t *ptep;
+ pte_t *ptep, *ptemap = NULL;
int idx, pid;
/*
- * Handle debugger faulting in for debugee.
+ * Handle debugger faulting in for debuggee.
*/
if (current->active_mm != vma->vm_mm)
return;
@@ -326,7 +328,7 @@ void __update_tlb(struct vm_area_struct * vma, unsigned long address, pte_t pte)
idx = read_c0_index();
#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
/* this could be a huge page */
- if (pmd_huge(*pmdp)) {
+ if (pmd_leaf(*pmdp)) {
unsigned long lo;
write_c0_pagemask(PM_HUGE_MASK);
ptep = (pte_t *)pmdp;
@@ -344,7 +346,12 @@ void __update_tlb(struct vm_area_struct * vma, unsigned long address, pte_t pte)
} else
#endif
{
- ptep = pte_offset_map(pmdp, address);
+ ptemap = ptep = pte_offset_map(pmdp, address);
+ /*
+ * update_mmu_cache() is called between pte_offset_map_lock()
+ * and pte_unmap_unlock(), so we can assume that ptep is not
+ * NULL here: and what should be done below if it were NULL?
+ */
#if defined(CONFIG_PHYS_ADDR_T_64BIT) && defined(CONFIG_CPU_MIPS32)
#ifdef CONFIG_XPA
@@ -373,6 +380,9 @@ void __update_tlb(struct vm_area_struct * vma, unsigned long address, pte_t pte)
tlbw_use_hazard();
htw_start();
flush_micro_tlb_vm(vma);
+
+ if (ptemap)
+ pte_unmap(ptemap);
local_irq_restore(flags);
}
@@ -450,6 +460,7 @@ EXPORT_SYMBOL(has_transparent_hugepage);
int temp_tlb_entry;
+#ifndef CONFIG_64BIT
__init int add_temporary_entry(unsigned long entrylo0, unsigned long entrylo1,
unsigned long entryhi, unsigned long pagemask)
{
@@ -488,6 +499,7 @@ out:
local_irq_restore(flags);
return ret;
}
+#endif
static int ntlb;
static int __init set_ntlb(char *str)
@@ -498,6 +510,97 @@ static int __init set_ntlb(char *str)
__setup("ntlb=", set_ntlb);
+
+/* Comparison function for EntryHi VPN fields. */
+static int r4k_vpn_cmp(const void *a, const void *b)
+{
+ long v = *(unsigned long *)a - *(unsigned long *)b;
+ int s = sizeof(long) > sizeof(int) ? sizeof(long) * 8 - 1: 0;
+ return s ? (v != 0) | v >> s : v;
+}
+
+/*
+ * Initialise all TLB entries with unique values that do not clash with
+ * what we have been handed over and what we'll be using ourselves.
+ */
+static void __ref r4k_tlb_uniquify(void)
+{
+ int tlbsize = current_cpu_data.tlbsize;
+ bool use_slab = slab_is_available();
+ int start = num_wired_entries();
+ phys_addr_t tlb_vpn_size;
+ unsigned long *tlb_vpns;
+ unsigned long vpn_mask;
+ int cnt, ent, idx, i;
+
+ vpn_mask = GENMASK(cpu_vmbits - 1, 13);
+ vpn_mask |= IS_ENABLED(CONFIG_64BIT) ? 3ULL << 62 : 1 << 31;
+
+ tlb_vpn_size = tlbsize * sizeof(*tlb_vpns);
+ tlb_vpns = (use_slab ?
+ kmalloc(tlb_vpn_size, GFP_KERNEL) :
+ memblock_alloc_raw(tlb_vpn_size, sizeof(*tlb_vpns)));
+ if (WARN_ON(!tlb_vpns))
+ return; /* Pray local_flush_tlb_all() is good enough. */
+
+ htw_stop();
+
+ for (i = start, cnt = 0; i < tlbsize; i++, cnt++) {
+ unsigned long vpn;
+
+ write_c0_index(i);
+ mtc0_tlbr_hazard();
+ tlb_read();
+ tlb_read_hazard();
+ vpn = read_c0_entryhi();
+ vpn &= vpn_mask & PAGE_MASK;
+ tlb_vpns[cnt] = vpn;
+
+ /* Prevent any large pages from overlapping regular ones. */
+ write_c0_pagemask(read_c0_pagemask() & PM_DEFAULT_MASK);
+ mtc0_tlbw_hazard();
+ tlb_write_indexed();
+ tlbw_use_hazard();
+ }
+
+ sort(tlb_vpns, cnt, sizeof(tlb_vpns[0]), r4k_vpn_cmp, NULL);
+
+ write_c0_pagemask(PM_DEFAULT_MASK);
+ write_c0_entrylo0(0);
+ write_c0_entrylo1(0);
+
+ idx = 0;
+ ent = tlbsize;
+ for (i = start; i < tlbsize; i++)
+ while (1) {
+ unsigned long entryhi, vpn;
+
+ entryhi = UNIQUE_ENTRYHI(ent);
+ vpn = entryhi & vpn_mask & PAGE_MASK;
+
+ if (idx >= cnt || vpn < tlb_vpns[idx]) {
+ write_c0_entryhi(entryhi);
+ write_c0_index(i);
+ mtc0_tlbw_hazard();
+ tlb_write_indexed();
+ ent++;
+ break;
+ } else if (vpn == tlb_vpns[idx]) {
+ ent++;
+ } else {
+ idx++;
+ }
+ }
+
+ tlbw_use_hazard();
+ htw_start();
+ flush_micro_tlb();
+ if (use_slab)
+ kfree(tlb_vpns);
+ else
+ memblock_free(tlb_vpns, tlb_vpn_size);
+}
+
/*
* Configure TLB (for init or after a CPU has been powered off).
*/
@@ -537,6 +640,7 @@ static void r4k_tlb_configure(void)
temp_tlb_entry = current_cpu_data.tlbsize - 1;
/* From this point on the ARC firmware is dead. */
+ r4k_tlb_uniquify();
local_flush_tlb_all();
/* Did I tell you that ARC SUCKS? */
diff --git a/arch/mips/mm/tlbex.c b/arch/mips/mm/tlbex.c
index 80e05ee98d62..69ea54bdc0c3 100644
--- a/arch/mips/mm/tlbex.c
+++ b/arch/mips/mm/tlbex.c
@@ -32,7 +32,9 @@
#include <asm/cacheflush.h>
#include <asm/cpu-type.h>
+#include <asm/mipsregs.h>
#include <asm/mmu_context.h>
+#include <asm/regdef.h>
#include <asm/uasm.h>
#include <asm/setup.h>
#include <asm/tlbex.h>
@@ -253,7 +255,7 @@ static void output_pgtable_bits_defines(void)
pr_define("_PAGE_GLOBAL_SHIFT %d\n", _PAGE_GLOBAL_SHIFT);
pr_define("_PAGE_VALID_SHIFT %d\n", _PAGE_VALID_SHIFT);
pr_define("_PAGE_DIRTY_SHIFT %d\n", _PAGE_DIRTY_SHIFT);
- pr_define("_PFN_SHIFT %d\n", _PFN_SHIFT);
+ pr_define("PFN_PTE_SHIFT %d\n", PFN_PTE_SHIFT);
pr_debug("\n");
}
@@ -276,27 +278,6 @@ static inline void dump_handler(const char *symbol, const void *start, const voi
pr_debug("\tEND(%s)\n", symbol);
}
-/* The only general purpose registers allowed in TLB handlers. */
-#define K0 26
-#define K1 27
-
-/* Some CP0 registers */
-#define C0_INDEX 0, 0
-#define C0_ENTRYLO0 2, 0
-#define C0_TCBIND 2, 2
-#define C0_ENTRYLO1 3, 0
-#define C0_CONTEXT 4, 0
-#define C0_PAGEMASK 5, 0
-#define C0_PWBASE 5, 5
-#define C0_PWFIELD 5, 6
-#define C0_PWSIZE 5, 7
-#define C0_PWCTL 6, 6
-#define C0_BADVADDR 8, 0
-#define C0_PGD 9, 7
-#define C0_ENTRYHI 10, 0
-#define C0_EPC 14, 0
-#define C0_XCONTEXT 20, 0
-
#ifdef CONFIG_64BIT
# define GET_CONTEXT(buf, reg) UASM_i_MFC0(buf, reg, C0_XCONTEXT)
#else
@@ -356,30 +337,30 @@ static struct work_registers build_get_work_registers(u32 **p)
if (scratch_reg >= 0) {
/* Save in CPU local C0_KScratch? */
UASM_i_MTC0(p, 1, c0_kscratch(), scratch_reg);
- r.r1 = K0;
- r.r2 = K1;
- r.r3 = 1;
+ r.r1 = GPR_K0;
+ r.r2 = GPR_K1;
+ r.r3 = GPR_AT;
return r;
}
if (num_possible_cpus() > 1) {
/* Get smp_processor_id */
- UASM_i_CPUID_MFC0(p, K0, SMP_CPUID_REG);
- UASM_i_SRL_SAFE(p, K0, K0, SMP_CPUID_REGSHIFT);
+ UASM_i_CPUID_MFC0(p, GPR_K0, SMP_CPUID_REG);
+ UASM_i_SRL_SAFE(p, GPR_K0, GPR_K0, SMP_CPUID_REGSHIFT);
- /* handler_reg_save index in K0 */
- UASM_i_SLL(p, K0, K0, ilog2(sizeof(struct tlb_reg_save)));
+ /* handler_reg_save index in GPR_K0 */
+ UASM_i_SLL(p, GPR_K0, GPR_K0, ilog2(sizeof(struct tlb_reg_save)));
- UASM_i_LA(p, K1, (long)&handler_reg_save);
- UASM_i_ADDU(p, K0, K0, K1);
+ UASM_i_LA(p, GPR_K1, (long)&handler_reg_save);
+ UASM_i_ADDU(p, GPR_K0, GPR_K0, GPR_K1);
} else {
- UASM_i_LA(p, K0, (long)&handler_reg_save);
+ UASM_i_LA(p, GPR_K0, (long)&handler_reg_save);
}
- /* K0 now points to save area, save $1 and $2 */
- UASM_i_SW(p, 1, offsetof(struct tlb_reg_save, a), K0);
- UASM_i_SW(p, 2, offsetof(struct tlb_reg_save, b), K0);
+ /* GPR_K0 now points to save area, save $1 and $2 */
+ UASM_i_SW(p, 1, offsetof(struct tlb_reg_save, a), GPR_K0);
+ UASM_i_SW(p, 2, offsetof(struct tlb_reg_save, b), GPR_K0);
- r.r1 = K1;
+ r.r1 = GPR_K1;
r.r2 = 1;
r.r3 = 2;
return r;
@@ -392,9 +373,9 @@ static void build_restore_work_registers(u32 **p)
UASM_i_MFC0(p, 1, c0_kscratch(), scratch_reg);
return;
}
- /* K0 already points to save area, restore $1 and $2 */
- UASM_i_LW(p, 1, offsetof(struct tlb_reg_save, a), K0);
- UASM_i_LW(p, 2, offsetof(struct tlb_reg_save, b), K0);
+ /* GPR_K0 already points to save area, restore $1 and $2 */
+ UASM_i_LW(p, 1, offsetof(struct tlb_reg_save, a), GPR_K0);
+ UASM_i_LW(p, 2, offsetof(struct tlb_reg_save, b), GPR_K0);
}
#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
@@ -413,22 +394,22 @@ static void build_r3000_tlb_refill_handler(void)
memset(tlb_handler, 0, sizeof(tlb_handler));
p = tlb_handler;
- uasm_i_mfc0(&p, K0, C0_BADVADDR);
- uasm_i_lui(&p, K1, uasm_rel_hi(pgdc)); /* cp0 delay */
- uasm_i_lw(&p, K1, uasm_rel_lo(pgdc), K1);
- uasm_i_srl(&p, K0, K0, 22); /* load delay */
- uasm_i_sll(&p, K0, K0, 2);
- uasm_i_addu(&p, K1, K1, K0);
- uasm_i_mfc0(&p, K0, C0_CONTEXT);
- uasm_i_lw(&p, K1, 0, K1); /* cp0 delay */
- uasm_i_andi(&p, K0, K0, 0xffc); /* load delay */
- uasm_i_addu(&p, K1, K1, K0);
- uasm_i_lw(&p, K0, 0, K1);
+ uasm_i_mfc0(&p, GPR_K0, C0_BADVADDR);
+ uasm_i_lui(&p, GPR_K1, uasm_rel_hi(pgdc)); /* cp0 delay */
+ uasm_i_lw(&p, GPR_K1, uasm_rel_lo(pgdc), GPR_K1);
+ uasm_i_srl(&p, GPR_K0, GPR_K0, 22); /* load delay */
+ uasm_i_sll(&p, GPR_K0, GPR_K0, 2);
+ uasm_i_addu(&p, GPR_K1, GPR_K1, GPR_K0);
+ uasm_i_mfc0(&p, GPR_K0, C0_CONTEXT);
+ uasm_i_lw(&p, GPR_K1, 0, GPR_K1); /* cp0 delay */
+ uasm_i_andi(&p, GPR_K0, GPR_K0, 0xffc); /* load delay */
+ uasm_i_addu(&p, GPR_K1, GPR_K1, GPR_K0);
+ uasm_i_lw(&p, GPR_K0, 0, GPR_K1);
uasm_i_nop(&p); /* load delay */
- uasm_i_mtc0(&p, K0, C0_ENTRYLO0);
- uasm_i_mfc0(&p, K1, C0_EPC); /* cp0 delay */
+ uasm_i_mtc0(&p, GPR_K0, C0_ENTRYLO0);
+ uasm_i_mfc0(&p, GPR_K1, C0_EPC); /* cp0 delay */
uasm_i_tlbwr(&p); /* cp0 delay */
- uasm_i_jr(&p, K1);
+ uasm_i_jr(&p, GPR_K1);
uasm_i_rfe(&p); /* branch delay */
if (p > tlb_handler + 32)
@@ -789,7 +770,7 @@ void build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
if (check_for_high_segbits) {
/*
- * The kernel currently implicitely assumes that the
+ * The kernel currently implicitly assumes that the
* MIPS SEGBITS parameter for the processor is
* (PGDIR_SHIFT+PGDIR_BITS) or less, and will never
* allocate virtual addresses outside the maximum
@@ -1276,11 +1257,11 @@ static void build_r4000_tlb_refill_handler(void)
memset(final_handler, 0, sizeof(final_handler));
if (IS_ENABLED(CONFIG_64BIT) && (scratch_reg >= 0 || scratchpad_available()) && use_bbit_insns()) {
- htlb_info = build_fast_tlb_refill_handler(&p, &l, &r, K0, K1,
+ htlb_info = build_fast_tlb_refill_handler(&p, &l, &r, GPR_K0, GPR_K1,
scratch_reg);
vmalloc_mode = refill_scratch;
} else {
- htlb_info.huge_pte = K0;
+ htlb_info.huge_pte = GPR_K0;
htlb_info.restore_scratch = 0;
htlb_info.need_reload_pte = true;
vmalloc_mode = refill_noscratch;
@@ -1290,29 +1271,29 @@ static void build_r4000_tlb_refill_handler(void)
if (bcm1250_m3_war()) {
unsigned int segbits = 44;
- uasm_i_dmfc0(&p, K0, C0_BADVADDR);
- uasm_i_dmfc0(&p, K1, C0_ENTRYHI);
- uasm_i_xor(&p, K0, K0, K1);
- uasm_i_dsrl_safe(&p, K1, K0, 62);
- uasm_i_dsrl_safe(&p, K0, K0, 12 + 1);
- uasm_i_dsll_safe(&p, K0, K0, 64 + 12 + 1 - segbits);
- uasm_i_or(&p, K0, K0, K1);
- uasm_il_bnez(&p, &r, K0, label_leave);
+ uasm_i_dmfc0(&p, GPR_K0, C0_BADVADDR);
+ uasm_i_dmfc0(&p, GPR_K1, C0_ENTRYHI);
+ uasm_i_xor(&p, GPR_K0, GPR_K0, GPR_K1);
+ uasm_i_dsrl_safe(&p, GPR_K1, GPR_K0, 62);
+ uasm_i_dsrl_safe(&p, GPR_K0, GPR_K0, 12 + 1);
+ uasm_i_dsll_safe(&p, GPR_K0, GPR_K0, 64 + 12 + 1 - segbits);
+ uasm_i_or(&p, GPR_K0, GPR_K0, GPR_K1);
+ uasm_il_bnez(&p, &r, GPR_K0, label_leave);
/* No need for uasm_i_nop */
}
#ifdef CONFIG_64BIT
- build_get_pmde64(&p, &l, &r, K0, K1); /* get pmd in K1 */
+ build_get_pmde64(&p, &l, &r, GPR_K0, GPR_K1); /* get pmd in GPR_K1 */
#else
- build_get_pgde32(&p, K0, K1); /* get pgd in K1 */
+ build_get_pgde32(&p, GPR_K0, GPR_K1); /* get pgd in GPR_K1 */
#endif
#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
- build_is_huge_pte(&p, &r, K0, K1, label_tlb_huge_update);
+ build_is_huge_pte(&p, &r, GPR_K0, GPR_K1, label_tlb_huge_update);
#endif
- build_get_ptep(&p, K0, K1);
- build_update_entries(&p, K0, K1);
+ build_get_ptep(&p, GPR_K0, GPR_K1);
+ build_update_entries(&p, GPR_K0, GPR_K1);
build_tlb_write_entry(&p, &l, &r, tlb_random);
uasm_l_leave(&l, p);
uasm_i_eret(&p); /* return from trap */
@@ -1320,14 +1301,14 @@ static void build_r4000_tlb_refill_handler(void)
#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
uasm_l_tlb_huge_update(&l, p);
if (htlb_info.need_reload_pte)
- UASM_i_LW(&p, htlb_info.huge_pte, 0, K1);
- build_huge_update_entries(&p, htlb_info.huge_pte, K1);
- build_huge_tlb_write_entry(&p, &l, &r, K0, tlb_random,
+ UASM_i_LW(&p, htlb_info.huge_pte, 0, GPR_K1);
+ build_huge_update_entries(&p, htlb_info.huge_pte, GPR_K1);
+ build_huge_tlb_write_entry(&p, &l, &r, GPR_K0, tlb_random,
htlb_info.restore_scratch);
#endif
#ifdef CONFIG_64BIT
- build_get_pgd_vmalloc64(&p, &l, &r, K0, K1, vmalloc_mode);
+ build_get_pgd_vmalloc64(&p, &l, &r, GPR_K0, GPR_K1, vmalloc_mode);
#endif
/*
@@ -1500,34 +1481,35 @@ static void build_loongson3_tlb_refill_handler(void)
memset(tlb_handler, 0, sizeof(tlb_handler));
if (check_for_high_segbits) {
- uasm_i_dmfc0(&p, K0, C0_BADVADDR);
- uasm_i_dsrl_safe(&p, K1, K0, PGDIR_SHIFT + PGD_TABLE_ORDER + PAGE_SHIFT - 3);
- uasm_il_beqz(&p, &r, K1, label_vmalloc);
+ uasm_i_dmfc0(&p, GPR_K0, C0_BADVADDR);
+ uasm_i_dsrl_safe(&p, GPR_K1, GPR_K0,
+ PGDIR_SHIFT + PGD_TABLE_ORDER + PAGE_SHIFT - 3);
+ uasm_il_beqz(&p, &r, GPR_K1, label_vmalloc);
uasm_i_nop(&p);
- uasm_il_bgez(&p, &r, K0, label_large_segbits_fault);
+ uasm_il_bgez(&p, &r, GPR_K0, label_large_segbits_fault);
uasm_i_nop(&p);
uasm_l_vmalloc(&l, p);
}
- uasm_i_dmfc0(&p, K1, C0_PGD);
+ uasm_i_dmfc0(&p, GPR_K1, C0_PGD);
- uasm_i_lddir(&p, K0, K1, 3); /* global page dir */
+ uasm_i_lddir(&p, GPR_K0, GPR_K1, 3); /* global page dir */
#ifndef __PAGETABLE_PMD_FOLDED
- uasm_i_lddir(&p, K1, K0, 1); /* middle page dir */
+ uasm_i_lddir(&p, GPR_K1, GPR_K0, 1); /* middle page dir */
#endif
- uasm_i_ldpte(&p, K1, 0); /* even */
- uasm_i_ldpte(&p, K1, 1); /* odd */
+ uasm_i_ldpte(&p, GPR_K1, 0); /* even */
+ uasm_i_ldpte(&p, GPR_K1, 1); /* odd */
uasm_i_tlbwr(&p);
/* restore page mask */
if (PM_DEFAULT_MASK >> 16) {
- uasm_i_lui(&p, K0, PM_DEFAULT_MASK >> 16);
- uasm_i_ori(&p, K0, K0, PM_DEFAULT_MASK & 0xffff);
- uasm_i_mtc0(&p, K0, C0_PAGEMASK);
+ uasm_i_lui(&p, GPR_K0, PM_DEFAULT_MASK >> 16);
+ uasm_i_ori(&p, GPR_K0, GPR_K0, PM_DEFAULT_MASK & 0xffff);
+ uasm_i_mtc0(&p, GPR_K0, C0_PAGEMASK);
} else if (PM_DEFAULT_MASK) {
- uasm_i_ori(&p, K0, 0, PM_DEFAULT_MASK);
- uasm_i_mtc0(&p, K0, C0_PAGEMASK);
+ uasm_i_ori(&p, GPR_K0, 0, PM_DEFAULT_MASK);
+ uasm_i_mtc0(&p, GPR_K0, C0_PAGEMASK);
} else {
uasm_i_mtc0(&p, 0, C0_PAGEMASK);
}
@@ -1536,8 +1518,8 @@ static void build_loongson3_tlb_refill_handler(void)
if (check_for_high_segbits) {
uasm_l_large_segbits_fault(&l, p);
- UASM_i_LA(&p, K1, (unsigned long)tlb_do_page_fault_0);
- uasm_i_jr(&p, K1);
+ UASM_i_LA(&p, GPR_K1, (unsigned long)tlb_do_page_fault_0);
+ uasm_i_jr(&p, GPR_K1);
uasm_i_nop(&p);
}
@@ -1715,7 +1697,7 @@ iPTE_SW(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr,
/*
* Check if PTE is present, if not then jump to LABEL. PTR points to
* the page table where this PTE is located, PTE will be re-loaded
- * with it's original value.
+ * with its original value.
*/
static void
build_pte_present(u32 **p, struct uasm_reloc **r,
@@ -1903,11 +1885,11 @@ static void build_r3000_tlb_load_handler(void)
memset(labels, 0, sizeof(labels));
memset(relocs, 0, sizeof(relocs));
- build_r3000_tlbchange_handler_head(&p, K0, K1);
- build_pte_present(&p, &r, K0, K1, -1, label_nopage_tlbl);
+ build_r3000_tlbchange_handler_head(&p, GPR_K0, GPR_K1);
+ build_pte_present(&p, &r, GPR_K0, GPR_K1, -1, label_nopage_tlbl);
uasm_i_nop(&p); /* load delay */
- build_make_valid(&p, &r, K0, K1, -1);
- build_r3000_tlb_reload_write(&p, &l, &r, K0, K1);
+ build_make_valid(&p, &r, GPR_K0, GPR_K1, -1);
+ build_r3000_tlb_reload_write(&p, &l, &r, GPR_K0, GPR_K1);
uasm_l_nopage_tlbl(&l, p);
uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff);
@@ -1933,11 +1915,11 @@ static void build_r3000_tlb_store_handler(void)
memset(labels, 0, sizeof(labels));
memset(relocs, 0, sizeof(relocs));
- build_r3000_tlbchange_handler_head(&p, K0, K1);
- build_pte_writable(&p, &r, K0, K1, -1, label_nopage_tlbs);
+ build_r3000_tlbchange_handler_head(&p, GPR_K0, GPR_K1);
+ build_pte_writable(&p, &r, GPR_K0, GPR_K1, -1, label_nopage_tlbs);
uasm_i_nop(&p); /* load delay */
- build_make_write(&p, &r, K0, K1, -1);
- build_r3000_tlb_reload_write(&p, &l, &r, K0, K1);
+ build_make_write(&p, &r, GPR_K0, GPR_K1, -1);
+ build_r3000_tlb_reload_write(&p, &l, &r, GPR_K0, GPR_K1);
uasm_l_nopage_tlbs(&l, p);
uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
@@ -1963,11 +1945,11 @@ static void build_r3000_tlb_modify_handler(void)
memset(labels, 0, sizeof(labels));
memset(relocs, 0, sizeof(relocs));
- build_r3000_tlbchange_handler_head(&p, K0, K1);
- build_pte_modifiable(&p, &r, K0, K1, -1, label_nopage_tlbm);
+ build_r3000_tlbchange_handler_head(&p, GPR_K0, GPR_K1);
+ build_pte_modifiable(&p, &r, GPR_K0, GPR_K1, -1, label_nopage_tlbm);
uasm_i_nop(&p); /* load delay */
- build_make_write(&p, &r, K0, K1, -1);
- build_r3000_pte_reload_tlbwi(&p, K0, K1);
+ build_make_write(&p, &r, GPR_K0, GPR_K1, -1);
+ build_r3000_pte_reload_tlbwi(&p, GPR_K0, GPR_K1);
uasm_l_nopage_tlbm(&l, p);
uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
@@ -2083,14 +2065,14 @@ static void build_r4000_tlb_load_handler(void)
if (bcm1250_m3_war()) {
unsigned int segbits = 44;
- uasm_i_dmfc0(&p, K0, C0_BADVADDR);
- uasm_i_dmfc0(&p, K1, C0_ENTRYHI);
- uasm_i_xor(&p, K0, K0, K1);
- uasm_i_dsrl_safe(&p, K1, K0, 62);
- uasm_i_dsrl_safe(&p, K0, K0, 12 + 1);
- uasm_i_dsll_safe(&p, K0, K0, 64 + 12 + 1 - segbits);
- uasm_i_or(&p, K0, K0, K1);
- uasm_il_bnez(&p, &r, K0, label_leave);
+ uasm_i_dmfc0(&p, GPR_K0, C0_BADVADDR);
+ uasm_i_dmfc0(&p, GPR_K1, C0_ENTRYHI);
+ uasm_i_xor(&p, GPR_K0, GPR_K0, GPR_K1);
+ uasm_i_dsrl_safe(&p, GPR_K1, GPR_K0, 62);
+ uasm_i_dsrl_safe(&p, GPR_K0, GPR_K0, 12 + 1);
+ uasm_i_dsll_safe(&p, GPR_K0, GPR_K0, 64 + 12 + 1 - segbits);
+ uasm_i_or(&p, GPR_K0, GPR_K0, GPR_K1);
+ uasm_il_bnez(&p, &r, GPR_K0, label_leave);
/* No need for uasm_i_nop */
}
@@ -2123,16 +2105,8 @@ static void build_r4000_tlb_load_handler(void)
uasm_i_tlbr(&p);
- switch (current_cpu_type()) {
- case CPU_CAVIUM_OCTEON:
- case CPU_CAVIUM_OCTEON_PLUS:
- case CPU_CAVIUM_OCTEON2:
- break;
- default:
- if (cpu_has_mips_r2_exec_hazard)
- uasm_i_ehb(&p);
- break;
- }
+ if (cpu_has_mips_r2_exec_hazard)
+ uasm_i_ehb(&p);
/* Examine entrylo 0 or 1 based on ptr. */
if (use_bbit_insns()) {
@@ -2197,16 +2171,8 @@ static void build_r4000_tlb_load_handler(void)
uasm_i_tlbr(&p);
- switch (current_cpu_type()) {
- case CPU_CAVIUM_OCTEON:
- case CPU_CAVIUM_OCTEON_PLUS:
- case CPU_CAVIUM_OCTEON2:
- break;
- default:
- if (cpu_has_mips_r2_exec_hazard)
- uasm_i_ehb(&p);
- break;
- }
+ if (cpu_has_mips_r2_exec_hazard)
+ uasm_i_ehb(&p);
/* Examine entrylo 0 or 1 based on ptr. */
if (use_bbit_insns()) {
@@ -2249,9 +2215,9 @@ static void build_r4000_tlb_load_handler(void)
build_restore_work_registers(&p);
#ifdef CONFIG_CPU_MICROMIPS
if ((unsigned long)tlb_do_page_fault_0 & 1) {
- uasm_i_lui(&p, K0, uasm_rel_hi((long)tlb_do_page_fault_0));
- uasm_i_addiu(&p, K0, K0, uasm_rel_lo((long)tlb_do_page_fault_0));
- uasm_i_jr(&p, K0);
+ uasm_i_lui(&p, GPR_K0, uasm_rel_hi((long)tlb_do_page_fault_0));
+ uasm_i_addiu(&p, GPR_K0, GPR_K0, uasm_rel_lo((long)tlb_do_page_fault_0));
+ uasm_i_jr(&p, GPR_K0);
} else
#endif
uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff);
@@ -2305,9 +2271,9 @@ static void build_r4000_tlb_store_handler(void)
build_restore_work_registers(&p);
#ifdef CONFIG_CPU_MICROMIPS
if ((unsigned long)tlb_do_page_fault_1 & 1) {
- uasm_i_lui(&p, K0, uasm_rel_hi((long)tlb_do_page_fault_1));
- uasm_i_addiu(&p, K0, K0, uasm_rel_lo((long)tlb_do_page_fault_1));
- uasm_i_jr(&p, K0);
+ uasm_i_lui(&p, GPR_K0, uasm_rel_hi((long)tlb_do_page_fault_1));
+ uasm_i_addiu(&p, GPR_K0, GPR_K0, uasm_rel_lo((long)tlb_do_page_fault_1));
+ uasm_i_jr(&p, GPR_K0);
} else
#endif
uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
@@ -2362,9 +2328,9 @@ static void build_r4000_tlb_modify_handler(void)
build_restore_work_registers(&p);
#ifdef CONFIG_CPU_MICROMIPS
if ((unsigned long)tlb_do_page_fault_1 & 1) {
- uasm_i_lui(&p, K0, uasm_rel_hi((long)tlb_do_page_fault_1));
- uasm_i_addiu(&p, K0, K0, uasm_rel_lo((long)tlb_do_page_fault_1));
- uasm_i_jr(&p, K0);
+ uasm_i_lui(&p, GPR_K0, uasm_rel_hi((long)tlb_do_page_fault_1));
+ uasm_i_addiu(&p, GPR_K0, GPR_K0, uasm_rel_lo((long)tlb_do_page_fault_1));
+ uasm_i_jr(&p, GPR_K0);
} else
#endif
uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
generated by cgit (git 2.34.1) at 2025-12-24 21:20:14 +0000