// SPDX-License-Identifier: GPL-2.0 /* * PARISC64 Huge TLB page support. * * This parisc implementation is heavily based on the SPARC and x86 code. * * Copyright (C) 2015 Helge Deller */ #include #include #include #include #include #include #include #include #include #include #include #include unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags) { struct hstate *h = hstate_file(file); if (len & ~huge_page_mask(h)) return -EINVAL; if (len > TASK_SIZE) return -ENOMEM; if (flags & MAP_FIXED) if (prepare_hugepage_range(file, addr, len)) return -EINVAL; if (addr) addr = ALIGN(addr, huge_page_size(h)); /* we need to make sure the colouring is OK */ return arch_get_unmapped_area(file, addr, len, pgoff, flags); } pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz) { pgd_t *pgd; pud_t *pud; pmd_t *pmd; pte_t *pte = NULL; /* We must align the address, because our caller will run * set_huge_pte_at() on whatever we return, which writes out * all of the sub-ptes for the hugepage range. So we have * to give it the first such sub-pte. */ addr &= HPAGE_MASK; pgd = pgd_offset(mm, addr); pud = pud_alloc(mm, pgd, addr); if (pud) { pmd = pmd_alloc(mm, pud, addr); if (pmd) pte = pte_alloc_map(mm, pmd, addr); } return pte; } pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr, unsigned long sz) { pgd_t *pgd; pud_t *pud; pmd_t *pmd; pte_t *pte = NULL; addr &= HPAGE_MASK; pgd = pgd_offset(mm, addr); if (!pgd_none(*pgd)) { pud = pud_offset(pgd, addr); if (!pud_none(*pud)) { pmd = pmd_offset(pud, addr); if (!pmd_none(*pmd)) pte = pte_offset_map(pmd, addr); } } return pte; } /* Purge data and instruction TLB entries. Must be called holding * the pa_tlb_lock. The TLB purge instructions are slow on SMP * machines since the purge must be broadcast to all CPUs. */ static inline void purge_tlb_entries_huge(struct mm_struct *mm, unsigned long addr) { int i; /* We may use multiple physical huge pages (e.g. 2x1 MB) to emulate * Linux standard huge pages (e.g. 2 MB) */ BUILD_BUG_ON(REAL_HPAGE_SHIFT > HPAGE_SHIFT); addr &= HPAGE_MASK; addr |= _HUGE_PAGE_SIZE_ENCODING_DEFAULT; for (i = 0; i < (1 << (HPAGE_SHIFT-REAL_HPAGE_SHIFT)); i++) { purge_tlb_entries(mm, addr); addr += (1UL << REAL_HPAGE_SHIFT); } } /* __set_huge_pte_at() must be called holding the pa_tlb_lock. */ static void __set_huge_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t entry) { unsigned long addr_start; int i; addr &= HPAGE_MASK; addr_start = addr; for (i = 0; i < (1 << HUGETLB_PAGE_ORDER); i++) { set_pte(ptep, entry); ptep++; addr += PAGE_SIZE; pte_val(entry) += PAGE_SIZE; } purge_tlb_entries_huge(mm, addr_start); } void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t entry) { unsigned long flags; purge_tlb_start(flags); __set_huge_pte_at(mm, addr, ptep, entry); purge_tlb_end(flags); } pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { unsigned long flags; pte_t entry; purge_tlb_start(flags); entry = *ptep; __set_huge_pte_at(mm, addr, ptep, __pte(0)); purge_tlb_end(flags); return entry; } void huge_ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { unsigned long flags; pte_t old_pte; purge_tlb_start(flags); old_pte = *ptep; __set_huge_pte_at(mm, addr, ptep, pte_wrprotect(old_pte)); purge_tlb_end(flags); } int huge_ptep_set_access_flags(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, pte_t pte, int dirty) { unsigned long flags; int changed; purge_tlb_start(flags); changed = !pte_same(*ptep, pte); if (changed) { __set_huge_pte_at(vma->vm_mm, addr, ptep, pte); } purge_tlb_end(flags); return changed; } int pmd_huge(pmd_t pmd) { return 0; } int pud_huge(pud_t pud) { return 0; }