diff options
Diffstat (limited to 'drivers/gpu/drm/i915/i915_gem_gtt.c')
| -rw-r--r-- | drivers/gpu/drm/i915/i915_gem_gtt.c | 3383 |
1 files changed, 56 insertions, 3327 deletions
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c index 61fc7e90a7da..25295eb626dc 100644 --- a/drivers/gpu/drm/i915/i915_gem_gtt.c +++ b/drivers/gpu/drm/i915/i915_gem_gtt.c @@ -1,26 +1,7 @@ +// SPDX-License-Identifier: MIT /* * Copyright © 2010 Daniel Vetter - * Copyright © 2011-2014 Intel Corporation - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - * + * Copyright © 2020 Intel Corporation */ #include <linux/slab.h> /* fault-inject.h is not standalone! */ @@ -32,3333 +13,65 @@ #include <linux/stop_machine.h> #include <asm/set_memory.h> +#include <asm/smp.h> -#include <drm/drmP.h> -#include <drm/i915_drm.h> +#include "gt/intel_gt.h" +#include "gt/intel_gt_requests.h" #include "i915_drv.h" -#include "i915_vgpu.h" +#include "i915_gem_evict.h" +#include "i915_scatterlist.h" #include "i915_trace.h" -#include "intel_drv.h" -#include "intel_frontbuffer.h" - -#define I915_GFP_DMA (GFP_KERNEL | __GFP_HIGHMEM) - -/** - * DOC: Global GTT views - * - * Background and previous state - * - * Historically objects could exists (be bound) in global GTT space only as - * singular instances with a view representing all of the object's backing pages - * in a linear fashion. This view will be called a normal view. - * - * To support multiple views of the same object, where the number of mapped - * pages is not equal to the backing store, or where the layout of the pages - * is not linear, concept of a GGTT view was added. - * - * One example of an alternative view is a stereo display driven by a single - * image. In this case we would have a framebuffer looking like this - * (2x2 pages): - * - * 12 - * 34 - * - * Above would represent a normal GGTT view as normally mapped for GPU or CPU - * rendering. In contrast, fed to the display engine would be an alternative - * view which could look something like this: - * - * 1212 - * 3434 - * - * In this example both the size and layout of pages in the alternative view is - * different from the normal view. - * - * Implementation and usage - * - * GGTT views are implemented using VMAs and are distinguished via enum - * i915_ggtt_view_type and struct i915_ggtt_view. - * - * A new flavour of core GEM functions which work with GGTT bound objects were - * added with the _ggtt_ infix, and sometimes with _view postfix to avoid - * renaming in large amounts of code. They take the struct i915_ggtt_view - * parameter encapsulating all metadata required to implement a view. - * - * As a helper for callers which are only interested in the normal view, - * globally const i915_ggtt_view_normal singleton instance exists. All old core - * GEM API functions, the ones not taking the view parameter, are operating on, - * or with the normal GGTT view. - * - * Code wanting to add or use a new GGTT view needs to: - * - * 1. Add a new enum with a suitable name. - * 2. Extend the metadata in the i915_ggtt_view structure if required. - * 3. Add support to i915_get_vma_pages(). - * - * New views are required to build a scatter-gather table from within the - * i915_get_vma_pages function. This table is stored in the vma.ggtt_view and - * exists for the lifetime of an VMA. - * - * Core API is designed to have copy semantics which means that passed in - * struct i915_ggtt_view does not need to be persistent (left around after - * calling the core API functions). - * - */ - -static int -i915_get_ggtt_vma_pages(struct i915_vma *vma); - -static void gen6_ggtt_invalidate(struct drm_i915_private *dev_priv) -{ - /* Note that as an uncached mmio write, this should flush the - * WCB of the writes into the GGTT before it triggers the invalidate. - */ - I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN); -} - -static void guc_ggtt_invalidate(struct drm_i915_private *dev_priv) -{ - gen6_ggtt_invalidate(dev_priv); - I915_WRITE(GEN8_GTCR, GEN8_GTCR_INVALIDATE); -} - -static void gmch_ggtt_invalidate(struct drm_i915_private *dev_priv) -{ - intel_gtt_chipset_flush(); -} - -static inline void i915_ggtt_invalidate(struct drm_i915_private *i915) -{ - i915->ggtt.invalidate(i915); -} - -int intel_sanitize_enable_ppgtt(struct drm_i915_private *dev_priv, - int enable_ppgtt) -{ - bool has_aliasing_ppgtt; - bool has_full_ppgtt; - bool has_full_48bit_ppgtt; - - has_aliasing_ppgtt = dev_priv->info.has_aliasing_ppgtt; - has_full_ppgtt = dev_priv->info.has_full_ppgtt; - has_full_48bit_ppgtt = dev_priv->info.has_full_48bit_ppgtt; - - if (intel_vgpu_active(dev_priv)) { - /* emulation is too hard */ - has_full_ppgtt = false; - has_full_48bit_ppgtt = false; - } - - if (!has_aliasing_ppgtt) - return 0; - - /* - * We don't allow disabling PPGTT for gen9+ as it's a requirement for - * execlists, the sole mechanism available to submit work. - */ - if (enable_ppgtt == 0 && INTEL_GEN(dev_priv) < 9) - return 0; - - if (enable_ppgtt == 1) - return 1; - - if (enable_ppgtt == 2 && has_full_ppgtt) - return 2; - - if (enable_ppgtt == 3 && has_full_48bit_ppgtt) - return 3; - - /* Disable ppgtt on SNB if VT-d is on. */ - if (IS_GEN6(dev_priv) && intel_vtd_active()) { - DRM_INFO("Disabling PPGTT because VT-d is on\n"); - return 0; - } - - /* Early VLV doesn't have this */ - if (IS_VALLEYVIEW(dev_priv) && dev_priv->drm.pdev->revision < 0xb) { - DRM_DEBUG_DRIVER("disabling PPGTT on pre-B3 step VLV\n"); - return 0; - } - - if (INTEL_GEN(dev_priv) >= 8 && i915.enable_execlists && has_full_ppgtt) - return has_full_48bit_ppgtt ? 3 : 2; - else - return has_aliasing_ppgtt ? 1 : 0; -} - -static int ppgtt_bind_vma(struct i915_vma *vma, - enum i915_cache_level cache_level, - u32 unused) -{ - u32 pte_flags; - int ret; - - if (!(vma->flags & I915_VMA_LOCAL_BIND)) { - ret = vma->vm->allocate_va_range(vma->vm, vma->node.start, - vma->size); - if (ret) - return ret; - } - - vma->pages = vma->obj->mm.pages; - - /* Currently applicable only to VLV */ - pte_flags = 0; - if (vma->obj->gt_ro) - pte_flags |= PTE_READ_ONLY; - - vma->vm->insert_entries(vma->vm, vma->pages, vma->node.start, - cache_level, pte_flags); - - return 0; -} - -static void ppgtt_unbind_vma(struct i915_vma *vma) -{ - vma->vm->clear_range(vma->vm, vma->node.start, vma->size); -} - -static gen8_pte_t gen8_pte_encode(dma_addr_t addr, - enum i915_cache_level level) -{ - gen8_pte_t pte = _PAGE_PRESENT | _PAGE_RW; - pte |= addr; - - switch (level) { - case I915_CACHE_NONE: - pte |= PPAT_UNCACHED_INDEX; - break; - case I915_CACHE_WT: - pte |= PPAT_DISPLAY_ELLC_INDEX; - break; - default: - pte |= PPAT_CACHED_INDEX; - break; - } - - return pte; -} - -static gen8_pde_t gen8_pde_encode(const dma_addr_t addr, - const enum i915_cache_level level) -{ - gen8_pde_t pde = _PAGE_PRESENT | _PAGE_RW; - pde |= addr; - if (level != I915_CACHE_NONE) - pde |= PPAT_CACHED_PDE_INDEX; - else - pde |= PPAT_UNCACHED_INDEX; - return pde; -} - -#define gen8_pdpe_encode gen8_pde_encode -#define gen8_pml4e_encode gen8_pde_encode - -static gen6_pte_t snb_pte_encode(dma_addr_t addr, - enum i915_cache_level level, - u32 unused) -{ - gen6_pte_t pte = GEN6_PTE_VALID; - pte |= GEN6_PTE_ADDR_ENCODE(addr); - - switch (level) { - case I915_CACHE_L3_LLC: - case I915_CACHE_LLC: - pte |= GEN6_PTE_CACHE_LLC; - break; - case I915_CACHE_NONE: - pte |= GEN6_PTE_UNCACHED; - break; - default: - MISSING_CASE(level); - } - - return pte; -} - -static gen6_pte_t ivb_pte_encode(dma_addr_t addr, - enum i915_cache_level level, - u32 unused) -{ - gen6_pte_t pte = GEN6_PTE_VALID; - pte |= GEN6_PTE_ADDR_ENCODE(addr); - - switch (level) { - case I915_CACHE_L3_LLC: - pte |= GEN7_PTE_CACHE_L3_LLC; - break; - case I915_CACHE_LLC: - pte |= GEN6_PTE_CACHE_LLC; - break; - case I915_CACHE_NONE: - pte |= GEN6_PTE_UNCACHED; - break; - default: - MISSING_CASE(level); - } - - return pte; -} - -static gen6_pte_t byt_pte_encode(dma_addr_t addr, - enum i915_cache_level level, - u32 flags) -{ - gen6_pte_t pte = GEN6_PTE_VALID; - pte |= GEN6_PTE_ADDR_ENCODE(addr); - - if (!(flags & PTE_READ_ONLY)) - pte |= BYT_PTE_WRITEABLE; - - if (level != I915_CACHE_NONE) - pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES; - - return pte; -} - -static gen6_pte_t hsw_pte_encode(dma_addr_t addr, - enum i915_cache_level level, - u32 unused) -{ - gen6_pte_t pte = GEN6_PTE_VALID; - pte |= HSW_PTE_ADDR_ENCODE(addr); - - if (level != I915_CACHE_NONE) - pte |= HSW_WB_LLC_AGE3; - - return pte; -} - -static gen6_pte_t iris_pte_encode(dma_addr_t addr, - enum i915_cache_level level, - u32 unused) -{ - gen6_pte_t pte = GEN6_PTE_VALID; - pte |= HSW_PTE_ADDR_ENCODE(addr); - - switch (level) { - case I915_CACHE_NONE: - break; - case I915_CACHE_WT: - pte |= HSW_WT_ELLC_LLC_AGE3; - break; - default: - pte |= HSW_WB_ELLC_LLC_AGE3; - break; - } - - return pte; -} - -static struct page *vm_alloc_page(struct i915_address_space *vm, gfp_t gfp) -{ - struct page *page; - - if (I915_SELFTEST_ONLY(should_fail(&vm->fault_attr, 1))) - i915_gem_shrink_all(vm->i915); - - if (vm->free_pages.nr) - return vm->free_pages.pages[--vm->free_pages.nr]; - - page = alloc_page(gfp); - if (!page) - return NULL; - - if (vm->pt_kmap_wc) - set_pages_array_wc(&page, 1); - - return page; -} - -static void vm_free_pages_release(struct i915_address_space *vm) -{ - GEM_BUG_ON(!pagevec_count(&vm->free_pages)); - - if (vm->pt_kmap_wc) - set_pages_array_wb(vm->free_pages.pages, - pagevec_count(&vm->free_pages)); - - __pagevec_release(&vm->free_pages); -} - -static void vm_free_page(struct i915_address_space *vm, struct page *page) -{ - if (!pagevec_add(&vm->free_pages, page)) - vm_free_pages_release(vm); -} - -static int __setup_page_dma(struct i915_address_space *vm, - struct i915_page_dma *p, - gfp_t gfp) -{ - p->page = vm_alloc_page(vm, gfp | __GFP_NOWARN | __GFP_NORETRY); - if (unlikely(!p->page)) - return -ENOMEM; - - p->daddr = dma_map_page(vm->dma, p->page, 0, PAGE_SIZE, - PCI_DMA_BIDIRECTIONAL); - if (unlikely(dma_mapping_error(vm->dma, p->daddr))) { - vm_free_page(vm, p->page); - return -ENOMEM; - } - - return 0; -} - -static int setup_page_dma(struct i915_address_space *vm, - struct i915_page_dma *p) -{ - return __setup_page_dma(vm, p, I915_GFP_DMA); -} - -static void cleanup_page_dma(struct i915_address_space *vm, - struct i915_page_dma *p) -{ - dma_unmap_page(vm->dma, p->daddr, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); - vm_free_page(vm, p->page); -} - -#define kmap_atomic_px(px) kmap_atomic(px_base(px)->page) - -#define setup_px(vm, px) setup_page_dma((vm), px_base(px)) -#define cleanup_px(vm, px) cleanup_page_dma((vm), px_base(px)) -#define fill_px(ppgtt, px, v) fill_page_dma((vm), px_base(px), (v)) -#define fill32_px(ppgtt, px, v) fill_page_dma_32((vm), px_base(px), (v)) - -static void fill_page_dma(struct i915_address_space *vm, - struct i915_page_dma *p, - const u64 val) -{ - u64 * const vaddr = kmap_atomic(p->page); - int i; - - for (i = 0; i < 512; i++) - vaddr[i] = val; - - kunmap_atomic(vaddr); -} - -static void fill_page_dma_32(struct i915_address_space *vm, - struct i915_page_dma *p, - const u32 v) -{ - fill_page_dma(vm, p, (u64)v << 32 | v); -} - -static int -setup_scratch_page(struct i915_address_space *vm, gfp_t gfp) -{ - return __setup_page_dma(vm, &vm->scratch_page, gfp | __GFP_ZERO); -} - -static void cleanup_scratch_page(struct i915_address_space *vm) -{ - cleanup_page_dma(vm, &vm->scratch_page); -} - -static struct i915_page_table *alloc_pt(struct i915_address_space *vm) -{ - struct i915_page_table *pt; - - pt = kmalloc(sizeof(*pt), GFP_KERNEL | __GFP_NOWARN); - if (unlikely(!pt)) - return ERR_PTR(-ENOMEM); - - if (unlikely(setup_px(vm, pt))) { - kfree(pt); - return ERR_PTR(-ENOMEM); - } - - pt->used_ptes = 0; - return pt; -} - -static void free_pt(struct i915_address_space *vm, struct i915_page_table *pt) -{ - cleanup_px(vm, pt); - kfree(pt); -} - -static void gen8_initialize_pt(struct i915_address_space *vm, - struct i915_page_table *pt) -{ - fill_px(vm, pt, - gen8_pte_encode(vm->scratch_page.daddr, I915_CACHE_LLC)); -} - -static void gen6_initialize_pt(struct i915_address_space *vm, - struct i915_page_table *pt) -{ - fill32_px(vm, pt, - vm->pte_encode(vm->scratch_page.daddr, I915_CACHE_LLC, 0)); -} - -static struct i915_page_directory *alloc_pd(struct i915_address_space *vm) -{ - struct i915_page_directory *pd; - - pd = kzalloc(sizeof(*pd), GFP_KERNEL | __GFP_NOWARN); - if (unlikely(!pd)) - return ERR_PTR(-ENOMEM); - - if (unlikely(setup_px(vm, pd))) { - kfree(pd); - return ERR_PTR(-ENOMEM); - } - - pd->used_pdes = 0; - return pd; -} - -static void free_pd(struct i915_address_space *vm, - struct i915_page_directory *pd) -{ - cleanup_px(vm, pd); - kfree(pd); -} - -static void gen8_initialize_pd(struct i915_address_space *vm, - struct i915_page_directory *pd) -{ - unsigned int i; - - fill_px(vm, pd, - gen8_pde_encode(px_dma(vm->scratch_pt), I915_CACHE_LLC)); - for (i = 0; i < I915_PDES; i++) - pd->page_table[i] = vm->scratch_pt; -} - -static int __pdp_init(struct i915_address_space *vm, - struct i915_page_directory_pointer *pdp) -{ - const unsigned int pdpes = i915_pdpes_per_pdp(vm); - unsigned int i; - - pdp->page_directory = kmalloc_array(pdpes, sizeof(*pdp->page_directory), - GFP_KERNEL | __GFP_NOWARN); - if (unlikely(!pdp->page_directory)) - return -ENOMEM; - - for (i = 0; i < pdpes; i++) - pdp->page_directory[i] = vm->scratch_pd; - - return 0; -} - -static void __pdp_fini(struct i915_page_directory_pointer *pdp) -{ - kfree(pdp->page_directory); - pdp->page_directory = NULL; -} - -static inline bool use_4lvl(const struct i915_address_space *vm) -{ - return i915_vm_is_48bit(vm); -} - -static struct i915_page_directory_pointer * -alloc_pdp(struct i915_address_space *vm) -{ - struct i915_page_directory_pointer *pdp; - int ret = -ENOMEM; - - WARN_ON(!use_4lvl(vm)); - - pdp = kzalloc(sizeof(*pdp), GFP_KERNEL); - if (!pdp) - return ERR_PTR(-ENOMEM); - - ret = __pdp_init(vm, pdp); - if (ret) - goto fail_bitmap; - - ret = setup_px(vm, pdp); - if (ret) - goto fail_page_m; - - return pdp; - -fail_page_m: - __pdp_fini(pdp); -fail_bitmap: - kfree(pdp); - - return ERR_PTR(ret); -} - -static void free_pdp(struct i915_address_space *vm, - struct i915_page_directory_pointer *pdp) -{ - __pdp_fini(pdp); - - if (!use_4lvl(vm)) - return; - - cleanup_px(vm, pdp); - kfree(pdp); -} - -static void gen8_initialize_pdp(struct i915_address_space *vm, - struct i915_page_directory_pointer *pdp) -{ - gen8_ppgtt_pdpe_t scratch_pdpe; - - scratch_pdpe = gen8_pdpe_encode(px_dma(vm->scratch_pd), I915_CACHE_LLC); - - fill_px(vm, pdp, scratch_pdpe); -} - -static void gen8_initialize_pml4(struct i915_address_space *vm, - struct i915_pml4 *pml4) -{ - unsigned int i; - - fill_px(vm, pml4, - gen8_pml4e_encode(px_dma(vm->scratch_pdp), I915_CACHE_LLC)); - for (i = 0; i < GEN8_PML4ES_PER_PML4; i++) - pml4->pdps[i] = vm->scratch_pdp; -} - -/* Broadwell Page Directory Pointer Descriptors */ -static int gen8_write_pdp(struct drm_i915_gem_request *req, - unsigned entry, - dma_addr_t addr) -{ - struct intel_engine_cs *engine = req->engine; - u32 *cs; - - BUG_ON(entry >= 4); - - cs = intel_ring_begin(req, 6); - if (IS_ERR(cs)) - return PTR_ERR(cs); - - *cs++ = MI_LOAD_REGISTER_IMM(1); - *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_UDW(engine, entry)); - *cs++ = upper_32_bits(addr); - *cs++ = MI_LOAD_REGISTER_IMM(1); - *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(engine, entry)); - *cs++ = lower_32_bits(addr); - intel_ring_advance(req, cs); - - return 0; -} - -static int gen8_mm_switch_3lvl(struct i915_hw_ppgtt *ppgtt, - struct drm_i915_gem_request *req) -{ - int i, ret; - - for (i = GEN8_3LVL_PDPES - 1; i >= 0; i--) { - const dma_addr_t pd_daddr = i915_page_dir_dma_addr(ppgtt, i); - - ret = gen8_write_pdp(req, i, pd_daddr); - if (ret) - return ret; - } - - return 0; -} - -static int gen8_mm_switch_4lvl(struct i915_hw_ppgtt *ppgtt, - struct drm_i915_gem_request *req) -{ - return gen8_write_pdp(req, 0, px_dma(&ppgtt->pml4)); -} - -/* PDE TLBs are a pain to invalidate on GEN8+. When we modify - * the page table structures, we mark them dirty so that - * context switching/execlist queuing code takes extra steps - * to ensure that tlbs are flushed. - */ -static void mark_tlbs_dirty(struct i915_hw_ppgtt *ppgtt) -{ - ppgtt->pd_dirty_rings = INTEL_INFO(ppgtt->base.i915)->ring_mask; -} - -/* Removes entries from a single page table, releasing it if it's empty. - * Caller can use the return value to update higher-level entries. - */ -static bool gen8_ppgtt_clear_pt(struct i915_address_space *vm, - struct i915_page_table *pt, - u64 start, u64 length) -{ - unsigned int num_entries = gen8_pte_count(start, length); - unsigned int pte = gen8_pte_index(start); - unsigned int pte_end = pte + num_entries; - const gen8_pte_t scratch_pte = - gen8_pte_encode(vm->scratch_page.daddr, I915_CACHE_LLC); - gen8_pte_t *vaddr; - - GEM_BUG_ON(num_entries > pt->used_ptes); - - pt->used_ptes -= num_entries; - if (!pt->used_ptes) - return true; - - vaddr = kmap_atomic_px(pt); - while (pte < pte_end) - vaddr[pte++] = scratch_pte; - kunmap_atomic(vaddr); - - return false; -} - -static void gen8_ppgtt_set_pde(struct i915_address_space *vm, - struct i915_page_directory *pd, - struct i915_page_table *pt, - unsigned int pde) -{ - gen8_pde_t *vaddr; - - pd->page_table[pde] = pt; - - vaddr = kmap_atomic_px(pd); - vaddr[pde] = gen8_pde_encode(px_dma(pt), I915_CACHE_LLC); - kunmap_atomic(vaddr); -} - -static bool gen8_ppgtt_clear_pd(struct i915_address_space *vm, - struct i915_page_directory *pd, - u64 start, u64 length) -{ - struct i915_page_table *pt; - u32 pde; - - gen8_for_each_pde(pt, pd, start, length, pde) { - GEM_BUG_ON(pt == vm->scratch_pt); - - if (!gen8_ppgtt_clear_pt(vm, pt, start, length)) - continue; - - gen8_ppgtt_set_pde(vm, pd, vm->scratch_pt, pde); - GEM_BUG_ON(!pd->used_pdes); - pd->used_pdes--; - - free_pt(vm, pt); - } - - return !pd->used_pdes; -} - -static void gen8_ppgtt_set_pdpe(struct i915_address_space *vm, - struct i915_page_directory_pointer *pdp, - struct i915_page_directory *pd, - unsigned int pdpe) -{ - gen8_ppgtt_pdpe_t *vaddr; - - pdp->page_directory[pdpe] = pd; - if (!use_4lvl(vm)) - return; - - vaddr = kmap_atomic_px(pdp); - vaddr[pdpe] = gen8_pdpe_encode(px_dma(pd), I915_CACHE_LLC); - kunmap_atomic(vaddr); -} - -/* Removes entries from a single page dir pointer, releasing it if it's empty. - * Caller can use the return value to update higher-level entries - */ -static bool gen8_ppgtt_clear_pdp(struct i915_address_space *vm, - struct i915_page_directory_pointer *pdp, - u64 start, u64 length) -{ - struct i915_page_directory *pd; - unsigned int pdpe; - - gen8_for_each_pdpe(pd, pdp, start, length, pdpe) { - GEM_BUG_ON(pd == vm->scratch_pd); - - if (!gen8_ppgtt_clear_pd(vm, pd, start, length)) - continue; - - gen8_ppgtt_set_pdpe(vm, pdp, vm->scratch_pd, pdpe); - GEM_BUG_ON(!pdp->used_pdpes); - pdp->used_pdpes--; - - free_pd(vm, pd); - } - - return !pdp->used_pdpes; -} - -static void gen8_ppgtt_clear_3lvl(struct i915_address_space *vm, - u64 start, u64 length) -{ - gen8_ppgtt_clear_pdp(vm, &i915_vm_to_ppgtt(vm)->pdp, start, length); -} - -static void gen8_ppgtt_set_pml4e(struct i915_pml4 *pml4, - struct i915_page_directory_pointer *pdp, - unsigned int pml4e) -{ - gen8_ppgtt_pml4e_t *vaddr; - - pml4->pdps[pml4e] = pdp; - - vaddr = kmap_atomic_px(pml4); - vaddr[pml4e] = gen8_pml4e_encode(px_dma(pdp), I915_CACHE_LLC); - kunmap_atomic(vaddr); -} - -/* Removes entries from a single pml4. - * This is the top-level structure in 4-level page tables used on gen8+. - * Empty entries are always scratch pml4e. - */ -static void gen8_ppgtt_clear_4lvl(struct i915_address_space *vm, - u64 start, u64 length) -{ - struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); - struct i915_pml4 *pml4 = &ppgtt->pml4; - struct i915_page_directory_pointer *pdp; - unsigned int pml4e; - - GEM_BUG_ON(!use_4lvl(vm)); - - gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) { - GEM_BUG_ON(pdp == vm->scratch_pdp); - - if (!gen8_ppgtt_clear_pdp(vm, pdp, start, length)) - continue; - - gen8_ppgtt_set_pml4e(pml4, vm->scratch_pdp, pml4e); - - free_pdp(vm, pdp); - } -} - -struct sgt_dma { - struct scatterlist *sg; - dma_addr_t dma, max; -}; - -struct gen8_insert_pte { - u16 pml4e; - u16 pdpe; - u16 pde; - u16 pte; -}; - -static __always_inline struct gen8_insert_pte gen8_insert_pte(u64 start) -{ - return (struct gen8_insert_pte) { - gen8_pml4e_index(start), - gen8_pdpe_index(start), - gen8_pde_index(start), - gen8_pte_index(start), - }; -} - -static __always_inline bool -gen8_ppgtt_insert_pte_entries(struct i915_hw_ppgtt *ppgtt, - struct i915_page_directory_pointer *pdp, - struct sgt_dma *iter, - struct gen8_insert_pte *idx, - enum i915_cache_level cache_level) -{ - struct i915_page_directory *pd; - const gen8_pte_t pte_encode = gen8_pte_encode(0, cache_level); - gen8_pte_t *vaddr; - bool ret; - - GEM_BUG_ON(idx->pdpe >= i915_pdpes_per_pdp(&ppgtt->base)); - pd = pdp->page_directory[idx->pdpe]; - vaddr = kmap_atomic_px(pd->page_table[idx->pde]); - do { - vaddr[idx->pte] = pte_encode | iter->dma; - - iter->dma += PAGE_SIZE; - if (iter->dma >= iter->max) { - iter->sg = __sg_next(iter->sg); - if (!iter->sg) { - ret = false; - break; - } - - iter->dma = sg_dma_address(iter->sg); - iter->max = iter->dma + iter->sg->length; - } - - if (++idx->pte == GEN8_PTES) { - idx->pte = 0; - - if (++idx->pde == I915_PDES) { - idx->pde = 0; - - /* Limited by sg length for 3lvl */ - if (++idx->pdpe == GEN8_PML4ES_PER_PML4) { - idx->pdpe = 0; - ret = true; - break; - } - - GEM_BUG_ON(idx->pdpe >= i915_pdpes_per_pdp(&ppgtt->base)); - pd = pdp->page_directory[idx->pdpe]; - } - - kunmap_atomic(vaddr); - vaddr = kmap_atomic_px(pd->page_table[idx->pde]); - } - } while (1); - kunmap_atomic(vaddr); - - return ret; -} - -static void gen8_ppgtt_insert_3lvl(struct i915_address_space *vm, - struct sg_table *pages, - u64 start, - enum i915_cache_level cache_level, - u32 unused) -{ - struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); - struct sgt_dma iter = { - .sg = pages->sgl, - .dma = sg_dma_address(iter.sg), - .max = iter.dma + iter.sg->length, - }; - struct gen8_insert_pte idx = gen8_insert_pte(start); - - gen8_ppgtt_insert_pte_entries(ppgtt, &ppgtt->pdp, &iter, &idx, - cache_level); -} - -static void gen8_ppgtt_insert_4lvl(struct i915_address_space *vm, - struct sg_table *pages, - u64 start, - enum i915_cache_level cache_level, - u32 unused) -{ - struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); - struct sgt_dma iter = { - .sg = pages->sgl, - .dma = sg_dma_address(iter.sg), - .max = iter.dma + iter.sg->length, - }; - struct i915_page_directory_pointer **pdps = ppgtt->pml4.pdps; - struct gen8_insert_pte idx = gen8_insert_pte(start); - - while (gen8_ppgtt_insert_pte_entries(ppgtt, pdps[idx.pml4e++], &iter, - &idx, cache_level)) - GEM_BUG_ON(idx.pml4e >= GEN8_PML4ES_PER_PML4); -} - -static void gen8_free_page_tables(struct i915_address_space *vm, - struct i915_page_directory *pd) -{ - int i; - - if (!px_page(pd)) - return; - - for (i = 0; i < I915_PDES; i++) { - if (pd->page_table[i] != vm->scratch_pt) - free_pt(vm, pd->page_table[i]); - } -} - -static int gen8_init_scratch(struct i915_address_space *vm) -{ - int ret; - - ret = setup_scratch_page(vm, I915_GFP_DMA); - if (ret) - return ret; - - vm->scratch_pt = alloc_pt(vm); - if (IS_ERR(vm->scratch_pt)) { - ret = PTR_ERR(vm->scratch_pt); - goto free_scratch_page; - } - - vm->scratch_pd = alloc_pd(vm); - if (IS_ERR(vm->scratch_pd)) { - ret = PTR_ERR(vm->scratch_pd); - goto free_pt; - } - - if (use_4lvl(vm)) { - vm->scratch_pdp = alloc_pdp(vm); - if (IS_ERR(vm->scratch_pdp)) { - ret = PTR_ERR(vm->scratch_pdp); - goto free_pd; - } - } - - gen8_initialize_pt(vm, vm->scratch_pt); - gen8_initialize_pd(vm, vm->scratch_pd); - if (use_4lvl(vm)) - gen8_initialize_pdp(vm, vm->scratch_pdp); - - return 0; - -free_pd: - free_pd(vm, vm->scratch_pd); -free_pt: - free_pt(vm, vm->scratch_pt); -free_scratch_page: - cleanup_scratch_page(vm); - - return ret; -} - -static int gen8_ppgtt_notify_vgt(struct i915_hw_ppgtt *ppgtt, bool create) -{ - struct i915_address_space *vm = &ppgtt->base; - struct drm_i915_private *dev_priv = vm->i915; - enum vgt_g2v_type msg; - int i; - - if (use_4lvl(vm)) { - const u64 daddr = px_dma(&ppgtt->pml4); - - I915_WRITE(vgtif_reg(pdp[0].lo), lower_32_bits(daddr)); - I915_WRITE(vgtif_reg(pdp[0].hi), upper_32_bits(daddr)); - - msg = (create ? VGT_G2V_PPGTT_L4_PAGE_TABLE_CREATE : - VGT_G2V_PPGTT_L4_PAGE_TABLE_DESTROY); - } else { - for (i = 0; i < GEN8_3LVL_PDPES; i++) { - const u64 daddr = i915_page_dir_dma_addr(ppgtt, i); - - I915_WRITE(vgtif_reg(pdp[i].lo), lower_32_bits(daddr)); - I915_WRITE(vgtif_reg(pdp[i].hi), upper_32_bits(daddr)); - } - - msg = (create ? VGT_G2V_PPGTT_L3_PAGE_TABLE_CREATE : - VGT_G2V_PPGTT_L3_PAGE_TABLE_DESTROY); - } - - I915_WRITE(vgtif_reg(g2v_notify), msg); - - return 0; -} - -static void gen8_free_scratch(struct i915_address_space *vm) -{ - if (use_4lvl(vm)) - free_pdp(vm, vm->scratch_pdp); - free_pd(vm, vm->scratch_pd); - free_pt(vm, vm->scratch_pt); - cleanup_scratch_page(vm); -} - -static void gen8_ppgtt_cleanup_3lvl(struct i915_address_space *vm, - struct i915_page_directory_pointer *pdp) -{ - const unsigned int pdpes = i915_pdpes_per_pdp(vm); - int i; - - for (i = 0; i < pdpes; i++) { - if (pdp->page_directory[i] == vm->scratch_pd) - continue; - - gen8_free_page_tables(vm, pdp->page_directory[i]); - free_pd(vm, pdp->page_directory[i]); - } - - free_pdp(vm, pdp); -} - -static void gen8_ppgtt_cleanup_4lvl(struct i915_hw_ppgtt *ppgtt) -{ - int i; - - for (i = 0; i < GEN8_PML4ES_PER_PML4; i++) { - if (ppgtt->pml4.pdps[i] == ppgtt->base.scratch_pdp) - continue; - - gen8_ppgtt_cleanup_3lvl(&ppgtt->base, ppgtt->pml4.pdps[i]); - } - - cleanup_px(&ppgtt->base, &ppgtt->pml4); -} - -static void gen8_ppgtt_cleanup(struct i915_address_space *vm) -{ - struct drm_i915_private *dev_priv = vm->i915; - struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); - - if (intel_vgpu_active(dev_priv)) - gen8_ppgtt_notify_vgt(ppgtt, false); - - if (use_4lvl(vm)) - gen8_ppgtt_cleanup_4lvl(ppgtt); - else - gen8_ppgtt_cleanup_3lvl(&ppgtt->base, &ppgtt->pdp); - - gen8_free_scratch(vm); -} - -static int gen8_ppgtt_alloc_pd(struct i915_address_space *vm, - struct i915_page_directory *pd, - u64 start, u64 length) -{ - struct i915_page_table *pt; - u64 from = start; - unsigned int pde; - - gen8_for_each_pde(pt, pd, start, length, pde) { - if (pt == vm->scratch_pt) { - pt = alloc_pt(vm); - if (IS_ERR(pt)) - goto unwind; - - gen8_initialize_pt(vm, pt); - - gen8_ppgtt_set_pde(vm, pd, pt, pde); - pd->used_pdes++; - GEM_BUG_ON(pd->used_pdes > I915_PDES); - } - - pt->used_ptes += gen8_pte_count(start, length); - } - return 0; - -unwind: - gen8_ppgtt_clear_pd(vm, pd, from, start - from); - return -ENOMEM; -} - -static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm, - struct i915_page_directory_pointer *pdp, - u64 start, u64 length) -{ - struct i915_page_directory *pd; - u64 from = start; - unsigned int pdpe; - int ret; - - gen8_for_each_pdpe(pd, pdp, start, length, pdpe) { - if (pd == vm->scratch_pd) { - pd = alloc_pd(vm); - if (IS_ERR(pd)) - goto unwind; - - gen8_initialize_pd(vm, pd); - gen8_ppgtt_set_pdpe(vm, pdp, pd, pdpe); - pdp->used_pdpes++; - GEM_BUG_ON(pdp->used_pdpes > i915_pdpes_per_pdp(vm)); - - mark_tlbs_dirty(i915_vm_to_ppgtt(vm)); - } - - ret = gen8_ppgtt_alloc_pd(vm, pd, start, length); - if (unlikely(ret)) - goto unwind_pd; - } - - return 0; - -unwind_pd: - if (!pd->used_pdes) { - gen8_ppgtt_set_pdpe(vm, pdp, vm->scratch_pd, pdpe); - GEM_BUG_ON(!pdp->used_pdpes); - pdp->used_pdpes--; - free_pd(vm, pd); - } -unwind: - gen8_ppgtt_clear_pdp(vm, pdp, from, start - from); - return -ENOMEM; -} - -static int gen8_ppgtt_alloc_3lvl(struct i915_address_space *vm, - u64 start, u64 length) -{ - return gen8_ppgtt_alloc_pdp(vm, - &i915_vm_to_ppgtt(vm)->pdp, start, length); -} - -static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm, - u64 start, u64 length) -{ - struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); - struct i915_pml4 *pml4 = &ppgtt->pml4; - struct i915_page_directory_pointer *pdp; - u64 from = start; - u32 pml4e; - int ret; - - gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) { - if (pml4->pdps[pml4e] == vm->scratch_pdp) { - pdp = alloc_pdp(vm); - if (IS_ERR(pdp)) - goto unwind; - - gen8_initialize_pdp(vm, pdp); - gen8_ppgtt_set_pml4e(pml4, pdp, pml4e); - } - - ret = gen8_ppgtt_alloc_pdp(vm, pdp, start, length); - if (unlikely(ret)) - goto unwind_pdp; - } - - return 0; - -unwind_pdp: - if (!pdp->used_pdpes) { - gen8_ppgtt_set_pml4e(pml4, vm->scratch_pdp, pml4e); - free_pdp(vm, pdp); - } -unwind: - gen8_ppgtt_clear_4lvl(vm, from, start - from); - return -ENOMEM; -} - -static void gen8_dump_pdp(struct i915_hw_ppgtt *ppgtt, - struct i915_page_directory_pointer *pdp, - u64 start, u64 length, - gen8_pte_t scratch_pte, - struct seq_file *m) -{ - struct i915_address_space *vm = &ppgtt->base; - struct i915_page_directory *pd; - u32 pdpe; - - gen8_for_each_pdpe(pd, pdp, start, length, pdpe) { - struct i915_page_table *pt; - u64 pd_len = length; - u64 pd_start = start; - u32 pde; - - if (pdp->page_directory[pdpe] == ppgtt->base.scratch_pd) - continue; - - seq_printf(m, "\tPDPE #%d\n", pdpe); - gen8_for_each_pde(pt, pd, pd_start, pd_len, pde) { - u32 pte; - gen8_pte_t *pt_vaddr; - - if (pd->page_table[pde] == ppgtt->base.scratch_pt) - continue; - - pt_vaddr = kmap_atomic_px(pt); - for (pte = 0; pte < GEN8_PTES; pte += 4) { - u64 va = (pdpe << GEN8_PDPE_SHIFT | - pde << GEN8_PDE_SHIFT | - pte << GEN8_PTE_SHIFT); - int i; - bool found = false; - - for (i = 0; i < 4; i++) - if (pt_vaddr[pte + i] != scratch_pte) - found = true; - if (!found) - continue; - - seq_printf(m, "\t\t0x%llx [%03d,%03d,%04d]: =", va, pdpe, pde, pte); - for (i = 0; i < 4; i++) { - if (pt_vaddr[pte + i] != scratch_pte) - seq_printf(m, " %llx", pt_vaddr[pte + i]); - else - seq_puts(m, " SCRATCH "); - } - seq_puts(m, "\n"); - } - kunmap_atomic(pt_vaddr); - } - } -} - -static void gen8_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m) -{ - struct i915_address_space *vm = &ppgtt->base; - const gen8_pte_t scratch_pte = - gen8_pte_encode(vm->scratch_page.daddr, I915_CACHE_LLC); - u64 start = 0, length = ppgtt->base.total; - - if (use_4lvl(vm)) { - u64 pml4e; - struct i915_pml4 *pml4 = &ppgtt->pml4; - struct i915_page_directory_pointer *pdp; - - gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) { - if (pml4->pdps[pml4e] == ppgtt->base.scratch_pdp) - continue; - - seq_printf(m, " PML4E #%llu\n", pml4e); - gen8_dump_pdp(ppgtt, pdp, start, length, scratch_pte, m); - } - } else { - gen8_dump_pdp(ppgtt, &ppgtt->pdp, start, length, scratch_pte, m); - } -} - -static int gen8_preallocate_top_level_pdp(struct i915_hw_ppgtt *ppgtt) -{ - struct i915_address_space *vm = &ppgtt->base; - struct i915_page_directory_pointer *pdp = &ppgtt->pdp; - struct i915_page_directory *pd; - u64 start = 0, length = ppgtt->base.total; - u64 from = start; - unsigned int pdpe; - - gen8_for_each_pdpe(pd, pdp, start, length, pdpe) { - pd = alloc_pd(vm); - if (IS_ERR(pd)) - goto unwind; - - gen8_initialize_pd(vm, pd); - gen8_ppgtt_set_pdpe(vm, pdp, pd, pdpe); - pdp->used_pdpes++; - } - - pdp->used_pdpes++; /* never remove */ - return 0; - -unwind: - start -= from; - gen8_for_each_pdpe(pd, pdp, from, start, pdpe) { - gen8_ppgtt_set_pdpe(vm, pdp, vm->scratch_pd, pdpe); - free_pd(vm, pd); - } - pdp->used_pdpes = 0; - return -ENOMEM; -} - -/* - * GEN8 legacy ppgtt programming is accomplished through a max 4 PDP registers - * with a net effect resembling a 2-level page table in normal x86 terms. Each - * PDP represents 1GB of memory 4 * 512 * 512 * 4096 = 4GB legacy 32b address - * space. - * - */ -static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt) -{ - struct i915_address_space *vm = &ppgtt->base; - struct drm_i915_private *dev_priv = vm->i915; - int ret; - - ppgtt->base.total = USES_FULL_48BIT_PPGTT(dev_priv) ? - 1ULL << 48 : - 1ULL << 32; - - ret = gen8_init_scratch(&ppgtt->base); - if (ret) { - ppgtt->base.total = 0; - return ret; - } - - /* There are only few exceptions for gen >=6. chv and bxt. - * And we are not sure about the latter so play safe for now. - */ - if (IS_CHERRYVIEW(dev_priv) || IS_BROXTON(dev_priv)) - ppgtt->base.pt_kmap_wc = true; - - if (use_4lvl(vm)) { - ret = setup_px(&ppgtt->base, &ppgtt->pml4); - if (ret) - goto free_scratch; - - gen8_initialize_pml4(&ppgtt->base, &ppgtt->pml4); - - ppgtt->switch_mm = gen8_mm_switch_4lvl; - ppgtt->base.allocate_va_range = gen8_ppgtt_alloc_4lvl; - ppgtt->base.insert_entries = gen8_ppgtt_insert_4lvl; - ppgtt->base.clear_range = gen8_ppgtt_clear_4lvl; - } else { - ret = __pdp_init(&ppgtt->base, &ppgtt->pdp); - if (ret) - goto free_scratch; - - if (intel_vgpu_active(dev_priv)) { - ret = gen8_preallocate_top_level_pdp(ppgtt); - if (ret) { - __pdp_fini(&ppgtt->pdp); - goto free_scratch; - } - } - - ppgtt->switch_mm = gen8_mm_switch_3lvl; - ppgtt->base.allocate_va_range = gen8_ppgtt_alloc_3lvl; - ppgtt->base.insert_entries = gen8_ppgtt_insert_3lvl; - ppgtt->base.clear_range = gen8_ppgtt_clear_3lvl; - } - - if (intel_vgpu_active(dev_priv)) - gen8_ppgtt_notify_vgt(ppgtt, true); - - ppgtt->base.cleanup = gen8_ppgtt_cleanup; - ppgtt->base.unbind_vma = ppgtt_unbind_vma; - ppgtt->base.bind_vma = ppgtt_bind_vma; - ppgtt->debug_dump = gen8_dump_ppgtt; - - return 0; - -free_scratch: - gen8_free_scratch(&ppgtt->base); - return ret; -} - -static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m) -{ - struct i915_address_space *vm = &ppgtt->base; - struct i915_page_table *unused; - gen6_pte_t scratch_pte; - u32 pd_entry, pte, pde; - u32 start = 0, length = ppgtt->base.total; - - scratch_pte = vm->pte_encode(vm->scratch_page.daddr, - I915_CACHE_LLC, 0); - - gen6_for_each_pde(unused, &ppgtt->pd, start, length, pde) { - u32 expected; - gen6_pte_t *pt_vaddr; - const dma_addr_t pt_addr = px_dma(ppgtt->pd.page_table[pde]); - pd_entry = readl(ppgtt->pd_addr + pde); - expected = (GEN6_PDE_ADDR_ENCODE(pt_addr) | GEN6_PDE_VALID); - - if (pd_entry != expected) - seq_printf(m, "\tPDE #%d mismatch: Actual PDE: %x Expected PDE: %x\n", - pde, - pd_entry, - expected); - seq_printf(m, "\tPDE: %x\n", pd_entry); - - pt_vaddr = kmap_atomic_px(ppgtt->pd.page_table[pde]); - - for (pte = 0; pte < GEN6_PTES; pte+=4) { - unsigned long va = - (pde * PAGE_SIZE * GEN6_PTES) + - (pte * PAGE_SIZE); - int i; - bool found = false; - for (i = 0; i < 4; i++) - if (pt_vaddr[pte + i] != scratch_pte) - found = true; - if (!found) - continue; - - seq_printf(m, "\t\t0x%lx [%03d,%04d]: =", va, pde, pte); - for (i = 0; i < 4; i++) { - if (pt_vaddr[pte + i] != scratch_pte) - seq_printf(m, " %08x", pt_vaddr[pte + i]); - else - seq_puts(m, " SCRATCH "); - } - seq_puts(m, "\n"); - } - kunmap_atomic(pt_vaddr); - } -} - -/* Write pde (index) from the page directory @pd to the page table @pt */ -static inline void gen6_write_pde(const struct i915_hw_ppgtt *ppgtt, - const unsigned int pde, - const struct i915_page_table *pt) -{ - /* Caller needs to make sure the write completes if necessary */ - writel_relaxed(GEN6_PDE_ADDR_ENCODE(px_dma(pt)) | GEN6_PDE_VALID, - ppgtt->pd_addr + pde); -} - -/* Write all the page tables found in the ppgtt structure to incrementing page - * directories. */ -static void gen6_write_page_range(struct i915_hw_ppgtt *ppgtt, - u32 start, u32 length) -{ - struct i915_page_table *pt; - unsigned int pde; - - gen6_for_each_pde(pt, &ppgtt->pd, start, length, pde) - gen6_write_pde(ppgtt, pde, pt); - - mark_tlbs_dirty(ppgtt); - wmb(); -} - -static inline u32 get_pd_offset(struct i915_hw_ppgtt *ppgtt) -{ - GEM_BUG_ON(ppgtt->pd.base.ggtt_offset & 0x3f); - return ppgtt->pd.base.ggtt_offset << 10; -} - -static int hsw_mm_switch(struct i915_hw_ppgtt *ppgtt, - struct drm_i915_gem_request *req) -{ - struct intel_engine_cs *engine = req->engine; - u32 *cs; - - /* NB: TLBs must be flushed and invalidated before a switch */ - cs = intel_ring_begin(req, 6); - if (IS_ERR(cs)) - return PTR_ERR(cs); - - *cs++ = MI_LOAD_REGISTER_IMM(2); - *cs++ = i915_mmio_reg_offset(RING_PP_DIR_DCLV(engine)); - *cs++ = PP_DIR_DCLV_2G; - *cs++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine)); - *cs++ = get_pd_offset(ppgtt); - *cs++ = MI_NOOP; - intel_ring_advance(req, cs); - - return 0; -} - -static int gen7_mm_switch(struct i915_hw_ppgtt *ppgtt, - struct drm_i915_gem_request *req) -{ - struct intel_engine_cs *engine = req->engine; - u32 *cs; - - /* NB: TLBs must be flushed and invalidated before a switch */ - cs = intel_ring_begin(req, 6); - if (IS_ERR(cs)) - return PTR_ERR(cs); - - *cs++ = MI_LOAD_REGISTER_IMM(2); - *cs++ = i915_mmio_reg_offset(RING_PP_DIR_DCLV(engine)); - *cs++ = PP_DIR_DCLV_2G; - *cs++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine)); - *cs++ = get_pd_offset(ppgtt); - *cs++ = MI_NOOP; - intel_ring_advance(req, cs); - - return 0; -} - -static int gen6_mm_switch(struct i915_hw_ppgtt *ppgtt, - struct drm_i915_gem_request *req) -{ - struct intel_engine_cs *engine = req->engine; - struct drm_i915_private *dev_priv = req->i915; - - I915_WRITE(RING_PP_DIR_DCLV(engine), PP_DIR_DCLV_2G); - I915_WRITE(RING_PP_DIR_BASE(engine), get_pd_offset(ppgtt)); - return 0; -} - -static void gen8_ppgtt_enable(struct drm_i915_private *dev_priv) -{ - struct intel_engine_cs *engine; - enum intel_engine_id id; - - for_each_engine(engine, dev_priv, id) { - u32 four_level = USES_FULL_48BIT_PPGTT(dev_priv) ? - GEN8_GFX_PPGTT_48B : 0; - I915_WRITE(RING_MODE_GEN7(engine), - _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE | four_level)); - } -} - -static void gen7_ppgtt_enable(struct drm_i915_private *dev_priv) -{ - struct intel_engine_cs *engine; - u32 ecochk, ecobits; - enum intel_engine_id id; - - ecobits = I915_READ(GAC_ECO_BITS); - I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B); - - ecochk = I915_READ(GAM_ECOCHK); - if (IS_HASWELL(dev_priv)) { - ecochk |= ECOCHK_PPGTT_WB_HSW; - } else { - ecochk |= ECOCHK_PPGTT_LLC_IVB; - ecochk &= ~ECOCHK_PPGTT_GFDT_IVB; - } - I915_WRITE(GAM_ECOCHK, ecochk); - - for_each_engine(engine, dev_priv, id) { - /* GFX_MODE is per-ring on gen7+ */ - I915_WRITE(RING_MODE_GEN7(engine), - _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); - } -} - -static void gen6_ppgtt_enable(struct drm_i915_private *dev_priv) -{ - u32 ecochk, gab_ctl, ecobits; - - ecobits = I915_READ(GAC_ECO_BITS); - I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_SNB_BIT | - ECOBITS_PPGTT_CACHE64B); - - gab_ctl = I915_READ(GAB_CTL); - I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT); - - ecochk = I915_READ(GAM_ECOCHK); - I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | ECOCHK_PPGTT_CACHE64B); - - I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); -} - -/* PPGTT support for Sandybdrige/Gen6 and later */ -static void gen6_ppgtt_clear_range(struct i915_address_space *vm, - u64 start, u64 length) -{ - struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); - unsigned int first_entry = start >> PAGE_SHIFT; - unsigned int pde = first_entry / GEN6_PTES; - unsigned int pte = first_entry % GEN6_PTES; - unsigned int num_entries = length >> PAGE_SHIFT; - gen6_pte_t scratch_pte = - vm->pte_encode(vm->scratch_page.daddr, I915_CACHE_LLC, 0); - - while (num_entries) { - struct i915_page_table *pt = ppgtt->pd.page_table[pde++]; - unsigned int end = min(pte + num_entries, GEN6_PTES); - gen6_pte_t *vaddr; - - num_entries -= end - pte; - - /* Note that the hw doesn't support removing PDE on the fly - * (they are cached inside the context with no means to - * invalidate the cache), so we can only reset the PTE - * entries back to scratch. - */ - - vaddr = kmap_atomic_px(pt); - do { - vaddr[pte++] = scratch_pte; - } while (pte < end); - kunmap_atomic(vaddr); - - pte = 0; - } -} - -static void gen6_ppgtt_insert_entries(struct i915_address_space *vm, - struct sg_table *pages, - u64 start, - enum i915_cache_level cache_level, - u32 flags) -{ - struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); - unsigned first_entry = start >> PAGE_SHIFT; - unsigned act_pt = first_entry / GEN6_PTES; - unsigned act_pte = first_entry % GEN6_PTES; - const u32 pte_encode = vm->pte_encode(0, cache_level, flags); - struct sgt_dma iter; - gen6_pte_t *vaddr; - - vaddr = kmap_atomic_px(ppgtt->pd.page_table[act_pt]); - iter.sg = pages->sgl; - iter.dma = sg_dma_address(iter.sg); - iter.max = iter.dma + iter.sg->length; - do { - vaddr[act_pte] = pte_encode | GEN6_PTE_ADDR_ENCODE(iter.dma); - - iter.dma += PAGE_SIZE; - if (iter.dma == iter.max) { - iter.sg = __sg_next(iter.sg); - if (!iter.sg) - break; - - iter.dma = sg_dma_address(iter.sg); - iter.max = iter.dma + iter.sg->length; - } - - if (++act_pte == GEN6_PTES) { - kunmap_atomic(vaddr); - vaddr = kmap_atomic_px(ppgtt->pd.page_table[++act_pt]); - act_pte = 0; - } - } while (1); - kunmap_atomic(vaddr); -} - -static int gen6_alloc_va_range(struct i915_address_space *vm, - u64 start, u64 length) -{ - struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); - struct i915_page_table *pt; - u64 from = start; - unsigned int pde; - bool flush = false; - - gen6_for_each_pde(pt, &ppgtt->pd, start, length, pde) { - if (pt == vm->scratch_pt) { - pt = alloc_pt(vm); - if (IS_ERR(pt)) - goto unwind_out; - - gen6_initialize_pt(vm, pt); - ppgtt->pd.page_table[pde] = pt; - gen6_write_pde(ppgtt, pde, pt); - flush = true; - } - } - - if (flush) { - mark_tlbs_dirty(ppgtt); - wmb(); - } - - return 0; - -unwind_out: - gen6_ppgtt_clear_range(vm, from, start); - return -ENOMEM; -} - -static int gen6_init_scratch(struct i915_address_space *vm) -{ - int ret; - - ret = setup_scratch_page(vm, I915_GFP_DMA); - if (ret) - return ret; - - vm->scratch_pt = alloc_pt(vm); - if (IS_ERR(vm->scratch_pt)) { - cleanup_scratch_page(vm); - return PTR_ERR(vm->scratch_pt); - } - - gen6_initialize_pt(vm, vm->scratch_pt); - - return 0; -} - -static void gen6_free_scratch(struct i915_address_space *vm) -{ - free_pt(vm, vm->scratch_pt); - cleanup_scratch_page(vm); -} - -static void gen6_ppgtt_cleanup(struct i915_address_space *vm) -{ - struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); - struct i915_page_directory *pd = &ppgtt->pd; - struct i915_page_table *pt; - u32 pde; - - drm_mm_remove_node(&ppgtt->node); - - gen6_for_all_pdes(pt, pd, pde) - if (pt != vm->scratch_pt) - free_pt(vm, pt); - - gen6_free_scratch(vm); -} - -static int gen6_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt) -{ - struct i915_address_space *vm = &ppgtt->base; - struct drm_i915_private *dev_priv = ppgtt->base.i915; - struct i915_ggtt *ggtt = &dev_priv->ggtt; - int ret; - - /* PPGTT PDEs reside in the GGTT and consists of 512 entries. The - * allocator works in address space sizes, so it's multiplied by page - * size. We allocate at the top of the GTT to avoid fragmentation. - */ - BUG_ON(!drm_mm_initialized(&ggtt->base.mm)); - - ret = gen6_init_scratch(vm); - if (ret) - return ret; - - ret = i915_gem_gtt_insert(&ggtt->base, &ppgtt->node, - GEN6_PD_SIZE, GEN6_PD_ALIGN, - I915_COLOR_UNEVICTABLE, - 0, ggtt->base.total, - PIN_HIGH); - if (ret) - goto err_out; - - if (ppgtt->node.start < ggtt->mappable_end) - DRM_DEBUG("Forced to use aperture for PDEs\n"); - - ppgtt->pd.base.ggtt_offset = - ppgtt->node.start / PAGE_SIZE * sizeof(gen6_pte_t); - - ppgtt->pd_addr = (gen6_pte_t __iomem *)ggtt->gsm + - ppgtt->pd.base.ggtt_offset / sizeof(gen6_pte_t); - - return 0; - -err_out: - gen6_free_scratch(vm); - return ret; -} - -static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt) -{ - return gen6_ppgtt_allocate_page_directories(ppgtt); -} - -static void gen6_scratch_va_range(struct i915_hw_ppgtt *ppgtt, - u64 start, u64 length) -{ - struct i915_page_table *unused; - u32 pde; - - gen6_for_each_pde(unused, &ppgtt->pd, start, length, pde) - ppgtt->pd.page_table[pde] = ppgtt->base.scratch_pt; -} - -static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt) -{ - struct drm_i915_private *dev_priv = ppgtt->base.i915; - struct i915_ggtt *ggtt = &dev_priv->ggtt; - int ret; - - ppgtt->base.pte_encode = ggtt->base.pte_encode; - if (intel_vgpu_active(dev_priv) || IS_GEN6(dev_priv)) - ppgtt->switch_mm = gen6_mm_switch; - else if (IS_HASWELL(dev_priv)) - ppgtt->switch_mm = hsw_mm_switch; - else if (IS_GEN7(dev_priv)) - ppgtt->switch_mm = gen7_mm_switch; - else - BUG(); - - ret = gen6_ppgtt_alloc(ppgtt); - if (ret) - return ret; - - ppgtt->base.total = I915_PDES * GEN6_PTES * PAGE_SIZE; - - gen6_scratch_va_range(ppgtt, 0, ppgtt->base.total); - gen6_write_page_range(ppgtt, 0, ppgtt->base.total); - - ret = gen6_alloc_va_range(&ppgtt->base, 0, ppgtt->base.total); - if (ret) { - gen6_ppgtt_cleanup(&ppgtt->base); - return ret; - } - - ppgtt->base.clear_range = gen6_ppgtt_clear_range; - ppgtt->base.insert_entries = gen6_ppgtt_insert_entries; - ppgtt->base.unbind_vma = ppgtt_unbind_vma; - ppgtt->base.bind_vma = ppgtt_bind_vma; - ppgtt->base.cleanup = gen6_ppgtt_cleanup; - ppgtt->debug_dump = gen6_dump_ppgtt; - - DRM_DEBUG_DRIVER("Allocated pde space (%lldM) at GTT entry: %llx\n", - ppgtt->node.size >> 20, - ppgtt->node.start / PAGE_SIZE); - - DRM_DEBUG_DRIVER("Adding PPGTT at offset %x\n", - ppgtt->pd.base.ggtt_offset << 10); - - return 0; -} - -static int __hw_ppgtt_init(struct i915_hw_ppgtt *ppgtt, - struct drm_i915_private *dev_priv) -{ - ppgtt->base.i915 = dev_priv; - ppgtt->base.dma = &dev_priv->drm.pdev->dev; - - if (INTEL_INFO(dev_priv)->gen < 8) - return gen6_ppgtt_init(ppgtt); - else - return gen8_ppgtt_init(ppgtt); -} - -static void i915_address_space_init(struct i915_address_space *vm, - struct drm_i915_private *dev_priv, - const char *name) -{ - i915_gem_timeline_init(dev_priv, &vm->timeline, name); - - drm_mm_init(&vm->mm, 0, vm->total); - vm->mm.head_node.color = I915_COLOR_UNEVICTABLE; - - INIT_LIST_HEAD(&vm->active_list); - INIT_LIST_HEAD(&vm->inactive_list); - INIT_LIST_HEAD(&vm->unbound_list); - - list_add_tail(&vm->global_link, &dev_priv->vm_list); - pagevec_init(&vm->free_pages, false); -} - -static void i915_address_space_fini(struct i915_address_space *vm) -{ - if (pagevec_count(&vm->free_pages)) - vm_free_pages_release(vm); - - i915_gem_timeline_fini(&vm->timeline); - drm_mm_takedown(&vm->mm); - list_del(&vm->global_link); -} - -static void gtt_write_workarounds(struct drm_i915_private *dev_priv) -{ - /* This function is for gtt related workarounds. This function is - * called on driver load and after a GPU reset, so you can place - * workarounds here even if they get overwritten by GPU reset. - */ - /* WaIncreaseDefaultTLBEntries:chv,bdw,skl,bxt,kbl,glk,cfl */ - if (IS_BROADWELL(dev_priv)) - I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN8_L3_LRA_1_GPGPU_DEFAULT_VALUE_BDW); - else if (IS_CHERRYVIEW(dev_priv)) - I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN8_L3_LRA_1_GPGPU_DEFAULT_VALUE_CHV); - else if (IS_GEN9_BC(dev_priv)) - I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN9_L3_LRA_1_GPGPU_DEFAULT_VALUE_SKL); - else if (IS_GEN9_LP(dev_priv)) - I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN9_L3_LRA_1_GPGPU_DEFAULT_VALUE_BXT); -} - -int i915_ppgtt_init_hw(struct drm_i915_private *dev_priv) -{ - gtt_write_workarounds(dev_priv); - - /* In the case of execlists, PPGTT is enabled by the context descriptor - * and the PDPs are contained within the context itself. We don't - * need to do anything here. */ - if (i915.enable_execlists) - return 0; - - if (!USES_PPGTT(dev_priv)) - return 0; - - if (IS_GEN6(dev_priv)) - gen6_ppgtt_enable(dev_priv); - else if (IS_GEN7(dev_priv)) - gen7_ppgtt_enable(dev_priv); - else if (INTEL_GEN(dev_priv) >= 8) - gen8_ppgtt_enable(dev_priv); - else - MISSING_CASE(INTEL_GEN(dev_priv)); - - return 0; -} - -struct i915_hw_ppgtt * -i915_ppgtt_create(struct drm_i915_private *dev_priv, - struct drm_i915_file_private *fpriv, - const char *name) -{ - struct i915_hw_ppgtt *ppgtt; - int ret; - - ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL); - if (!ppgtt) - return ERR_PTR(-ENOMEM); - - ret = __hw_ppgtt_init(ppgtt, dev_priv); - if (ret) { - kfree(ppgtt); - return ERR_PTR(ret); - } - - kref_init(&ppgtt->ref); - i915_address_space_init(&ppgtt->base, dev_priv, name); - ppgtt->base.file = fpriv; - - trace_i915_ppgtt_create(&ppgtt->base); - - return ppgtt; -} - -void i915_ppgtt_close(struct i915_address_space *vm) -{ - struct list_head *phases[] = { - &vm->active_list, - &vm->inactive_list, - &vm->unbound_list, - NULL, - }, **phase; - - GEM_BUG_ON(vm->closed); - vm->closed = true; - - for (phase = phases; *phase; phase++) { - struct i915_vma *vma, *vn; - - list_for_each_entry_safe(vma, vn, *phase, vm_link) - if (!i915_vma_is_closed(vma)) - i915_vma_close(vma); - } -} - -void i915_ppgtt_release(struct kref *kref) -{ - struct i915_hw_ppgtt *ppgtt = - container_of(kref, struct i915_hw_ppgtt, ref); - - trace_i915_ppgtt_release(&ppgtt->base); - - /* vmas should already be unbound and destroyed */ - WARN_ON(!list_empty(&ppgtt->base.active_list)); - WARN_ON(!list_empty(&ppgtt->base.inactive_list)); - WARN_ON(!list_empty(&ppgtt->base.unbound_list)); - - ppgtt->base.cleanup(&ppgtt->base); - i915_address_space_fini(&ppgtt->base); - kfree(ppgtt); -} - -/* Certain Gen5 chipsets require require idling the GPU before - * unmapping anything from the GTT when VT-d is enabled. - */ -static bool needs_idle_maps(struct drm_i915_private *dev_priv) -{ - /* Query intel_iommu to see if we need the workaround. Presumably that - * was loaded first. - */ - return IS_GEN5(dev_priv) && IS_MOBILE(dev_priv) && intel_vtd_active(); -} - -void i915_check_and_clear_faults(struct drm_i915_private *dev_priv) -{ - struct intel_engine_cs *engine; - enum intel_engine_id id; - - if (INTEL_INFO(dev_priv)->gen < 6) - return; - - for_each_engine(engine, dev_priv, id) { - u32 fault_reg; - fault_reg = I915_READ(RING_FAULT_REG(engine)); - if (fault_reg & RING_FAULT_VALID) { - DRM_DEBUG_DRIVER("Unexpected fault\n" - "\tAddr: 0x%08lx\n" - "\tAddress space: %s\n" - "\tSource ID: %d\n" - "\tType: %d\n", - fault_reg & PAGE_MASK, - fault_reg & RING_FAULT_GTTSEL_MASK ? "GGTT" : "PPGTT", - RING_FAULT_SRCID(fault_reg), - RING_FAULT_FAULT_TYPE(fault_reg)); - I915_WRITE(RING_FAULT_REG(engine), - fault_reg & ~RING_FAULT_VALID); - } - } - - /* Engine specific init may not have been done till this point. */ - if (dev_priv->engine[RCS]) - POSTING_READ(RING_FAULT_REG(dev_priv->engine[RCS])); -} - -void i915_gem_suspend_gtt_mappings(struct drm_i915_private *dev_priv) -{ - struct i915_ggtt *ggtt = &dev_priv->ggtt; - - /* Don't bother messing with faults pre GEN6 as we have little - * documentation supporting that it's a good idea. - */ - if (INTEL_GEN(dev_priv) < 6) - return; - - i915_check_and_clear_faults(dev_priv); - - ggtt->base.clear_range(&ggtt->base, 0, ggtt->base.total); - - i915_ggtt_invalidate(dev_priv); -} +#include "i915_vgpu.h" int i915_gem_gtt_prepare_pages(struct drm_i915_gem_object *obj, struct sg_table *pages) { do { - if (dma_map_sg(&obj->base.dev->pdev->dev, - pages->sgl, pages->nents, - PCI_DMA_BIDIRECTIONAL)) + if (dma_map_sg_attrs(obj->base.dev->dev, + pages->sgl, pages->nents, + DMA_BIDIRECTIONAL, + DMA_ATTR_SKIP_CPU_SYNC | + DMA_ATTR_NO_KERNEL_MAPPING | + DMA_ATTR_NO_WARN)) return 0; - /* If the DMA remap fails, one cause can be that we have + /* + * If the DMA remap fails, one cause can be that we have * too many objects pinned in a small remapping table, * such as swiotlb. Incrementally purge all other objects and * try again - if there are no more pages to remove from * the DMA remapper, i915_gem_shrink will return 0. */ GEM_BUG_ON(obj->mm.pages == pages); - } while (i915_gem_shrink(to_i915(obj->base.dev), - obj->base.size >> PAGE_SHIFT, + } while (i915_gem_shrink(NULL, to_i915(obj->base.dev), + obj->base.size >> PAGE_SHIFT, NULL, I915_SHRINK_BOUND | - I915_SHRINK_UNBOUND | - I915_SHRINK_ACTIVE)); + I915_SHRINK_UNBOUND)); return -ENOSPC; } -static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte) -{ - writeq(pte, addr); -} - -static void gen8_ggtt_insert_page(struct i915_address_space *vm, - dma_addr_t addr, - u64 offset, - enum i915_cache_level level, - u32 unused) -{ - struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); - gen8_pte_t __iomem *pte = - (gen8_pte_t __iomem *)ggtt->gsm + (offset >> PAGE_SHIFT); - - gen8_set_pte(pte, gen8_pte_encode(addr, level)); - - ggtt->invalidate(vm->i915); -} - -static void gen8_ggtt_insert_entries(struct i915_address_space *vm, - struct sg_table *st, - u64 start, - enum i915_cache_level level, - u32 unused) -{ - struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); - struct sgt_iter sgt_iter; - gen8_pte_t __iomem *gtt_entries; - const gen8_pte_t pte_encode = gen8_pte_encode(0, level); - dma_addr_t addr; - - gtt_entries = (gen8_pte_t __iomem *)ggtt->gsm; - gtt_entries += start >> PAGE_SHIFT; - for_each_sgt_dma(addr, sgt_iter, st) - gen8_set_pte(gtt_entries++, pte_encode | addr); - - wmb(); - - /* This next bit makes the above posting read even more important. We - * want to flush the TLBs only after we're certain all the PTE updates - * have finished. - */ - ggtt->invalidate(vm->i915); -} - -static void gen6_ggtt_insert_page(struct i915_address_space *vm, - dma_addr_t addr, - u64 offset, - enum i915_cache_level level, - u32 flags) -{ - struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); - gen6_pte_t __iomem *pte = - (gen6_pte_t __iomem *)ggtt->gsm + (offset >> PAGE_SHIFT); - - iowrite32(vm->pte_encode(addr, level, flags), pte); - - ggtt->invalidate(vm->i915); -} - -/* - * Binds an object into the global gtt with the specified cache level. The object - * will be accessible to the GPU via commands whose operands reference offsets - * within the global GTT as well as accessible by the GPU through the GMADR - * mapped BAR (dev_priv->mm.gtt->gtt). - */ -static void gen6_ggtt_insert_entries(struct i915_address_space *vm, - struct sg_table *st, - u64 start, - enum i915_cache_level level, - u32 flags) -{ - struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); - gen6_pte_t __iomem *entries = (gen6_pte_t __iomem *)ggtt->gsm; - unsigned int i = start >> PAGE_SHIFT; - struct sgt_iter iter; - dma_addr_t addr; - for_each_sgt_dma(addr, iter, st) - iowrite32(vm->pte_encode(addr, level, flags), &entries[i++]); - wmb(); - - /* This next bit makes the above posting read even more important. We - * want to flush the TLBs only after we're certain all the PTE updates - * have finished. - */ - ggtt->invalidate(vm->i915); -} - -static void nop_clear_range(struct i915_address_space *vm, - u64 start, u64 length) -{ -} - -static void gen8_ggtt_clear_range(struct i915_address_space *vm, - u64 start, u64 length) -{ - struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); - unsigned first_entry = start >> PAGE_SHIFT; - unsigned num_entries = length >> PAGE_SHIFT; - const gen8_pte_t scratch_pte = - gen8_pte_encode(vm->scratch_page.daddr, I915_CACHE_LLC); - gen8_pte_t __iomem *gtt_base = - (gen8_pte_t __iomem *)ggtt->gsm + first_entry; - const int max_entries = ggtt_total_entries(ggtt) - first_entry; - int i; - - if (WARN(num_entries > max_entries, - "First entry = %d; Num entries = %d (max=%d)\n", - first_entry, num_entries, max_entries)) - num_entries = max_entries; - - for (i = 0; i < num_entries; i++) - gen8_set_pte(>t_base[i], scratch_pte); -} - -static void bxt_vtd_ggtt_wa(struct i915_address_space *vm) -{ - struct drm_i915_private *dev_priv = vm->i915; - - /* - * Make sure the internal GAM fifo has been cleared of all GTT - * writes before exiting stop_machine(). This guarantees that - * any aperture accesses waiting to start in another process - * cannot back up behind the GTT writes causing a hang. - * The register can be any arbitrary GAM register. - */ - POSTING_READ(GFX_FLSH_CNTL_GEN6); -} - -struct insert_page { - struct i915_address_space *vm; - dma_addr_t addr; - u64 offset; - enum i915_cache_level level; -}; - -static int bxt_vtd_ggtt_insert_page__cb(void *_arg) -{ - struct insert_page *arg = _arg; - - gen8_ggtt_insert_page(arg->vm, arg->addr, arg->offset, arg->level, 0); - bxt_vtd_ggtt_wa(arg->vm); - - return 0; -} - -static void bxt_vtd_ggtt_insert_page__BKL(struct i915_address_space *vm, - dma_addr_t addr, - u64 offset, - enum i915_cache_level level, - u32 unused) -{ - struct insert_page arg = { vm, addr, offset, level }; - - stop_machine(bxt_vtd_ggtt_insert_page__cb, &arg, NULL); -} - -struct insert_entries { - struct i915_address_space *vm; - struct sg_table *st; - u64 start; - enum i915_cache_level level; -}; - -static int bxt_vtd_ggtt_insert_entries__cb(void *_arg) -{ - struct insert_entries *arg = _arg; - - gen8_ggtt_insert_entries(arg->vm, arg->st, arg->start, arg->level, 0); - bxt_vtd_ggtt_wa(arg->vm); - - return 0; -} - -static void bxt_vtd_ggtt_insert_entries__BKL(struct i915_address_space *vm, - struct sg_table *st, - u64 start, - enum i915_cache_level level, - u32 unused) -{ - struct insert_entries arg = { vm, st, start, level }; - - stop_machine(bxt_vtd_ggtt_insert_entries__cb, &arg, NULL); -} - -struct clear_range { - struct i915_address_space *vm; - u64 start; - u64 length; -}; - -static int bxt_vtd_ggtt_clear_range__cb(void *_arg) -{ - struct clear_range *arg = _arg; - - gen8_ggtt_clear_range(arg->vm, arg->start, arg->length); - bxt_vtd_ggtt_wa(arg->vm); - - return 0; -} - -static void bxt_vtd_ggtt_clear_range__BKL(struct i915_address_space *vm, - u64 start, - u64 length) -{ - struct clear_range arg = { vm, start, length }; - - stop_machine(bxt_vtd_ggtt_clear_range__cb, &arg, NULL); -} - -static void gen6_ggtt_clear_range(struct i915_address_space *vm, - u64 start, u64 length) -{ - struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); - unsigned first_entry = start >> PAGE_SHIFT; - unsigned num_entries = length >> PAGE_SHIFT; - gen6_pte_t scratch_pte, __iomem *gtt_base = - (gen6_pte_t __iomem *)ggtt->gsm + first_entry; - const int max_entries = ggtt_total_entries(ggtt) - first_entry; - int i; - - if (WARN(num_entries > max_entries, - "First entry = %d; Num entries = %d (max=%d)\n", - first_entry, num_entries, max_entries)) - num_entries = max_entries; - - scratch_pte = vm->pte_encode(vm->scratch_page.daddr, - I915_CACHE_LLC, 0); - - for (i = 0; i < num_entries; i++) - iowrite32(scratch_pte, >t_base[i]); -} - -static void i915_ggtt_insert_page(struct i915_address_space *vm, - dma_addr_t addr, - u64 offset, - enum i915_cache_level cache_level, - u32 unused) -{ - unsigned int flags = (cache_level == I915_CACHE_NONE) ? - AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY; - - intel_gtt_insert_page(addr, offset >> PAGE_SHIFT, flags); -} - -static void i915_ggtt_insert_entries(struct i915_address_space *vm, - struct sg_table *pages, - u64 start, - enum i915_cache_level cache_level, - u32 unused) -{ - unsigned int flags = (cache_level == I915_CACHE_NONE) ? - AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY; - - intel_gtt_insert_sg_entries(pages, start >> PAGE_SHIFT, flags); -} - -static void i915_ggtt_clear_range(struct i915_address_space *vm, - u64 start, u64 length) -{ - intel_gtt_clear_range(start >> PAGE_SHIFT, length >> PAGE_SHIFT); -} - -static int ggtt_bind_vma(struct i915_vma *vma, - enum i915_cache_level cache_level, - u32 flags) -{ - struct drm_i915_private *i915 = vma->vm->i915; - struct drm_i915_gem_object *obj = vma->obj; - u32 pte_flags; - - if (unlikely(!vma->pages)) { - int ret = i915_get_ggtt_vma_pages(vma); - if (ret) - return ret; - } - - /* Currently applicable only to VLV */ - pte_flags = 0; - if (obj->gt_ro) - pte_flags |= PTE_READ_ONLY; - - intel_runtime_pm_get(i915); - vma->vm->insert_entries(vma->vm, vma->pages, vma->node.start, - cache_level, pte_flags); - intel_runtime_pm_put(i915); - - /* - * Without aliasing PPGTT there's no difference between - * GLOBAL/LOCAL_BIND, it's all the same ptes. Hence unconditionally - * upgrade to both bound if we bind either to avoid double-binding. - */ - vma->flags |= I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND; - - return 0; -} - -static void ggtt_unbind_vma(struct i915_vma *vma) -{ - struct drm_i915_private *i915 = vma->vm->i915; - - intel_runtime_pm_get(i915); - vma->vm->clear_range(vma->vm, vma->node.start, vma->size); - intel_runtime_pm_put(i915); -} - -static int aliasing_gtt_bind_vma(struct i915_vma *vma, - enum i915_cache_level cache_level, - u32 flags) -{ - struct drm_i915_private *i915 = vma->vm->i915; - u32 pte_flags; - int ret; - - if (unlikely(!vma->pages)) { - ret = i915_get_ggtt_vma_pages(vma); - if (ret) - return ret; - } - - /* Currently applicable only to VLV */ - pte_flags = 0; - if (vma->obj->gt_ro) - pte_flags |= PTE_READ_ONLY; - - if (flags & I915_VMA_LOCAL_BIND) { - struct i915_hw_ppgtt *appgtt = i915->mm.aliasing_ppgtt; - - if (!(vma->flags & I915_VMA_LOCAL_BIND) && - appgtt->base.allocate_va_range) { - ret = appgtt->base.allocate_va_range(&appgtt->base, - vma->node.start, - vma->size); - if (ret) - goto err_pages; - } - - appgtt->base.insert_entries(&appgtt->base, - vma->pages, vma->node.start, - cache_level, pte_flags); - } - - if (flags & I915_VMA_GLOBAL_BIND) { - intel_runtime_pm_get(i915); - vma->vm->insert_entries(vma->vm, - vma->pages, vma->node.start, - cache_level, pte_flags); - intel_runtime_pm_put(i915); - } - - return 0; - -err_pages: - if (!(vma->flags & (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND))) { - if (vma->pages != vma->obj->mm.pages) { - GEM_BUG_ON(!vma->pages); - sg_free_table(vma->pages); - kfree(vma->pages); - } - vma->pages = NULL; - } - return ret; -} - -static void aliasing_gtt_unbind_vma(struct i915_vma *vma) -{ - struct drm_i915_private *i915 = vma->vm->i915; - - if (vma->flags & I915_VMA_GLOBAL_BIND) { - intel_runtime_pm_get(i915); - vma->vm->clear_range(vma->vm, vma->node.start, vma->size); - intel_runtime_pm_put(i915); - } - - if (vma->flags & I915_VMA_LOCAL_BIND) { - struct i915_address_space *vm = &i915->mm.aliasing_ppgtt->base; - - vm->clear_range(vm, vma->node.start, vma->size); - } -} - void i915_gem_gtt_finish_pages(struct drm_i915_gem_object *obj, struct sg_table *pages) { - struct drm_i915_private *dev_priv = to_i915(obj->base.dev); - struct device *kdev = &dev_priv->drm.pdev->dev; - struct i915_ggtt *ggtt = &dev_priv->ggtt; - - if (unlikely(ggtt->do_idle_maps)) { - if (i915_gem_wait_for_idle(dev_priv, 0)) { - DRM_ERROR("Failed to wait for idle; VT'd may hang.\n"); - /* Wait a bit, in hopes it avoids the hang */ - udelay(10); - } - } - - dma_unmap_sg(kdev, pages->sgl, pages->nents, PCI_DMA_BIDIRECTIONAL); -} - -static void i915_gtt_color_adjust(const struct drm_mm_node *node, - unsigned long color, - u64 *start, - u64 *end) -{ - if (node->allocated && node->color != color) - *start += I915_GTT_PAGE_SIZE; - - /* Also leave a space between the unallocated reserved node after the - * GTT and any objects within the GTT, i.e. we use the color adjustment - * to insert a guard page to prevent prefetches crossing over the - * GTT boundary. - */ - node = list_next_entry(node, node_list); - if (node->color != color) - *end -= I915_GTT_PAGE_SIZE; -} - -int i915_gem_init_aliasing_ppgtt(struct drm_i915_private *i915) -{ - struct i915_ggtt *ggtt = &i915->ggtt; - struct i915_hw_ppgtt *ppgtt; - int err; - - ppgtt = i915_ppgtt_create(i915, ERR_PTR(-EPERM), "[alias]"); - if (IS_ERR(ppgtt)) - return PTR_ERR(ppgtt); - - if (WARN_ON(ppgtt->base.total < ggtt->base.total)) { - err = -ENODEV; - goto err_ppgtt; - } - - if (ppgtt->base.allocate_va_range) { - /* Note we only pre-allocate as far as the end of the global - * GTT. On 48b / 4-level page-tables, the difference is very, - * very significant! We have to preallocate as GVT/vgpu does - * not like the page directory disappearing. - */ - err = ppgtt->base.allocate_va_range(&ppgtt->base, - 0, ggtt->base.total); - if (err) - goto err_ppgtt; - } - - i915->mm.aliasing_ppgtt = ppgtt; - - WARN_ON(ggtt->base.bind_vma != ggtt_bind_vma); - ggtt->base.bind_vma = aliasing_gtt_bind_vma; - - WARN_ON(ggtt->base.unbind_vma != ggtt_unbind_vma); - ggtt->base.unbind_vma = aliasing_gtt_unbind_vma; - - return 0; - -err_ppgtt: - i915_ppgtt_put(ppgtt); - return err; -} - -void i915_gem_fini_aliasing_ppgtt(struct drm_i915_private *i915) -{ - struct i915_ggtt *ggtt = &i915->ggtt; - struct i915_hw_ppgtt *ppgtt; - - ppgtt = fetch_and_zero(&i915->mm.aliasing_ppgtt); - if (!ppgtt) - return; - - i915_ppgtt_put(ppgtt); - - ggtt->base.bind_vma = ggtt_bind_vma; - ggtt->base.unbind_vma = ggtt_unbind_vma; -} - -int i915_gem_init_ggtt(struct drm_i915_private *dev_priv) -{ - /* Let GEM Manage all of the aperture. - * - * However, leave one page at the end still bound to the scratch page. - * There are a number of places where the hardware apparently prefetches - * past the end of the object, and we've seen multiple hangs with the - * GPU head pointer stuck in a batchbuffer bound at the last page of the - * aperture. One page should be enough to keep any prefetching inside - * of the aperture. - */ - struct i915_ggtt *ggtt = &dev_priv->ggtt; - unsigned long hole_start, hole_end; - struct drm_mm_node *entry; - int ret; - - ret = intel_vgt_balloon(dev_priv); - if (ret) - return ret; - - /* Reserve a mappable slot for our lockless error capture */ - ret = drm_mm_insert_node_in_range(&ggtt->base.mm, &ggtt->error_capture, - PAGE_SIZE, 0, I915_COLOR_UNEVICTABLE, - 0, ggtt->mappable_end, - DRM_MM_INSERT_LOW); - if (ret) - return ret; - - /* Clear any non-preallocated blocks */ - drm_mm_for_each_hole(entry, &ggtt->base.mm, hole_start, hole_end) { - DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n", - hole_start, hole_end); - ggtt->base.clear_range(&ggtt->base, hole_start, - hole_end - hole_start); - } - - /* And finally clear the reserved guard page */ - ggtt->base.clear_range(&ggtt->base, - ggtt->base.total - PAGE_SIZE, PAGE_SIZE); - - if (USES_PPGTT(dev_priv) && !USES_FULL_PPGTT(dev_priv)) { - ret = i915_gem_init_aliasing_ppgtt(dev_priv); - if (ret) - goto err; - } - - return 0; - -err: - drm_mm_remove_node(&ggtt->error_capture); - return ret; -} - -/** - * i915_ggtt_cleanup_hw - Clean up GGTT hardware initialization - * @dev_priv: i915 device - */ -void i915_ggtt_cleanup_hw(struct drm_i915_private *dev_priv) -{ - struct i915_ggtt *ggtt = &dev_priv->ggtt; - struct i915_vma *vma, *vn; - - ggtt->base.closed = true; - - mutex_lock(&dev_priv->drm.struct_mutex); - WARN_ON(!list_empty(&ggtt->base.active_list)); - list_for_each_entry_safe(vma, vn, &ggtt->base.inactive_list, vm_link) - WARN_ON(i915_vma_unbind(vma)); - mutex_unlock(&dev_priv->drm.struct_mutex); - - i915_gem_cleanup_stolen(&dev_priv->drm); - - mutex_lock(&dev_priv->drm.struct_mutex); - i915_gem_fini_aliasing_ppgtt(dev_priv); - - if (drm_mm_node_allocated(&ggtt->error_capture)) - drm_mm_remove_node(&ggtt->error_capture); - - if (drm_mm_initialized(&ggtt->base.mm)) { - intel_vgt_deballoon(dev_priv); - i915_address_space_fini(&ggtt->base); - } - - ggtt->base.cleanup(&ggtt->base); - mutex_unlock(&dev_priv->drm.struct_mutex); - - arch_phys_wc_del(ggtt->mtrr); - io_mapping_fini(&ggtt->mappable); -} - -static unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl) -{ - snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT; - snb_gmch_ctl &= SNB_GMCH_GGMS_MASK; - return snb_gmch_ctl << 20; -} - -static unsigned int gen8_get_total_gtt_size(u16 bdw_gmch_ctl) -{ - bdw_gmch_ctl >>= BDW_GMCH_GGMS_SHIFT; - bdw_gmch_ctl &= BDW_GMCH_GGMS_MASK; - if (bdw_gmch_ctl) - bdw_gmch_ctl = 1 << bdw_gmch_ctl; - -#ifdef CONFIG_X86_32 - /* Limit 32b platforms to a 2GB GGTT: 4 << 20 / pte size * PAGE_SIZE */ - if (bdw_gmch_ctl > 4) - bdw_gmch_ctl = 4; -#endif - - return bdw_gmch_ctl << 20; -} - -static unsigned int chv_get_total_gtt_size(u16 gmch_ctrl) -{ - gmch_ctrl >>= SNB_GMCH_GGMS_SHIFT; - gmch_ctrl &= SNB_GMCH_GGMS_MASK; - - if (gmch_ctrl) - return 1 << (20 + gmch_ctrl); - - return 0; -} - -static size_t gen6_get_stolen_size(u16 snb_gmch_ctl) -{ - snb_gmch_ctl >>= SNB_GMCH_GMS_SHIFT; - snb_gmch_ctl &= SNB_GMCH_GMS_MASK; - return (size_t)snb_gmch_ctl << 25; /* 32 MB units */ -} - -static size_t gen8_get_stolen_size(u16 bdw_gmch_ctl) -{ - bdw_gmch_ctl >>= BDW_GMCH_GMS_SHIFT; - bdw_gmch_ctl &= BDW_GMCH_GMS_MASK; - return (size_t)bdw_gmch_ctl << 25; /* 32 MB units */ -} - -static size_t chv_get_stolen_size(u16 gmch_ctrl) -{ - gmch_ctrl >>= SNB_GMCH_GMS_SHIFT; - gmch_ctrl &= SNB_GMCH_GMS_MASK; - - /* - * 0x0 to 0x10: 32MB increments starting at 0MB - * 0x11 to 0x16: 4MB increments starting at 8MB - * 0x17 to 0x1d: 4MB increments start at 36MB - */ - if (gmch_ctrl < 0x11) - return (size_t)gmch_ctrl << 25; - else if (gmch_ctrl < 0x17) - return (size_t)(gmch_ctrl - 0x11 + 2) << 22; - else - return (size_t)(gmch_ctrl - 0x17 + 9) << 22; -} - -static size_t gen9_get_stolen_size(u16 gen9_gmch_ctl) -{ - gen9_gmch_ctl >>= BDW_GMCH_GMS_SHIFT; - gen9_gmch_ctl &= BDW_GMCH_GMS_MASK; - - if (gen9_gmch_ctl < 0xf0) - return (size_t)gen9_gmch_ctl << 25; /* 32 MB units */ - else - /* 4MB increments starting at 0xf0 for 4MB */ - return (size_t)(gen9_gmch_ctl - 0xf0 + 1) << 22; -} - -static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size) -{ - struct drm_i915_private *dev_priv = ggtt->base.i915; - struct pci_dev *pdev = dev_priv->drm.pdev; - phys_addr_t phys_addr; - int ret; - - /* For Modern GENs the PTEs and register space are split in the BAR */ - phys_addr = pci_resource_start(pdev, 0) + pci_resource_len(pdev, 0) / 2; - - /* - * On BXT writes larger than 64 bit to the GTT pagetable range will be - * dropped. For WC mappings in general we have 64 byte burst writes - * when the WC buffer is flushed, so we can't use it, but have to - * resort to an uncached mapping. The WC issue is easily caught by the - * readback check when writing GTT PTE entries. - */ - if (IS_GEN9_LP(dev_priv)) - ggtt->gsm = ioremap_nocache(phys_addr, size); - else - ggtt->gsm = ioremap_wc(phys_addr, size); - if (!ggtt->gsm) { - DRM_ERROR("Failed to map the ggtt page table\n"); - return -ENOMEM; - } - - ret = setup_scratch_page(&ggtt->base, GFP_DMA32); - if (ret) { - DRM_ERROR("Scratch setup failed\n"); - /* iounmap will also get called at remove, but meh */ - iounmap(ggtt->gsm); - return ret; - } - - return 0; -} - -/* The GGTT and PPGTT need a private PPAT setup in order to handle cacheability - * bits. When using advanced contexts each context stores its own PAT, but - * writing this data shouldn't be harmful even in those cases. */ -static void bdw_setup_private_ppat(struct drm_i915_private *dev_priv) -{ - u64 pat; - - pat = GEN8_PPAT(0, GEN8_PPAT_WB | GEN8_PPAT_LLC) | /* for normal objects, no eLLC */ - GEN8_PPAT(1, GEN8_PPAT_WC | GEN8_PPAT_LLCELLC) | /* for something pointing to ptes? */ - GEN8_PPAT(2, GEN8_PPAT_WT | GEN8_PPAT_LLCELLC) | /* for scanout with eLLC */ - GEN8_PPAT(3, GEN8_PPAT_UC) | /* Uncached objects, mostly for scanout */ - GEN8_PPAT(4, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(0)) | - GEN8_PPAT(5, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(1)) | - GEN8_PPAT(6, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(2)) | - GEN8_PPAT(7, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3)); - - if (!USES_PPGTT(dev_priv)) - /* Spec: "For GGTT, there is NO pat_sel[2:0] from the entry, - * so RTL will always use the value corresponding to - * pat_sel = 000". - * So let's disable cache for GGTT to avoid screen corruptions. - * MOCS still can be used though. - * - System agent ggtt writes (i.e. cpu gtt mmaps) already work - * before this patch, i.e. the same uncached + snooping access - * like on gen6/7 seems to be in effect. - * - So this just fixes blitter/render access. Again it looks - * like it's not just uncached access, but uncached + snooping. - * So we can still hold onto all our assumptions wrt cpu - * clflushing on LLC machines. - */ - pat = GEN8_PPAT(0, GEN8_PPAT_UC); - - /* XXX: spec defines this as 2 distinct registers. It's unclear if a 64b - * write would work. */ - I915_WRITE(GEN8_PRIVATE_PAT_LO, pat); - I915_WRITE(GEN8_PRIVATE_PAT_HI, pat >> 32); -} - -static void chv_setup_private_ppat(struct drm_i915_private *dev_priv) -{ - u64 pat; - - /* - * Map WB on BDW to snooped on CHV. - * - * Only the snoop bit has meaning for CHV, the rest is - * ignored. - * - * The hardware will never snoop for certain types of accesses: - * - CPU GTT (GMADR->GGTT->no snoop->memory) - * - PPGTT page tables - * - some other special cycles - * - * As with BDW, we also need to consider the following for GT accesses: - * "For GGTT, there is NO pat_sel[2:0] from the entry, - * so RTL will always use the value corresponding to - * pat_sel = 000". - * Which means we must set the snoop bit in PAT entry 0 - * in order to keep the global status page working. - */ - pat = GEN8_PPAT(0, CHV_PPAT_SNOOP) | - GEN8_PPAT(1, 0) | - GEN8_PPAT(2, 0) | - GEN8_PPAT(3, 0) | - GEN8_PPAT(4, CHV_PPAT_SNOOP) | - GEN8_PPAT(5, CHV_PPAT_SNOOP) | - GEN8_PPAT(6, CHV_PPAT_SNOOP) | - GEN8_PPAT(7, CHV_PPAT_SNOOP); - - I915_WRITE(GEN8_PRIVATE_PAT_LO, pat); - I915_WRITE(GEN8_PRIVATE_PAT_HI, pat >> 32); -} - -static void gen6_gmch_remove(struct i915_address_space *vm) -{ - struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); - - iounmap(ggtt->gsm); - cleanup_scratch_page(vm); -} - -static int gen8_gmch_probe(struct i915_ggtt *ggtt) -{ - struct drm_i915_private *dev_priv = ggtt->base.i915; - struct pci_dev *pdev = dev_priv->drm.pdev; - unsigned int size; - u16 snb_gmch_ctl; - int err; - - /* TODO: We're not aware of mappable constraints on gen8 yet */ - ggtt->mappable_base = pci_resource_start(pdev, 2); - ggtt->mappable_end = pci_resource_len(pdev, 2); - - err = pci_set_dma_mask(pdev, DMA_BIT_MASK(39)); - if (!err) - err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(39)); - if (err) - DRM_ERROR("Can't set DMA mask/consistent mask (%d)\n", err); - - pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl); - - if (INTEL_GEN(dev_priv) >= 9) { - ggtt->stolen_size = gen9_get_stolen_size(snb_gmch_ctl); - size = gen8_get_total_gtt_size(snb_gmch_ctl); - } else if (IS_CHERRYVIEW(dev_priv)) { - ggtt->stolen_size = chv_get_stolen_size(snb_gmch_ctl); - size = chv_get_total_gtt_size(snb_gmch_ctl); - } else { - ggtt->stolen_size = gen8_get_stolen_size(snb_gmch_ctl); - size = gen8_get_total_gtt_size(snb_gmch_ctl); - } - - ggtt->base.total = (size / sizeof(gen8_pte_t)) << PAGE_SHIFT; - - if (IS_CHERRYVIEW(dev_priv) || IS_GEN9_LP(dev_priv)) - chv_setup_private_ppat(dev_priv); - else - bdw_setup_private_ppat(dev_priv); - - ggtt->base.cleanup = gen6_gmch_remove; - ggtt->base.bind_vma = ggtt_bind_vma; - ggtt->base.unbind_vma = ggtt_unbind_vma; - ggtt->base.insert_page = gen8_ggtt_insert_page; - ggtt->base.clear_range = nop_clear_range; - if (!USES_FULL_PPGTT(dev_priv) || intel_scanout_needs_vtd_wa(dev_priv)) - ggtt->base.clear_range = gen8_ggtt_clear_range; - - ggtt->base.insert_entries = gen8_ggtt_insert_entries; - - /* Serialize GTT updates with aperture access on BXT if VT-d is on. */ - if (intel_ggtt_update_needs_vtd_wa(dev_priv)) { - ggtt->base.insert_entries = bxt_vtd_ggtt_insert_entries__BKL; - ggtt->base.insert_page = bxt_vtd_ggtt_insert_page__BKL; - if (ggtt->base.clear_range != nop_clear_range) - ggtt->base.clear_range = bxt_vtd_ggtt_clear_range__BKL; - } - - ggtt->invalidate = gen6_ggtt_invalidate; - - return ggtt_probe_common(ggtt, size); -} - -static int gen6_gmch_probe(struct i915_ggtt *ggtt) -{ - struct drm_i915_private *dev_priv = ggtt->base.i915; - struct pci_dev *pdev = dev_priv->drm.pdev; - unsigned int size; - u16 snb_gmch_ctl; - int err; - - ggtt->mappable_base = pci_resource_start(pdev, 2); - ggtt->mappable_end = pci_resource_len(pdev, 2); - - /* 64/512MB is the current min/max we actually know of, but this is just - * a coarse sanity check. - */ - if (ggtt->mappable_end < (64<<20) || ggtt->mappable_end > (512<<20)) { - DRM_ERROR("Unknown GMADR size (%llx)\n", ggtt->mappable_end); - return -ENXIO; - } - - err = pci_set_dma_mask(pdev, DMA_BIT_MASK(40)); - if (!err) - err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(40)); - if (err) - DRM_ERROR("Can't set DMA mask/consistent mask (%d)\n", err); - pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl); - - ggtt->stolen_size = gen6_get_stolen_size(snb_gmch_ctl); - - size = gen6_get_total_gtt_size(snb_gmch_ctl); - ggtt->base.total = (size / sizeof(gen6_pte_t)) << PAGE_SHIFT; - - ggtt->base.clear_range = gen6_ggtt_clear_range; - ggtt->base.insert_page = gen6_ggtt_insert_page; - ggtt->base.insert_entries = gen6_ggtt_insert_entries; - ggtt->base.bind_vma = ggtt_bind_vma; - ggtt->base.unbind_vma = ggtt_unbind_vma; - ggtt->base.cleanup = gen6_gmch_remove; - - ggtt->invalidate = gen6_ggtt_invalidate; - - if (HAS_EDRAM(dev_priv)) - ggtt->base.pte_encode = iris_pte_encode; - else if (IS_HASWELL(dev_priv)) - ggtt->base.pte_encode = hsw_pte_encode; - else if (IS_VALLEYVIEW(dev_priv)) - ggtt->base.pte_encode = byt_pte_encode; - else if (INTEL_GEN(dev_priv) >= 7) - ggtt->base.pte_encode = ivb_pte_encode; - else - ggtt->base.pte_encode = snb_pte_encode; - - return ggtt_probe_common(ggtt, size); -} - -static void i915_gmch_remove(struct i915_address_space *vm) -{ - intel_gmch_remove(); -} - -static int i915_gmch_probe(struct i915_ggtt *ggtt) -{ - struct drm_i915_private *dev_priv = ggtt->base.i915; - int ret; - - ret = intel_gmch_probe(dev_priv->bridge_dev, dev_priv->drm.pdev, NULL); - if (!ret) { - DRM_ERROR("failed to set up gmch\n"); - return -EIO; - } - - intel_gtt_get(&ggtt->base.total, - &ggtt->stolen_size, - &ggtt->mappable_base, - &ggtt->mappable_end); - - ggtt->do_idle_maps = needs_idle_maps(dev_priv); - ggtt->base.insert_page = i915_ggtt_insert_page; - ggtt->base.insert_entries = i915_ggtt_insert_entries; - ggtt->base.clear_range = i915_ggtt_clear_range; - ggtt->base.bind_vma = ggtt_bind_vma; - ggtt->base.unbind_vma = ggtt_unbind_vma; - ggtt->base.cleanup = i915_gmch_remove; - - ggtt->invalidate = gmch_ggtt_invalidate; + struct drm_i915_private *i915 = to_i915(obj->base.dev); + struct i915_ggtt *ggtt = to_gt(i915)->ggtt; + /* XXX This does not prevent more requests being submitted! */ if (unlikely(ggtt->do_idle_maps)) - DRM_INFO("applying Ironlake quirks for intel_iommu\n"); + /* Wait a bit, in the hope it avoids the hang */ + usleep_range(100, 250); - return 0; -} - -/** - * i915_ggtt_probe_hw - Probe GGTT hardware location - * @dev_priv: i915 device - */ -int i915_ggtt_probe_hw(struct drm_i915_private *dev_priv) -{ - struct i915_ggtt *ggtt = &dev_priv->ggtt; - int ret; - - ggtt->base.i915 = dev_priv; - ggtt->base.dma = &dev_priv->drm.pdev->dev; - - if (INTEL_GEN(dev_priv) <= 5) - ret = i915_gmch_probe(ggtt); - else if (INTEL_GEN(dev_priv) < 8) - ret = gen6_gmch_probe(ggtt); - else - ret = gen8_gmch_probe(ggtt); - if (ret) - return ret; - - /* Trim the GGTT to fit the GuC mappable upper range (when enabled). - * This is easier than doing range restriction on the fly, as we - * currently don't have any bits spare to pass in this upper - * restriction! - */ - if (HAS_GUC(dev_priv) && i915.enable_guc_loading) { - ggtt->base.total = min_t(u64, ggtt->base.total, GUC_GGTT_TOP); - ggtt->mappable_end = min(ggtt->mappable_end, ggtt->base.total); - } - - if ((ggtt->base.total - 1) >> 32) { - DRM_ERROR("We never expected a Global GTT with more than 32bits" - " of address space! Found %lldM!\n", - ggtt->base.total >> 20); - ggtt->base.total = 1ULL << 32; - ggtt->mappable_end = min(ggtt->mappable_end, ggtt->base.total); - } - - if (ggtt->mappable_end > ggtt->base.total) { - DRM_ERROR("mappable aperture extends past end of GGTT," - " aperture=%llx, total=%llx\n", - ggtt->mappable_end, ggtt->base.total); - ggtt->mappable_end = ggtt->base.total; - } - - /* GMADR is the PCI mmio aperture into the global GTT. */ - DRM_INFO("Memory usable by graphics device = %lluM\n", - ggtt->base.total >> 20); - DRM_DEBUG_DRIVER("GMADR size = %lldM\n", ggtt->mappable_end >> 20); - DRM_DEBUG_DRIVER("GTT stolen size = %uM\n", ggtt->stolen_size >> 20); - if (intel_vtd_active()) - DRM_INFO("VT-d active for gfx access\n"); - - return 0; -} - -/** - * i915_ggtt_init_hw - Initialize GGTT hardware - * @dev_priv: i915 device - */ -int i915_ggtt_init_hw(struct drm_i915_private *dev_priv) -{ - struct i915_ggtt *ggtt = &dev_priv->ggtt; - int ret; - - INIT_LIST_HEAD(&dev_priv->vm_list); - - /* Note that we use page colouring to enforce a guard page at the - * end of the address space. This is required as the CS may prefetch - * beyond the end of the batch buffer, across the page boundary, - * and beyond the end of the GTT if we do not provide a guard. - */ - mutex_lock(&dev_priv->drm.struct_mutex); - i915_address_space_init(&ggtt->base, dev_priv, "[global]"); - if (!HAS_LLC(dev_priv) && !USES_PPGTT(dev_priv)) - ggtt->base.mm.color_adjust = i915_gtt_color_adjust; - mutex_unlock(&dev_priv->drm.struct_mutex); - - if (!io_mapping_init_wc(&dev_priv->ggtt.mappable, - dev_priv->ggtt.mappable_base, - dev_priv->ggtt.mappable_end)) { - ret = -EIO; - goto out_gtt_cleanup; - } - - ggtt->mtrr = arch_phys_wc_add(ggtt->mappable_base, ggtt->mappable_end); - - /* - * Initialise stolen early so that we may reserve preallocated - * objects for the BIOS to KMS transition. - */ - ret = i915_gem_init_stolen(dev_priv); - if (ret) - goto out_gtt_cleanup; - - return 0; - -out_gtt_cleanup: - ggtt->base.cleanup(&ggtt->base); - return ret; -} - -int i915_ggtt_enable_hw(struct drm_i915_private *dev_priv) -{ - if (INTEL_GEN(dev_priv) < 6 && !intel_enable_gtt()) - return -EIO; - - return 0; -} - -void i915_ggtt_enable_guc(struct drm_i915_private *i915) -{ - GEM_BUG_ON(i915->ggtt.invalidate != gen6_ggtt_invalidate); - - i915->ggtt.invalidate = guc_ggtt_invalidate; -} - -void i915_ggtt_disable_guc(struct drm_i915_private *i915) -{ - /* We should only be called after i915_ggtt_enable_guc() */ - GEM_BUG_ON(i915->ggtt.invalidate != guc_ggtt_invalidate); - - i915->ggtt.invalidate = gen6_ggtt_invalidate; -} - -void i915_gem_restore_gtt_mappings(struct drm_i915_private *dev_priv) -{ - struct i915_ggtt *ggtt = &dev_priv->ggtt; - struct drm_i915_gem_object *obj, *on; - - i915_check_and_clear_faults(dev_priv); - - /* First fill our portion of the GTT with scratch pages */ - ggtt->base.clear_range(&ggtt->base, 0, ggtt->base.total); - - ggtt->base.closed = true; /* skip rewriting PTE on VMA unbind */ - - /* clflush objects bound into the GGTT and rebind them. */ - list_for_each_entry_safe(obj, on, - &dev_priv->mm.bound_list, global_link) { - bool ggtt_bound = false; - struct i915_vma *vma; - - list_for_each_entry(vma, &obj->vma_list, obj_link) { - if (vma->vm != &ggtt->base) - continue; - - if (!i915_vma_unbind(vma)) - continue; - - WARN_ON(i915_vma_bind(vma, obj->cache_level, - PIN_UPDATE)); - ggtt_bound = true; - } - - if (ggtt_bound) - WARN_ON(i915_gem_object_set_to_gtt_domain(obj, false)); - } - - ggtt->base.closed = false; - - if (INTEL_GEN(dev_priv) >= 8) { - if (IS_CHERRYVIEW(dev_priv) || IS_GEN9_LP(dev_priv)) - chv_setup_private_ppat(dev_priv); - else - bdw_setup_private_ppat(dev_priv); - - return; - } - - if (USES_PPGTT(dev_priv)) { - struct i915_address_space *vm; - - list_for_each_entry(vm, &dev_priv->vm_list, global_link) { - struct i915_hw_ppgtt *ppgtt; - - if (i915_is_ggtt(vm)) - ppgtt = dev_priv->mm.aliasing_ppgtt; - else - ppgtt = i915_vm_to_ppgtt(vm); - - gen6_write_page_range(ppgtt, 0, ppgtt->base.total); - } - } - - i915_ggtt_invalidate(dev_priv); -} - -static struct scatterlist * -rotate_pages(const dma_addr_t *in, unsigned int offset, - unsigned int width, unsigned int height, - unsigned int stride, - struct sg_table *st, struct scatterlist *sg) -{ - unsigned int column, row; - unsigned int src_idx; - - for (column = 0; column < width; column++) { - src_idx = stride * (height - 1) + column; - for (row = 0; row < height; row++) { - st->nents++; - /* We don't need the pages, but need to initialize - * the entries so the sg list can be happily traversed. - * The only thing we need are DMA addresses. - */ - sg_set_page(sg, NULL, PAGE_SIZE, 0); - sg_dma_address(sg) = in[offset + src_idx]; - sg_dma_len(sg) = PAGE_SIZE; - sg = sg_next(sg); - src_idx -= stride; - } - } - - return sg; -} - -static noinline struct sg_table * -intel_rotate_pages(struct intel_rotation_info *rot_info, - struct drm_i915_gem_object *obj) -{ - const unsigned long n_pages = obj->base.size / PAGE_SIZE; - unsigned int size = intel_rotation_info_size(rot_info); - struct sgt_iter sgt_iter; - dma_addr_t dma_addr; - unsigned long i; - dma_addr_t *page_addr_list; - struct sg_table *st; - struct scatterlist *sg; - int ret = -ENOMEM; - - /* Allocate a temporary list of source pages for random access. */ - page_addr_list = kvmalloc_array(n_pages, - sizeof(dma_addr_t), - GFP_TEMPORARY); - if (!page_addr_list) - return ERR_PTR(ret); - - /* Allocate target SG list. */ - st = kmalloc(sizeof(*st), GFP_KERNEL); - if (!st) - goto err_st_alloc; - - ret = sg_alloc_table(st, size, GFP_KERNEL); - if (ret) - goto err_sg_alloc; - - /* Populate source page list from the object. */ - i = 0; - for_each_sgt_dma(dma_addr, sgt_iter, obj->mm.pages) - page_addr_list[i++] = dma_addr; - - GEM_BUG_ON(i != n_pages); - st->nents = 0; - sg = st->sgl; - - for (i = 0 ; i < ARRAY_SIZE(rot_info->plane); i++) { - sg = rotate_pages(page_addr_list, rot_info->plane[i].offset, - rot_info->plane[i].width, rot_info->plane[i].height, - rot_info->plane[i].stride, st, sg); - } - - DRM_DEBUG_KMS("Created rotated page mapping for object size %zu (%ux%u tiles, %u pages)\n", - obj->base.size, rot_info->plane[0].width, rot_info->plane[0].height, size); - - kvfree(page_addr_list); - - return st; - -err_sg_alloc: - kfree(st); -err_st_alloc: - kvfree(page_addr_list); - - DRM_DEBUG_KMS("Failed to create rotated mapping for object size %zu! (%ux%u tiles, %u pages)\n", - obj->base.size, rot_info->plane[0].width, rot_info->plane[0].height, size); - - return ERR_PTR(ret); -} - -static noinline struct sg_table * -intel_partial_pages(const struct i915_ggtt_view *view, - struct drm_i915_gem_object *obj) -{ - struct sg_table *st; - struct scatterlist *sg, *iter; - unsigned int count = view->partial.size; - unsigned int offset; - int ret = -ENOMEM; - - st = kmalloc(sizeof(*st), GFP_KERNEL); - if (!st) - goto err_st_alloc; - - ret = sg_alloc_table(st, count, GFP_KERNEL); - if (ret) - goto err_sg_alloc; - - iter = i915_gem_object_get_sg(obj, view->partial.offset, &offset); - GEM_BUG_ON(!iter); - - sg = st->sgl; - st->nents = 0; - do { - unsigned int len; - - len = min(iter->length - (offset << PAGE_SHIFT), - count << PAGE_SHIFT); - sg_set_page(sg, NULL, len, 0); - sg_dma_address(sg) = - sg_dma_address(iter) + (offset << PAGE_SHIFT); - sg_dma_len(sg) = len; - - st->nents++; - count -= len >> PAGE_SHIFT; - if (count == 0) { - sg_mark_end(sg); - return st; - } - - sg = __sg_next(sg); - iter = __sg_next(iter); - offset = 0; - } while (1); - -err_sg_alloc: - kfree(st); -err_st_alloc: - return ERR_PTR(ret); -} - -static int -i915_get_ggtt_vma_pages(struct i915_vma *vma) -{ - int ret; - - /* The vma->pages are only valid within the lifespan of the borrowed - * obj->mm.pages. When the obj->mm.pages sg_table is regenerated, so - * must be the vma->pages. A simple rule is that vma->pages must only - * be accessed when the obj->mm.pages are pinned. - */ - GEM_BUG_ON(!i915_gem_object_has_pinned_pages(vma->obj)); - - switch (vma->ggtt_view.type) { - case I915_GGTT_VIEW_NORMAL: - vma->pages = vma->obj->mm.pages; - return 0; - - case I915_GGTT_VIEW_ROTATED: - vma->pages = - intel_rotate_pages(&vma->ggtt_view.rotated, vma->obj); - break; - - case I915_GGTT_VIEW_PARTIAL: - vma->pages = intel_partial_pages(&vma->ggtt_view, vma->obj); - break; - - default: - WARN_ONCE(1, "GGTT view %u not implemented!\n", - vma->ggtt_view.type); - return -EINVAL; - } - - ret = 0; - if (unlikely(IS_ERR(vma->pages))) { - ret = PTR_ERR(vma->pages); - vma->pages = NULL; - DRM_ERROR("Failed to get pages for VMA view type %u (%d)!\n", - vma->ggtt_view.type, ret); - } - return ret; + dma_unmap_sg(i915->drm.dev, pages->sgl, pages->nents, + DMA_BIDIRECTIONAL); } /** * i915_gem_gtt_reserve - reserve a node in an address_space (GTT) * @vm: the &struct i915_address_space - * @node: the &struct drm_mm_node (typically i915_vma.mode) + * @ww: An optional struct i915_gem_ww_ctx. + * @node: the &struct drm_mm_node (typically i915_vma.node) * @size: how much space to allocate inside the GTT, * must be #I915_GTT_PAGE_SIZE aligned * @offset: where to insert inside the GTT, @@ -3381,6 +94,7 @@ i915_get_ggtt_vma_pages(struct i915_vma *vma) * asked to wait for eviction and interrupted. */ int i915_gem_gtt_reserve(struct i915_address_space *vm, + struct i915_gem_ww_ctx *ww, struct drm_mm_node *node, u64 size, u64 offset, unsigned long color, unsigned int flags) @@ -3391,7 +105,7 @@ int i915_gem_gtt_reserve(struct i915_address_space *vm, GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE)); GEM_BUG_ON(!IS_ALIGNED(offset, I915_GTT_MIN_ALIGNMENT)); GEM_BUG_ON(range_overflows(offset, size, vm->total)); - GEM_BUG_ON(vm == &vm->i915->mm.aliasing_ppgtt->base); + GEM_BUG_ON(vm == &to_gt(vm->i915)->ggtt->alias->vm); GEM_BUG_ON(drm_mm_node_allocated(node)); node->size = size; @@ -3405,7 +119,7 @@ int i915_gem_gtt_reserve(struct i915_address_space *vm, if (flags & PIN_NOEVICT) return -ENOSPC; - err = i915_gem_evict_for_node(vm, node, flags); + err = i915_gem_evict_for_node(vm, ww, node, flags); if (err == 0) err = drm_mm_reserve_node(&vm->mm, node); @@ -3422,12 +136,12 @@ static u64 random_offset(u64 start, u64 end, u64 len, u64 align) range = round_down(end - len, align) - round_up(start, align); if (range) { if (sizeof(unsigned long) == sizeof(u64)) { - addr = get_random_long(); + addr = get_random_u64(); } else { - addr = get_random_int(); + addr = get_random_u32(); if (range > U32_MAX) { addr <<= 32; - addr |= get_random_int(); + addr |= get_random_u32(); } } div64_u64_rem(addr, range, &addr); @@ -3440,6 +154,7 @@ static u64 random_offset(u64 start, u64 end, u64 len, u64 align) /** * i915_gem_gtt_insert - insert a node into an address_space (GTT) * @vm: the &struct i915_address_space + * @ww: An optional struct i915_gem_ww_ctx. * @node: the &struct drm_mm_node (typically i915_vma.node) * @size: how much space to allocate inside the GTT, * must be #I915_GTT_PAGE_SIZE aligned @@ -3472,6 +187,7 @@ static u64 random_offset(u64 start, u64 end, u64 len, u64 align) * asked to wait for eviction and interrupted. */ int i915_gem_gtt_insert(struct i915_address_space *vm, + struct i915_gem_ww_ctx *ww, struct drm_mm_node *node, u64 size, u64 alignment, unsigned long color, u64 start, u64 end, unsigned int flags) @@ -3480,7 +196,8 @@ int i915_gem_gtt_insert(struct i915_address_space *vm, u64 offset; int err; - lockdep_assert_held(&vm->i915->drm.struct_mutex); + lockdep_assert_held(&vm->mutex); + GEM_BUG_ON(!size); GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE)); GEM_BUG_ON(alignment && !is_power_of_2(alignment)); @@ -3488,7 +205,7 @@ int i915_gem_gtt_insert(struct i915_address_space *vm, GEM_BUG_ON(start >= end); GEM_BUG_ON(start > 0 && !IS_ALIGNED(start, I915_GTT_PAGE_SIZE)); GEM_BUG_ON(end < U64_MAX && !IS_ALIGNED(end, I915_GTT_PAGE_SIZE)); - GEM_BUG_ON(vm == &vm->i915->mm.aliasing_ppgtt->base); + GEM_BUG_ON(vm == &to_gt(vm->i915)->ggtt->alias->vm); GEM_BUG_ON(drm_mm_node_allocated(node)); if (unlikely(range_overflows(start, size, end))) @@ -3499,7 +216,7 @@ int i915_gem_gtt_insert(struct i915_address_space *vm, mode = DRM_MM_INSERT_BEST; if (flags & PIN_HIGH) - mode = DRM_MM_INSERT_HIGH; + mode = DRM_MM_INSERT_HIGHEST; if (flags & PIN_MAPPABLE) mode = DRM_MM_INSERT_LOW; @@ -3519,10 +236,20 @@ int i915_gem_gtt_insert(struct i915_address_space *vm, if (err != -ENOSPC) return err; + if (mode & DRM_MM_INSERT_ONCE) { + err = drm_mm_insert_node_in_range(&vm->mm, node, + size, alignment, color, + start, end, + DRM_MM_INSERT_BEST); + if (err != -ENOSPC) + return err; + } + if (flags & PIN_NOEVICT) return -ENOSPC; - /* No free space, pick a slot at random. + /* + * No free space, pick a slot at random. * * There is a pathological case here using a GTT shared between * mmap and GPU (i.e. ggtt/aliasing_ppgtt but not full-ppgtt): @@ -3546,12 +273,15 @@ int i915_gem_gtt_insert(struct i915_address_space *vm, */ offset = random_offset(start, end, size, alignment ?: I915_GTT_MIN_ALIGNMENT); - err = i915_gem_gtt_reserve(vm, node, size, offset, color, flags); + err = i915_gem_gtt_reserve(vm, ww, node, size, offset, color, flags); if (err != -ENOSPC) return err; + if (flags & PIN_NOSEARCH) + return -ENOSPC; + /* Randomly selected placement is pinned, do a search */ - err = i915_gem_evict_something(vm, size, alignment, color, + err = i915_gem_evict_something(vm, ww, size, alignment, color, start, end, flags); if (err) return err; @@ -3562,6 +292,5 @@ int i915_gem_gtt_insert(struct i915_address_space *vm, } #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) -#include "selftests/mock_gtt.c" #include "selftests/i915_gem_gtt.c" #endif |