diff options
Diffstat (limited to 'drivers/gpu/drm/i915/gem/i915_gem_object.c')
| -rw-r--r-- | drivers/gpu/drm/i915/gem/i915_gem_object.c | 322 |
1 files changed, 262 insertions, 60 deletions
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object.c b/drivers/gpu/drm/i915/gem/i915_gem_object.c index d87b508b59b1..3f6f040c359d 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_object.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_object.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: MIT /* * Copyright © 2017 Intel Corporation * @@ -22,15 +23,23 @@ * */ +#include <linux/highmem.h> #include <linux/sched/mm.h> +#include <drm/drm_cache.h> +#include <drm/drm_print.h> + #include "display/intel_frontbuffer.h" #include "pxp/intel_pxp.h" + #include "i915_drv.h" +#include "i915_file_private.h" #include "i915_gem_clflush.h" #include "i915_gem_context.h" +#include "i915_gem_dmabuf.h" #include "i915_gem_mman.h" #include "i915_gem_object.h" +#include "i915_gem_object_frontbuffer.h" #include "i915_gem_ttm.h" #include "i915_memcpy.h" #include "i915_trace.h" @@ -39,6 +48,33 @@ static struct kmem_cache *slab_objects; static const struct drm_gem_object_funcs i915_gem_object_funcs; +unsigned int i915_gem_get_pat_index(struct drm_i915_private *i915, + enum i915_cache_level level) +{ + if (drm_WARN_ON(&i915->drm, level >= I915_MAX_CACHE_LEVEL)) + return 0; + + return INTEL_INFO(i915)->cachelevel_to_pat[level]; +} + +bool i915_gem_object_has_cache_level(const struct drm_i915_gem_object *obj, + enum i915_cache_level lvl) +{ + /* + * In case the pat_index is set by user space, this kernel mode + * driver should leave the coherency to be managed by user space, + * simply return true here. + */ + if (obj->pat_set_by_user) + return true; + + /* + * Otherwise the pat_index should have been converted from cache_level + * so that the following comparison is valid. + */ + return obj->pat_index == i915_gem_get_pat_index(obj_to_i915(obj), lvl); +} + struct drm_i915_gem_object *i915_gem_object_alloc(void) { struct drm_i915_gem_object *obj; @@ -72,6 +108,10 @@ void i915_gem_object_init(struct drm_i915_gem_object *obj, INIT_LIST_HEAD(&obj->mm.link); +#ifdef CONFIG_PROC_FS + INIT_LIST_HEAD(&obj->client_link); +#endif + INIT_LIST_HEAD(&obj->lut_list); spin_lock_init(&obj->lut_lock); @@ -118,7 +158,7 @@ void i915_gem_object_set_cache_coherency(struct drm_i915_gem_object *obj, { struct drm_i915_private *i915 = to_i915(obj->base.dev); - obj->cache_level = cache_level; + obj->pat_index = i915_gem_get_pat_index(i915, cache_level); if (cache_level != I915_CACHE_NONE) obj->cache_coherent = (I915_BO_CACHE_COHERENT_FOR_READ | @@ -133,6 +173,37 @@ void i915_gem_object_set_cache_coherency(struct drm_i915_gem_object *obj, !IS_DGFX(i915); } +/** + * i915_gem_object_set_pat_index - set PAT index to be used in PTE encode + * @obj: #drm_i915_gem_object + * @pat_index: PAT index + * + * This is a clone of i915_gem_object_set_cache_coherency taking pat index + * instead of cache_level as its second argument. + */ +void i915_gem_object_set_pat_index(struct drm_i915_gem_object *obj, + unsigned int pat_index) +{ + struct drm_i915_private *i915 = to_i915(obj->base.dev); + + if (obj->pat_index == pat_index) + return; + + obj->pat_index = pat_index; + + if (pat_index != i915_gem_get_pat_index(i915, I915_CACHE_NONE)) + obj->cache_coherent = (I915_BO_CACHE_COHERENT_FOR_READ | + I915_BO_CACHE_COHERENT_FOR_WRITE); + else if (HAS_LLC(i915)) + obj->cache_coherent = I915_BO_CACHE_COHERENT_FOR_READ; + else + obj->cache_coherent = 0; + + obj->cache_dirty = + !(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE) && + !IS_DGFX(i915); +} + bool i915_gem_object_can_bypass_llc(struct drm_i915_gem_object *obj) { struct drm_i915_private *i915 = to_i915(obj->base.dev); @@ -145,6 +216,12 @@ bool i915_gem_object_can_bypass_llc(struct drm_i915_gem_object *obj) return false; /* + * Always flush cache for UMD objects at creation time. + */ + if (obj->pat_set_by_user) + return true; + + /* * EHL and JSL add the 'Bypass LLC' MOCS entry, which should make it * possible for userspace to bypass the GTT caching bits set by the * kernel, as per the given object cache_level. This is troublesome @@ -156,7 +233,7 @@ bool i915_gem_object_can_bypass_llc(struct drm_i915_gem_object *obj) * it, but since i915 takes the stance of always zeroing memory before * handing it to userspace, we need to prevent this. */ - return IS_JSL_EHL(i915); + return (IS_JASPERLAKE(i915) || IS_ELKHARTLAKE(i915)); } static void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file) @@ -222,6 +299,10 @@ void __i915_gem_free_object_rcu(struct rcu_head *head) container_of(head, typeof(*obj), rcu); struct drm_i915_private *i915 = to_i915(obj->base.dev); + /* We need to keep this alive for RCU read access from fdinfo. */ + if (obj->mm.n_placements > 1) + kfree(obj->mm.placements); + i915_gem_object_free(obj); GEM_BUG_ON(!atomic_read(&i915->mm.free_count)); @@ -232,7 +313,7 @@ static void __i915_gem_object_free_mmaps(struct drm_i915_gem_object *obj) { /* Skip serialisation and waking the device if known to be not used. */ - if (obj->userfault_count) + if (obj->userfault_count && !IS_DGFX(to_i915(obj->base.dev))) i915_gem_object_release_mmap_gtt(obj); if (!RB_EMPTY_ROOT(&obj->mmo.offsets)) { @@ -262,17 +343,11 @@ static void __i915_gem_object_free_mmaps(struct drm_i915_gem_object *obj) */ void __i915_gem_object_pages_fini(struct drm_i915_gem_object *obj) { - assert_object_held(obj); + assert_object_held_shared(obj); if (!list_empty(&obj->vma.list)) { struct i915_vma *vma; - /* - * Note that the vma keeps an object reference while - * it is active, so it *should* not sleep while we - * destroy it. Our debug code errs insits it *might*. - * For the moment, play along. - */ spin_lock(&obj->vma.lock); while ((vma = list_first_entry_or_null(&obj->vma.list, struct i915_vma, @@ -280,7 +355,7 @@ void __i915_gem_object_pages_fini(struct drm_i915_gem_object *obj) GEM_BUG_ON(vma->obj != obj); spin_unlock(&obj->vma.lock); - __i915_vma_put(vma); + i915_vma_destroy(vma); spin_lock(&obj->vma.lock); } @@ -290,7 +365,21 @@ void __i915_gem_object_pages_fini(struct drm_i915_gem_object *obj) __i915_gem_object_free_mmaps(obj); atomic_set(&obj->mm.pages_pin_count, 0); + + /* + * dma_buf_unmap_attachment() requires reservation to be + * locked. The imported GEM shouldn't share reservation lock + * and ttm_bo_cleanup_memtype_use() shouldn't be invoked for + * dma-buf, so it's safe to take the lock. + */ + if (obj->base.import_attach) + i915_gem_object_lock(obj, NULL); + __i915_gem_object_put_pages(obj); + + if (obj->base.import_attach) + i915_gem_object_unlock(obj); + GEM_BUG_ON(i915_gem_object_has_pages(obj)); } @@ -310,9 +399,6 @@ void __i915_gem_free_object(struct drm_i915_gem_object *obj) if (obj->ops->release) obj->ops->release(obj); - if (obj->mm.n_placements > 1) - kfree(obj->mm.placements); - if (obj->shares_resv_from) i915_vm_resv_put(obj->shares_resv_from); @@ -331,15 +417,7 @@ static void __i915_gem_free_objects(struct drm_i915_private *i915, continue; } - if (!i915_gem_object_trylock(obj, NULL)) { - /* busy, toss it back to the pile */ - if (llist_add(&obj->freed, &i915->mm.free_list)) - queue_delayed_work(i915->wq, &i915->mm.free_work, msecs_to_jiffies(10)); - continue; - } - __i915_gem_object_pages_fini(obj); - i915_gem_object_unlock(obj); __i915_gem_free_object(obj); /* But keep the pointer alive for RCU-protected lookups */ @@ -359,7 +437,7 @@ void i915_gem_flush_free_objects(struct drm_i915_private *i915) static void __i915_gem_free_work(struct work_struct *work) { struct drm_i915_private *i915 = - container_of(work, struct drm_i915_private, mm.free_work.work); + container_of(work, struct drm_i915_private, mm.free_work); i915_gem_flush_free_objects(i915); } @@ -371,6 +449,8 @@ static void i915_gem_free_object(struct drm_gem_object *gem_obj) GEM_BUG_ON(i915_gem_object_is_framebuffer(obj)); + i915_drm_client_remove_object(obj); + /* * Before we free the object, make sure any pure RCU-only * read-side critical sections are complete, e.g. @@ -380,8 +460,8 @@ static void i915_gem_free_object(struct drm_gem_object *gem_obj) atomic_inc(&i915->mm.free_count); /* - * Since we require blocking on struct_mutex to unbind the freed - * object from the GPU before releasing resources back to the + * Since we require blocking on drm_i915_gem_object->vma.lock to unbind + * the freed object from the GPU before releasing resources back to the * system, we can not do that directly from the RCU callback (which may * be a softirq context), but must instead then defer that work onto a * kthread. We use the RCU callback rather than move the freed object @@ -391,55 +471,55 @@ static void i915_gem_free_object(struct drm_gem_object *gem_obj) */ if (llist_add(&obj->freed, &i915->mm.free_list)) - queue_delayed_work(i915->wq, &i915->mm.free_work, 0); + queue_work(i915->wq, &i915->mm.free_work); } void __i915_gem_object_flush_frontbuffer(struct drm_i915_gem_object *obj, enum fb_op_origin origin) { - struct intel_frontbuffer *front; + struct i915_frontbuffer *front; - front = __intel_frontbuffer_get(obj); + front = i915_gem_object_frontbuffer_lookup(obj); if (front) { - intel_frontbuffer_flush(front, origin); - intel_frontbuffer_put(front); + intel_frontbuffer_flush(&front->base, origin); + i915_gem_object_frontbuffer_put(front); } } void __i915_gem_object_invalidate_frontbuffer(struct drm_i915_gem_object *obj, enum fb_op_origin origin) { - struct intel_frontbuffer *front; + struct i915_frontbuffer *front; - front = __intel_frontbuffer_get(obj); + front = i915_gem_object_frontbuffer_lookup(obj); if (front) { - intel_frontbuffer_invalidate(front, origin); - intel_frontbuffer_put(front); + intel_frontbuffer_invalidate(&front->base, origin); + i915_gem_object_frontbuffer_put(front); } } static void i915_gem_object_read_from_page_kmap(struct drm_i915_gem_object *obj, u64 offset, void *dst, int size) { - void *src_map; + pgoff_t idx = offset >> PAGE_SHIFT; void *src_ptr; - src_map = kmap_atomic(i915_gem_object_get_page(obj, offset >> PAGE_SHIFT)); - - src_ptr = src_map + offset_in_page(offset); + src_ptr = kmap_local_page(i915_gem_object_get_page(obj, idx)) + + offset_in_page(offset); if (!(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ)) drm_clflush_virt_range(src_ptr, size); memcpy(dst, src_ptr, size); - kunmap_atomic(src_map); + kunmap_local(src_ptr); } static void i915_gem_object_read_from_page_iomap(struct drm_i915_gem_object *obj, u64 offset, void *dst, int size) { + pgoff_t idx = offset >> PAGE_SHIFT; + dma_addr_t dma = i915_gem_object_get_dma_address(obj, idx); void __iomem *src_map; void __iomem *src_ptr; - dma_addr_t dma = i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT); src_map = io_mapping_map_wc(&obj->mm.region->iomap, dma - obj->mm.region->region.start, @@ -452,6 +532,16 @@ i915_gem_object_read_from_page_iomap(struct drm_i915_gem_object *obj, u64 offset io_mapping_unmap(src_map); } +static bool object_has_mappable_iomem(struct drm_i915_gem_object *obj) +{ + GEM_BUG_ON(!i915_gem_object_has_iomem(obj)); + + if (IS_DGFX(to_i915(obj->base.dev))) + return i915_ttm_resource_mappable(i915_gem_to_ttm(obj)->resource); + + return true; +} + /** * i915_gem_object_read_from_page - read data from the page of a GEM object * @obj: GEM object to read from @@ -468,13 +558,14 @@ i915_gem_object_read_from_page_iomap(struct drm_i915_gem_object *obj, u64 offset */ int i915_gem_object_read_from_page(struct drm_i915_gem_object *obj, u64 offset, void *dst, int size) { + GEM_BUG_ON(overflows_type(offset >> PAGE_SHIFT, pgoff_t)); GEM_BUG_ON(offset >= obj->base.size); GEM_BUG_ON(offset_in_page(offset) > PAGE_SIZE - size); GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); if (i915_gem_object_has_struct_page(obj)) i915_gem_object_read_from_page_kmap(obj, offset, dst, size); - else if (i915_gem_object_has_iomem(obj)) + else if (i915_gem_object_has_iomem(obj) && object_has_mappable_iomem(obj)) i915_gem_object_read_from_page_iomap(obj, offset, dst, size); else return -ENODEV; @@ -606,6 +697,9 @@ bool i915_gem_object_can_migrate(struct drm_i915_gem_object *obj, if (!mr) return false; + if (!IS_ALIGNED(obj->base.size, mr->min_page_size)) + return false; + if (obj->mm.region == mr) return true; @@ -658,6 +752,41 @@ int i915_gem_object_migrate(struct drm_i915_gem_object *obj, struct i915_gem_ww_ctx *ww, enum intel_region_id id) { + return __i915_gem_object_migrate(obj, ww, id, obj->flags); +} + +/** + * __i915_gem_object_migrate - Migrate an object to the desired region id, with + * control of the extra flags + * @obj: The object to migrate. + * @ww: An optional struct i915_gem_ww_ctx. If NULL, the backend may + * not be successful in evicting other objects to make room for this object. + * @id: The region id to migrate to. + * @flags: The object flags. Normally just obj->flags. + * + * Attempt to migrate the object to the desired memory region. The + * object backend must support migration and the object may not be + * pinned, (explicitly pinned pages or pinned vmas). The object must + * be locked. + * On successful completion, the object will have pages pointing to + * memory in the new region, but an async migration task may not have + * completed yet, and to accomplish that, i915_gem_object_wait_migration() + * must be called. + * + * Note: the @ww parameter is not used yet, but included to make sure + * callers put some effort into obtaining a valid ww ctx if one is + * available. + * + * Return: 0 on success. Negative error code on failure. In particular may + * return -ENXIO on lack of region space, -EDEADLK for deadlock avoidance + * if @ww is set, -EINTR or -ERESTARTSYS if signal pending, and + * -EBUSY if the object is pinned. + */ +int __i915_gem_object_migrate(struct drm_i915_gem_object *obj, + struct i915_gem_ww_ctx *ww, + enum intel_region_id id, + unsigned int flags) +{ struct drm_i915_private *i915 = to_i915(obj->base.dev); struct intel_memory_region *mr; @@ -677,7 +806,7 @@ int i915_gem_object_migrate(struct drm_i915_gem_object *obj, return 0; } - return obj->ops->migrate(obj, mr); + return obj->ops->migrate(obj, mr, flags); } /** @@ -714,9 +843,65 @@ bool i915_gem_object_placement_possible(struct drm_i915_gem_object *obj, return false; } +/** + * i915_gem_object_needs_ccs_pages - Check whether the object requires extra + * pages when placed in system-memory, in order to save and later restore the + * flat-CCS aux state when the object is moved between local-memory and + * system-memory + * @obj: Pointer to the object + * + * Return: True if the object needs extra ccs pages. False otherwise. + */ +bool i915_gem_object_needs_ccs_pages(struct drm_i915_gem_object *obj) +{ + bool lmem_placement = false; + int i; + + if (!HAS_FLAT_CCS(to_i915(obj->base.dev))) + return false; + + if (obj->flags & I915_BO_ALLOC_CCS_AUX) + return true; + + for (i = 0; i < obj->mm.n_placements; i++) { + /* Compression is not allowed for the objects with smem placement */ + if (obj->mm.placements[i]->type == INTEL_MEMORY_SYSTEM) + return false; + if (!lmem_placement && + obj->mm.placements[i]->type == INTEL_MEMORY_LOCAL) + lmem_placement = true; + } + + return lmem_placement; +} + +static int i915_gem_vmap_object(struct drm_gem_object *gem_obj, + struct iosys_map *map) +{ + struct drm_i915_gem_object *obj = to_intel_bo(gem_obj); + void *vaddr; + + vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB); + if (IS_ERR(vaddr)) + return PTR_ERR(vaddr); + + iosys_map_set_vaddr(map, vaddr); + + return 0; +} + +static void i915_gem_vunmap_object(struct drm_gem_object *gem_obj, + struct iosys_map *map) +{ + struct drm_i915_gem_object *obj = to_intel_bo(gem_obj); + + i915_gem_object_flush_map(obj); + i915_gem_object_unpin_map(obj); +} + void i915_gem_init__objects(struct drm_i915_private *i915) { - INIT_DELAYED_WORK(&i915->mm.free_work, __i915_gem_free_work); + INIT_WORK(&i915->mm.free_work, __i915_gem_free_work); } void i915_objects_module_exit(void) @@ -737,23 +922,26 @@ static const struct drm_gem_object_funcs i915_gem_object_funcs = { .free = i915_gem_free_object, .close = i915_gem_close_object, .export = i915_gem_prime_export, + .vmap = i915_gem_vmap_object, + .vunmap = i915_gem_vunmap_object, }; /** * i915_gem_object_get_moving_fence - Get the object's moving fence if any * @obj: The object whose moving fence to get. + * @fence: The resulting fence * * A non-signaled moving fence means that there is an async operation * pending on the object that needs to be waited on before setting up * any GPU- or CPU PTEs to the object's pages. * - * Return: A refcounted pointer to the object's moving fence if any, - * NULL otherwise. + * Return: Negative error code or 0 for success. */ -struct dma_fence * -i915_gem_object_get_moving_fence(struct drm_i915_gem_object *obj) +int i915_gem_object_get_moving_fence(struct drm_i915_gem_object *obj, + struct dma_fence **fence) { - return dma_fence_get(i915_gem_to_ttm(obj)->moving); + return dma_resv_get_singleton(obj->base.resv, DMA_RESV_USAGE_KERNEL, + fence); } /** @@ -771,23 +959,37 @@ i915_gem_object_get_moving_fence(struct drm_i915_gem_object *obj) int i915_gem_object_wait_moving_fence(struct drm_i915_gem_object *obj, bool intr) { - struct dma_fence *fence = i915_gem_to_ttm(obj)->moving; - int ret; + long ret; assert_object_held(obj); - if (!fence) - return 0; - ret = dma_fence_wait(fence, intr); - if (ret) - return ret; + ret = dma_resv_wait_timeout(obj->base. resv, DMA_RESV_USAGE_KERNEL, + intr, MAX_SCHEDULE_TIMEOUT); + if (!ret) + ret = -ETIME; + else if (ret > 0 && i915_gem_object_has_unknown_state(obj)) + ret = -EIO; - if (fence->error) - return fence->error; + return ret < 0 ? ret : 0; +} - i915_gem_to_ttm(obj)->moving = NULL; - dma_fence_put(fence); - return 0; +/* + * i915_gem_object_has_unknown_state - Return true if the object backing pages are + * in an unknown_state. This means that userspace must NEVER be allowed to touch + * the pages, with either the GPU or CPU. + * + * ONLY valid to be called after ensuring that all kernel fences have signalled + * (in particular the fence for moving/clearing the object). + */ +bool i915_gem_object_has_unknown_state(struct drm_i915_gem_object *obj) +{ + /* + * The below barrier pairs with the dma_fence_signal() in + * __memcpy_work(). We should only sample the unknown_state after all + * the kernel fences have signalled. + */ + smp_rmb(); + return obj->mm.unknown_state; } #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) |