diff options
Diffstat (limited to 'drivers/gpu/drm/i915/gem')
29 files changed, 336 insertions, 687 deletions
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_client_blt.c b/drivers/gpu/drm/i915/gem/i915_gem_client_blt.c index d3a86a4d5c04..278664f831e7 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_client_blt.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_client_blt.c @@ -32,16 +32,17 @@ static void vma_clear_pages(struct i915_vma *vma) vma->pages = NULL; } -static int vma_bind(struct i915_vma *vma, +static int vma_bind(struct i915_address_space *vm, + struct i915_vma *vma, enum i915_cache_level cache_level, u32 flags) { - return vma->vm->vma_ops.bind_vma(vma, cache_level, flags); + return vm->vma_ops.bind_vma(vm, vma, cache_level, flags); } -static void vma_unbind(struct i915_vma *vma) +static void vma_unbind(struct i915_address_space *vm, struct i915_vma *vma) { - vma->vm->vma_ops.unbind_vma(vma); + vm->vma_ops.unbind_vma(vm, vma); } static const struct i915_vma_ops proxy_vma_ops = { diff --git a/drivers/gpu/drm/i915/gem/i915_gem_context.c b/drivers/gpu/drm/i915/gem/i915_gem_context.c index 30c229fcb404..d0bdb6d447ed 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_context.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_context.c @@ -101,8 +101,7 @@ static void lut_close(struct i915_gem_context *ctx) struct radix_tree_iter iter; void __rcu **slot; - lockdep_assert_held(&ctx->mutex); - + mutex_lock(&ctx->lut_mutex); rcu_read_lock(); radix_tree_for_each_slot(slot, &ctx->handles_vma, &iter, 0) { struct i915_vma *vma = rcu_dereference_raw(*slot); @@ -112,8 +111,7 @@ static void lut_close(struct i915_gem_context *ctx) if (!kref_get_unless_zero(&obj->base.refcount)) continue; - rcu_read_unlock(); - i915_gem_object_lock(obj); + spin_lock(&obj->lut_lock); list_for_each_entry(lut, &obj->lut_list, obj_link) { if (lut->ctx != ctx) continue; @@ -124,8 +122,7 @@ static void lut_close(struct i915_gem_context *ctx) list_del(&lut->obj_link); break; } - i915_gem_object_unlock(obj); - rcu_read_lock(); + spin_unlock(&obj->lut_lock); if (&lut->obj_link != &obj->lut_list) { i915_lut_handle_free(lut); @@ -137,6 +134,7 @@ static void lut_close(struct i915_gem_context *ctx) i915_gem_object_put(obj); } rcu_read_unlock(); + mutex_unlock(&ctx->lut_mutex); } static struct intel_context * @@ -344,6 +342,7 @@ static void i915_gem_context_free(struct i915_gem_context *ctx) spin_unlock(&ctx->i915->gem.contexts.lock); mutex_destroy(&ctx->engines_mutex); + mutex_destroy(&ctx->lut_mutex); if (ctx->timeline) intel_timeline_put(ctx->timeline); @@ -650,7 +649,7 @@ static void context_close(struct i915_gem_context *ctx) * context close. */ if (!i915_gem_context_is_persistent(ctx) || - !i915_modparams.enable_hangcheck) + !ctx->i915->params.enable_hangcheck) kill_context(ctx); i915_gem_context_put(ctx); @@ -667,7 +666,7 @@ static int __context_set_persistence(struct i915_gem_context *ctx, bool state) * reset] are allowed to survive past termination. We require * hangcheck to ensure that the persistent requests are healthy. */ - if (!i915_modparams.enable_hangcheck) + if (!ctx->i915->params.enable_hangcheck) return -EINVAL; i915_gem_context_set_persistence(ctx); @@ -727,6 +726,7 @@ __create_context(struct drm_i915_private *i915) RCU_INIT_POINTER(ctx->engines, e); INIT_RADIX_TREE(&ctx->handles_vma, GFP_KERNEL); + mutex_init(&ctx->lut_mutex); /* NB: Mark all slices as needing a remap so that when the context first * loads it will restore whatever remap state already exists. If there @@ -1314,11 +1314,11 @@ static int set_ppgtt(struct drm_i915_file_private *file_priv, if (vm == rcu_access_pointer(ctx->vm)) goto unlock; + old = __set_ppgtt(ctx, vm); + /* Teardown the existing obj:vma cache, it will have to be rebuilt. */ lut_close(ctx); - old = __set_ppgtt(ctx, vm); - /* * We need to flush any requests using the current ppgtt before * we release it as the requests do not hold a reference themselves, @@ -1332,6 +1332,7 @@ static int set_ppgtt(struct drm_i915_file_private *file_priv, if (err) { i915_vm_close(__set_ppgtt(ctx, old)); i915_vm_close(old); + lut_close(ctx); /* force a rebuild of the old obj:vma cache */ } unlock: @@ -1399,11 +1400,12 @@ static int get_ringsize(struct i915_gem_context *ctx, } int -i915_gem_user_to_context_sseu(struct drm_i915_private *i915, +i915_gem_user_to_context_sseu(struct intel_gt *gt, const struct drm_i915_gem_context_param_sseu *user, struct intel_sseu *context) { - const struct sseu_dev_info *device = &RUNTIME_INFO(i915)->sseu; + const struct sseu_dev_info *device = >->info.sseu; + struct drm_i915_private *i915 = gt->i915; /* No zeros in any field. */ if (!user->slice_mask || !user->subslice_mask || @@ -1536,7 +1538,7 @@ static int set_sseu(struct i915_gem_context *ctx, goto out_ce; } - ret = i915_gem_user_to_context_sseu(i915, &user_sseu, &sseu); + ret = i915_gem_user_to_context_sseu(ce->engine->gt, &user_sseu, &sseu); if (ret) goto out_ce; diff --git a/drivers/gpu/drm/i915/gem/i915_gem_context.h b/drivers/gpu/drm/i915/gem/i915_gem_context.h index 3702b2fb27ab..a133f92bbedb 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_context.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_context.h @@ -225,7 +225,7 @@ i915_gem_engines_iter_next(struct i915_gem_engines_iter *it); struct i915_lut_handle *i915_lut_handle_alloc(void); void i915_lut_handle_free(struct i915_lut_handle *lut); -int i915_gem_user_to_context_sseu(struct drm_i915_private *i915, +int i915_gem_user_to_context_sseu(struct intel_gt *gt, const struct drm_i915_gem_context_param_sseu *user, struct intel_sseu *context); diff --git a/drivers/gpu/drm/i915/gem/i915_gem_context_types.h b/drivers/gpu/drm/i915/gem/i915_gem_context_types.h index 28760bd03265..ae14ca24a11f 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_context_types.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_context_types.h @@ -170,6 +170,7 @@ struct i915_gem_context { * per vm, which may be one per context or shared with the global GTT) */ struct radix_tree_root handles_vma; + struct mutex lut_mutex; /** * @name: arbitrary name, used for user debug diff --git a/drivers/gpu/drm/i915/gem/i915_gem_dmabuf.c b/drivers/gpu/drm/i915/gem/i915_gem_dmabuf.c index 7db5a793739d..2679380159fc 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_dmabuf.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_dmabuf.c @@ -217,6 +217,7 @@ static void i915_gem_object_put_pages_dmabuf(struct drm_i915_gem_object *obj, } static const struct drm_i915_gem_object_ops i915_gem_object_dmabuf_ops = { + .name = "i915_gem_object_dmabuf", .get_pages = i915_gem_object_get_pages_dmabuf, .put_pages = i915_gem_object_put_pages_dmabuf, }; diff --git a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c index db8eb1c6afe9..6b4ec66cb558 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c @@ -45,13 +45,6 @@ struct eb_vma_array { struct eb_vma vma[]; }; -enum { - FORCE_CPU_RELOC = 1, - FORCE_GTT_RELOC, - FORCE_GPU_RELOC, -#define DBG_FORCE_RELOC 0 /* choose one of the above! */ -}; - #define __EXEC_OBJECT_HAS_PIN BIT(31) #define __EXEC_OBJECT_HAS_FENCE BIT(30) #define __EXEC_OBJECT_NEEDS_MAP BIT(29) @@ -260,8 +253,6 @@ struct i915_execbuffer { */ struct reloc_cache { struct drm_mm_node node; /** temporary GTT binding */ - unsigned long vaddr; /** Current kmap address */ - unsigned long page; /** Currently mapped page index */ unsigned int gen; /** Cached value of INTEL_GEN */ bool use_64bit_reloc : 1; bool has_llc : 1; @@ -605,23 +596,6 @@ eb_add_vma(struct i915_execbuffer *eb, } } -static inline int use_cpu_reloc(const struct reloc_cache *cache, - const struct drm_i915_gem_object *obj) -{ - if (!i915_gem_object_has_struct_page(obj)) - return false; - - if (DBG_FORCE_RELOC == FORCE_CPU_RELOC) - return true; - - if (DBG_FORCE_RELOC == FORCE_GTT_RELOC) - return false; - - return (cache->has_llc || - obj->cache_dirty || - obj->cache_level != I915_CACHE_NONE); -} - static int eb_reserve_vma(const struct i915_execbuffer *eb, struct eb_vma *ev, u64 pin_flags) @@ -808,23 +782,28 @@ static int __eb_add_lut(struct i915_execbuffer *eb, /* Check that the context hasn't been closed in the meantime */ err = -EINTR; - if (!mutex_lock_interruptible(&ctx->mutex)) { - err = -ENOENT; - if (likely(!i915_gem_context_is_closed(ctx))) + if (!mutex_lock_interruptible(&ctx->lut_mutex)) { + struct i915_address_space *vm = rcu_access_pointer(ctx->vm); + + if (unlikely(vm && vma->vm != vm)) + err = -EAGAIN; /* user racing with ctx set-vm */ + else if (likely(!i915_gem_context_is_closed(ctx))) err = radix_tree_insert(&ctx->handles_vma, handle, vma); + else + err = -ENOENT; if (err == 0) { /* And nor has this handle */ struct drm_i915_gem_object *obj = vma->obj; - i915_gem_object_lock(obj); + spin_lock(&obj->lut_lock); if (idr_find(&eb->file->object_idr, handle) == obj) { list_add(&lut->obj_link, &obj->lut_list); } else { radix_tree_delete(&ctx->handles_vma, handle); err = -ENOENT; } - i915_gem_object_unlock(obj); + spin_unlock(&obj->lut_lock); } - mutex_unlock(&ctx->mutex); + mutex_unlock(&ctx->lut_mutex); } if (unlikely(err)) goto err; @@ -840,6 +819,8 @@ err: static struct i915_vma *eb_lookup_vma(struct i915_execbuffer *eb, u32 handle) { + struct i915_address_space *vm = eb->context->vm; + do { struct drm_i915_gem_object *obj; struct i915_vma *vma; @@ -847,7 +828,7 @@ static struct i915_vma *eb_lookup_vma(struct i915_execbuffer *eb, u32 handle) rcu_read_lock(); vma = radix_tree_lookup(&eb->gem_context->handles_vma, handle); - if (likely(vma)) + if (likely(vma && vma->vm == vm)) vma = i915_vma_tryget(vma); rcu_read_unlock(); if (likely(vma)) @@ -857,7 +838,7 @@ static struct i915_vma *eb_lookup_vma(struct i915_execbuffer *eb, u32 handle) if (unlikely(!obj)) return ERR_PTR(-ENOENT); - vma = i915_vma_instance(obj, eb->context->vm, NULL); + vma = i915_vma_instance(obj, vm, NULL); if (IS_ERR(vma)) { i915_gem_object_put(obj); return vma; @@ -945,8 +926,6 @@ relocation_target(const struct drm_i915_gem_relocation_entry *reloc, static void reloc_cache_init(struct reloc_cache *cache, struct drm_i915_private *i915) { - cache->page = -1; - cache->vaddr = 0; /* Must be a variable in the struct to allow GCC to unroll. */ cache->gen = INTEL_GEN(i915); cache->has_llc = HAS_LLC(i915); @@ -958,25 +937,6 @@ static void reloc_cache_init(struct reloc_cache *cache, cache->target = NULL; } -static inline void *unmask_page(unsigned long p) -{ - return (void *)(uintptr_t)(p & PAGE_MASK); -} - -static inline unsigned int unmask_flags(unsigned long p) -{ - return p & ~PAGE_MASK; -} - -#define KMAP 0x4 /* after CLFLUSH_FLAGS */ - -static inline struct i915_ggtt *cache_to_ggtt(struct reloc_cache *cache) -{ - struct drm_i915_private *i915 = - container_of(cache, struct i915_execbuffer, reloc_cache)->i915; - return &i915->ggtt; -} - #define RELOC_TAIL 4 static int reloc_gpu_chain(struct reloc_cache *cache) @@ -1089,181 +1049,6 @@ static int reloc_gpu_flush(struct reloc_cache *cache) return err; } -static void reloc_cache_reset(struct reloc_cache *cache) -{ - void *vaddr; - - if (!cache->vaddr) - return; - - vaddr = unmask_page(cache->vaddr); - if (cache->vaddr & KMAP) { - if (cache->vaddr & CLFLUSH_AFTER) - mb(); - - kunmap_atomic(vaddr); - i915_gem_object_finish_access((struct drm_i915_gem_object *)cache->node.mm); - } else { - struct i915_ggtt *ggtt = cache_to_ggtt(cache); - - intel_gt_flush_ggtt_writes(ggtt->vm.gt); - io_mapping_unmap_atomic((void __iomem *)vaddr); - - if (drm_mm_node_allocated(&cache->node)) { - ggtt->vm.clear_range(&ggtt->vm, - cache->node.start, - cache->node.size); - mutex_lock(&ggtt->vm.mutex); - drm_mm_remove_node(&cache->node); - mutex_unlock(&ggtt->vm.mutex); - } else { - i915_vma_unpin((struct i915_vma *)cache->node.mm); - } - } - - cache->vaddr = 0; - cache->page = -1; -} - -static void *reloc_kmap(struct drm_i915_gem_object *obj, - struct reloc_cache *cache, - unsigned long page) -{ - void *vaddr; - - if (cache->vaddr) { - kunmap_atomic(unmask_page(cache->vaddr)); - } else { - unsigned int flushes; - int err; - - err = i915_gem_object_prepare_write(obj, &flushes); - if (err) - return ERR_PTR(err); - - BUILD_BUG_ON(KMAP & CLFLUSH_FLAGS); - BUILD_BUG_ON((KMAP | CLFLUSH_FLAGS) & PAGE_MASK); - - cache->vaddr = flushes | KMAP; - cache->node.mm = (void *)obj; - if (flushes) - mb(); - } - - vaddr = kmap_atomic(i915_gem_object_get_dirty_page(obj, page)); - cache->vaddr = unmask_flags(cache->vaddr) | (unsigned long)vaddr; - cache->page = page; - - return vaddr; -} - -static void *reloc_iomap(struct drm_i915_gem_object *obj, - struct reloc_cache *cache, - unsigned long page) -{ - struct i915_ggtt *ggtt = cache_to_ggtt(cache); - unsigned long offset; - void *vaddr; - - if (cache->vaddr) { - intel_gt_flush_ggtt_writes(ggtt->vm.gt); - io_mapping_unmap_atomic((void __force __iomem *) unmask_page(cache->vaddr)); - } else { - struct i915_vma *vma; - int err; - - if (i915_gem_object_is_tiled(obj)) - return ERR_PTR(-EINVAL); - - if (use_cpu_reloc(cache, obj)) - return NULL; - - i915_gem_object_lock(obj); - err = i915_gem_object_set_to_gtt_domain(obj, true); - i915_gem_object_unlock(obj); - if (err) - return ERR_PTR(err); - - vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, - PIN_MAPPABLE | - PIN_NONBLOCK /* NOWARN */ | - PIN_NOEVICT); - if (IS_ERR(vma)) { - memset(&cache->node, 0, sizeof(cache->node)); - mutex_lock(&ggtt->vm.mutex); - err = drm_mm_insert_node_in_range - (&ggtt->vm.mm, &cache->node, - PAGE_SIZE, 0, I915_COLOR_UNEVICTABLE, - 0, ggtt->mappable_end, - DRM_MM_INSERT_LOW); - mutex_unlock(&ggtt->vm.mutex); - if (err) /* no inactive aperture space, use cpu reloc */ - return NULL; - } else { - cache->node.start = vma->node.start; - cache->node.mm = (void *)vma; - } - } - - offset = cache->node.start; - if (drm_mm_node_allocated(&cache->node)) { - ggtt->vm.insert_page(&ggtt->vm, - i915_gem_object_get_dma_address(obj, page), - offset, I915_CACHE_NONE, 0); - } else { - offset += page << PAGE_SHIFT; - } - - vaddr = (void __force *)io_mapping_map_atomic_wc(&ggtt->iomap, - offset); - cache->page = page; - cache->vaddr = (unsigned long)vaddr; - - return vaddr; -} - -static void *reloc_vaddr(struct drm_i915_gem_object *obj, - struct reloc_cache *cache, - unsigned long page) -{ - void *vaddr; - - if (cache->page == page) { - vaddr = unmask_page(cache->vaddr); - } else { - vaddr = NULL; - if ((cache->vaddr & KMAP) == 0) - vaddr = reloc_iomap(obj, cache, page); - if (!vaddr) - vaddr = reloc_kmap(obj, cache, page); - } - - return vaddr; -} - -static void clflush_write32(u32 *addr, u32 value, unsigned int flushes) -{ - if (unlikely(flushes & (CLFLUSH_BEFORE | CLFLUSH_AFTER))) { - if (flushes & CLFLUSH_BEFORE) { - clflushopt(addr); - mb(); - } - - *addr = value; - - /* - * Writes to the same cacheline are serialised by the CPU - * (including clflush). On the write path, we only require - * that it hits memory in an orderly fashion and place - * mb barriers at the start and end of the relocation phase - * to ensure ordering of clflush wrt to the system. - */ - if (flushes & CLFLUSH_AFTER) - clflushopt(addr); - } else - *addr = value; -} - static int reloc_move_to_gpu(struct i915_request *rq, struct i915_vma *vma) { struct drm_i915_gem_object *obj = vma->obj; @@ -1429,17 +1214,6 @@ static u32 *reloc_gpu(struct i915_execbuffer *eb, return cmd; } -static inline bool use_reloc_gpu(struct i915_vma *vma) -{ - if (DBG_FORCE_RELOC == FORCE_GPU_RELOC) - return true; - - if (DBG_FORCE_RELOC) - return false; - - return !dma_resv_test_signaled_rcu(vma->resv, true); -} - static unsigned long vma_phys_addr(struct i915_vma *vma, u32 offset) { struct page *page; @@ -1454,10 +1228,10 @@ static unsigned long vma_phys_addr(struct i915_vma *vma, u32 offset) return addr + offset_in_page(offset); } -static bool __reloc_entry_gpu(struct i915_execbuffer *eb, - struct i915_vma *vma, - u64 offset, - u64 target_addr) +static int __reloc_entry_gpu(struct i915_execbuffer *eb, + struct i915_vma *vma, + u64 offset, + u64 target_addr) { const unsigned int gen = eb->reloc_cache.gen; unsigned int len; @@ -1473,7 +1247,7 @@ static bool __reloc_entry_gpu(struct i915_execbuffer *eb, batch = reloc_gpu(eb, vma, len); if (IS_ERR(batch)) - return false; + return PTR_ERR(batch); addr = gen8_canonical_addr(vma->node.start + offset); if (gen >= 8) { @@ -1522,55 +1296,21 @@ static bool __reloc_entry_gpu(struct i915_execbuffer *eb, *batch++ = target_addr; } - return true; -} - -static bool reloc_entry_gpu(struct i915_execbuffer *eb, - struct i915_vma *vma, - u64 offset, - u64 target_addr) -{ - if (eb->reloc_cache.vaddr) - return false; - - if (!use_reloc_gpu(vma)) - return false; - - return __reloc_entry_gpu(eb, vma, offset, target_addr); + return 0; } static u64 -relocate_entry(struct i915_vma *vma, +relocate_entry(struct i915_execbuffer *eb, + struct i915_vma *vma, const struct drm_i915_gem_relocation_entry *reloc, - struct i915_execbuffer *eb, const struct i915_vma *target) { u64 target_addr = relocation_target(reloc, target); - u64 offset = reloc->offset; - - if (!reloc_entry_gpu(eb, vma, offset, target_addr)) { - bool wide = eb->reloc_cache.use_64bit_reloc; - void *vaddr; - -repeat: - vaddr = reloc_vaddr(vma->obj, - &eb->reloc_cache, - offset >> PAGE_SHIFT); - if (IS_ERR(vaddr)) - return PTR_ERR(vaddr); - - GEM_BUG_ON(!IS_ALIGNED(offset, sizeof(u32))); - clflush_write32(vaddr + offset_in_page(offset), - lower_32_bits(target_addr), - eb->reloc_cache.vaddr); - - if (wide) { - offset += sizeof(u32); - target_addr >>= 32; - wide = false; - goto repeat; - } - } + int err; + + err = __reloc_entry_gpu(eb, vma, reloc->offset, target_addr); + if (err) + return err; return target->node.start | UPDATE; } @@ -1626,8 +1366,7 @@ eb_relocate_entry(struct i915_execbuffer *eb, err = i915_vma_bind(target->vma, target->vma->obj->cache_level, PIN_GLOBAL, NULL); - if (WARN_ONCE(err, - "Unexpected failure to bind target VMA!")) + if (err) return err; } } @@ -1636,8 +1375,7 @@ eb_relocate_entry(struct i915_execbuffer *eb, * If the relocation already has the right value in it, no * more work needs to be done. */ - if (!DBG_FORCE_RELOC && - gen8_canonical_addr(target->vma->node.start) == reloc->presumed_offset) + if (gen8_canonical_addr(target->vma->node.start) == reloc->presumed_offset) return 0; /* Check that the relocation address is valid... */ @@ -1669,7 +1407,7 @@ eb_relocate_entry(struct i915_execbuffer *eb, ev->flags &= ~EXEC_OBJECT_ASYNC; /* and update the user's relocation entry */ - return relocate_entry(ev->vma, reloc, eb, target->vma); + return relocate_entry(eb, ev->vma, reloc, target->vma); } static int eb_relocate_vma(struct i915_execbuffer *eb, struct eb_vma *ev) @@ -1707,10 +1445,8 @@ static int eb_relocate_vma(struct i915_execbuffer *eb, struct eb_vma *ev) * this is bad and so lockdep complains vehemently. */ copied = __copy_from_user(r, urelocs, count * sizeof(r[0])); - if (unlikely(copied)) { - remain = -EFAULT; - goto out; - } + if (unlikely(copied)) + return -EFAULT; remain -= count; do { @@ -1718,8 +1454,7 @@ static int eb_relocate_vma(struct i915_execbuffer *eb, struct eb_vma *ev) if (likely(offset == 0)) { } else if ((s64)offset < 0) { - remain = (int)offset; - goto out; + return (int)offset; } else { /* * Note that reporting an error now @@ -1749,9 +1484,8 @@ static int eb_relocate_vma(struct i915_execbuffer *eb, struct eb_vma *ev) } while (r++, --count); urelocs += ARRAY_SIZE(stack); } while (remain); -out: - reloc_cache_reset(&eb->reloc_cache); - return remain; + + return 0; } static int eb_relocate(struct i915_execbuffer *eb) @@ -1911,8 +1645,8 @@ static int i915_reset_gen7_sol_offsets(struct i915_request *rq) u32 *cs; int i; - if (!IS_GEN(rq->i915, 7) || rq->engine->id != RCS0) { - drm_dbg(&rq->i915->drm, "sol reset is gen7/rcs only\n"); + if (!IS_GEN(rq->engine->i915, 7) || rq->engine->id != RCS0) { + drm_dbg(&rq->engine->i915->drm, "sol reset is gen7/rcs only\n"); return -EINVAL; } @@ -2246,8 +1980,7 @@ static int eb_submit(struct i915_execbuffer *eb, struct i915_vma *batch) static int num_vcs_engines(const struct drm_i915_private *i915) { - return hweight64(INTEL_INFO(i915)->engine_mask & - GENMASK_ULL(VCS0 + I915_MAX_VCS - 1, VCS0)); + return hweight64(VDBOX_MASK(&i915->gt)); } /* @@ -2659,7 +2392,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, eb.i915 = i915; eb.file = file; eb.args = args; - if (DBG_FORCE_RELOC || !(args->flags & I915_EXEC_NO_RELOC)) + if (!(args->flags & I915_EXEC_NO_RELOC)) args->flags |= __EXEC_HAS_RELOC; eb.exec = exec; diff --git a/drivers/gpu/drm/i915/gem/i915_gem_internal.c b/drivers/gpu/drm/i915/gem/i915_gem_internal.c index cbbff81aa0af..ad22f42541bd 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_internal.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_internal.c @@ -137,6 +137,7 @@ static void i915_gem_object_put_pages_internal(struct drm_i915_gem_object *obj, } static const struct drm_i915_gem_object_ops i915_gem_object_internal_ops = { + .name = "i915_gem_object_internal", .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE | I915_GEM_OBJECT_IS_SHRINKABLE, .get_pages = i915_gem_object_get_pages_internal, diff --git a/drivers/gpu/drm/i915/gem/i915_gem_lmem.c b/drivers/gpu/drm/i915/gem/i915_gem_lmem.c index 70543c83df06..932ee21e6609 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_lmem.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_lmem.c @@ -9,6 +9,7 @@ #include "i915_drv.h" const struct drm_i915_gem_object_ops i915_gem_lmem_obj_ops = { + .name = "i915_gem_object_lmem", .flags = I915_GEM_OBJECT_HAS_IOMEM, .get_pages = i915_gem_object_get_pages_buddy, diff --git a/drivers/gpu/drm/i915/gem/i915_gem_mman.c b/drivers/gpu/drm/i915/gem/i915_gem_mman.c index fe45bd4d63a5..b23368529a40 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_mman.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_mman.c @@ -216,12 +216,12 @@ static vm_fault_t i915_error_to_vmf_fault(int err) case -ENXIO: /* unable to access backing store (on device) */ return VM_FAULT_SIGBUS; - case -ENOSPC: /* shmemfs allocation failure */ case -ENOMEM: /* our allocation failure */ return VM_FAULT_OOM; case 0: case -EAGAIN: + case -ENOSPC: /* transient failure to evict? */ case -ERESTARTSYS: case -EINTR: case -EBUSY: @@ -448,7 +448,7 @@ void __i915_gem_object_release_mmap_gtt(struct drm_i915_gem_object *obj) * mapping will then trigger a page fault on the next user access, allowing * fixup by vm_fault_gtt(). */ -static void i915_gem_object_release_mmap_gtt(struct drm_i915_gem_object *obj) +void i915_gem_object_release_mmap_gtt(struct drm_i915_gem_object *obj) { struct drm_i915_private *i915 = to_i915(obj->base.dev); intel_wakeref_t wakeref; @@ -507,19 +507,6 @@ void i915_gem_object_release_mmap_offset(struct drm_i915_gem_object *obj) spin_unlock(&obj->mmo.lock); } -/** - * i915_gem_object_release_mmap - remove physical page mappings - * @obj: obj in question - * - * Preserve the reservation of the mmapping with the DRM core code, but - * relinquish ownership of the pages back to the system. - */ -void i915_gem_object_release_mmap(struct drm_i915_gem_object *obj) -{ - i915_gem_object_release_mmap_gtt(obj); - i915_gem_object_release_mmap_offset(obj); -} - static struct i915_mmap_offset * lookup_mmo(struct drm_i915_gem_object *obj, enum i915_mmap_type mmap_type) diff --git a/drivers/gpu/drm/i915/gem/i915_gem_mman.h b/drivers/gpu/drm/i915/gem/i915_gem_mman.h index 862e01b7cb69..efee9e0d2508 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_mman.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_mman.h @@ -25,7 +25,8 @@ int i915_gem_dumb_mmap_offset(struct drm_file *file_priv, u32 handle, u64 *offset); void __i915_gem_object_release_mmap_gtt(struct drm_i915_gem_object *obj); -void i915_gem_object_release_mmap(struct drm_i915_gem_object *obj); +void i915_gem_object_release_mmap_gtt(struct drm_i915_gem_object *obj); + void i915_gem_object_release_mmap_offset(struct drm_i915_gem_object *obj); #endif diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object.c b/drivers/gpu/drm/i915/gem/i915_gem_object.c index 99356c00c19e..c8421fd9d2dc 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_object.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_object.c @@ -53,7 +53,7 @@ void i915_gem_object_init(struct drm_i915_gem_object *obj, const struct drm_i915_gem_object_ops *ops, struct lock_class_key *key) { - __mutex_init(&obj->mm.lock, "obj->mm.lock", key); + __mutex_init(&obj->mm.lock, ops->name ?: "obj->mm.lock", key); spin_lock_init(&obj->vma.lock); INIT_LIST_HEAD(&obj->vma.list); @@ -61,6 +61,7 @@ void i915_gem_object_init(struct drm_i915_gem_object *obj, INIT_LIST_HEAD(&obj->mm.link); INIT_LIST_HEAD(&obj->lut_list); + spin_lock_init(&obj->lut_lock); spin_lock_init(&obj->mmo.lock); obj->mmo.offsets = RB_ROOT; @@ -72,6 +73,10 @@ void i915_gem_object_init(struct drm_i915_gem_object *obj, obj->mm.madv = I915_MADV_WILLNEED; INIT_RADIX_TREE(&obj->mm.get_page.radix, GFP_KERNEL | __GFP_NOWARN); mutex_init(&obj->mm.get_page.lock); + + if (IS_ENABLED(CONFIG_LOCKDEP) && i915_gem_object_is_shrinkable(obj)) + i915_gem_shrinker_taints_mutex(to_i915(obj->base.dev), + &obj->mm.lock); } /** @@ -100,21 +105,29 @@ void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file) { struct drm_i915_gem_object *obj = to_intel_bo(gem); struct drm_i915_file_private *fpriv = file->driver_priv; + struct i915_lut_handle bookmark = {}; struct i915_mmap_offset *mmo, *mn; struct i915_lut_handle *lut, *ln; LIST_HEAD(close); - i915_gem_object_lock(obj); + spin_lock(&obj->lut_lock); list_for_each_entry_safe(lut, ln, &obj->lut_list, obj_link) { struct i915_gem_context *ctx = lut->ctx; - if (ctx->file_priv != fpriv) - continue; + if (ctx && ctx->file_priv == fpriv) { + i915_gem_context_get(ctx); + list_move(&lut->obj_link, &close); + } - i915_gem_context_get(ctx); - list_move(&lut->obj_link, &close); + /* Break long locks, and carefully continue on from this spot */ + if (&ln->obj_link != &obj->lut_list) { + list_add_tail(&bookmark.obj_link, &ln->obj_link); + if (cond_resched_lock(&obj->lut_lock)) + list_safe_reset_next(&bookmark, ln, obj_link); + __list_del_entry(&bookmark.obj_link); + } } - i915_gem_object_unlock(obj); + spin_unlock(&obj->lut_lock); spin_lock(&obj->mmo.lock); rbtree_postorder_for_each_entry_safe(mmo, mn, &obj->mmo.offsets, offset) @@ -130,14 +143,14 @@ void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file) * vma, in the same fd namespace, by virtue of flink/open. */ - mutex_lock(&ctx->mutex); + mutex_lock(&ctx->lut_mutex); vma = radix_tree_delete(&ctx->handles_vma, lut->handle); if (vma) { GEM_BUG_ON(vma->obj != obj); GEM_BUG_ON(!atomic_read(&vma->open_count)); i915_vma_close(vma); } - mutex_unlock(&ctx->mutex); + mutex_unlock(&ctx->lut_mutex); i915_gem_context_put(lut->ctx); i915_lut_handle_free(lut); @@ -158,14 +171,35 @@ static void __i915_gem_free_object_rcu(struct rcu_head *head) atomic_dec(&i915->mm.free_count); } +static void __i915_gem_object_free_mmaps(struct drm_i915_gem_object *obj) +{ + /* Skip serialisation and waking the device if known to be not used. */ + + if (obj->userfault_count) + i915_gem_object_release_mmap_gtt(obj); + + if (!RB_EMPTY_ROOT(&obj->mmo.offsets)) { + struct i915_mmap_offset *mmo, *mn; + + i915_gem_object_release_mmap_offset(obj); + + rbtree_postorder_for_each_entry_safe(mmo, mn, + &obj->mmo.offsets, + offset) { + drm_vma_offset_remove(obj->base.dev->vma_offset_manager, + &mmo->vma_node); + kfree(mmo); + } + obj->mmo.offsets = RB_ROOT; + } +} + static void __i915_gem_free_objects(struct drm_i915_private *i915, struct llist_node *freed) { struct drm_i915_gem_object *obj, *on; llist_for_each_entry_safe(obj, on, freed, freed) { - struct i915_mmap_offset *mmo, *mn; - trace_i915_gem_object_destroy(obj); if (!list_empty(&obj->vma.list)) { @@ -191,18 +225,8 @@ static void __i915_gem_free_objects(struct drm_i915_private *i915, spin_unlock(&obj->vma.lock); } - i915_gem_object_release_mmap(obj); - - rbtree_postorder_for_each_entry_safe(mmo, mn, - &obj->mmo.offsets, - offset) { - drm_vma_offset_remove(obj->base.dev->vma_offset_manager, - &mmo->vma_node); - kfree(mmo); - } - obj->mmo.offsets = RB_ROOT; + __i915_gem_object_free_mmaps(obj); - GEM_BUG_ON(obj->userfault_count); GEM_BUG_ON(!list_empty(&obj->lut_list)); atomic_set(&obj->mm.pages_pin_count, 0); diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object.h b/drivers/gpu/drm/i915/gem/i915_gem_object.h index 2faa481cc18f..e5b9276d254c 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_object.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_object.h @@ -258,10 +258,6 @@ struct page * i915_gem_object_get_page(struct drm_i915_gem_object *obj, unsigned int n); -struct page * -i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, - unsigned int n); - dma_addr_t i915_gem_object_get_dma_address_len(struct drm_i915_gem_object *obj, unsigned long n, @@ -394,6 +390,8 @@ static inline void i915_gem_object_unpin_map(struct drm_i915_gem_object *obj) i915_gem_object_unpin_pages(obj); } +void __i915_gem_object_release_map(struct drm_i915_gem_object *obj); + void i915_gem_object_flush_write_domain(struct drm_i915_gem_object *obj, unsigned int flush_domains); diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object_blt.c b/drivers/gpu/drm/i915/gem/i915_gem_object_blt.c index f457d7130491..bfdb32d46877 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_object_blt.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_object_blt.c @@ -126,6 +126,17 @@ void intel_emit_vma_release(struct intel_context *ce, struct i915_vma *vma) intel_engine_pm_put(ce->engine); } +static int +move_obj_to_gpu(struct drm_i915_gem_object *obj, + struct i915_request *rq, + bool write) +{ + if (obj->cache_dirty & ~obj->cache_coherent) + i915_gem_clflush_object(obj, 0); + + return i915_request_await_object(rq, obj, write); +} + int i915_gem_object_fill_blt(struct drm_i915_gem_object *obj, struct intel_context *ce, u32 value) @@ -143,12 +154,6 @@ int i915_gem_object_fill_blt(struct drm_i915_gem_object *obj, if (unlikely(err)) return err; - if (obj->cache_dirty & ~obj->cache_coherent) { - i915_gem_object_lock(obj); - i915_gem_clflush_object(obj, 0); - i915_gem_object_unlock(obj); - } - batch = intel_emit_vma_fill_blt(ce, vma, value); if (IS_ERR(batch)) { err = PTR_ERR(batch); @@ -165,27 +170,22 @@ int i915_gem_object_fill_blt(struct drm_i915_gem_object *obj, if (unlikely(err)) goto out_request; - err = i915_request_await_object(rq, obj, true); - if (unlikely(err)) - goto out_request; - - if (ce->engine->emit_init_breadcrumb) { - err = ce->engine->emit_init_breadcrumb(rq); - if (unlikely(err)) - goto out_request; - } - i915_vma_lock(vma); - err = i915_request_await_object(rq, vma->obj, true); + err = move_obj_to_gpu(vma->obj, rq, true); if (err == 0) err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE); i915_vma_unlock(vma); if (unlikely(err)) goto out_request; - err = ce->engine->emit_bb_start(rq, - batch->node.start, batch->node.size, - 0); + if (ce->engine->emit_init_breadcrumb) + err = ce->engine->emit_init_breadcrumb(rq); + + if (likely(!err)) + err = ce->engine->emit_bb_start(rq, + batch->node.start, + batch->node.size, + 0); out_request: if (unlikely(err)) i915_request_set_error_once(rq, err); @@ -317,16 +317,6 @@ out_pm: return ERR_PTR(err); } -static int move_to_gpu(struct i915_vma *vma, struct i915_request *rq, bool write) -{ - struct drm_i915_gem_object *obj = vma->obj; - - if (obj->cache_dirty & ~obj->cache_coherent) - i915_gem_clflush_object(obj, 0); - - return i915_request_await_object(rq, obj, write); -} - int i915_gem_object_copy_blt(struct drm_i915_gem_object *src, struct drm_i915_gem_object *dst, struct intel_context *ce) @@ -375,7 +365,7 @@ int i915_gem_object_copy_blt(struct drm_i915_gem_object *src, goto out_request; for (i = 0; i < ARRAY_SIZE(vma); i++) { - err = move_to_gpu(vma[i], rq, i); + err = move_obj_to_gpu(vma[i]->obj, rq, i); if (unlikely(err)) goto out_unlock; } diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object_types.h b/drivers/gpu/drm/i915/gem/i915_gem_object_types.h index 54ee658bb168..5335f799b548 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_object_types.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_object_types.h @@ -61,6 +61,8 @@ struct drm_i915_gem_object_ops { int (*dmabuf_export)(struct drm_i915_gem_object *obj); void (*release)(struct drm_i915_gem_object *obj); + + const char *name; /* friendly name for debug, e.g. lockdep classes */ }; enum i915_mmap_type { @@ -119,6 +121,7 @@ struct drm_i915_gem_object { * this translation from object to context->handles_vma. */ struct list_head lut_list; + spinlock_t lut_lock; /* guards lut_list */ /** Stolen memory for this object, instead of being backed by shmem. */ struct drm_mm_node *stolen; diff --git a/drivers/gpu/drm/i915/gem/i915_gem_pages.c b/drivers/gpu/drm/i915/gem/i915_gem_pages.c index af9e48ee4a33..7050519c87a4 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_pages.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_pages.c @@ -408,6 +408,21 @@ void __i915_gem_object_flush_map(struct drm_i915_gem_object *obj, } } +void __i915_gem_object_release_map(struct drm_i915_gem_object *obj) +{ + GEM_BUG_ON(!obj->mm.mapping); + + /* + * We allow removing the mapping from underneath pinned pages! + * + * Furthermore, since this is an unsafe operation reserved only + * for construction time manipulation, we ignore locking prudence. + */ + unmap_object(obj, page_mask_bits(fetch_and_zero(&obj->mm.mapping))); + + i915_gem_object_unpin_map(obj); +} + struct scatterlist * i915_gem_object_get_sg(struct drm_i915_gem_object *obj, unsigned int n, @@ -533,20 +548,6 @@ i915_gem_object_get_page(struct drm_i915_gem_object *obj, unsigned int n) return nth_page(sg_page(sg), offset); } -/* Like i915_gem_object_get_page(), but mark the returned page dirty */ -struct page * -i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, - unsigned int n) -{ - struct page *page; - - page = i915_gem_object_get_page(obj, n); - if (!obj->mm.dirty) - set_page_dirty(page); - - return page; -} - dma_addr_t i915_gem_object_get_dma_address_len(struct drm_i915_gem_object *obj, unsigned long n, diff --git a/drivers/gpu/drm/i915/gem/i915_gem_phys.c b/drivers/gpu/drm/i915/gem/i915_gem_phys.c index 7fe9831aa9ba..28147aab47b9 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_phys.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_phys.c @@ -27,7 +27,7 @@ static int i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj) void *dst; int i; - if (WARN_ON(i915_gem_object_needs_bit17_swizzle(obj))) + if (GEM_WARN_ON(i915_gem_object_needs_bit17_swizzle(obj))) return -EINVAL; /* @@ -140,6 +140,7 @@ static void phys_release(struct drm_i915_gem_object *obj) } static const struct drm_i915_gem_object_ops i915_gem_phys_ops = { + .name = "i915_gem_object_phys", .get_pages = i915_gem_object_get_pages_phys, .put_pages = i915_gem_object_put_pages_phys, diff --git a/drivers/gpu/drm/i915/gem/i915_gem_shmem.c b/drivers/gpu/drm/i915/gem/i915_gem_shmem.c index 7aff3514d97a..38113d3c0138 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_shmem.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_shmem.c @@ -147,8 +147,7 @@ rebuild_st: last_pfn = page_to_pfn(page); /* Check that the i965g/gm workaround works. */ - drm_WARN_ON(&i915->drm, - (gfp & __GFP_DMA32) && (last_pfn >= 0x00100000UL)); + GEM_BUG_ON(gfp & __GFP_DMA32 && last_pfn >= 0x00100000UL); } if (sg) { /* loop terminated early; short sg table */ sg_page_sizes |= sg->length; @@ -430,6 +429,7 @@ static void shmem_release(struct drm_i915_gem_object *obj) } const struct drm_i915_gem_object_ops i915_gem_shmem_ops = { + .name = "i915_gem_object_shmem", .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE | I915_GEM_OBJECT_IS_SHRINKABLE, diff --git a/drivers/gpu/drm/i915/gem/i915_gem_shrinker.c b/drivers/gpu/drm/i915/gem/i915_gem_shrinker.c index 5b65ce738b16..dc8f052a0ffe 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_shrinker.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_shrinker.c @@ -13,6 +13,8 @@ #include <linux/dma-buf.h> #include <linux/vmalloc.h> +#include "gt/intel_gt_requests.h" + #include "i915_trace.h" static bool swap_available(void) @@ -111,15 +113,6 @@ i915_gem_shrink(struct drm_i915_private *i915, unsigned long count = 0; unsigned long scanned = 0; - /* - * When shrinking the active list, we should also consider active - * contexts. Active contexts are pinned until they are retired, and - * so can not be simply unbound to retire and unpin their pages. To - * shrink the contexts, we must wait until the gpu is idle and - * completed its switch to the kernel context. In short, we do - * not have a good mechanism for idling a specific context. - */ - trace_i915_gem_shrink(i915, target, shrink); /* @@ -134,6 +127,20 @@ i915_gem_shrink(struct drm_i915_private *i915, } /* + * When shrinking the active list, we should also consider active + * contexts. Active contexts are pinned until they are retired, and + * so can not be simply unbound to retire and unpin their pages. To + * shrink the contexts, we must wait until the gpu is idle and + * completed its switch to the kernel context. In short, we do + * not have a good mechanism for idling a specific context, but + * what we can do is give them a kick so that we do not keep idle + * contexts around longer than is necessary. + */ + if (shrink & I915_SHRINK_ACTIVE) + /* Retire requests to unpin all idle contexts */ + intel_gt_retire_requests(&i915->gt); + + /* * As we may completely rewrite the (un)bound list whilst unbinding * (due to retiring requests) we have to strictly process only * one element of the list at the time, and recheck the list @@ -408,26 +415,15 @@ void i915_gem_driver_unregister__shrinker(struct drm_i915_private *i915) void i915_gem_shrinker_taints_mutex(struct drm_i915_private *i915, struct mutex *mutex) { - bool unlock = false; - if (!IS_ENABLED(CONFIG_LOCKDEP)) return; - if (!lockdep_is_held_type(&i915->drm.struct_mutex, -1)) { - mutex_acquire(&i915->drm.struct_mutex.dep_map, - I915_MM_NORMAL, 0, _RET_IP_); - unlock = true; - } - fs_reclaim_acquire(GFP_KERNEL); mutex_acquire(&mutex->dep_map, 0, 0, _RET_IP_); mutex_release(&mutex->dep_map, _RET_IP_); fs_reclaim_release(GFP_KERNEL); - - if (unlock) - mutex_release(&i915->drm.struct_mutex.dep_map, _RET_IP_); } #define obj_to_i915(obj__) to_i915((obj__)->base.dev) diff --git a/drivers/gpu/drm/i915/gem/i915_gem_stolen.c b/drivers/gpu/drm/i915/gem/i915_gem_stolen.c index dc250278bd2c..e0f21f12d3ce 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_stolen.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_stolen.c @@ -566,6 +566,7 @@ i915_gem_object_release_stolen(struct drm_i915_gem_object *obj) } static const struct drm_i915_gem_object_ops i915_gem_object_stolen_ops = { + .name = "i915_gem_object_stolen", .get_pages = i915_gem_object_get_pages_stolen, .put_pages = i915_gem_object_put_pages_stolen, .release = i915_gem_object_release_stolen, diff --git a/drivers/gpu/drm/i915/gem/i915_gem_tiling.c b/drivers/gpu/drm/i915/gem/i915_gem_tiling.c index 0158e49bf9bb..ff72ee2fd9cd 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_tiling.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_tiling.c @@ -299,7 +299,7 @@ i915_gem_object_set_tiling(struct drm_i915_gem_object *obj, i915_gem_object_unlock(obj); /* Force the fence to be reacquired for GTT access */ - i915_gem_object_release_mmap(obj); + i915_gem_object_release_mmap_gtt(obj); /* Try to preallocate memory required to save swizzling on put-pages */ if (i915_gem_object_needs_bit17_swizzle(obj)) { diff --git a/drivers/gpu/drm/i915/gem/i915_gem_userptr.c b/drivers/gpu/drm/i915/gem/i915_gem_userptr.c index c31a6744daee..e946032b13e4 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_userptr.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_userptr.c @@ -21,7 +21,7 @@ struct i915_mm_struct { struct i915_mmu_notifier *mn; struct hlist_node node; struct kref kref; - struct work_struct work; + struct rcu_work work; }; #if defined(CONFIG_MMU_NOTIFIER) @@ -189,40 +189,31 @@ i915_gem_userptr_release__mmu_notifier(struct drm_i915_gem_object *obj) static struct i915_mmu_notifier * i915_mmu_notifier_find(struct i915_mm_struct *mm) { - struct i915_mmu_notifier *mn; - int err = 0; + struct i915_mmu_notifier *mn, *old; + int err; - mn = mm->mn; - if (mn) + mn = READ_ONCE(mm->mn); + if (likely(mn)) return mn; mn = i915_mmu_notifier_create(mm); if (IS_ERR(mn)) - err = PTR_ERR(mn); - - mmap_write_lock(mm->mm); - mutex_lock(&mm->i915->mm_lock); - if (mm->mn == NULL && !err) { - /* Protected by mmap_lock (write-lock) */ - err = __mmu_notifier_register(&mn->mn, mm->mm); - if (!err) { - /* Protected by mm_lock */ - mm->mn = fetch_and_zero(&mn); - } - } else if (mm->mn) { - /* - * Someone else raced and successfully installed the mmu - * notifier, we can cancel our own errors. - */ - err = 0; + return mn; + + err = mmu_notifier_register(&mn->mn, mm->mm); + if (err) { + kfree(mn); + return ERR_PTR(err); } - mutex_unlock(&mm->i915->mm_lock); - mmap_write_unlock(mm->mm); - if (mn && !IS_ERR(mn)) + old = cmpxchg(&mm->mn, NULL, mn); + if (old) { + mmu_notifier_unregister(&mn->mn, mm->mm); kfree(mn); + mn = old; + } - return err ? ERR_PTR(err) : mm->mn; + return mn; } static int @@ -235,7 +226,7 @@ i915_gem_userptr_init__mmu_notifier(struct drm_i915_gem_object *obj, if (flags & I915_USERPTR_UNSYNCHRONIZED) return capable(CAP_SYS_ADMIN) ? 0 : -EPERM; - if (WARN_ON(obj->userptr.mm == NULL)) + if (GEM_WARN_ON(!obj->userptr.mm)) return -EINVAL; mn = i915_mmu_notifier_find(obj->userptr.mm); @@ -301,23 +292,28 @@ i915_mmu_notifier_free(struct i915_mmu_notifier *mn, #endif static struct i915_mm_struct * -__i915_mm_struct_find(struct drm_i915_private *dev_priv, struct mm_struct *real) +__i915_mm_struct_find(struct drm_i915_private *i915, struct mm_struct *real) { - struct i915_mm_struct *mm; - - /* Protected by dev_priv->mm_lock */ - hash_for_each_possible(dev_priv->mm_structs, mm, node, (unsigned long)real) - if (mm->mm == real) - return mm; + struct i915_mm_struct *it, *mm = NULL; + + rcu_read_lock(); + hash_for_each_possible_rcu(i915->mm_structs, + it, node, + (unsigned long)real) + if (it->mm == real && kref_get_unless_zero(&it->kref)) { + mm = it; + break; + } + rcu_read_unlock(); - return NULL; + return mm; } static int i915_gem_userptr_init__mm_struct(struct drm_i915_gem_object *obj) { - struct drm_i915_private *dev_priv = to_i915(obj->base.dev); - struct i915_mm_struct *mm; + struct drm_i915_private *i915 = to_i915(obj->base.dev); + struct i915_mm_struct *mm, *new; int ret = 0; /* During release of the GEM object we hold the struct_mutex. This @@ -330,39 +326,42 @@ i915_gem_userptr_init__mm_struct(struct drm_i915_gem_object *obj) * struct_mutex, i.e. we need to schedule a worker to do the clean * up. */ - mutex_lock(&dev_priv->mm_lock); - mm = __i915_mm_struct_find(dev_priv, current->mm); - if (mm == NULL) { - mm = kmalloc(sizeof(*mm), GFP_KERNEL); - if (mm == NULL) { - ret = -ENOMEM; - goto out; - } + mm = __i915_mm_struct_find(i915, current->mm); + if (mm) + goto out; - kref_init(&mm->kref); - mm->i915 = to_i915(obj->base.dev); + new = kmalloc(sizeof(*mm), GFP_KERNEL); + if (!new) + return -ENOMEM; - mm->mm = current->mm; + kref_init(&new->kref); + new->i915 = to_i915(obj->base.dev); + new->mm = current->mm; + new->mn = NULL; + + spin_lock(&i915->mm_lock); + mm = __i915_mm_struct_find(i915, current->mm); + if (!mm) { + hash_add_rcu(i915->mm_structs, + &new->node, + (unsigned long)new->mm); mmgrab(current->mm); + mm = new; + } + spin_unlock(&i915->mm_lock); + if (mm != new) + kfree(new); - mm->mn = NULL; - - /* Protected by dev_priv->mm_lock */ - hash_add(dev_priv->mm_structs, - &mm->node, (unsigned long)mm->mm); - } else - kref_get(&mm->kref); - - obj->userptr.mm = mm; out: - mutex_unlock(&dev_priv->mm_lock); + obj->userptr.mm = mm; return ret; } static void __i915_mm_struct_free__worker(struct work_struct *work) { - struct i915_mm_struct *mm = container_of(work, typeof(*mm), work); + struct i915_mm_struct *mm = container_of(work, typeof(*mm), work.work); + i915_mmu_notifier_free(mm->mn, mm->mm); mmdrop(mm->mm); kfree(mm); @@ -373,12 +372,12 @@ __i915_mm_struct_free(struct kref *kref) { struct i915_mm_struct *mm = container_of(kref, typeof(*mm), kref); - /* Protected by dev_priv->mm_lock */ - hash_del(&mm->node); - mutex_unlock(&mm->i915->mm_lock); + spin_lock(&mm->i915->mm_lock); + hash_del_rcu(&mm->node); + spin_unlock(&mm->i915->mm_lock); - INIT_WORK(&mm->work, __i915_mm_struct_free__worker); - queue_work(mm->i915->mm.userptr_wq, &mm->work); + INIT_RCU_WORK(&mm->work, __i915_mm_struct_free__worker); + queue_rcu_work(system_wq, &mm->work); } static void @@ -387,9 +386,7 @@ i915_gem_userptr_release__mm_struct(struct drm_i915_gem_object *obj) if (obj->userptr.mm == NULL) return; - kref_put_mutex(&obj->userptr.mm->kref, - __i915_mm_struct_free, - &to_i915(obj->base.dev)->mm_lock); + kref_put(&obj->userptr.mm->kref, __i915_mm_struct_free); obj->userptr.mm = NULL; } @@ -712,6 +709,7 @@ i915_gem_userptr_dmabuf_export(struct drm_i915_gem_object *obj) } static const struct drm_i915_gem_object_ops i915_gem_userptr_ops = { + .name = "i915_gem_object_userptr", .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE | I915_GEM_OBJECT_IS_SHRINKABLE | I915_GEM_OBJECT_NO_MMAP | @@ -850,7 +848,7 @@ i915_gem_userptr_ioctl(struct drm_device *dev, int i915_gem_init_userptr(struct drm_i915_private *dev_priv) { - mutex_init(&dev_priv->mm_lock); + spin_lock_init(&dev_priv->mm_lock); hash_init(dev_priv->mm_structs); dev_priv->mm.userptr_wq = diff --git a/drivers/gpu/drm/i915/gem/selftests/huge_gem_object.c b/drivers/gpu/drm/i915/gem/selftests/huge_gem_object.c index 2b46c6530da9..a768ec61e966 100644 --- a/drivers/gpu/drm/i915/gem/selftests/huge_gem_object.c +++ b/drivers/gpu/drm/i915/gem/selftests/huge_gem_object.c @@ -88,6 +88,7 @@ static void huge_put_pages(struct drm_i915_gem_object *obj, } static const struct drm_i915_gem_object_ops huge_ops = { + .name = "huge-gem", .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE, .get_pages = huge_get_pages, .put_pages = huge_put_pages, diff --git a/drivers/gpu/drm/i915/gem/selftests/huge_pages.c b/drivers/gpu/drm/i915/gem/selftests/huge_pages.c index c9988b6d5c88..8291ede6902c 100644 --- a/drivers/gpu/drm/i915/gem/selftests/huge_pages.c +++ b/drivers/gpu/drm/i915/gem/selftests/huge_pages.c @@ -139,6 +139,7 @@ static void put_huge_pages(struct drm_i915_gem_object *obj, } static const struct drm_i915_gem_object_ops huge_page_ops = { + .name = "huge-gem", .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE | I915_GEM_OBJECT_IS_SHRINKABLE, .get_pages = get_huge_pages, @@ -283,12 +284,14 @@ static void fake_put_huge_pages(struct drm_i915_gem_object *obj, } static const struct drm_i915_gem_object_ops fake_ops = { + .name = "fake-gem", .flags = I915_GEM_OBJECT_IS_SHRINKABLE, .get_pages = fake_get_huge_pages, .put_pages = fake_put_huge_pages, }; static const struct drm_i915_gem_object_ops fake_ops_single = { + .name = "fake-gem", .flags = I915_GEM_OBJECT_IS_SHRINKABLE, .get_pages = fake_get_huge_pages_single, .put_pages = fake_put_huge_pages, @@ -1409,147 +1412,6 @@ out: return err; } -static int igt_ppgtt_pin_update(void *arg) -{ - struct i915_gem_context *ctx = arg; - struct drm_i915_private *dev_priv = ctx->i915; - unsigned long supported = INTEL_INFO(dev_priv)->page_sizes; - struct drm_i915_gem_object *obj; - struct i915_gem_engines_iter it; - struct i915_address_space *vm; - struct intel_context *ce; - struct i915_vma *vma; - unsigned int flags = PIN_USER | PIN_OFFSET_FIXED; - unsigned int n; - int first, last; - int err = 0; - - /* - * Make sure there's no funny business when doing a PIN_UPDATE -- in the - * past we had a subtle issue with being able to incorrectly do multiple - * alloc va ranges on the same object when doing a PIN_UPDATE, which - * resulted in some pretty nasty bugs, though only when using - * huge-gtt-pages. - */ - - vm = i915_gem_context_get_vm_rcu(ctx); - if (!i915_vm_is_4lvl(vm)) { - pr_info("48b PPGTT not supported, skipping\n"); - goto out_vm; - } - - first = ilog2(I915_GTT_PAGE_SIZE_64K); - last = ilog2(I915_GTT_PAGE_SIZE_2M); - - for_each_set_bit_from(first, &supported, last + 1) { - unsigned int page_size = BIT(first); - - obj = i915_gem_object_create_internal(dev_priv, page_size); - if (IS_ERR(obj)) { - err = PTR_ERR(obj); - goto out_vm; - } - - vma = i915_vma_instance(obj, vm, NULL); - if (IS_ERR(vma)) { - err = PTR_ERR(vma); - goto out_put; - } - - err = i915_vma_pin(vma, SZ_2M, 0, flags); - if (err) - goto out_put; - - if (vma->page_sizes.sg < page_size) { - pr_info("Unable to allocate page-size %x, finishing test early\n", - page_size); - goto out_unpin; - } - - err = igt_check_page_sizes(vma); - if (err) - goto out_unpin; - - if (vma->page_sizes.gtt != page_size) { - dma_addr_t addr = i915_gem_object_get_dma_address(obj, 0); - - /* - * The only valid reason for this to ever fail would be - * if the dma-mapper screwed us over when we did the - * dma_map_sg(), since it has the final say over the dma - * address. - */ - if (IS_ALIGNED(addr, page_size)) { - pr_err("page_sizes.gtt=%u, expected=%u\n", - vma->page_sizes.gtt, page_size); - err = -EINVAL; - } else { - pr_info("dma address misaligned, finishing test early\n"); - } - - goto out_unpin; - } - - err = i915_vma_bind(vma, I915_CACHE_NONE, PIN_UPDATE, NULL); - if (err) - goto out_unpin; - - i915_vma_unpin(vma); - i915_gem_object_put(obj); - } - - obj = i915_gem_object_create_internal(dev_priv, PAGE_SIZE); - if (IS_ERR(obj)) { - err = PTR_ERR(obj); - goto out_vm; - } - - vma = i915_vma_instance(obj, vm, NULL); - if (IS_ERR(vma)) { - err = PTR_ERR(vma); - goto out_put; - } - - err = i915_vma_pin(vma, 0, 0, flags); - if (err) - goto out_put; - - /* - * Make sure we don't end up with something like where the pde is still - * pointing to the 2M page, and the pt we just filled-in is dangling -- - * we can check this by writing to the first page where it would then - * land in the now stale 2M page. - */ - - n = 0; - for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) { - if (!intel_engine_can_store_dword(ce->engine)) - continue; - - err = gpu_write(ce, vma, n++, 0xdeadbeaf); - if (err) - break; - } - i915_gem_context_unlock_engines(ctx); - if (err) - goto out_unpin; - - while (n--) { - err = cpu_check(obj, n, 0xdeadbeaf); - if (err) - goto out_unpin; - } - -out_unpin: - i915_vma_unpin(vma); -out_put: - i915_gem_object_put(obj); -out_vm: - i915_vm_put(vm); - - return err; -} - static int igt_tmpfs_fallback(void *arg) { struct i915_gem_context *ctx = arg; @@ -1760,7 +1622,6 @@ int i915_gem_huge_page_live_selftests(struct drm_i915_private *i915) { static const struct i915_subtest tests[] = { SUBTEST(igt_shrink_thp), - SUBTEST(igt_ppgtt_pin_update), SUBTEST(igt_tmpfs_fallback), SUBTEST(igt_ppgtt_smoke_huge), SUBTEST(igt_ppgtt_sanity_check), diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_client_blt.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_client_blt.c index 8fe3ad2ee34e..299c29e9ad86 100644 --- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_client_blt.c +++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_client_blt.c @@ -702,8 +702,5 @@ int i915_gem_client_blt_live_selftests(struct drm_i915_private *i915) if (intel_gt_is_wedged(&i915->gt)) return 0; - if (!HAS_ENGINE(i915, BCS0)) - return 0; - return i915_live_subtests(tests, i915); } diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c index b81978890641..7ffc3c751432 100644 --- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c +++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c @@ -1229,7 +1229,7 @@ __igt_ctx_sseu(struct drm_i915_private *i915, int inst = 0; int ret = 0; - if (INTEL_GEN(i915) < 9 || !RUNTIME_INFO(i915)->sseu.has_slice_pg) + if (INTEL_GEN(i915) < 9) return 0; if (flags & TEST_RESET) @@ -1255,6 +1255,9 @@ __igt_ctx_sseu(struct drm_i915_private *i915, if (hweight32(engine->sseu.slice_mask) < 2) continue; + if (!engine->gt->info.sseu.has_slice_pg) + continue; + /* * Gen11 VME friendly power-gated configuration with * half enabled sub-slices. diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_execbuffer.c index a49016f8ee0d..57c14d3340cd 100644 --- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_execbuffer.c +++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_execbuffer.c @@ -37,20 +37,14 @@ static int __igt_gpu_reloc(struct i915_execbuffer *eb, return err; /* 8-Byte aligned */ - if (!__reloc_entry_gpu(eb, vma, - offsets[0] * sizeof(u32), - 0)) { - err = -EIO; + err = __reloc_entry_gpu(eb, vma, offsets[0] * sizeof(u32), 0); + if (err) goto unpin_vma; - } /* !8-Byte aligned */ - if (!__reloc_entry_gpu(eb, vma, - offsets[1] * sizeof(u32), - 1)) { - err = -EIO; + err = __reloc_entry_gpu(eb, vma, offsets[1] * sizeof(u32), 1); + if (err) goto unpin_vma; - } /* Skip to the end of the cmd page */ i = PAGE_SIZE / sizeof(u32) - RELOC_TAIL - 1; @@ -60,12 +54,9 @@ static int __igt_gpu_reloc(struct i915_execbuffer *eb, eb->reloc_cache.rq_size += i; /* Force batch chaining */ - if (!__reloc_entry_gpu(eb, vma, - offsets[2] * sizeof(u32), - 2)) { - err = -EIO; + err = __reloc_entry_gpu(eb, vma, offsets[2] * sizeof(u32), 2); + if (err) goto unpin_vma; - } GEM_BUG_ON(!eb->reloc_cache.rq); rq = i915_request_get(eb->reloc_cache.rq); diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_object_blt.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_object_blt.c index 31549ad83fa6..23b6e11bbc3e 100644 --- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_object_blt.c +++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_object_blt.c @@ -193,7 +193,7 @@ err_src: } struct igt_thread_arg { - struct drm_i915_private *i915; + struct intel_engine_cs *engine; struct i915_gem_context *ctx; struct file *file; struct rnd_state prng; @@ -203,7 +203,7 @@ struct igt_thread_arg { static int igt_fill_blt_thread(void *arg) { struct igt_thread_arg *thread = arg; - struct drm_i915_private *i915 = thread->i915; + struct intel_engine_cs *engine = thread->engine; struct rnd_state *prng = &thread->prng; struct drm_i915_gem_object *obj; struct i915_gem_context *ctx; @@ -215,7 +215,7 @@ static int igt_fill_blt_thread(void *arg) ctx = thread->ctx; if (!ctx) { - ctx = live_context(i915, thread->file); + ctx = live_context_for_engine(engine, thread->file); if (IS_ERR(ctx)) return PTR_ERR(ctx); @@ -223,7 +223,7 @@ static int igt_fill_blt_thread(void *arg) ctx->sched.priority = I915_USER_PRIORITY(prio); } - ce = i915_gem_context_get_engine(ctx, BCS0); + ce = i915_gem_context_get_engine(ctx, 0); GEM_BUG_ON(IS_ERR(ce)); /* @@ -256,7 +256,7 @@ static int igt_fill_blt_thread(void *arg) pr_debug("%s with phys_sz= %x, sz=%x, val=%x\n", __func__, phys_sz, sz, val); - obj = huge_gem_object(i915, phys_sz, sz); + obj = huge_gem_object(engine->i915, phys_sz, sz); if (IS_ERR(obj)) { err = PTR_ERR(obj); goto err_flush; @@ -321,7 +321,7 @@ err_flush: static int igt_copy_blt_thread(void *arg) { struct igt_thread_arg *thread = arg; - struct drm_i915_private *i915 = thread->i915; + struct intel_engine_cs *engine = thread->engine; struct rnd_state *prng = &thread->prng; struct drm_i915_gem_object *src, *dst; struct i915_gem_context *ctx; @@ -333,7 +333,7 @@ static int igt_copy_blt_thread(void *arg) ctx = thread->ctx; if (!ctx) { - ctx = live_context(i915, thread->file); + ctx = live_context_for_engine(engine, thread->file); if (IS_ERR(ctx)) return PTR_ERR(ctx); @@ -341,7 +341,7 @@ static int igt_copy_blt_thread(void *arg) ctx->sched.priority = I915_USER_PRIORITY(prio); } - ce = i915_gem_context_get_engine(ctx, BCS0); + ce = i915_gem_context_get_engine(ctx, 0); GEM_BUG_ON(IS_ERR(ce)); /* @@ -374,7 +374,7 @@ static int igt_copy_blt_thread(void *arg) pr_debug("%s with phys_sz= %x, sz=%x, val=%x\n", __func__, phys_sz, sz, val); - src = huge_gem_object(i915, phys_sz, sz); + src = huge_gem_object(engine->i915, phys_sz, sz); if (IS_ERR(src)) { err = PTR_ERR(src); goto err_flush; @@ -394,7 +394,7 @@ static int igt_copy_blt_thread(void *arg) if (!(src->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ)) src->cache_dirty = true; - dst = huge_gem_object(i915, phys_sz, sz); + dst = huge_gem_object(engine->i915, phys_sz, sz); if (IS_ERR(dst)) { err = PTR_ERR(dst); goto err_put_src; @@ -456,7 +456,7 @@ err_flush: return err; } -static int igt_threaded_blt(struct drm_i915_private *i915, +static int igt_threaded_blt(struct intel_engine_cs *engine, int (*blt_fn)(void *arg), unsigned int flags) #define SINGLE_CTX BIT(0) @@ -477,14 +477,14 @@ static int igt_threaded_blt(struct drm_i915_private *i915, if (!thread) goto out_tsk; - thread[0].file = mock_file(i915); + thread[0].file = mock_file(engine->i915); if (IS_ERR(thread[0].file)) { err = PTR_ERR(thread[0].file); goto out_thread; } if (flags & SINGLE_CTX) { - thread[0].ctx = live_context(i915, thread[0].file); + thread[0].ctx = live_context_for_engine(engine, thread[0].file); if (IS_ERR(thread[0].ctx)) { err = PTR_ERR(thread[0].ctx); goto out_file; @@ -492,7 +492,7 @@ static int igt_threaded_blt(struct drm_i915_private *i915, } for (i = 0; i < n_cpus; ++i) { - thread[i].i915 = i915; + thread[i].engine = engine; thread[i].file = thread[0].file; thread[i].ctx = thread[0].ctx; thread[i].n_cpus = n_cpus; @@ -532,24 +532,40 @@ out_tsk: return err; } +static int test_copy_engines(struct drm_i915_private *i915, + int (*fn)(void *arg), + unsigned int flags) +{ + struct intel_engine_cs *engine; + int ret; + + for_each_uabi_class_engine(engine, I915_ENGINE_CLASS_COPY, i915) { + ret = igt_threaded_blt(engine, fn, flags); + if (ret) + return ret; + } + + return 0; +} + static int igt_fill_blt(void *arg) { - return igt_threaded_blt(arg, igt_fill_blt_thread, 0); + return test_copy_engines(arg, igt_fill_blt_thread, 0); } static int igt_fill_blt_ctx0(void *arg) { - return igt_threaded_blt(arg, igt_fill_blt_thread, SINGLE_CTX); + return test_copy_engines(arg, igt_fill_blt_thread, SINGLE_CTX); } static int igt_copy_blt(void *arg) { - return igt_threaded_blt(arg, igt_copy_blt_thread, 0); + return test_copy_engines(arg, igt_copy_blt_thread, 0); } static int igt_copy_blt_ctx0(void *arg) { - return igt_threaded_blt(arg, igt_copy_blt_thread, SINGLE_CTX); + return test_copy_engines(arg, igt_copy_blt_thread, SINGLE_CTX); } int i915_gem_object_blt_live_selftests(struct drm_i915_private *i915) @@ -564,9 +580,6 @@ int i915_gem_object_blt_live_selftests(struct drm_i915_private *i915) if (intel_gt_is_wedged(&i915->gt)) return 0; - if (!HAS_ENGINE(i915, BCS0)) - return 0; - return i915_live_subtests(tests, i915); } diff --git a/drivers/gpu/drm/i915/gem/selftests/mock_context.c b/drivers/gpu/drm/i915/gem/selftests/mock_context.c index e7e3c620f542..51b5a3421b40 100644 --- a/drivers/gpu/drm/i915/gem/selftests/mock_context.c +++ b/drivers/gpu/drm/i915/gem/selftests/mock_context.c @@ -23,6 +23,8 @@ mock_context(struct drm_i915_private *i915, INIT_LIST_HEAD(&ctx->link); ctx->i915 = i915; + mutex_init(&ctx->mutex); + spin_lock_init(&ctx->stale.lock); INIT_LIST_HEAD(&ctx->stale.engines); @@ -35,7 +37,7 @@ mock_context(struct drm_i915_private *i915, RCU_INIT_POINTER(ctx->engines, e); INIT_RADIX_TREE(&ctx->handles_vma, GFP_KERNEL); - mutex_init(&ctx->mutex); + mutex_init(&ctx->lut_mutex); if (name) { struct i915_ppgtt *ppgtt; @@ -100,6 +102,43 @@ err_ctx: } struct i915_gem_context * +live_context_for_engine(struct intel_engine_cs *engine, struct file *file) +{ + struct i915_gem_engines *engines; + struct i915_gem_context *ctx; + struct intel_context *ce; + + engines = alloc_engines(1); + if (!engines) + return ERR_PTR(-ENOMEM); + + ctx = live_context(engine->i915, file); + if (IS_ERR(ctx)) { + __free_engines(engines, 0); + return ctx; + } + + ce = intel_context_create(engine); + if (IS_ERR(ce)) { + __free_engines(engines, 0); + return ERR_CAST(ce); + } + + intel_context_set_gem(ce, ctx); + engines->engines[0] = ce; + engines->num_engines = 1; + + mutex_lock(&ctx->engines_mutex); + i915_gem_context_set_user_engines(ctx); + engines = rcu_replace_pointer(ctx->engines, engines, 1); + mutex_unlock(&ctx->engines_mutex); + + engines_idle_release(ctx, engines); + + return ctx; +} + +struct i915_gem_context * kernel_context(struct drm_i915_private *i915) { struct i915_gem_context *ctx; diff --git a/drivers/gpu/drm/i915/gem/selftests/mock_context.h b/drivers/gpu/drm/i915/gem/selftests/mock_context.h index fb83d2f09212..2a6121d33352 100644 --- a/drivers/gpu/drm/i915/gem/selftests/mock_context.h +++ b/drivers/gpu/drm/i915/gem/selftests/mock_context.h @@ -9,6 +9,7 @@ struct file; struct drm_i915_private; +struct intel_engine_cs; void mock_init_contexts(struct drm_i915_private *i915); @@ -21,6 +22,9 @@ void mock_context_close(struct i915_gem_context *ctx); struct i915_gem_context * live_context(struct drm_i915_private *i915, struct file *file); +struct i915_gem_context * +live_context_for_engine(struct intel_engine_cs *engine, struct file *file); + struct i915_gem_context *kernel_context(struct drm_i915_private *i915); void kernel_context_close(struct i915_gem_context *ctx); |