diff options
Diffstat (limited to 'drivers/gpu/drm/i915/selftests')
35 files changed, 1809 insertions, 1018 deletions
diff --git a/drivers/gpu/drm/i915/selftests/i915_active.c b/drivers/gpu/drm/i915/selftests/i915_active.c index 61bf4560d8af..36c3a5460221 100644 --- a/drivers/gpu/drm/i915/selftests/i915_active.c +++ b/drivers/gpu/drm/i915/selftests/i915_active.c @@ -5,6 +5,9 @@ */ #include <linux/kref.h> +#include <linux/string_helpers.h> + +#include <drm/drm_print.h> #include "gem/i915_gem_pm.h" #include "gt/intel_gt.h" @@ -155,9 +158,9 @@ static int live_active_wait(void *arg) __i915_active_wait(&active->base, TASK_UNINTERRUPTIBLE); if (!READ_ONCE(active->retired)) { - struct drm_printer p = drm_err_printer(__func__); + struct drm_printer p = drm_err_printer(&i915->drm, __func__); - pr_err("i915_active not retired after waiting!\n"); + drm_printf(&p, "i915_active not retired after waiting!\n"); i915_active_print(&active->base, &p); err = -EINVAL; @@ -188,9 +191,9 @@ static int live_active_retire(void *arg) err = -EIO; if (!READ_ONCE(active->retired)) { - struct drm_printer p = drm_err_printer(__func__); + struct drm_printer p = drm_err_printer(&i915->drm, __func__); - pr_err("i915_active not retired after flushing!\n"); + drm_printf(&p, "i915_active not retired after flushing!\n"); i915_active_print(&active->base, &p); err = -EINVAL; @@ -254,7 +257,7 @@ int i915_active_live_selftests(struct drm_i915_private *i915) SUBTEST(live_active_barrier), }; - if (intel_gt_is_wedged(&i915->gt)) + if (intel_gt_is_wedged(to_gt(i915))) return 0; return i915_subtests(tests, i915); @@ -280,7 +283,7 @@ void i915_active_print(struct i915_active *ref, struct drm_printer *m) drm_printf(m, "active %ps:%ps\n", ref->active, ref->retire); drm_printf(m, "\tcount: %d\n", atomic_read(&ref->count)); drm_printf(m, "\tpreallocated barriers? %s\n", - yesno(!llist_empty(&ref->preallocated_barriers))); + str_yes_no(!llist_empty(&ref->preallocated_barriers))); if (i915_active_acquire_if_busy(ref)) { struct active_node *it, *n; diff --git a/drivers/gpu/drm/i915/selftests/i915_gem.c b/drivers/gpu/drm/i915/selftests/i915_gem.c index 152d9ab135b1..ad650f67114a 100644 --- a/drivers/gpu/drm/i915/selftests/i915_gem.c +++ b/drivers/gpu/drm/i915/selftests/i915_gem.c @@ -4,11 +4,12 @@ * Copyright © 2018 Intel Corporation */ -#include <linux/random.h> +#include <linux/prandom.h> +#include "gem/i915_gem_internal.h" +#include "gem/i915_gem_pm.h" #include "gem/selftests/igt_gem_utils.h" #include "gem/selftests/mock_context.h" -#include "gem/i915_gem_pm.h" #include "gt/intel_gt.h" #include "gt/intel_gt_pm.h" @@ -41,27 +42,31 @@ static int switch_to_context(struct i915_gem_context *ctx) static void trash_stolen(struct drm_i915_private *i915) { - struct i915_ggtt *ggtt = &i915->ggtt; + struct i915_ggtt *ggtt = to_gt(i915)->ggtt; const u64 slot = ggtt->error_capture.start; - const resource_size_t size = resource_size(&i915->dsm); + const resource_size_t size = resource_size(&i915->dsm.stolen); + struct rnd_state prng; unsigned long page; - u32 prng = 0x12345678; /* XXX: fsck. needs some more thought... */ if (!i915_ggtt_has_aperture(ggtt)) return; + prandom_seed_state(&prng, 0x12345678); + for (page = 0; page < size; page += PAGE_SIZE) { - const dma_addr_t dma = i915->dsm.start + page; + const dma_addr_t dma = i915->dsm.stolen.start + page; u32 __iomem *s; int x; - ggtt->vm.insert_page(&ggtt->vm, dma, slot, I915_CACHE_NONE, 0); + ggtt->vm.insert_page(&ggtt->vm, dma, slot, + i915_gem_get_pat_index(i915, + I915_CACHE_NONE), + 0); s = io_mapping_map_atomic_wc(&ggtt->iomap, slot); for (x = 0; x < PAGE_SIZE / sizeof(u32); x++) { - prng = next_pseudo_random32(prng); - iowrite32(prng, &s[x]); + iowrite32(prandom_u32_state(&prng), &s[x]); } io_mapping_unmap_atomic(s); } @@ -76,7 +81,7 @@ static void simulate_hibernate(struct drm_i915_private *i915) wakeref = intel_runtime_pm_get(&i915->runtime_pm); /* - * As a final sting in the tail, invalidate stolen. Under a real S4, + * As a final string in the tail, invalidate stolen. Under a real S4, * stolen is lost and needs to be refilled on resume. However, under * CI we merely do S4-device testing (as full S4 is too unreliable * for automated testing across a cluster), so to simulate the effect @@ -99,7 +104,7 @@ static void igt_pm_suspend(struct drm_i915_private *i915) intel_wakeref_t wakeref; with_intel_runtime_pm(&i915->runtime_pm, wakeref) { - i915_ggtt_suspend(&i915->ggtt); + i915_ggtt_suspend(to_gt(i915)->ggtt); i915_gem_suspend_late(i915); } } @@ -109,7 +114,7 @@ static void igt_pm_hibernate(struct drm_i915_private *i915) intel_wakeref_t wakeref; with_intel_runtime_pm(&i915->runtime_pm, wakeref) { - i915_ggtt_suspend(&i915->ggtt); + i915_ggtt_suspend(to_gt(i915)->ggtt); i915_gem_freeze(i915); i915_gem_freeze_late(i915); @@ -125,7 +130,9 @@ static void igt_pm_resume(struct drm_i915_private *i915) * that runtime-pm just works. */ with_intel_runtime_pm(&i915->runtime_pm, wakeref) { - i915_ggtt_resume(&i915->ggtt); + i915_ggtt_resume(to_gt(i915)->ggtt); + if (GRAPHICS_VER(i915) >= 8) + setup_private_pat(to_gt(i915)); i915_gem_resume(i915); } } @@ -248,7 +255,7 @@ int i915_gem_live_selftests(struct drm_i915_private *i915) SUBTEST(igt_gem_ww_ctx), }; - if (intel_gt_is_wedged(&i915->gt)) + if (intel_gt_is_wedged(to_gt(i915))) return 0; return i915_live_subtests(tests, i915); diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_evict.c b/drivers/gpu/drm/i915/selftests/i915_gem_evict.c index f99bb0113726..f8fe3681c3dc 100644 --- a/drivers/gpu/drm/i915/selftests/i915_gem_evict.c +++ b/drivers/gpu/drm/i915/selftests/i915_gem_evict.c @@ -22,10 +22,12 @@ * */ +#include "gem/i915_gem_internal.h" #include "gem/i915_gem_pm.h" #include "gem/selftests/igt_gem_utils.h" #include "gem/selftests/mock_context.h" #include "gt/intel_gt.h" +#include "gt/intel_gt_print.h" #include "i915_selftest.h" @@ -117,7 +119,7 @@ static int igt_evict_something(void *arg) /* Everything is pinned, nothing should happen */ mutex_lock(&ggtt->vm.mutex); - err = i915_gem_evict_something(&ggtt->vm, + err = i915_gem_evict_something(&ggtt->vm, NULL, I915_GTT_PAGE_SIZE, 0, 0, 0, U64_MAX, 0); @@ -132,7 +134,7 @@ static int igt_evict_something(void *arg) /* Everything is unpinned, we should be able to evict something */ mutex_lock(&ggtt->vm.mutex); - err = i915_gem_evict_something(&ggtt->vm, + err = i915_gem_evict_something(&ggtt->vm, NULL, I915_GTT_PAGE_SIZE, 0, 0, 0, U64_MAX, 0); @@ -204,7 +206,7 @@ static int igt_evict_for_vma(void *arg) /* Everything is pinned, nothing should happen */ mutex_lock(&ggtt->vm.mutex); - err = i915_gem_evict_for_node(&ggtt->vm, &target, 0); + err = i915_gem_evict_for_node(&ggtt->vm, NULL, &target, 0); mutex_unlock(&ggtt->vm.mutex); if (err != -ENOSPC) { pr_err("i915_gem_evict_for_node on a full GGTT returned err=%d\n", @@ -216,7 +218,7 @@ static int igt_evict_for_vma(void *arg) /* Everything is unpinned, we should be able to evict the node */ mutex_lock(&ggtt->vm.mutex); - err = i915_gem_evict_for_node(&ggtt->vm, &target, 0); + err = i915_gem_evict_for_node(&ggtt->vm, NULL, &target, 0); mutex_unlock(&ggtt->vm.mutex); if (err) { pr_err("i915_gem_evict_for_node returned err=%d\n", @@ -244,7 +246,7 @@ static int igt_evict_for_cache_color(void *arg) struct drm_mm_node target = { .start = I915_GTT_PAGE_SIZE * 2, .size = I915_GTT_PAGE_SIZE, - .color = I915_CACHE_LLC, + .color = i915_gem_get_pat_index(gt->i915, I915_CACHE_LLC), }; struct drm_i915_gem_object *obj; struct i915_vma *vma; @@ -297,7 +299,7 @@ static int igt_evict_for_cache_color(void *arg) /* Remove just the second vma */ mutex_lock(&ggtt->vm.mutex); - err = i915_gem_evict_for_node(&ggtt->vm, &target, 0); + err = i915_gem_evict_for_node(&ggtt->vm, NULL, &target, 0); mutex_unlock(&ggtt->vm.mutex); if (err) { pr_err("[0]i915_gem_evict_for_node returned err=%d\n", err); @@ -307,10 +309,10 @@ static int igt_evict_for_cache_color(void *arg) /* Attempt to remove the first *pinned* vma, by removing the (empty) * neighbour -- this should fail. */ - target.color = I915_CACHE_L3_LLC; + target.color = i915_gem_get_pat_index(gt->i915, I915_CACHE_L3_LLC); mutex_lock(&ggtt->vm.mutex); - err = i915_gem_evict_for_node(&ggtt->vm, &target, 0); + err = i915_gem_evict_for_node(&ggtt->vm, NULL, &target, 0); mutex_unlock(&ggtt->vm.mutex); if (!err) { pr_err("[1]i915_gem_evict_for_node returned err=%d\n", err); @@ -331,6 +333,7 @@ static int igt_evict_vm(void *arg) { struct intel_gt *gt = arg; struct i915_ggtt *ggtt = gt->ggtt; + struct i915_gem_ww_ctx ww; LIST_HEAD(objects); int err; @@ -342,7 +345,7 @@ static int igt_evict_vm(void *arg) /* Everything is pinned, nothing should happen */ mutex_lock(&ggtt->vm.mutex); - err = i915_gem_evict_vm(&ggtt->vm); + err = i915_gem_evict_vm(&ggtt->vm, NULL, NULL); mutex_unlock(&ggtt->vm.mutex); if (err) { pr_err("i915_gem_evict_vm on a full GGTT returned err=%d]\n", @@ -352,9 +355,12 @@ static int igt_evict_vm(void *arg) unpin_ggtt(ggtt); - mutex_lock(&ggtt->vm.mutex); - err = i915_gem_evict_vm(&ggtt->vm); - mutex_unlock(&ggtt->vm.mutex); + for_i915_gem_ww(&ww, err, false) { + mutex_lock(&ggtt->vm.mutex); + err = i915_gem_evict_vm(&ggtt->vm, &ww, NULL); + mutex_unlock(&ggtt->vm.mutex); + } + if (err) { pr_err("i915_gem_evict_vm on a full GGTT returned err=%d]\n", err); @@ -402,7 +408,7 @@ static int igt_evict_contexts(void *arg) /* Reserve a block so that we know we have enough to fit a few rq */ memset(&hole, 0, sizeof(hole)); mutex_lock(&ggtt->vm.mutex); - err = i915_gem_gtt_insert(&ggtt->vm, &hole, + err = i915_gem_gtt_insert(&ggtt->vm, NULL, &hole, PRETEND_GGTT_SIZE, 0, I915_COLOR_UNEVICTABLE, 0, ggtt->vm.total, PIN_NOEVICT); @@ -422,7 +428,7 @@ static int igt_evict_contexts(void *arg) goto out_locked; } - if (i915_gem_gtt_insert(&ggtt->vm, &r->node, + if (i915_gem_gtt_insert(&ggtt->vm, NULL, &r->node, 1ul << 20, 0, I915_COLOR_UNEVICTABLE, 0, ggtt->vm.total, PIN_NOEVICT)) { @@ -442,6 +448,7 @@ static int igt_evict_contexts(void *arg) /* Overfill the GGTT with context objects and so try to evict one. */ for_each_engine(engine, gt, id) { struct i915_sw_fence fence; + struct i915_request *last = NULL; count = 0; onstack_fence_init(&fence); @@ -479,6 +486,9 @@ static int igt_evict_contexts(void *arg) i915_request_add(rq); count++; + if (last) + i915_request_put(last); + last = i915_request_get(rq); err = 0; } while(1); onstack_fence_fini(&fence); @@ -486,6 +496,22 @@ static int igt_evict_contexts(void *arg) count, engine->name); if (err) break; + if (last) { + if (i915_request_wait(last, 0, HZ) < 0) { + err = -EIO; + i915_request_put(last); + pr_err("Failed waiting for last request (on %s)", + engine->name); + break; + } + i915_request_put(last); + } + err = intel_gt_wait_for_idle(engine->gt, HZ * 3); + if (err) { + gt_err(engine->gt, "Failed to idle GT (on %s)", + engine->name); + break; + } } mutex_lock(&ggtt->vm.mutex); @@ -526,7 +552,7 @@ int i915_gem_evict_mock_selftests(void) return -ENOMEM; with_intel_runtime_pm(&i915->runtime_pm, wakeref) - err = i915_subtests(tests, &i915->gt); + err = i915_subtests(tests, to_gt(i915)); mock_destroy_device(i915); return err; @@ -538,8 +564,8 @@ int i915_gem_evict_live_selftests(struct drm_i915_private *i915) SUBTEST(igt_evict_contexts), }; - if (intel_gt_is_wedged(&i915->gt)) + if (intel_gt_is_wedged(to_gt(i915))) return 0; - return intel_gt_live_subtests(tests, &i915->gt); + return intel_gt_live_subtests(tests, to_gt(i915)); } diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c b/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c index f843a5040706..0a86e4857539 100644 --- a/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c +++ b/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c @@ -26,12 +26,17 @@ #include <linux/prime_numbers.h> #include "gem/i915_gem_context.h" +#include "gem/i915_gem_internal.h" +#include "gem/i915_gem_lmem.h" +#include "gem/i915_gem_region.h" #include "gem/selftests/mock_context.h" #include "gt/intel_context.h" #include "gt/intel_gpu_commands.h" +#include "gt/intel_gtt.h" #include "i915_random.h" #include "i915_selftest.h" +#include "i915_vma_resource.h" #include "mock_drm.h" #include "mock_gem_device.h" @@ -56,7 +61,6 @@ static int fake_get_pages(struct drm_i915_gem_object *obj) #define PFN_BIAS 0x1000 struct sg_table *pages; struct scatterlist *sg; - unsigned int sg_page_sizes; typeof(obj->base.size) rem; pages = kmalloc(sizeof(*pages), GFP); @@ -64,12 +68,17 @@ static int fake_get_pages(struct drm_i915_gem_object *obj) return -ENOMEM; rem = round_up(obj->base.size, BIT(31)) >> 31; + /* restricted by sg_alloc_table */ + if (overflows_type(rem, unsigned int)) { + kfree(pages); + return -E2BIG; + } + if (sg_alloc_table(pages, rem, GFP)) { kfree(pages); return -ENOMEM; } - sg_page_sizes = 0; rem = obj->base.size; for (sg = pages->sgl; sg; sg = sg_next(sg)) { unsigned long len = min_t(typeof(rem), rem, BIT(31)); @@ -78,13 +87,12 @@ static int fake_get_pages(struct drm_i915_gem_object *obj) sg_set_page(sg, pfn_to_page(PFN_BIAS), len, 0); sg_dma_address(sg) = page_to_phys(sg_page(sg)); sg_dma_len(sg) = len; - sg_page_sizes |= len; rem -= len; } GEM_BUG_ON(rem); - __i915_gem_object_set_pages(obj, pages, sg_page_sizes); + __i915_gem_object_set_pages(obj, pages); return 0; #undef GFP @@ -127,7 +135,7 @@ fake_dma_object(struct drm_i915_private *i915, u64 size) obj->write_domain = I915_GEM_DOMAIN_CPU; obj->read_domains = I915_GEM_DOMAIN_CPU; - obj->cache_level = I915_CACHE_NONE; + obj->pat_index = i915_gem_get_pat_index(i915, I915_CACHE_NONE); /* Preallocate the "backing storage" */ if (i915_gem_object_pin_pages_unlocked(obj)) @@ -155,12 +163,12 @@ static int igt_ppgtt_alloc(void *arg) if (!HAS_PPGTT(dev_priv)) return 0; - ppgtt = i915_ppgtt_create(&dev_priv->gt); + ppgtt = i915_ppgtt_create(to_gt(dev_priv), 0); if (IS_ERR(ppgtt)) return PTR_ERR(ppgtt); if (!ppgtt->vm.allocate_va_range) - goto err_ppgtt_cleanup; + goto ppgtt_vm_put; /* * While we only allocate the page tables here and so we could @@ -228,7 +236,7 @@ err_ppgtt_cleanup: goto retry; } i915_gem_ww_ctx_fini(&ww); - +ppgtt_vm_put: i915_vm_put(&ppgtt->vm); return err; } @@ -237,12 +245,14 @@ static int lowlevel_hole(struct i915_address_space *vm, u64 hole_start, u64 hole_end, unsigned long end_time) { + const unsigned int min_alignment = + i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM); I915_RND_STATE(seed_prng); - struct i915_vma *mock_vma; + struct i915_vma_resource *mock_vma_res; unsigned int size; - mock_vma = kzalloc(sizeof(*mock_vma), GFP_KERNEL); - if (!mock_vma) + mock_vma_res = kzalloc(sizeof(*mock_vma_res), GFP_KERNEL); + if (!mock_vma_res) return -ENOMEM; /* Keep creating larger objects until one cannot fit into the hole */ @@ -250,9 +260,10 @@ static int lowlevel_hole(struct i915_address_space *vm, I915_RND_SUBSTATE(prng, seed_prng); struct drm_i915_gem_object *obj; unsigned int *order, count, n; - u64 hole_size; + u64 hole_size, aligned_size; - hole_size = (hole_end - hole_start) >> size; + aligned_size = max_t(u32, ilog2(min_alignment), size); + hole_size = (hole_end - hole_start) >> aligned_size; if (hole_size > KMALLOC_MAX_SIZE / sizeof(u32)) hole_size = KMALLOC_MAX_SIZE / sizeof(u32); count = hole_size >> 1; @@ -268,15 +279,16 @@ static int lowlevel_hole(struct i915_address_space *vm, break; } while (count >>= 1); if (!count) { - kfree(mock_vma); + kfree(mock_vma_res); return -ENOMEM; } GEM_BUG_ON(!order); - GEM_BUG_ON(count * BIT_ULL(size) > vm->total); - GEM_BUG_ON(hole_start + count * BIT_ULL(size) > hole_end); + GEM_BUG_ON(count * BIT_ULL(aligned_size) > vm->total); + GEM_BUG_ON(hole_start + count * BIT_ULL(aligned_size) > hole_end); - /* Ignore allocation failures (i.e. don't report them as + /* + * Ignore allocation failures (i.e. don't report them as * a test failure) as we are purposefully allocating very * large objects without checking that we have sufficient * memory. We expect to hit -ENOMEM. @@ -297,10 +309,10 @@ static int lowlevel_hole(struct i915_address_space *vm, } for (n = 0; n < count; n++) { - u64 addr = hole_start + order[n] * BIT_ULL(size); + u64 addr = hole_start + order[n] * BIT_ULL(aligned_size); intel_wakeref_t wakeref; - GEM_BUG_ON(addr + BIT_ULL(size) > vm->total); + GEM_BUG_ON(addr + BIT_ULL(aligned_size) > vm->total); if (igt_timeout(end_time, "%s timed out before %d/%d\n", @@ -342,19 +354,21 @@ alloc_vm_end: break; } - mock_vma->pages = obj->mm.pages; - mock_vma->node.size = BIT_ULL(size); - mock_vma->node.start = addr; + mock_vma_res->bi.pages = obj->mm.pages; + mock_vma_res->node_size = BIT_ULL(aligned_size); + mock_vma_res->start = addr; with_intel_runtime_pm(vm->gt->uncore->rpm, wakeref) - vm->insert_entries(vm, mock_vma, - I915_CACHE_NONE, 0); + vm->insert_entries(vm, mock_vma_res, + i915_gem_get_pat_index(vm->i915, + I915_CACHE_NONE), + 0); } count = n; i915_random_reorder(order, count, &prng); for (n = 0; n < count; n++) { - u64 addr = hole_start + order[n] * BIT_ULL(size); + u64 addr = hole_start + order[n] * BIT_ULL(aligned_size); intel_wakeref_t wakeref; GEM_BUG_ON(addr + BIT_ULL(size) > vm->total); @@ -370,7 +384,7 @@ alloc_vm_end: cleanup_freed_objects(vm->i915); } - kfree(mock_vma); + kfree(mock_vma_res); return 0; } @@ -378,14 +392,14 @@ static void close_object_list(struct list_head *objects, struct i915_address_space *vm) { struct drm_i915_gem_object *obj, *on; - int ignored; + int __maybe_unused ignored; list_for_each_entry_safe(obj, on, objects, st_link) { struct i915_vma *vma; vma = i915_vma_instance(obj, vm, NULL); if (!IS_ERR(vma)) - ignored = i915_vma_unbind(vma); + ignored = i915_vma_unbind_unlocked(vma); list_del(&obj->st_link); i915_gem_object_put(obj); @@ -398,8 +412,10 @@ static int fill_hole(struct i915_address_space *vm, { const u64 hole_size = hole_end - hole_start; struct drm_i915_gem_object *obj; + const unsigned int min_alignment = + i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM); const unsigned long max_pages = - min_t(u64, ULONG_MAX - 1, hole_size/2 >> PAGE_SHIFT); + min_t(u64, ULONG_MAX - 1, (hole_size / 2) >> ilog2(min_alignment)); const unsigned long max_step = max(int_sqrt(max_pages), 2UL); unsigned long npages, prime, flags; struct i915_vma *vma; @@ -431,7 +447,8 @@ static int fill_hole(struct i915_address_space *vm, list_add(&obj->st_link, &objects); - /* Align differing sized objects against the edges, and + /* + * Align differing sized objects against the edges, and * check we don't walk off into the void when binding * them into the GTT. */ @@ -440,14 +457,17 @@ static int fill_hole(struct i915_address_space *vm, offset = p->offset; list_for_each_entry(obj, &objects, st_link) { + u64 aligned_size = round_up(obj->base.size, + min_alignment); + vma = i915_vma_instance(obj, vm, NULL); if (IS_ERR(vma)) continue; if (p->step < 0) { - if (offset < hole_start + obj->base.size) + if (offset < hole_start + aligned_size) break; - offset -= obj->base.size; + offset -= aligned_size; } err = i915_vma_pin(vma, 0, 0, offset | flags); @@ -469,22 +489,25 @@ static int fill_hole(struct i915_address_space *vm, i915_vma_unpin(vma); if (p->step > 0) { - if (offset + obj->base.size > hole_end) + if (offset + aligned_size > hole_end) break; - offset += obj->base.size; + offset += aligned_size; } } offset = p->offset; list_for_each_entry(obj, &objects, st_link) { + u64 aligned_size = round_up(obj->base.size, + min_alignment); + vma = i915_vma_instance(obj, vm, NULL); if (IS_ERR(vma)) continue; if (p->step < 0) { - if (offset < hole_start + obj->base.size) + if (offset < hole_start + aligned_size) break; - offset -= obj->base.size; + offset -= aligned_size; } if (!drm_mm_node_allocated(&vma->node) || @@ -496,7 +519,7 @@ static int fill_hole(struct i915_address_space *vm, goto err; } - err = i915_vma_unbind(vma); + err = i915_vma_unbind_unlocked(vma); if (err) { pr_err("%s(%s) (forward) unbind of vma.node=%llx + %llx failed with err=%d\n", __func__, p->name, vma->node.start, vma->node.size, @@ -505,22 +528,25 @@ static int fill_hole(struct i915_address_space *vm, } if (p->step > 0) { - if (offset + obj->base.size > hole_end) + if (offset + aligned_size > hole_end) break; - offset += obj->base.size; + offset += aligned_size; } } offset = p->offset; list_for_each_entry_reverse(obj, &objects, st_link) { + u64 aligned_size = round_up(obj->base.size, + min_alignment); + vma = i915_vma_instance(obj, vm, NULL); if (IS_ERR(vma)) continue; if (p->step < 0) { - if (offset < hole_start + obj->base.size) + if (offset < hole_start + aligned_size) break; - offset -= obj->base.size; + offset -= aligned_size; } err = i915_vma_pin(vma, 0, 0, offset | flags); @@ -542,22 +568,25 @@ static int fill_hole(struct i915_address_space *vm, i915_vma_unpin(vma); if (p->step > 0) { - if (offset + obj->base.size > hole_end) + if (offset + aligned_size > hole_end) break; - offset += obj->base.size; + offset += aligned_size; } } offset = p->offset; list_for_each_entry_reverse(obj, &objects, st_link) { + u64 aligned_size = round_up(obj->base.size, + min_alignment); + vma = i915_vma_instance(obj, vm, NULL); if (IS_ERR(vma)) continue; if (p->step < 0) { - if (offset < hole_start + obj->base.size) + if (offset < hole_start + aligned_size) break; - offset -= obj->base.size; + offset -= aligned_size; } if (!drm_mm_node_allocated(&vma->node) || @@ -569,7 +598,7 @@ static int fill_hole(struct i915_address_space *vm, goto err; } - err = i915_vma_unbind(vma); + err = i915_vma_unbind_unlocked(vma); if (err) { pr_err("%s(%s) (backward) unbind of vma.node=%llx + %llx failed with err=%d\n", __func__, p->name, vma->node.start, vma->node.size, @@ -578,9 +607,9 @@ static int fill_hole(struct i915_address_space *vm, } if (p->step > 0) { - if (offset + obj->base.size > hole_end) + if (offset + aligned_size > hole_end) break; - offset += obj->base.size; + offset += aligned_size; } } } @@ -610,6 +639,7 @@ static int walk_hole(struct i915_address_space *vm, const u64 hole_size = hole_end - hole_start; const unsigned long max_pages = min_t(u64, ULONG_MAX - 1, hole_size >> PAGE_SHIFT); + unsigned long min_alignment; unsigned long flags; u64 size; @@ -619,6 +649,8 @@ static int walk_hole(struct i915_address_space *vm, if (i915_is_ggtt(vm)) flags |= PIN_GLOBAL; + min_alignment = i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM); + for_each_prime_number_from(size, 1, max_pages) { struct drm_i915_gem_object *obj; struct i915_vma *vma; @@ -637,7 +669,7 @@ static int walk_hole(struct i915_address_space *vm, for (addr = hole_start; addr + obj->base.size < hole_end; - addr += obj->base.size) { + addr += round_up(obj->base.size, min_alignment)) { err = i915_vma_pin(vma, 0, 0, addr | flags); if (err) { pr_err("%s bind failed at %llx + %llx [hole %llx- %llx] with err=%d\n", @@ -655,7 +687,7 @@ static int walk_hole(struct i915_address_space *vm, goto err_put; } - err = i915_vma_unbind(vma); + err = i915_vma_unbind_unlocked(vma); if (err) { pr_err("%s unbind failed at %llx + %llx with err=%d\n", __func__, addr, vma->size, err); @@ -689,6 +721,7 @@ static int pot_hole(struct i915_address_space *vm, { struct drm_i915_gem_object *obj; struct i915_vma *vma; + unsigned int min_alignment; unsigned long flags; unsigned int pot; int err = 0; @@ -697,6 +730,8 @@ static int pot_hole(struct i915_address_space *vm, if (i915_is_ggtt(vm)) flags |= PIN_GLOBAL; + min_alignment = i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM); + obj = i915_gem_object_create_internal(vm->i915, 2 * I915_GTT_PAGE_SIZE); if (IS_ERR(obj)) return PTR_ERR(obj); @@ -709,13 +744,13 @@ static int pot_hole(struct i915_address_space *vm, /* Insert a pair of pages across every pot boundary within the hole */ for (pot = fls64(hole_end - 1) - 1; - pot > ilog2(2 * I915_GTT_PAGE_SIZE); + pot > ilog2(2 * min_alignment); pot--) { u64 step = BIT_ULL(pot); u64 addr; - for (addr = round_up(hole_start + I915_GTT_PAGE_SIZE, step) - I915_GTT_PAGE_SIZE; - addr <= round_down(hole_end - 2*I915_GTT_PAGE_SIZE, step) - I915_GTT_PAGE_SIZE; + for (addr = round_up(hole_start + min_alignment, step) - min_alignment; + hole_end > addr && hole_end - addr >= 2 * min_alignment; addr += step) { err = i915_vma_pin(vma, 0, 0, addr | flags); if (err) { @@ -732,13 +767,13 @@ static int pot_hole(struct i915_address_space *vm, pr_err("%s incorrect at %llx + %llx\n", __func__, addr, vma->size); i915_vma_unpin(vma); - err = i915_vma_unbind(vma); + err = i915_vma_unbind_unlocked(vma); err = -EINVAL; goto err_obj; } i915_vma_unpin(vma); - err = i915_vma_unbind(vma); + err = i915_vma_unbind_unlocked(vma); GEM_BUG_ON(err); } @@ -760,6 +795,7 @@ static int drunk_hole(struct i915_address_space *vm, unsigned long end_time) { I915_RND_STATE(prng); + unsigned int min_alignment; unsigned int size; unsigned long flags; @@ -767,15 +803,18 @@ static int drunk_hole(struct i915_address_space *vm, if (i915_is_ggtt(vm)) flags |= PIN_GLOBAL; + min_alignment = i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM); + /* Keep creating larger objects until one cannot fit into the hole */ for (size = 12; (hole_end - hole_start) >> size; size++) { struct drm_i915_gem_object *obj; unsigned int *order, count, n; struct i915_vma *vma; - u64 hole_size; + u64 hole_size, aligned_size; int err = -ENODEV; - hole_size = (hole_end - hole_start) >> size; + aligned_size = max_t(u32, ilog2(min_alignment), size); + hole_size = (hole_end - hole_start) >> aligned_size; if (hole_size > KMALLOC_MAX_SIZE / sizeof(u32)) hole_size = KMALLOC_MAX_SIZE / sizeof(u32); count = hole_size >> 1; @@ -794,7 +833,8 @@ static int drunk_hole(struct i915_address_space *vm, return -ENOMEM; GEM_BUG_ON(!order); - /* Ignore allocation failures (i.e. don't report them as + /* + * Ignore allocation failures (i.e. don't report them as * a test failure) as we are purposefully allocating very * large objects without checking that we have sufficient * memory. We expect to hit -ENOMEM. @@ -815,7 +855,7 @@ static int drunk_hole(struct i915_address_space *vm, GEM_BUG_ON(vma->size != BIT_ULL(size)); for (n = 0; n < count; n++) { - u64 addr = hole_start + order[n] * BIT_ULL(size); + u64 addr = hole_start + order[n] * BIT_ULL(aligned_size); err = i915_vma_pin(vma, 0, 0, addr | flags); if (err) { @@ -832,13 +872,13 @@ static int drunk_hole(struct i915_address_space *vm, pr_err("%s incorrect at %llx + %llx\n", __func__, addr, BIT_ULL(size)); i915_vma_unpin(vma); - err = i915_vma_unbind(vma); + err = i915_vma_unbind_unlocked(vma); err = -EINVAL; goto err_obj; } i915_vma_unpin(vma); - err = i915_vma_unbind(vma); + err = i915_vma_unbind_unlocked(vma); GEM_BUG_ON(err); if (igt_timeout(end_time, @@ -867,11 +907,14 @@ static int __shrink_hole(struct i915_address_space *vm, { struct drm_i915_gem_object *obj; unsigned long flags = PIN_OFFSET_FIXED | PIN_USER; + unsigned int min_alignment; unsigned int order = 12; LIST_HEAD(objects); int err = 0; u64 addr; + min_alignment = i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM); + /* Keep creating larger objects until one cannot fit into the hole */ for (addr = hole_start; addr < hole_end; ) { struct i915_vma *vma; @@ -906,13 +949,13 @@ static int __shrink_hole(struct i915_address_space *vm, pr_err("%s incorrect at %llx + %llx\n", __func__, addr, size); i915_vma_unpin(vma); - err = i915_vma_unbind(vma); + err = i915_vma_unbind_unlocked(vma); err = -EINVAL; break; } i915_vma_unpin(vma); - addr += size; + addr += round_up(size, min_alignment); /* * Since we are injecting allocation faults at random intervals, @@ -924,7 +967,7 @@ static int __shrink_hole(struct i915_address_space *vm, break; if (igt_timeout(end_time, - "%s timed out at ofset %llx [%llx - %llx]\n", + "%s timed out at offset %llx [%llx - %llx]\n", __func__, addr, hole_start, hole_end)) { err = -EINTR; break; @@ -971,7 +1014,7 @@ static int shrink_boom(struct i915_address_space *vm, /* * Catch the case which shrink_hole seems to miss. The setup here * requires invoking the shrinker as we do the alloc_pt/alloc_pd, while - * ensuring that all vma assiocated with the respective pd/pdp are + * ensuring that all vma associated with the respective pd/pdp are * unpinned at the time. */ @@ -1036,6 +1079,121 @@ err_purge: return err; } +static int misaligned_case(struct i915_address_space *vm, struct intel_memory_region *mr, + u64 addr, u64 size, unsigned long flags) +{ + struct drm_i915_gem_object *obj; + struct i915_vma *vma; + int err = 0; + u64 expected_vma_size, expected_node_size; + bool is_stolen = mr->type == INTEL_MEMORY_STOLEN_SYSTEM || + mr->type == INTEL_MEMORY_STOLEN_LOCAL; + + obj = i915_gem_object_create_region(mr, size, 0, I915_BO_ALLOC_GPU_ONLY); + if (IS_ERR(obj)) { + /* if iGVT-g or DMAR is active, stolen mem will be uninitialized */ + if (PTR_ERR(obj) == -ENODEV && is_stolen) + return 0; + return PTR_ERR(obj); + } + + vma = i915_vma_instance(obj, vm, NULL); + if (IS_ERR(vma)) { + err = PTR_ERR(vma); + goto err_put; + } + + err = i915_vma_pin(vma, 0, 0, addr | flags); + if (err) + goto err_put; + i915_vma_unpin(vma); + + if (!drm_mm_node_allocated(&vma->node)) { + err = -EINVAL; + goto err_put; + } + + if (i915_vma_misplaced(vma, 0, 0, addr | flags)) { + err = -EINVAL; + goto err_put; + } + + /* make sure page_sizes_gtt has been populated before use */ + if (i915_is_ggtt(vm) && intel_vm_no_concurrent_access_wa(vm->i915)) + i915_vma_wait_for_bind(vma); + + expected_vma_size = round_up(size, 1 << (ffs(vma->resource->page_sizes_gtt) - 1)); + expected_node_size = expected_vma_size; + + if (HAS_64K_PAGES(vm->i915) && i915_gem_object_is_lmem(obj)) { + expected_vma_size = round_up(size, I915_GTT_PAGE_SIZE_64K); + expected_node_size = round_up(size, I915_GTT_PAGE_SIZE_64K); + } + + if (vma->size != expected_vma_size || vma->node.size != expected_node_size) { + err = i915_vma_unbind_unlocked(vma); + err = -EBADSLT; + goto err_put; + } + + err = i915_vma_unbind_unlocked(vma); + if (err) + goto err_put; + + GEM_BUG_ON(drm_mm_node_allocated(&vma->node)); + +err_put: + i915_gem_object_put(obj); + cleanup_freed_objects(vm->i915); + return err; +} + +static int misaligned_pin(struct i915_address_space *vm, + u64 hole_start, u64 hole_end, + unsigned long end_time) +{ + struct intel_memory_region *mr; + enum intel_region_id id; + unsigned long flags = PIN_OFFSET_FIXED | PIN_USER; + int err = 0; + u64 hole_size = hole_end - hole_start; + + if (i915_is_ggtt(vm)) + flags |= PIN_GLOBAL; + + for_each_memory_region(mr, vm->i915, id) { + u64 min_alignment = i915_vm_min_alignment(vm, mr->type); + u64 size = min_alignment; + u64 addr = round_down(hole_start + (hole_size / 2), min_alignment); + + /* avoid -ENOSPC on very small hole setups */ + if (hole_size < 3 * min_alignment) + continue; + + /* we can't test < 4k alignment due to flags being encoded in lower bits */ + if (min_alignment != I915_GTT_PAGE_SIZE_4K) { + err = misaligned_case(vm, mr, addr + (min_alignment / 2), size, flags); + /* misaligned should error with -EINVAL*/ + if (!err) + err = -EBADSLT; + if (err != -EINVAL) + return err; + } + + /* test for vma->size expansion to min page size */ + err = misaligned_case(vm, mr, addr, PAGE_SIZE, flags); + if (err) + return err; + + /* test for intermediate size not expanding vma->size for large alignments */ + err = misaligned_case(vm, mr, addr, size / 2, flags); + if (err) + return err; + } + + return 0; +} + static int exercise_ppgtt(struct drm_i915_private *dev_priv, int (*func)(struct i915_address_space *vm, u64 hole_start, u64 hole_end, @@ -1053,13 +1211,13 @@ static int exercise_ppgtt(struct drm_i915_private *dev_priv, if (IS_ERR(file)) return PTR_ERR(file); - ppgtt = i915_ppgtt_create(&dev_priv->gt); + ppgtt = i915_ppgtt_create(to_gt(dev_priv), 0); if (IS_ERR(ppgtt)) { err = PTR_ERR(ppgtt); goto out_free; } GEM_BUG_ON(offset_in_page(ppgtt->vm.total)); - GEM_BUG_ON(!atomic_read(&ppgtt->vm.open)); + assert_vm_alive(&ppgtt->vm); err = func(&ppgtt->vm, 0, ppgtt->vm.total, end_time); @@ -1105,6 +1263,11 @@ static int igt_ppgtt_shrink_boom(void *arg) return exercise_ppgtt(arg, shrink_boom); } +static int igt_ppgtt_misaligned_pin(void *arg) +{ + return exercise_ppgtt(arg, misaligned_pin); +} + static int sort_holes(void *priv, const struct list_head *A, const struct list_head *B) { @@ -1122,7 +1285,7 @@ static int exercise_ggtt(struct drm_i915_private *i915, u64 hole_start, u64 hole_end, unsigned long end_time)) { - struct i915_ggtt *ggtt = &i915->ggtt; + struct i915_ggtt *ggtt = to_gt(i915)->ggtt; u64 hole_start, hole_end, last = 0; struct drm_mm_node *node; IGT_TIMEOUT(end_time); @@ -1177,12 +1340,17 @@ static int igt_ggtt_lowlevel(void *arg) return exercise_ggtt(arg, lowlevel_hole); } +static int igt_ggtt_misaligned_pin(void *arg) +{ + return exercise_ggtt(arg, misaligned_pin); +} + static int igt_ggtt_page(void *arg) { const unsigned int count = PAGE_SIZE/sizeof(u32); I915_RND_STATE(prng); struct drm_i915_private *i915 = arg; - struct i915_ggtt *ggtt = &i915->ggtt; + struct i915_ggtt *ggtt = to_gt(i915)->ggtt; struct drm_i915_gem_object *obj; intel_wakeref_t wakeref; struct drm_mm_node tmp; @@ -1218,7 +1386,10 @@ static int igt_ggtt_page(void *arg) ggtt->vm.insert_page(&ggtt->vm, i915_gem_object_get_dma_address(obj, 0), - offset, I915_CACHE_NONE, 0); + offset, + i915_gem_get_pat_index(i915, + I915_CACHE_NONE), + 0); } order = i915_random_order(count, &prng); @@ -1275,13 +1446,14 @@ static void track_vma_bind(struct i915_vma *vma) __i915_gem_object_pin_pages(obj); - GEM_BUG_ON(vma->pages); + GEM_BUG_ON(atomic_read(&vma->pages_count)); atomic_set(&vma->pages_count, I915_VMA_PAGES_ACTIVE); __i915_gem_object_pin_pages(obj); vma->pages = obj->mm.pages; + vma->resource->bi.pages = vma->pages; mutex_lock(&vma->vm->mutex); - list_add_tail(&vma->vm_link, &vma->vm->bound_list); + list_move_tail(&vma->vm_link, &vma->vm->bound_list); mutex_unlock(&vma->vm->mutex); } @@ -1300,7 +1472,7 @@ static int exercise_mock(struct drm_i915_private *i915, if (!ctx) return -ENOMEM; - vm = i915_gem_context_get_vm_rcu(ctx); + vm = i915_gem_context_get_eb_vm(ctx); err = func(vm, 0, min(vm->total, limit), end_time); i915_vm_put(vm); @@ -1336,6 +1508,33 @@ static int igt_mock_drunk(void *arg) return exercise_mock(ggtt->vm.i915, drunk_hole); } +static int reserve_gtt_with_resource(struct i915_vma *vma, u64 offset) +{ + struct i915_address_space *vm = vma->vm; + struct i915_vma_resource *vma_res; + struct drm_i915_gem_object *obj = vma->obj; + int err; + + vma_res = i915_vma_resource_alloc(); + if (IS_ERR(vma_res)) + return PTR_ERR(vma_res); + + mutex_lock(&vm->mutex); + err = i915_gem_gtt_reserve(vm, NULL, &vma->node, obj->base.size, + offset, + obj->pat_index, + 0); + if (!err) { + i915_vma_resource_init_from_vma(vma_res, vma); + vma->resource = vma_res; + } else { + kfree(vma_res); + } + mutex_unlock(&vm->mutex); + + return err; +} + static int igt_gtt_reserve(void *arg) { struct i915_ggtt *ggtt = arg; @@ -1345,9 +1544,10 @@ static int igt_gtt_reserve(void *arg) u64 total; int err = -ENODEV; - /* i915_gem_gtt_reserve() tries to reserve the precise range + /* + * i915_gem_gtt_reserve() tries to reserve the precise range * for the node, and evicts if it has to. So our test checks that - * it can give us the requsted space and prevent overlaps. + * it can give us the requested space and prevent overlaps. */ /* Start by filling the GGTT */ @@ -1370,20 +1570,13 @@ static int igt_gtt_reserve(void *arg) } list_add(&obj->st_link, &objects); - vma = i915_vma_instance(obj, &ggtt->vm, NULL); if (IS_ERR(vma)) { err = PTR_ERR(vma); goto out; } - mutex_lock(&ggtt->vm.mutex); - err = i915_gem_gtt_reserve(&ggtt->vm, &vma->node, - obj->base.size, - total, - obj->cache_level, - 0); - mutex_unlock(&ggtt->vm.mutex); + err = reserve_gtt_with_resource(vma, total); if (err) { pr_err("i915_gem_gtt_reserve (pass 1) failed at %llu/%llu with err=%d\n", total, ggtt->vm.total, err); @@ -1429,13 +1622,7 @@ static int igt_gtt_reserve(void *arg) goto out; } - mutex_lock(&ggtt->vm.mutex); - err = i915_gem_gtt_reserve(&ggtt->vm, &vma->node, - obj->base.size, - total, - obj->cache_level, - 0); - mutex_unlock(&ggtt->vm.mutex); + err = reserve_gtt_with_resource(vma, total); if (err) { pr_err("i915_gem_gtt_reserve (pass 2) failed at %llu/%llu with err=%d\n", total, ggtt->vm.total, err); @@ -1465,7 +1652,7 @@ static int igt_gtt_reserve(void *arg) goto out; } - err = i915_vma_unbind(vma); + err = i915_vma_unbind_unlocked(vma); if (err) { pr_err("i915_vma_unbind failed with err=%d!\n", err); goto out; @@ -1476,13 +1663,7 @@ static int igt_gtt_reserve(void *arg) 2 * I915_GTT_PAGE_SIZE, I915_GTT_MIN_ALIGNMENT); - mutex_lock(&ggtt->vm.mutex); - err = i915_gem_gtt_reserve(&ggtt->vm, &vma->node, - obj->base.size, - offset, - obj->cache_level, - 0); - mutex_unlock(&ggtt->vm.mutex); + err = reserve_gtt_with_resource(vma, offset); if (err) { pr_err("i915_gem_gtt_reserve (pass 3) failed at %llu/%llu with err=%d\n", total, ggtt->vm.total, err); @@ -1509,6 +1690,31 @@ out: return err; } +static int insert_gtt_with_resource(struct i915_vma *vma) +{ + struct i915_address_space *vm = vma->vm; + struct i915_vma_resource *vma_res; + struct drm_i915_gem_object *obj = vma->obj; + int err; + + vma_res = i915_vma_resource_alloc(); + if (IS_ERR(vma_res)) + return PTR_ERR(vma_res); + + mutex_lock(&vm->mutex); + err = i915_gem_gtt_insert(vm, NULL, &vma->node, obj->base.size, 0, + obj->pat_index, 0, vm->total, 0); + if (!err) { + i915_vma_resource_init_from_vma(vma_res, vma); + vma->resource = vma_res; + } else { + kfree(vma_res); + } + mutex_unlock(&vm->mutex); + + return err; +} + static int igt_gtt_insert(void *arg) { struct i915_ggtt *ggtt = arg; @@ -1545,14 +1751,15 @@ static int igt_gtt_insert(void *arg) u64 total; int err = -ENODEV; - /* i915_gem_gtt_insert() tries to allocate some free space in the GTT + /* + * i915_gem_gtt_insert() tries to allocate some free space in the GTT * to the node, evicting if required. */ /* Check a couple of obviously invalid requests */ for (ii = invalid_insert; ii->size; ii++) { mutex_lock(&ggtt->vm.mutex); - err = i915_gem_gtt_insert(&ggtt->vm, &tmp, + err = i915_gem_gtt_insert(&ggtt->vm, NULL, &tmp, ii->size, ii->alignment, I915_COLOR_UNEVICTABLE, ii->start, ii->end, @@ -1593,12 +1800,7 @@ static int igt_gtt_insert(void *arg) goto out; } - mutex_lock(&ggtt->vm.mutex); - err = i915_gem_gtt_insert(&ggtt->vm, &vma->node, - obj->base.size, 0, obj->cache_level, - 0, ggtt->vm.total, - 0); - mutex_unlock(&ggtt->vm.mutex); + err = insert_gtt_with_resource(vma); if (err == -ENOSPC) { /* maxed out the GGTT space */ i915_gem_object_put(obj); @@ -1647,18 +1849,13 @@ static int igt_gtt_insert(void *arg) GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); offset = vma->node.start; - err = i915_vma_unbind(vma); + err = i915_vma_unbind_unlocked(vma); if (err) { pr_err("i915_vma_unbind failed with err=%d!\n", err); goto out; } - mutex_lock(&ggtt->vm.mutex); - err = i915_gem_gtt_insert(&ggtt->vm, &vma->node, - obj->base.size, 0, obj->cache_level, - 0, ggtt->vm.total, - 0); - mutex_unlock(&ggtt->vm.mutex); + err = insert_gtt_with_resource(vma); if (err) { pr_err("i915_gem_gtt_insert (pass 2) failed at %llu/%llu with err=%d\n", total, ggtt->vm.total, err); @@ -1702,12 +1899,7 @@ static int igt_gtt_insert(void *arg) goto out; } - mutex_lock(&ggtt->vm.mutex); - err = i915_gem_gtt_insert(&ggtt->vm, &vma->node, - obj->base.size, 0, obj->cache_level, - 0, ggtt->vm.total, - 0); - mutex_unlock(&ggtt->vm.mutex); + err = insert_gtt_with_resource(vma); if (err) { pr_err("i915_gem_gtt_insert (pass 3) failed at %llu/%llu with err=%d\n", total, ggtt->vm.total, err); @@ -1737,359 +1929,33 @@ int i915_gem_gtt_mock_selftests(void) SUBTEST(igt_gtt_insert), }; struct drm_i915_private *i915; - struct i915_ggtt *ggtt; + struct intel_gt *gt; int err; i915 = mock_gem_device(); if (!i915) return -ENOMEM; - ggtt = kmalloc(sizeof(*ggtt), GFP_KERNEL); - if (!ggtt) { - err = -ENOMEM; + /* allocate the ggtt */ + err = intel_gt_assign_ggtt(to_gt(i915)); + if (err) goto out_put; - } - mock_init_ggtt(i915, ggtt); - err = i915_subtests(tests, ggtt); + gt = to_gt(i915); + + mock_init_ggtt(gt); + + err = i915_subtests(tests, gt->ggtt); mock_device_flush(i915); i915_gem_drain_freed_objects(i915); - mock_fini_ggtt(ggtt); - kfree(ggtt); + mock_fini_ggtt(gt->ggtt); + out_put: mock_destroy_device(i915); return err; } -static int context_sync(struct intel_context *ce) -{ - struct i915_request *rq; - long timeout; - - rq = intel_context_create_request(ce); - if (IS_ERR(rq)) - return PTR_ERR(rq); - - i915_request_get(rq); - i915_request_add(rq); - - timeout = i915_request_wait(rq, 0, HZ / 5); - i915_request_put(rq); - - return timeout < 0 ? -EIO : 0; -} - -static struct i915_request * -submit_batch(struct intel_context *ce, u64 addr) -{ - struct i915_request *rq; - int err; - - rq = intel_context_create_request(ce); - if (IS_ERR(rq)) - return rq; - - err = 0; - if (rq->engine->emit_init_breadcrumb) /* detect a hang */ - err = rq->engine->emit_init_breadcrumb(rq); - if (err == 0) - err = rq->engine->emit_bb_start(rq, addr, 0, 0); - - if (err == 0) - i915_request_get(rq); - i915_request_add(rq); - - return err ? ERR_PTR(err) : rq; -} - -static u32 *spinner(u32 *batch, int i) -{ - return batch + i * 64 / sizeof(*batch) + 4; -} - -static void end_spin(u32 *batch, int i) -{ - *spinner(batch, i) = MI_BATCH_BUFFER_END; - wmb(); -} - -static int igt_cs_tlb(void *arg) -{ - const unsigned int count = PAGE_SIZE / 64; - const unsigned int chunk_size = count * PAGE_SIZE; - struct drm_i915_private *i915 = arg; - struct drm_i915_gem_object *bbe, *act, *out; - struct i915_gem_engines_iter it; - struct i915_address_space *vm; - struct i915_gem_context *ctx; - struct intel_context *ce; - struct i915_vma *vma; - I915_RND_STATE(prng); - struct file *file; - unsigned int i; - u32 *result; - u32 *batch; - int err = 0; - - /* - * Our mission here is to fool the hardware to execute something - * from scratch as it has not seen the batch move (due to missing - * the TLB invalidate). - */ - - file = mock_file(i915); - if (IS_ERR(file)) - return PTR_ERR(file); - - ctx = live_context(i915, file); - if (IS_ERR(ctx)) { - err = PTR_ERR(ctx); - goto out_unlock; - } - - vm = i915_gem_context_get_vm_rcu(ctx); - if (i915_is_ggtt(vm)) - goto out_vm; - - /* Create two pages; dummy we prefill the TLB, and intended */ - bbe = i915_gem_object_create_internal(i915, PAGE_SIZE); - if (IS_ERR(bbe)) { - err = PTR_ERR(bbe); - goto out_vm; - } - - batch = i915_gem_object_pin_map_unlocked(bbe, I915_MAP_WC); - if (IS_ERR(batch)) { - err = PTR_ERR(batch); - goto out_put_bbe; - } - memset32(batch, MI_BATCH_BUFFER_END, PAGE_SIZE / sizeof(u32)); - i915_gem_object_flush_map(bbe); - i915_gem_object_unpin_map(bbe); - - act = i915_gem_object_create_internal(i915, PAGE_SIZE); - if (IS_ERR(act)) { - err = PTR_ERR(act); - goto out_put_bbe; - } - - /* Track the execution of each request by writing into different slot */ - batch = i915_gem_object_pin_map_unlocked(act, I915_MAP_WC); - if (IS_ERR(batch)) { - err = PTR_ERR(batch); - goto out_put_act; - } - for (i = 0; i < count; i++) { - u32 *cs = batch + i * 64 / sizeof(*cs); - u64 addr = (vm->total - PAGE_SIZE) + i * sizeof(u32); - - GEM_BUG_ON(GRAPHICS_VER(i915) < 6); - cs[0] = MI_STORE_DWORD_IMM_GEN4; - if (GRAPHICS_VER(i915) >= 8) { - cs[1] = lower_32_bits(addr); - cs[2] = upper_32_bits(addr); - cs[3] = i; - cs[4] = MI_NOOP; - cs[5] = MI_BATCH_BUFFER_START_GEN8; - } else { - cs[1] = 0; - cs[2] = lower_32_bits(addr); - cs[3] = i; - cs[4] = MI_NOOP; - cs[5] = MI_BATCH_BUFFER_START; - } - } - - out = i915_gem_object_create_internal(i915, PAGE_SIZE); - if (IS_ERR(out)) { - err = PTR_ERR(out); - goto out_put_batch; - } - i915_gem_object_set_cache_coherency(out, I915_CACHING_CACHED); - - vma = i915_vma_instance(out, vm, NULL); - if (IS_ERR(vma)) { - err = PTR_ERR(vma); - goto out_put_out; - } - - err = i915_vma_pin(vma, 0, 0, - PIN_USER | - PIN_OFFSET_FIXED | - (vm->total - PAGE_SIZE)); - if (err) - goto out_put_out; - GEM_BUG_ON(vma->node.start != vm->total - PAGE_SIZE); - - result = i915_gem_object_pin_map_unlocked(out, I915_MAP_WB); - if (IS_ERR(result)) { - err = PTR_ERR(result); - goto out_put_out; - } - - for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) { - IGT_TIMEOUT(end_time); - unsigned long pass = 0; - - if (!intel_engine_can_store_dword(ce->engine)) - continue; - - while (!__igt_timeout(end_time, NULL)) { - struct i915_vm_pt_stash stash = {}; - struct i915_request *rq; - struct i915_gem_ww_ctx ww; - u64 offset; - - offset = igt_random_offset(&prng, - 0, vm->total - PAGE_SIZE, - chunk_size, PAGE_SIZE); - - memset32(result, STACK_MAGIC, PAGE_SIZE / sizeof(u32)); - - vma = i915_vma_instance(bbe, vm, NULL); - if (IS_ERR(vma)) { - err = PTR_ERR(vma); - goto end; - } - - err = vma->ops->set_pages(vma); - if (err) - goto end; - - i915_gem_ww_ctx_init(&ww, false); -retry: - err = i915_vm_lock_objects(vm, &ww); - if (err) - goto end_ww; - - err = i915_vm_alloc_pt_stash(vm, &stash, chunk_size); - if (err) - goto end_ww; - - err = i915_vm_map_pt_stash(vm, &stash); - if (!err) - vm->allocate_va_range(vm, &stash, offset, chunk_size); - i915_vm_free_pt_stash(vm, &stash); -end_ww: - if (err == -EDEADLK) { - err = i915_gem_ww_ctx_backoff(&ww); - if (!err) - goto retry; - } - i915_gem_ww_ctx_fini(&ww); - if (err) - goto end; - - /* Prime the TLB with the dummy pages */ - for (i = 0; i < count; i++) { - vma->node.start = offset + i * PAGE_SIZE; - vm->insert_entries(vm, vma, I915_CACHE_NONE, 0); - - rq = submit_batch(ce, vma->node.start); - if (IS_ERR(rq)) { - err = PTR_ERR(rq); - goto end; - } - i915_request_put(rq); - } - - vma->ops->clear_pages(vma); - - err = context_sync(ce); - if (err) { - pr_err("%s: dummy setup timed out\n", - ce->engine->name); - goto end; - } - - vma = i915_vma_instance(act, vm, NULL); - if (IS_ERR(vma)) { - err = PTR_ERR(vma); - goto end; - } - - err = vma->ops->set_pages(vma); - if (err) - goto end; - - /* Replace the TLB with target batches */ - for (i = 0; i < count; i++) { - struct i915_request *rq; - u32 *cs = batch + i * 64 / sizeof(*cs); - u64 addr; - - vma->node.start = offset + i * PAGE_SIZE; - vm->insert_entries(vm, vma, I915_CACHE_NONE, 0); - - addr = vma->node.start + i * 64; - cs[4] = MI_NOOP; - cs[6] = lower_32_bits(addr); - cs[7] = upper_32_bits(addr); - wmb(); - - rq = submit_batch(ce, addr); - if (IS_ERR(rq)) { - err = PTR_ERR(rq); - goto end; - } - - /* Wait until the context chain has started */ - if (i == 0) { - while (READ_ONCE(result[i]) && - !i915_request_completed(rq)) - cond_resched(); - } else { - end_spin(batch, i - 1); - } - - i915_request_put(rq); - } - end_spin(batch, count - 1); - - vma->ops->clear_pages(vma); - - err = context_sync(ce); - if (err) { - pr_err("%s: writes timed out\n", - ce->engine->name); - goto end; - } - - for (i = 0; i < count; i++) { - if (result[i] != i) { - pr_err("%s: Write lost on pass %lu, at offset %llx, index %d, found %x, expected %x\n", - ce->engine->name, pass, - offset, i, result[i], i); - err = -EINVAL; - goto end; - } - } - - vm->clear_range(vm, offset, chunk_size); - pass++; - } - } -end: - if (igt_flush_test(i915)) - err = -EIO; - i915_gem_context_unlock_engines(ctx); - i915_gem_object_unpin_map(out); -out_put_out: - i915_gem_object_put(out); -out_put_batch: - i915_gem_object_unpin_map(act); -out_put_act: - i915_gem_object_put(act); -out_put_bbe: - i915_gem_object_put(bbe); -out_vm: - i915_vm_put(vm); -out_unlock: - fput(file); - return err; -} - int i915_gem_gtt_live_selftests(struct drm_i915_private *i915) { static const struct i915_subtest tests[] = { @@ -2101,16 +1967,17 @@ int i915_gem_gtt_live_selftests(struct drm_i915_private *i915) SUBTEST(igt_ppgtt_fill), SUBTEST(igt_ppgtt_shrink), SUBTEST(igt_ppgtt_shrink_boom), + SUBTEST(igt_ppgtt_misaligned_pin), SUBTEST(igt_ggtt_lowlevel), SUBTEST(igt_ggtt_drunk), SUBTEST(igt_ggtt_walk), SUBTEST(igt_ggtt_pot), SUBTEST(igt_ggtt_fill), SUBTEST(igt_ggtt_page), - SUBTEST(igt_cs_tlb), + SUBTEST(igt_ggtt_misaligned_pin), }; - GEM_BUG_ON(offset_in_page(i915->ggtt.vm.total)); + GEM_BUG_ON(offset_in_page(to_gt(i915)->ggtt->vm.total)); - return i915_subtests(tests, i915); + return i915_live_subtests(tests, i915); } diff --git a/drivers/gpu/drm/i915/selftests/i915_live_selftests.h b/drivers/gpu/drm/i915/selftests/i915_live_selftests.h index a92c0e9b7e6b..5aee6c9a8295 100644 --- a/drivers/gpu/drm/i915/selftests/i915_live_selftests.h +++ b/drivers/gpu/drm/i915/selftests/i915_live_selftests.h @@ -25,7 +25,9 @@ selftest(gt_lrc, intel_lrc_live_selftests) selftest(gt_mocs, intel_mocs_live_selftests) selftest(gt_pm, intel_gt_pm_live_selftests) selftest(gt_heartbeat, intel_heartbeat_live_selftests) +selftest(gt_tlb, intel_tlb_live_selftests) selftest(requests, i915_request_live_selftests) +selftest(migrate, intel_migrate_live_selftests) selftest(active, i915_active_live_selftests) selftest(objects, i915_gem_object_live_selftests) selftest(mman, i915_gem_mman_live_selftests) @@ -37,14 +39,17 @@ selftest(gem, i915_gem_live_selftests) selftest(evict, i915_gem_evict_live_selftests) selftest(hugepages, i915_gem_huge_page_live_selftests) selftest(gem_contexts, i915_gem_context_live_selftests) -selftest(gem_execbuf, i915_gem_execbuffer_live_selftests) -selftest(blt, i915_gem_object_blt_live_selftests) selftest(client, i915_gem_client_blt_live_selftests) +selftest(gem_migrate, i915_gem_migrate_live_selftests) selftest(reset, intel_reset_live_selftests) selftest(memory_region, intel_memory_region_live_selftests) selftest(hangcheck, intel_hangcheck_live_selftests) selftest(execlists, intel_execlists_live_selftests) selftest(ring_submission, intel_ring_submission_live_selftests) selftest(perf, i915_perf_live_selftests) +selftest(slpc, intel_slpc_live_selftests) +selftest(guc, intel_guc_live_selftests) +selftest(guc_multi_lrc, intel_guc_multi_lrc_live_selftests) +selftest(guc_hang, intel_guc_hang_check) /* Here be dragons: keep last to run last! */ selftest(late_gt_pm, intel_gt_pm_late_selftests) diff --git a/drivers/gpu/drm/i915/selftests/i915_mock_selftests.h b/drivers/gpu/drm/i915/selftests/i915_mock_selftests.h index 34e5caf38093..0c22e0fc9059 100644 --- a/drivers/gpu/drm/i915/selftests/i915_mock_selftests.h +++ b/drivers/gpu/drm/i915/selftests/i915_mock_selftests.h @@ -32,5 +32,4 @@ selftest(vma, i915_vma_mock_selftests) selftest(evict, i915_gem_evict_mock_selftests) selftest(gtt, i915_gem_gtt_mock_selftests) selftest(hugepages, i915_gem_huge_page_mock_selftests) -selftest(contexts, i915_gem_context_mock_selftests) selftest(memory_region, intel_memory_region_mock_selftests) diff --git a/drivers/gpu/drm/i915/selftests/i915_perf.c b/drivers/gpu/drm/i915/selftests/i915_perf.c index 9e9a6cb1d9e5..403134a7acec 100644 --- a/drivers/gpu/drm/i915/selftests/i915_perf.c +++ b/drivers/gpu/drm/i915/selftests/i915_perf.c @@ -28,7 +28,7 @@ alloc_empty_config(struct i915_perf *perf) oa_config->perf = perf; kref_init(&oa_config->ref); - strlcpy(oa_config->uuid, TEST_OA_CONFIG_UUID, sizeof(oa_config->uuid)); + strscpy(oa_config->uuid, TEST_OA_CONFIG_UUID, sizeof(oa_config->uuid)); mutex_lock(&perf->metrics_lock); @@ -102,6 +102,12 @@ test_stream(struct i915_perf *perf) I915_OA_FORMAT_A32u40_A4u32_B8_C8 : I915_OA_FORMAT_C4_B8, }; struct i915_perf_stream *stream; + struct intel_gt *gt; + + if (!props.engine) + return NULL; + + gt = props.engine->gt; if (!oa_config) return NULL; @@ -116,12 +122,12 @@ test_stream(struct i915_perf *perf) stream->perf = perf; - mutex_lock(&perf->lock); + mutex_lock(>->perf.lock); if (i915_oa_stream_init(stream, ¶m, &props)) { kfree(stream); stream = NULL; } - mutex_unlock(&perf->lock); + mutex_unlock(>->perf.lock); i915_oa_config_put(oa_config); @@ -130,11 +136,11 @@ test_stream(struct i915_perf *perf) static void stream_destroy(struct i915_perf_stream *stream) { - struct i915_perf *perf = stream->perf; + struct intel_gt *gt = stream->engine->gt; - mutex_lock(&perf->lock); + mutex_lock(>->perf.lock); i915_perf_destroy_locked(stream); - mutex_unlock(&perf->lock); + mutex_unlock(>->perf.lock); } static int live_sanitycheck(void *arg) @@ -162,7 +168,7 @@ static int write_timestamp(struct i915_request *rq, int slot) return PTR_ERR(cs); len = 5; - if (GRAPHICS_VER(rq->engine->i915) >= 8) + if (GRAPHICS_VER(rq->i915) >= 8) len++; *cs++ = GFX_OP_PIPE_CONTROL(len); @@ -424,14 +430,14 @@ int i915_perf_live_selftests(struct drm_i915_private *i915) if (!perf->metrics_kobj || !perf->ops.enable_metric_set) return 0; - if (intel_gt_is_wedged(&i915->gt)) + if (intel_gt_is_wedged(to_gt(i915))) return 0; err = alloc_empty_config(&i915->perf); if (err) return err; - err = i915_subtests(tests, i915); + err = i915_live_subtests(tests, i915); destroy_empty_config(&i915->perf); diff --git a/drivers/gpu/drm/i915/selftests/i915_perf_selftests.h b/drivers/gpu/drm/i915/selftests/i915_perf_selftests.h index c2389f8a257d..058450d351f7 100644 --- a/drivers/gpu/drm/i915/selftests/i915_perf_selftests.h +++ b/drivers/gpu/drm/i915/selftests/i915_perf_selftests.h @@ -17,5 +17,5 @@ */ selftest(engine_cs, intel_engine_cs_perf_selftests) selftest(request, i915_request_perf_selftests) -selftest(blt, i915_gem_object_blt_perf_selftests) +selftest(migrate, intel_migrate_perf_selftests) selftest(region, intel_memory_region_perf_selftests) diff --git a/drivers/gpu/drm/i915/selftests/i915_random.h b/drivers/gpu/drm/i915/selftests/i915_random.h index 05364eca20f7..70330a2e80f2 100644 --- a/drivers/gpu/drm/i915/selftests/i915_random.h +++ b/drivers/gpu/drm/i915/selftests/i915_random.h @@ -26,7 +26,7 @@ #define __I915_SELFTESTS_RANDOM_H__ #include <linux/math64.h> -#include <linux/random.h> +#include <linux/prandom.h> #include "../i915_selftest.h" diff --git a/drivers/gpu/drm/i915/selftests/i915_request.c b/drivers/gpu/drm/i915/selftests/i915_request.c index bd5c96a77ba3..1260601bda1f 100644 --- a/drivers/gpu/drm/i915/selftests/i915_request.c +++ b/drivers/gpu/drm/i915/selftests/i915_request.c @@ -22,13 +22,15 @@ * */ -#include <linux/prime_numbers.h> #include <linux/pm_qos.h> +#include <linux/prime_numbers.h> #include <linux/sort.h> +#include <drm/drm_print.h> + +#include "gem/i915_gem_internal.h" #include "gem/i915_gem_pm.h" #include "gem/selftests/mock_context.h" - #include "gt/intel_engine_heartbeat.h" #include "gt/intel_engine_pm.h" #include "gt/intel_engine_user.h" @@ -39,11 +41,11 @@ #include "i915_random.h" #include "i915_selftest.h" +#include "i915_wait_util.h" #include "igt_flush_test.h" #include "igt_live_test.h" #include "igt_spinner.h" #include "lib_sw_fence.h" - #include "mock_drm.h" #include "mock_gem_device.h" @@ -72,8 +74,8 @@ static int igt_add_request(void *arg) /* Basic preliminary test to create a request and let it loose! */ request = mock_request(rcs0(i915)->kernel_context, HZ / 10); - if (!request) - return -ENOMEM; + if (IS_ERR(request)) + return PTR_ERR(request); i915_request_add(request); @@ -90,8 +92,8 @@ static int igt_wait_request(void *arg) /* Submit a request, then wait upon it */ request = mock_request(rcs0(i915)->kernel_context, T); - if (!request) - return -ENOMEM; + if (IS_ERR(request)) + return PTR_ERR(request); i915_request_get(request); @@ -159,8 +161,8 @@ static int igt_fence_wait(void *arg) /* Submit a request, treat it as a fence and wait upon it */ request = mock_request(rcs0(i915)->kernel_context, T); - if (!request) - return -ENOMEM; + if (IS_ERR(request)) + return PTR_ERR(request); if (dma_fence_wait_timeout(&request->fence, false, T) != -ETIME) { pr_err("fence wait success before submit (expected timeout)!\n"); @@ -209,13 +211,17 @@ static int igt_request_rewind(void *arg) int err = -EINVAL; ctx[0] = mock_context(i915, "A"); + if (!ctx[0]) { + err = -ENOMEM; + goto err_ctx_0; + } ce = i915_gem_context_get_engine(ctx[0], RCS0); GEM_BUG_ON(IS_ERR(ce)); request = mock_request(ce, 2 * HZ); intel_context_put(ce); - if (!request) { - err = -ENOMEM; + if (IS_ERR(request)) { + err = PTR_ERR(request); goto err_context_0; } @@ -223,13 +229,17 @@ static int igt_request_rewind(void *arg) i915_request_add(request); ctx[1] = mock_context(i915, "B"); + if (!ctx[1]) { + err = -ENOMEM; + goto err_ctx_1; + } ce = i915_gem_context_get_engine(ctx[1], RCS0); GEM_BUG_ON(IS_ERR(ce)); vip = mock_request(ce, 0); intel_context_put(ce); - if (!vip) { - err = -ENOMEM; + if (IS_ERR(vip)) { + err = PTR_ERR(vip); goto err_context_1; } @@ -261,9 +271,11 @@ err: i915_request_put(vip); err_context_1: mock_context_close(ctx[1]); +err_ctx_1: i915_request_put(request); err_context_0: mock_context_close(ctx[0]); +err_ctx_0: mock_device_flush(i915); return err; } @@ -288,9 +300,18 @@ __live_request_alloc(struct intel_context *ce) return intel_context_create_request(ce); } -static int __igt_breadcrumbs_smoketest(void *arg) +struct smoke_thread { + struct kthread_worker *worker; + struct kthread_work work; + struct smoketest *t; + bool stop; + int result; +}; + +static void __igt_breadcrumbs_smoketest(struct kthread_work *work) { - struct smoketest *t = arg; + struct smoke_thread *thread = container_of(work, typeof(*thread), work); + struct smoketest *t = thread->t; const unsigned int max_batch = min(t->ncontexts, t->max_batch) - 1; const unsigned int total = 4 * t->ncontexts + 1; unsigned int num_waits = 0, num_fences = 0; @@ -309,8 +330,10 @@ static int __igt_breadcrumbs_smoketest(void *arg) */ requests = kcalloc(total, sizeof(*requests), GFP_KERNEL); - if (!requests) - return -ENOMEM; + if (!requests) { + thread->result = -ENOMEM; + return; + } order = i915_random_order(total, &prng); if (!order) { @@ -318,7 +341,7 @@ static int __igt_breadcrumbs_smoketest(void *arg) goto out_requests; } - while (!kthread_should_stop()) { + while (!READ_ONCE(thread->stop)) { struct i915_sw_fence *submit, *wait; unsigned int n, count; @@ -426,7 +449,7 @@ static int __igt_breadcrumbs_smoketest(void *arg) kfree(order); out_requests: kfree(requests); - return err; + thread->result = err; } static int mock_breadcrumbs_smoketest(void *arg) @@ -439,7 +462,7 @@ static int mock_breadcrumbs_smoketest(void *arg) .request_alloc = __mock_request_alloc }; unsigned int ncpus = num_online_cpus(); - struct task_struct **threads; + struct smoke_thread *threads; unsigned int n; int ret = 0; @@ -468,28 +491,37 @@ static int mock_breadcrumbs_smoketest(void *arg) } for (n = 0; n < ncpus; n++) { - threads[n] = kthread_run(__igt_breadcrumbs_smoketest, - &t, "igt/%d", n); - if (IS_ERR(threads[n])) { - ret = PTR_ERR(threads[n]); + struct kthread_worker *worker; + + worker = kthread_run_worker(0, "igt/%d", n); + if (IS_ERR(worker)) { + ret = PTR_ERR(worker); ncpus = n; break; } - get_task_struct(threads[n]); + threads[n].worker = worker; + threads[n].t = &t; + threads[n].stop = false; + threads[n].result = 0; + + kthread_init_work(&threads[n].work, + __igt_breadcrumbs_smoketest); + kthread_queue_work(worker, &threads[n].work); } - yield(); /* start all threads before we begin */ msleep(jiffies_to_msecs(i915_selftest.timeout_jiffies)); for (n = 0; n < ncpus; n++) { int err; - err = kthread_stop(threads[n]); + WRITE_ONCE(threads[n].stop, true); + kthread_flush_work(&threads[n].work); + err = READ_ONCE(threads[n].result); if (err < 0 && !ret) ret = err; - put_task_struct(threads[n]); + kthread_destroy_worker(threads[n].worker); } pr_info("Completed %lu waits for %lu fence across %d cpus\n", atomic_long_read(&t.num_waits), @@ -772,6 +804,115 @@ out_spin: return err; } +/* + * Test to prove a non-preemptable request can be cancelled and a subsequent + * request on the same context can successfully complete after cancellation. + * + * Testing methodology is to create a non-preemptible request and submit it, + * wait for spinner to start, create a NOP request and submit it, cancel the + * spinner, wait for spinner to complete and verify it failed with an error, + * finally wait for NOP request to complete verify it succeeded without an + * error. Preemption timeout also reduced / restored so test runs in a timely + * maner. + */ +static int __cancel_reset(struct drm_i915_private *i915, + struct intel_engine_cs *engine) +{ + struct intel_context *ce; + struct igt_spinner spin; + struct i915_request *rq, *nop; + unsigned long preempt_timeout_ms; + int err = 0; + + if (!CONFIG_DRM_I915_PREEMPT_TIMEOUT || + !intel_has_reset_engine(engine->gt)) + return 0; + + preempt_timeout_ms = engine->props.preempt_timeout_ms; + engine->props.preempt_timeout_ms = 100; + + if (igt_spinner_init(&spin, engine->gt)) + goto out_restore; + + ce = intel_context_create(engine); + if (IS_ERR(ce)) { + err = PTR_ERR(ce); + goto out_spin; + } + + rq = igt_spinner_create_request(&spin, ce, MI_NOOP); + if (IS_ERR(rq)) { + err = PTR_ERR(rq); + goto out_ce; + } + + pr_debug("%s: Cancelling active non-preemptable request\n", + engine->name); + i915_request_get(rq); + i915_request_add(rq); + if (!igt_wait_for_spinner(&spin, rq)) { + struct drm_printer p = drm_info_printer(engine->i915->drm.dev); + + pr_err("Failed to start spinner on %s\n", engine->name); + intel_engine_dump(engine, &p, "%s\n", engine->name); + err = -ETIME; + goto out_rq; + } + + nop = intel_context_create_request(ce); + if (IS_ERR(nop)) + goto out_rq; + i915_request_get(nop); + i915_request_add(nop); + + i915_request_cancel(rq, -EINTR); + + if (i915_request_wait(rq, 0, HZ) < 0) { + struct drm_printer p = drm_info_printer(engine->i915->drm.dev); + + pr_err("%s: Failed to cancel hung request\n", engine->name); + intel_engine_dump(engine, &p, "%s\n", engine->name); + err = -ETIME; + goto out_nop; + } + + if (rq->fence.error != -EINTR) { + pr_err("%s: fence not cancelled (%u)\n", + engine->name, rq->fence.error); + err = -EINVAL; + goto out_nop; + } + + if (i915_request_wait(nop, 0, HZ) < 0) { + struct drm_printer p = drm_info_printer(engine->i915->drm.dev); + + pr_err("%s: Failed to complete nop request\n", engine->name); + intel_engine_dump(engine, &p, "%s\n", engine->name); + err = -ETIME; + goto out_nop; + } + + if (nop->fence.error != 0) { + pr_err("%s: Nop request errored (%u)\n", + engine->name, nop->fence.error); + err = -EINVAL; + } + +out_nop: + i915_request_put(nop); +out_rq: + i915_request_put(rq); +out_ce: + intel_context_put(ce); +out_spin: + igt_spinner_fini(&spin); +out_restore: + engine->props.preempt_timeout_ms = preempt_timeout_ms; + if (err) + pr_err("%s: %s error %d\n", __func__, engine->name, err); + return err; +} + static int live_cancel_request(void *arg) { struct drm_i915_private *i915 = arg; @@ -804,23 +945,31 @@ static int live_cancel_request(void *arg) return err; if (err2) return err2; + + /* Expects reset so call outside of igt_live_test_* */ + err = __cancel_reset(i915, engine); + if (err) + return err; + + if (igt_flush_test(i915)) + return -EIO; } return 0; } -static struct i915_vma *empty_batch(struct drm_i915_private *i915) +static struct i915_vma *empty_batch(struct intel_gt *gt) { struct drm_i915_gem_object *obj; struct i915_vma *vma; u32 *cmd; int err; - obj = i915_gem_object_create_internal(i915, PAGE_SIZE); + obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE); if (IS_ERR(obj)) return ERR_CAST(obj); - cmd = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB); + cmd = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC); if (IS_ERR(cmd)) { err = PTR_ERR(cmd); goto err; @@ -831,19 +980,19 @@ static struct i915_vma *empty_batch(struct drm_i915_private *i915) __i915_gem_object_flush_map(obj, 0, 64); i915_gem_object_unpin_map(obj); - intel_gt_chipset_flush(&i915->gt); + intel_gt_chipset_flush(gt); - vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL); + vma = i915_vma_instance(obj, gt->vm, NULL); if (IS_ERR(vma)) { err = PTR_ERR(vma); goto err; } - err = i915_vma_pin(vma, 0, 0, PIN_USER | PIN_GLOBAL); + err = i915_vma_pin(vma, 0, 0, PIN_USER); if (err) goto err; - /* Force the wait wait now to avoid including it in the benchmark */ + /* Force the wait now to avoid including it in the benchmark */ err = i915_vma_sync(vma); if (err) goto err_pin; @@ -857,6 +1006,14 @@ err: return ERR_PTR(err); } +static int emit_bb_start(struct i915_request *rq, struct i915_vma *batch) +{ + return rq->engine->emit_bb_start(rq, + i915_vma_offset(batch), + i915_vma_size(batch), + 0); +} + static struct i915_request * empty_request(struct intel_engine_cs *engine, struct i915_vma *batch) @@ -868,10 +1025,7 @@ empty_request(struct intel_engine_cs *engine, if (IS_ERR(request)) return request; - err = engine->emit_bb_start(request, - batch->node.start, - batch->node.size, - I915_DISPATCH_SECURE); + err = emit_bb_start(request, batch); if (err) goto out_request; @@ -886,8 +1040,7 @@ static int live_empty_request(void *arg) struct drm_i915_private *i915 = arg; struct intel_engine_cs *engine; struct igt_live_test t; - struct i915_vma *batch; - int err = 0; + int err; /* * Submit various sized batches of empty requests, to each engine @@ -895,16 +1048,17 @@ static int live_empty_request(void *arg) * the overhead of submitting requests to the hardware. */ - batch = empty_batch(i915); - if (IS_ERR(batch)) - return PTR_ERR(batch); - for_each_uabi_engine(engine, i915) { IGT_TIMEOUT(end_time); struct i915_request *request; + struct i915_vma *batch; unsigned long n, prime; ktime_t times[2] = {}; + batch = empty_batch(engine->gt); + if (IS_ERR(batch)) + return PTR_ERR(batch); + err = igt_live_test_begin(&t, i915, __func__, engine->name); if (err) goto out_batch; @@ -952,27 +1106,29 @@ static int live_empty_request(void *arg) engine->name, ktime_to_ns(times[0]), prime, div64_u64(ktime_to_ns(times[1]), prime)); +out_batch: + i915_vma_unpin(batch); + i915_vma_put(batch); + if (err) + break; } -out_batch: - i915_vma_unpin(batch); - i915_vma_put(batch); return err; } -static struct i915_vma *recursive_batch(struct drm_i915_private *i915) +static struct i915_vma *recursive_batch(struct intel_gt *gt) { struct drm_i915_gem_object *obj; - const int ver = GRAPHICS_VER(i915); + const int ver = GRAPHICS_VER(gt->i915); struct i915_vma *vma; u32 *cmd; int err; - obj = i915_gem_object_create_internal(i915, PAGE_SIZE); + obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE); if (IS_ERR(obj)) return ERR_CAST(obj); - vma = i915_vma_instance(obj, i915->gt.vm, NULL); + vma = i915_vma_instance(obj, gt->vm, NULL); if (IS_ERR(vma)) { err = PTR_ERR(vma); goto err; @@ -990,21 +1146,21 @@ static struct i915_vma *recursive_batch(struct drm_i915_private *i915) if (ver >= 8) { *cmd++ = MI_BATCH_BUFFER_START | 1 << 8 | 1; - *cmd++ = lower_32_bits(vma->node.start); - *cmd++ = upper_32_bits(vma->node.start); + *cmd++ = lower_32_bits(i915_vma_offset(vma)); + *cmd++ = upper_32_bits(i915_vma_offset(vma)); } else if (ver >= 6) { *cmd++ = MI_BATCH_BUFFER_START | 1 << 8; - *cmd++ = lower_32_bits(vma->node.start); + *cmd++ = lower_32_bits(i915_vma_offset(vma)); } else { *cmd++ = MI_BATCH_BUFFER_START | MI_BATCH_GTT; - *cmd++ = lower_32_bits(vma->node.start); + *cmd++ = lower_32_bits(i915_vma_offset(vma)); } *cmd++ = MI_BATCH_BUFFER_END; /* terminate early in case of error */ __i915_gem_object_flush_map(obj, 0, 64); i915_gem_object_unpin_map(obj); - intel_gt_chipset_flush(&i915->gt); + intel_gt_chipset_flush(gt); return vma; @@ -1038,7 +1194,6 @@ static int live_all_engines(void *arg) struct intel_engine_cs *engine; struct i915_request **request; struct igt_live_test t; - struct i915_vma *batch; unsigned int idx; int err; @@ -1056,44 +1211,44 @@ static int live_all_engines(void *arg) if (err) goto out_free; - batch = recursive_batch(i915); - if (IS_ERR(batch)) { - err = PTR_ERR(batch); - pr_err("%s: Unable to create batch, err=%d\n", __func__, err); - goto out_free; - } - - i915_vma_lock(batch); - idx = 0; for_each_uabi_engine(engine, i915) { + struct i915_vma *batch; + + batch = recursive_batch(engine->gt); + if (IS_ERR(batch)) { + err = PTR_ERR(batch); + pr_err("%s: Unable to create batch, err=%d\n", + __func__, err); + goto out_free; + } + + i915_vma_lock(batch); request[idx] = intel_engine_create_kernel_request(engine); if (IS_ERR(request[idx])) { err = PTR_ERR(request[idx]); pr_err("%s: Request allocation failed with err=%d\n", __func__, err); - goto out_request; + goto out_unlock; } + GEM_BUG_ON(request[idx]->context->vm != batch->vm); - err = i915_request_await_object(request[idx], batch->obj, 0); - if (err == 0) - err = i915_vma_move_to_active(batch, request[idx], 0); + err = i915_vma_move_to_active(batch, request[idx], 0); GEM_BUG_ON(err); - err = engine->emit_bb_start(request[idx], - batch->node.start, - batch->node.size, - 0); + err = emit_bb_start(request[idx], batch); GEM_BUG_ON(err); request[idx]->batch = batch; i915_request_get(request[idx]); i915_request_add(request[idx]); idx++; +out_unlock: + i915_vma_unlock(batch); + if (err) + goto out_request; } - i915_vma_unlock(batch); - idx = 0; for_each_uabi_engine(engine, i915) { if (i915_request_completed(request[idx])) { @@ -1105,17 +1260,23 @@ static int live_all_engines(void *arg) idx++; } - err = recursive_batch_resolve(batch); - if (err) { - pr_err("%s: failed to resolve batch, err=%d\n", __func__, err); - goto out_request; + idx = 0; + for_each_uabi_engine(engine, i915) { + err = recursive_batch_resolve(request[idx]->batch); + if (err) { + pr_err("%s: failed to resolve batch, err=%d\n", + __func__, err); + goto out_request; + } + idx++; } idx = 0; for_each_uabi_engine(engine, i915) { + struct i915_request *rq = request[idx]; long timeout; - timeout = i915_request_wait(request[idx], 0, + timeout = i915_request_wait(rq, 0, MAX_SCHEDULE_TIMEOUT); if (timeout < 0) { err = timeout; @@ -1124,8 +1285,10 @@ static int live_all_engines(void *arg) goto out_request; } - GEM_BUG_ON(!i915_request_completed(request[idx])); - i915_request_put(request[idx]); + GEM_BUG_ON(!i915_request_completed(rq)); + i915_vma_unpin(rq->batch); + i915_vma_put(rq->batch); + i915_request_put(rq); request[idx] = NULL; idx++; } @@ -1135,12 +1298,18 @@ static int live_all_engines(void *arg) out_request: idx = 0; for_each_uabi_engine(engine, i915) { - if (request[idx]) - i915_request_put(request[idx]); + struct i915_request *rq = request[idx]; + + if (!rq) + continue; + + if (rq->batch) { + i915_vma_unpin(rq->batch); + i915_vma_put(rq->batch); + } + i915_request_put(rq); idx++; } - i915_vma_unpin(batch); - i915_vma_put(batch); out_free: kfree(request); return err; @@ -1176,7 +1345,7 @@ static int live_sequential_engines(void *arg) for_each_uabi_engine(engine, i915) { struct i915_vma *batch; - batch = recursive_batch(i915); + batch = recursive_batch(engine->gt); if (IS_ERR(batch)) { err = PTR_ERR(batch); pr_err("%s: Unable to create batch for %s, err=%d\n", @@ -1192,6 +1361,7 @@ static int live_sequential_engines(void *arg) __func__, engine->name, err); goto out_unlock; } + GEM_BUG_ON(request[idx]->context->vm != batch->vm); if (prev) { err = i915_request_await_dma_fence(request[idx], @@ -1204,16 +1374,10 @@ static int live_sequential_engines(void *arg) } } - err = i915_request_await_object(request[idx], - batch->obj, false); - if (err == 0) - err = i915_vma_move_to_active(batch, request[idx], 0); + err = i915_vma_move_to_active(batch, request[idx], 0); GEM_BUG_ON(err); - err = engine->emit_bb_start(request[idx], - batch->node.start, - batch->node.size, - 0); + err = emit_bb_start(request[idx], batch); GEM_BUG_ON(err); request[idx]->batch = batch; @@ -1291,9 +1455,18 @@ out_free: return err; } -static int __live_parallel_engine1(void *arg) +struct parallel_thread { + struct kthread_worker *worker; + struct kthread_work work; + struct intel_engine_cs *engine; + int result; +}; + +static void __live_parallel_engine1(struct kthread_work *work) { - struct intel_engine_cs *engine = arg; + struct parallel_thread *thread = + container_of(work, typeof(*thread), work); + struct intel_engine_cs *engine = thread->engine; IGT_TIMEOUT(end_time); unsigned long count; int err = 0; @@ -1313,7 +1486,7 @@ static int __live_parallel_engine1(void *arg) i915_request_add(rq); err = 0; - if (i915_request_wait(rq, 0, HZ / 5) < 0) + if (i915_request_wait(rq, 0, HZ) < 0) err = -ETIME; i915_request_put(rq); if (err) @@ -1324,12 +1497,14 @@ static int __live_parallel_engine1(void *arg) intel_engine_pm_put(engine); pr_info("%s: %lu request + sync\n", engine->name, count); - return err; + thread->result = err; } -static int __live_parallel_engineN(void *arg) +static void __live_parallel_engineN(struct kthread_work *work) { - struct intel_engine_cs *engine = arg; + struct parallel_thread *thread = + container_of(work, typeof(*thread), work); + struct intel_engine_cs *engine = thread->engine; IGT_TIMEOUT(end_time); unsigned long count; int err = 0; @@ -1351,7 +1526,7 @@ static int __live_parallel_engineN(void *arg) intel_engine_pm_put(engine); pr_info("%s: %lu requests\n", engine->name, count); - return err; + thread->result = err; } static bool wake_all(struct drm_i915_private *i915) @@ -1377,9 +1552,11 @@ static int wait_for_all(struct drm_i915_private *i915) return -ETIME; } -static int __live_parallel_spin(void *arg) +static void __live_parallel_spin(struct kthread_work *work) { - struct intel_engine_cs *engine = arg; + struct parallel_thread *thread = + container_of(work, typeof(*thread), work); + struct intel_engine_cs *engine = thread->engine; struct igt_spinner spin; struct i915_request *rq; int err = 0; @@ -1392,7 +1569,8 @@ static int __live_parallel_spin(void *arg) if (igt_spinner_init(&spin, engine->gt)) { wake_all(engine->i915); - return -ENOMEM; + thread->result = -ENOMEM; + return; } intel_engine_pm_get(engine); @@ -1419,28 +1597,28 @@ static int __live_parallel_spin(void *arg) } igt_spinner_end(&spin); - if (err == 0 && i915_request_wait(rq, 0, HZ / 5) < 0) + if (err == 0 && i915_request_wait(rq, 0, HZ) < 0) err = -EIO; i915_request_put(rq); out_spin: igt_spinner_fini(&spin); - return err; + thread->result = err; } static int live_parallel_engines(void *arg) { struct drm_i915_private *i915 = arg; - static int (* const func[])(void *arg) = { + static void (* const func[])(struct kthread_work *) = { __live_parallel_engine1, __live_parallel_engineN, __live_parallel_spin, NULL, }; const unsigned int nengines = num_uabi_engines(i915); + struct parallel_thread *threads; struct intel_engine_cs *engine; - int (* const *fn)(void *arg); - struct task_struct **tsk; + void (* const *fn)(struct kthread_work *); int err = 0; /* @@ -1448,8 +1626,8 @@ static int live_parallel_engines(void *arg) * tests that we load up the system maximally. */ - tsk = kcalloc(nengines, sizeof(*tsk), GFP_KERNEL); - if (!tsk) + threads = kcalloc(nengines, sizeof(*threads), GFP_KERNEL); + if (!threads) return -ENOMEM; for (fn = func; !err && *fn; fn++) { @@ -1466,37 +1644,44 @@ static int live_parallel_engines(void *arg) idx = 0; for_each_uabi_engine(engine, i915) { - tsk[idx] = kthread_run(*fn, engine, - "igt/parallel:%s", - engine->name); - if (IS_ERR(tsk[idx])) { - err = PTR_ERR(tsk[idx]); + struct kthread_worker *worker; + + worker = kthread_run_worker(0, "igt/parallel:%s", + engine->name); + if (IS_ERR(worker)) { + err = PTR_ERR(worker); break; } - get_task_struct(tsk[idx++]); - } - yield(); /* start all threads before we kthread_stop() */ + threads[idx].worker = worker; + threads[idx].result = 0; + threads[idx].engine = engine; + + kthread_init_work(&threads[idx].work, *fn); + kthread_queue_work(worker, &threads[idx].work); + idx++; + } idx = 0; for_each_uabi_engine(engine, i915) { int status; - if (IS_ERR(tsk[idx])) + if (!threads[idx].worker) break; - status = kthread_stop(tsk[idx]); + kthread_flush_work(&threads[idx].work); + status = READ_ONCE(threads[idx].result); if (status && !err) err = status; - put_task_struct(tsk[idx++]); + kthread_destroy_worker(threads[idx++].worker); } if (igt_live_test_end(&t)) err = -EIO; } - kfree(tsk); + kfree(threads); return err; } @@ -1541,10 +1726,11 @@ static int live_breadcrumbs_smoketest(void *arg) { struct drm_i915_private *i915 = arg; const unsigned int nengines = num_uabi_engines(i915); - const unsigned int ncpus = num_online_cpus(); + const unsigned int ncpus = /* saturate with nengines * ncpus */ + max_t(int, 2, DIV_ROUND_UP(num_online_cpus(), nengines)); unsigned long num_waits, num_fences; struct intel_engine_cs *engine; - struct task_struct **threads; + struct smoke_thread *threads; struct igt_live_test live; intel_wakeref_t wakeref; struct smoketest *smoke; @@ -1613,28 +1799,31 @@ static int live_breadcrumbs_smoketest(void *arg) goto out_flush; } /* One ring interleaved between requests from all cpus */ - smoke[idx].max_batch /= num_online_cpus() + 1; + smoke[idx].max_batch /= ncpus + 1; pr_debug("Limiting batches to %d requests on %s\n", smoke[idx].max_batch, engine->name); for (n = 0; n < ncpus; n++) { - struct task_struct *tsk; + unsigned int i = idx * ncpus + n; + struct kthread_worker *worker; - tsk = kthread_run(__igt_breadcrumbs_smoketest, - &smoke[idx], "igt/%d.%d", idx, n); - if (IS_ERR(tsk)) { - ret = PTR_ERR(tsk); + worker = kthread_run_worker(0, "igt/%d.%d", idx, n); + if (IS_ERR(worker)) { + ret = PTR_ERR(worker); goto out_flush; } - get_task_struct(tsk); - threads[idx * ncpus + n] = tsk; + threads[i].worker = worker; + threads[i].t = &smoke[idx]; + + kthread_init_work(&threads[i].work, + __igt_breadcrumbs_smoketest); + kthread_queue_work(worker, &threads[i].work); } idx++; } - yield(); /* start all threads before we begin */ msleep(jiffies_to_msecs(i915_selftest.timeout_jiffies)); out_flush: @@ -1643,17 +1832,19 @@ out_flush: num_fences = 0; for_each_uabi_engine(engine, i915) { for (n = 0; n < ncpus; n++) { - struct task_struct *tsk = threads[idx * ncpus + n]; + unsigned int i = idx * ncpus + n; int err; - if (!tsk) + if (!threads[i].worker) continue; - err = kthread_stop(tsk); + WRITE_ONCE(threads[i].stop, true); + kthread_flush_work(&threads[i].work); + err = READ_ONCE(threads[i].result); if (err < 0 && !ret) ret = err; - put_task_struct(tsk); + kthread_destroy_worker(threads[i].worker); } num_waits += atomic_long_read(&smoke[idx].num_waits); @@ -1690,10 +1881,10 @@ int i915_request_live_selftests(struct drm_i915_private *i915) SUBTEST(live_breadcrumbs_smoketest), }; - if (intel_gt_is_wedged(&i915->gt)) + if (intel_gt_is_wedged(to_gt(i915))) return 0; - return i915_subtests(tests, i915); + return i915_live_subtests(tests, i915); } static int switch_to_kernel_sync(struct intel_context *ce, int err) @@ -1734,7 +1925,7 @@ struct perf_stats { struct perf_series { struct drm_i915_private *i915; unsigned int nengines; - struct intel_context *ce[]; + struct intel_context *ce[] __counted_by(nengines); }; static int cmp_u32(const void *A, const void *B) @@ -2763,9 +2954,18 @@ out: return err; } -static int p_sync0(void *arg) +struct p_thread { + struct perf_stats p; + struct kthread_worker *worker; + struct kthread_work work; + struct intel_engine_cs *engine; + int result; +}; + +static void p_sync0(struct kthread_work *work) { - struct perf_stats *p = arg; + struct p_thread *thread = container_of(work, typeof(*thread), work); + struct perf_stats *p = &thread->p; struct intel_engine_cs *engine = p->engine; struct intel_context *ce; IGT_TIMEOUT(end_time); @@ -2774,13 +2974,16 @@ static int p_sync0(void *arg) int err = 0; ce = intel_context_create(engine); - if (IS_ERR(ce)) - return PTR_ERR(ce); + if (IS_ERR(ce)) { + thread->result = PTR_ERR(ce); + return; + } err = intel_context_pin(ce); if (err) { intel_context_put(ce); - return err; + thread->result = err; + return; } if (intel_engine_supports_stats(engine)) { @@ -2805,7 +3008,7 @@ static int p_sync0(void *arg) i915_request_add(rq); err = 0; - if (i915_request_wait(rq, 0, HZ / 5) < 0) + if (i915_request_wait(rq, 0, HZ) < 0) err = -ETIME; i915_request_put(rq); if (err) @@ -2830,12 +3033,13 @@ static int p_sync0(void *arg) intel_context_unpin(ce); intel_context_put(ce); - return err; + thread->result = err; } -static int p_sync1(void *arg) +static void p_sync1(struct kthread_work *work) { - struct perf_stats *p = arg; + struct p_thread *thread = container_of(work, typeof(*thread), work); + struct perf_stats *p = &thread->p; struct intel_engine_cs *engine = p->engine; struct i915_request *prev = NULL; struct intel_context *ce; @@ -2845,13 +3049,16 @@ static int p_sync1(void *arg) int err = 0; ce = intel_context_create(engine); - if (IS_ERR(ce)) - return PTR_ERR(ce); + if (IS_ERR(ce)) { + thread->result = PTR_ERR(ce); + return; + } err = intel_context_pin(ce); if (err) { intel_context_put(ce); - return err; + thread->result = err; + return; } if (intel_engine_supports_stats(engine)) { @@ -2876,7 +3083,7 @@ static int p_sync1(void *arg) i915_request_add(rq); err = 0; - if (prev && i915_request_wait(prev, 0, HZ / 5) < 0) + if (prev && i915_request_wait(prev, 0, HZ) < 0) err = -ETIME; i915_request_put(prev); prev = rq; @@ -2903,12 +3110,13 @@ static int p_sync1(void *arg) intel_context_unpin(ce); intel_context_put(ce); - return err; + thread->result = err; } -static int p_many(void *arg) +static void p_many(struct kthread_work *work) { - struct perf_stats *p = arg; + struct p_thread *thread = container_of(work, typeof(*thread), work); + struct perf_stats *p = &thread->p; struct intel_engine_cs *engine = p->engine; struct intel_context *ce; IGT_TIMEOUT(end_time); @@ -2917,13 +3125,16 @@ static int p_many(void *arg) bool busy; ce = intel_context_create(engine); - if (IS_ERR(ce)) - return PTR_ERR(ce); + if (IS_ERR(ce)) { + thread->result = PTR_ERR(ce); + return; + } err = intel_context_pin(ce); if (err) { intel_context_put(ce); - return err; + thread->result = err; + return; } if (intel_engine_supports_stats(engine)) { @@ -2964,26 +3175,23 @@ static int p_many(void *arg) intel_context_unpin(ce); intel_context_put(ce); - return err; + thread->result = err; } static int perf_parallel_engines(void *arg) { struct drm_i915_private *i915 = arg; - static int (* const func[])(void *arg) = { + static void (* const func[])(struct kthread_work *) = { p_sync0, p_sync1, p_many, NULL, }; const unsigned int nengines = num_uabi_engines(i915); + void (* const *fn)(struct kthread_work *); struct intel_engine_cs *engine; - int (* const *fn)(void *arg); struct pm_qos_request qos; - struct { - struct perf_stats p; - struct task_struct *tsk; - } *engines; + struct p_thread *engines; int err = 0; engines = kcalloc(nengines, sizeof(*engines), GFP_KERNEL); @@ -3006,36 +3214,45 @@ static int perf_parallel_engines(void *arg) idx = 0; for_each_uabi_engine(engine, i915) { + struct kthread_worker *worker; + intel_engine_pm_get(engine); memset(&engines[idx].p, 0, sizeof(engines[idx].p)); - engines[idx].p.engine = engine; - engines[idx].tsk = kthread_run(*fn, &engines[idx].p, - "igt:%s", engine->name); - if (IS_ERR(engines[idx].tsk)) { - err = PTR_ERR(engines[idx].tsk); + worker = kthread_run_worker(0, "igt:%s", + engine->name); + if (IS_ERR(worker)) { + err = PTR_ERR(worker); intel_engine_pm_put(engine); break; } - get_task_struct(engines[idx++].tsk); - } + engines[idx].worker = worker; + engines[idx].result = 0; + engines[idx].p.engine = engine; + engines[idx].engine = engine; - yield(); /* start all threads before we kthread_stop() */ + kthread_init_work(&engines[idx].work, *fn); + kthread_queue_work(worker, &engines[idx].work); + idx++; + } idx = 0; for_each_uabi_engine(engine, i915) { int status; - if (IS_ERR(engines[idx].tsk)) + if (!engines[idx].worker) break; - status = kthread_stop(engines[idx].tsk); + kthread_flush_work(&engines[idx].work); + status = READ_ONCE(engines[idx].result); if (status && !err) err = status; intel_engine_pm_put(engine); - put_task_struct(engines[idx++].tsk); + + kthread_destroy_worker(engines[idx].worker); + idx++; } if (igt_live_test_end(&t)) @@ -3081,7 +3298,7 @@ int i915_request_perf_selftests(struct drm_i915_private *i915) SUBTEST(perf_parallel_engines), }; - if (intel_gt_is_wedged(&i915->gt)) + if (intel_gt_is_wedged(to_gt(i915))) return 0; return i915_subtests(tests, i915); diff --git a/drivers/gpu/drm/i915/selftests/i915_selftest.c b/drivers/gpu/drm/i915/selftests/i915_selftest.c index 1bc11c09faef..8460f0a70d04 100644 --- a/drivers/gpu/drm/i915/selftests/i915_selftest.c +++ b/drivers/gpu/drm/i915/selftests/i915_selftest.c @@ -23,10 +23,16 @@ #include <linux/random.h> +#include "gt/intel_gt.h" #include "gt/intel_gt_pm.h" +#include "gt/intel_gt_regs.h" +#include "gt/uc/intel_gsc_fw.h" + +#include "i915_driver.h" #include "i915_drv.h" +#include "i915_jiffies.h" #include "i915_selftest.h" - +#include "i915_wait_util.h" #include "igt_flush_test.h" struct i915_selftest i915_selftest __read_mostly = { @@ -126,6 +132,55 @@ static void set_default_test_all(struct selftest *st, unsigned int count) st[i].enabled = true; } +static bool +__gsc_proxy_init_progressing(struct intel_gsc_uc *gsc) +{ + return intel_gsc_uc_fw_proxy_get_status(gsc) == -EAGAIN; +} + +static void +__wait_gsc_proxy_completed(struct drm_i915_private *i915) +{ + bool need_to_wait = (IS_ENABLED(CONFIG_INTEL_MEI_GSC_PROXY) && + i915->media_gt && + HAS_ENGINE(i915->media_gt, GSC0) && + intel_uc_fw_is_loadable(&i915->media_gt->uc.gsc.fw)); + /* + * The gsc proxy component depends on the kernel component driver load ordering + * and in corner cases (the first time after an IFWI flash), init-completion + * firmware flows take longer. + */ + unsigned long timeout_ms = 8000; + + if (need_to_wait && wait_for(!__gsc_proxy_init_progressing(&i915->media_gt->uc.gsc), + timeout_ms)) + pr_warn(DRIVER_NAME "Timed out waiting for gsc_proxy_completion!\n"); +} + +static void +__wait_gsc_huc_load_completed(struct drm_i915_private *i915) +{ + /* this only applies to DG2, so we only care about GT0 */ + struct intel_huc *huc = &to_gt(i915)->uc.huc; + bool need_to_wait = (IS_ENABLED(CONFIG_INTEL_MEI_PXP) && + intel_huc_wait_required(huc)); + /* + * The GSC and PXP mei bringup depends on the kernel boot ordering, so + * to account for the worst case scenario the HuC code waits for up to + * 10s for the GSC driver to load and then another 5s for the PXP + * component to bind before giving up, even though those steps normally + * complete in less than a second from the i915 load. We match that + * timeout here, but we expect to bail early due to the fence being + * signalled even in a failure case, as it is extremely unlikely that + * both components will use their full timeout. + */ + unsigned long timeout_ms = 15000; + + if (need_to_wait && + wait_for(i915_sw_fence_done(&huc->delayed_load.fence), timeout_ms)) + pr_warn(DRIVER_NAME "Timed out waiting for huc load via GSC!\n"); +} + static int __run_selftests(const char *name, struct selftest *st, unsigned int count, @@ -134,7 +189,7 @@ static int __run_selftests(const char *name, int err = 0; while (!i915_selftest.random_seed) - i915_selftest.random_seed = get_random_int(); + i915_selftest.random_seed = get_random_u32(); i915_selftest.timeout_jiffies = i915_selftest.timeout_ms ? @@ -187,7 +242,7 @@ int i915_mock_selftests(void) err = run_selftests(mock, NULL); if (err) { i915_selftest.mock = err; - return err; + return 1; } if (i915_selftest.mock < 0) { @@ -200,12 +255,32 @@ int i915_mock_selftests(void) int i915_live_selftests(struct pci_dev *pdev) { + struct drm_i915_private *i915 = pdev_to_i915(pdev); + struct intel_uncore *uncore = &i915->uncore; int err; + u32 pg_enable; + intel_wakeref_t wakeref; if (!i915_selftest.live) return 0; - err = run_selftests(live, pdev_to_i915(pdev)); + /* + * FIXME Disable render powergating, this is temporary wa and should be removed + * after fixing real cause of forcewake timeouts. + */ + with_intel_runtime_pm(uncore->rpm, wakeref) { + if (IS_GFX_GT_IP_RANGE(to_gt(i915), IP_VER(12, 00), IP_VER(12, 74))) { + pg_enable = intel_uncore_read(uncore, GEN9_PG_ENABLE); + if (pg_enable & GEN9_RENDER_PG_ENABLE) + intel_uncore_write_fw(uncore, GEN9_PG_ENABLE, + pg_enable & ~GEN9_RENDER_PG_ENABLE); + } + } + + __wait_gsc_proxy_completed(i915); + __wait_gsc_huc_load_completed(i915); + + err = run_selftests(live, i915); if (err) { i915_selftest.live = err; return err; @@ -221,12 +296,16 @@ int i915_live_selftests(struct pci_dev *pdev) int i915_perf_selftests(struct pci_dev *pdev) { + struct drm_i915_private *i915 = pdev_to_i915(pdev); int err; if (!i915_selftest.perf) return 0; - err = run_selftests(perf, pdev_to_i915(pdev)); + __wait_gsc_proxy_completed(i915); + __wait_gsc_huc_load_completed(i915); + + err = run_selftests(perf, i915); if (err) { i915_selftest.perf = err; return err; @@ -298,10 +377,10 @@ int __i915_live_setup(void *data) struct drm_i915_private *i915 = data; /* The selftests expect an idle system */ - if (intel_gt_pm_wait_for_idle(&i915->gt)) + if (intel_gt_pm_wait_for_idle(to_gt(i915))) return -EIO; - return intel_gt_terminally_wedged(&i915->gt); + return intel_gt_terminally_wedged(to_gt(i915)); } int __i915_live_teardown(int err, void *data) @@ -430,7 +509,7 @@ module_param_named(st_timeout, i915_selftest.timeout_ms, uint, 0400); module_param_named(st_filter, i915_selftest.filter, charp, 0400); module_param_named_unsafe(mock_selftests, i915_selftest.mock, int, 0400); -MODULE_PARM_DESC(mock_selftests, "Run selftests before loading, using mock hardware (0:disabled [default], 1:run tests then load driver, -1:run tests then exit module)"); +MODULE_PARM_DESC(mock_selftests, "Run selftests before loading, using mock hardware (0:disabled [default], 1:run tests then load driver, -1:run tests then leave dummy module)"); module_param_named_unsafe(live_selftests, i915_selftest.live, int, 0400); MODULE_PARM_DESC(live_selftests, "Run selftests after driver initialisation on the live system (0:disabled [default], 1:run tests then continue, -1:run tests then exit module)"); diff --git a/drivers/gpu/drm/i915/selftests/i915_sw_fence.c b/drivers/gpu/drm/i915/selftests/i915_sw_fence.c index cbf45d85cbff..8f5ce71fa453 100644 --- a/drivers/gpu/drm/i915/selftests/i915_sw_fence.c +++ b/drivers/gpu/drm/i915/selftests/i915_sw_fence.c @@ -28,7 +28,7 @@ #include "../i915_selftest.h" -static int __i915_sw_fence_call +static int fence_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state) { switch (state) { @@ -523,12 +523,19 @@ static void task_ipc(struct work_struct *work) static int test_ipc(void *arg) { struct task_ipc ipc; + struct workqueue_struct *wq; int ret = 0; + wq = alloc_workqueue("i1915-selftest", 0, 0); + if (wq == NULL) + return -ENOMEM; + /* Test use of i915_sw_fence as an interprocess signaling mechanism */ ipc.in = alloc_fence(); - if (!ipc.in) - return -ENOMEM; + if (!ipc.in) { + ret = -ENOMEM; + goto err_work; + } ipc.out = alloc_fence(); if (!ipc.out) { ret = -ENOMEM; @@ -540,7 +547,7 @@ static int test_ipc(void *arg) ipc.value = 0; INIT_WORK_ONSTACK(&ipc.work, task_ipc); - schedule_work(&ipc.work); + queue_work(wq, &ipc.work); wait_for_completion(&ipc.started); @@ -563,6 +570,9 @@ static int test_ipc(void *arg) free_fence(ipc.out); err_in: free_fence(ipc.in); +err_work: + destroy_workqueue(wq); + return ret; } diff --git a/drivers/gpu/drm/i915/selftests/i915_syncmap.c b/drivers/gpu/drm/i915/selftests/i915_syncmap.c index 47f4ae18a1ef..88fa845e9f4a 100644 --- a/drivers/gpu/drm/i915/selftests/i915_syncmap.c +++ b/drivers/gpu/drm/i915/selftests/i915_syncmap.c @@ -77,7 +77,7 @@ __sync_print(struct i915_syncmap *p, for_each_set_bit(i, (unsigned long *)&p->bitmap, KSYNCMAP) { buf = __sync_print(__sync_child(p)[i], buf, sz, depth + 1, - last << 1 | !!(p->bitmap >> (i + 1)), + last << 1 | ((p->bitmap >> (i + 1)) ? 1 : 0), i); } } diff --git a/drivers/gpu/drm/i915/selftests/i915_vma.c b/drivers/gpu/drm/i915/selftests/i915_vma.c index dd0607254a95..7c4111e60f2e 100644 --- a/drivers/gpu/drm/i915/selftests/i915_vma.c +++ b/drivers/gpu/drm/i915/selftests/i915_vma.c @@ -25,6 +25,7 @@ #include <linux/prime_numbers.h> #include "gem/i915_gem_context.h" +#include "gem/i915_gem_internal.h" #include "gem/selftests/mock_context.h" #include "i915_scatterlist.h" @@ -39,7 +40,7 @@ static bool assert_vma(struct i915_vma *vma, { bool ok = true; - if (vma->vm != rcu_access_pointer(ctx->vm)) { + if (vma->vm != ctx->vm) { pr_err("VMA created with wrong VM\n"); ok = false; } @@ -50,9 +51,9 @@ static bool assert_vma(struct i915_vma *vma, ok = false; } - if (vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL) { + if (vma->gtt_view.type != I915_GTT_VIEW_NORMAL) { pr_err("VMA created with wrong type [%d]\n", - vma->ggtt_view.type); + vma->gtt_view.type); ok = false; } @@ -62,7 +63,7 @@ static bool assert_vma(struct i915_vma *vma, static struct i915_vma * checked_vma_instance(struct drm_i915_gem_object *obj, struct i915_address_space *vm, - const struct i915_ggtt_view *view) + const struct i915_gtt_view *view) { struct i915_vma *vma; bool ok = true; @@ -90,7 +91,7 @@ checked_vma_instance(struct drm_i915_gem_object *obj, } if (i915_vma_compare(vma, vma->vm, - i915_vma_is_ggtt(vma) ? &vma->ggtt_view : NULL)) { + i915_vma_is_ggtt(vma) ? &vma->gtt_view : NULL)) { pr_err("i915_vma_compare failed with itself\n"); return ERR_PTR(-EINVAL); } @@ -118,7 +119,7 @@ static int create_vmas(struct drm_i915_private *i915, struct i915_vma *vma; int err; - vm = i915_gem_context_get_vm_rcu(ctx); + vm = i915_gem_context_get_eb_vm(ctx); vma = checked_vma_instance(obj, vm, NULL); i915_vm_put(vm); if (IS_ERR(vma)) @@ -158,7 +159,8 @@ static int igt_vma_create(void *arg) LIST_HEAD(objects); int err = -ENOMEM; - /* Exercise creating many vma amonst many objections, checking the + /* + * Exercise creating many vma amongst many objections, checking the * vma creation and lookup routines. */ @@ -291,7 +293,8 @@ static int igt_vma_pin1(void *arg) VALID(8192, PIN_GLOBAL | PIN_OFFSET_BIAS | (ggtt->mappable_end - 4096)), #if !IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM) - /* Misusing BIAS is a programming error (it is not controllable + /* + * Misusing BIAS is a programming error (it is not controllable * from userspace) so when debugging is enabled, it explodes. * However, the tests are still quite interesting for checking * variable start, end and size. @@ -311,7 +314,8 @@ static int igt_vma_pin1(void *arg) struct i915_vma *vma; int err = -EINVAL; - /* Exercise all the weird and wonderful i915_vma_pin requests, + /* + * Exercise all the weird and wonderful i915_vma_pin requests, * focusing on error handling of boundary conditions. */ @@ -340,7 +344,7 @@ static int igt_vma_pin1(void *arg) if (!err) { i915_vma_unpin(vma); - err = i915_vma_unbind(vma); + err = i915_vma_unbind_unlocked(vma); if (err) { pr_err("Failed to unbind single page from GGTT, err=%d\n", err); goto out; @@ -529,12 +533,12 @@ assert_remapped(struct drm_i915_gem_object *obj, return sg; } -static unsigned int remapped_size(enum i915_ggtt_view_type view_type, +static unsigned int remapped_size(enum i915_gtt_view_type view_type, const struct intel_remapped_plane_info *a, const struct intel_remapped_plane_info *b) { - if (view_type == I915_GGTT_VIEW_ROTATED) + if (view_type == I915_GTT_VIEW_ROTATED) return a->dst_stride * a->width + b->dst_stride * b->width; else return a->dst_stride * a->height + b->dst_stride * b->height; @@ -568,15 +572,16 @@ static int igt_vma_rotate_remap(void *arg) { } }, *a, *b; - enum i915_ggtt_view_type types[] = { - I915_GGTT_VIEW_ROTATED, - I915_GGTT_VIEW_REMAPPED, + enum i915_gtt_view_type types[] = { + I915_GTT_VIEW_ROTATED, + I915_GTT_VIEW_REMAPPED, 0, }, *t; const unsigned int max_pages = 64; int err = -ENOMEM; - /* Create VMA for many different combinations of planes and check + /* + * Create VMA for many different combinations of planes and check * that the page layout within the rotated VMA match our expectations. */ @@ -587,7 +592,7 @@ static int igt_vma_rotate_remap(void *arg) for (t = types; *t; t++) { for (a = planes; a->width; a++) { for (b = planes + ARRAY_SIZE(planes); b-- != planes; ) { - struct i915_ggtt_view view = { + struct i915_gtt_view view = { .type = *t, .remapped.plane[0] = *a, .remapped.plane[1] = *b, @@ -601,11 +606,11 @@ static int igt_vma_rotate_remap(void *arg) max_offset = max_pages - max_offset; if (!plane_info[0].dst_stride) - plane_info[0].dst_stride = view.type == I915_GGTT_VIEW_ROTATED ? + plane_info[0].dst_stride = view.type == I915_GTT_VIEW_ROTATED ? plane_info[0].height : plane_info[0].width; if (!plane_info[1].dst_stride) - plane_info[1].dst_stride = view.type == I915_GGTT_VIEW_ROTATED ? + plane_info[1].dst_stride = view.type == I915_GTT_VIEW_ROTATED ? plane_info[1].height : plane_info[1].width; @@ -629,7 +634,7 @@ static int igt_vma_rotate_remap(void *arg) expected_pages = remapped_size(view.type, &plane_info[0], &plane_info[1]); - if (view.type == I915_GGTT_VIEW_ROTATED && + if (view.type == I915_GTT_VIEW_ROTATED && vma->size != expected_pages * PAGE_SIZE) { pr_err("VMA is wrong size, expected %lu, found %llu\n", PAGE_SIZE * expected_pages, vma->size); @@ -637,7 +642,7 @@ static int igt_vma_rotate_remap(void *arg) goto out_object; } - if (view.type == I915_GGTT_VIEW_REMAPPED && + if (view.type == I915_GTT_VIEW_REMAPPED && vma->size > expected_pages * PAGE_SIZE) { pr_err("VMA is wrong size, expected %lu, found %llu\n", PAGE_SIZE * expected_pages, vma->size); @@ -667,13 +672,13 @@ static int igt_vma_rotate_remap(void *arg) sg = vma->pages->sgl; for (n = 0; n < ARRAY_SIZE(view.rotated.plane); n++) { - if (view.type == I915_GGTT_VIEW_ROTATED) + if (view.type == I915_GTT_VIEW_ROTATED) sg = assert_rotated(obj, &view.rotated, n, sg); else sg = assert_remapped(obj, &view.remapped, n, sg); if (IS_ERR(sg)) { pr_err("Inconsistent %s VMA pages for plane %d: [(%d, %d, %d, %d, %d), (%d, %d, %d, %d, %d)]\n", - view.type == I915_GGTT_VIEW_ROTATED ? + view.type == I915_GTT_VIEW_ROTATED ? "rotated" : "remapped", n, plane_info[0].width, plane_info[0].height, @@ -691,7 +696,11 @@ static int igt_vma_rotate_remap(void *arg) } i915_vma_unpin(vma); - + err = i915_vma_unbind_unlocked(vma); + if (err) { + pr_err("Unbinding returned %i\n", err); + goto out_object; + } cond_resched(); } } @@ -736,7 +745,7 @@ static bool assert_partial(struct drm_i915_gem_object *obj, } static bool assert_pin(struct i915_vma *vma, - struct i915_ggtt_view *view, + struct i915_gtt_view *view, u64 size, const char *name) { @@ -754,8 +763,8 @@ static bool assert_pin(struct i915_vma *vma, ok = false; } - if (view && view->type != I915_GGTT_VIEW_NORMAL) { - if (memcmp(&vma->ggtt_view, view, sizeof(*view))) { + if (view && view->type != I915_GTT_VIEW_NORMAL) { + if (memcmp(&vma->gtt_view, view, sizeof(*view))) { pr_err("(%s) VMA mismatch upon creation!\n", name); ok = false; @@ -767,9 +776,9 @@ static bool assert_pin(struct i915_vma *vma, ok = false; } } else { - if (vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL) { + if (vma->gtt_view.type != I915_GTT_VIEW_NORMAL) { pr_err("Not the normal ggtt view! Found %d\n", - vma->ggtt_view.type); + vma->gtt_view.type); ok = false; } @@ -799,7 +808,8 @@ static int igt_vma_partial(void *arg) struct i915_vma *vma; int err = -ENOMEM; - /* Create lots of different VMA for the object and check that + /* + * Create lots of different VMA for the object and check that * we are returned the same VMA when we later request the same range. */ @@ -813,14 +823,14 @@ static int igt_vma_partial(void *arg) nvma = 0; for_each_prime_number_from(sz, 1, npages) { for_each_prime_number_from(offset, 0, npages - sz) { - struct i915_ggtt_view view; + struct i915_gtt_view view; - view.type = I915_GGTT_VIEW_PARTIAL; + view.type = I915_GTT_VIEW_PARTIAL; view.partial.offset = offset; view.partial.size = sz; if (sz == npages) - view.type = I915_GGTT_VIEW_NORMAL; + view.type = I915_GTT_VIEW_NORMAL; vma = checked_vma_instance(obj, vm, &view); if (IS_ERR(vma)) { @@ -848,6 +858,11 @@ static int igt_vma_partial(void *arg) i915_vma_unpin(vma); nvma++; + err = i915_vma_unbind_unlocked(vma); + if (err) { + pr_err("Unbinding returned %i\n", err); + goto out_object; + } cond_resched(); } @@ -882,6 +897,12 @@ static int igt_vma_partial(void *arg) i915_vma_unpin(vma); + err = i915_vma_unbind_unlocked(vma); + if (err) { + pr_err("Unbinding returned %i\n", err); + goto out_object; + } + count = 0; list_for_each_entry(vma, &obj->vma.list, obj_link) count++; @@ -907,26 +928,28 @@ int i915_vma_mock_selftests(void) SUBTEST(igt_vma_partial), }; struct drm_i915_private *i915; - struct i915_ggtt *ggtt; + struct intel_gt *gt; int err; i915 = mock_gem_device(); if (!i915) return -ENOMEM; - ggtt = kmalloc(sizeof(*ggtt), GFP_KERNEL); - if (!ggtt) { - err = -ENOMEM; + /* allocate the ggtt */ + err = intel_gt_assign_ggtt(to_gt(i915)); + if (err) goto out_put; - } - mock_init_ggtt(i915, ggtt); - err = i915_subtests(tests, ggtt); + gt = to_gt(i915); + + mock_init_ggtt(gt); + + err = i915_subtests(tests, gt->ggtt); mock_device_flush(i915); i915_gem_drain_freed_objects(i915); - mock_fini_ggtt(ggtt); - kfree(ggtt); + mock_fini_ggtt(gt->ggtt); + out_put: mock_destroy_device(i915); return err; @@ -958,16 +981,16 @@ static int igt_vma_remapped_gtt(void *arg) { } }, *p; - enum i915_ggtt_view_type types[] = { - I915_GGTT_VIEW_ROTATED, - I915_GGTT_VIEW_REMAPPED, + enum i915_gtt_view_type types[] = { + I915_GTT_VIEW_ROTATED, + I915_GTT_VIEW_REMAPPED, 0, }, *t; struct drm_i915_gem_object *obj; intel_wakeref_t wakeref; int err = 0; - if (!i915_ggtt_has_aperture(&i915->ggtt)) + if (!i915_ggtt_has_aperture(to_gt(i915)->ggtt)) return 0; obj = i915_gem_object_create_internal(i915, 10 * 10 * PAGE_SIZE); @@ -978,7 +1001,7 @@ static int igt_vma_remapped_gtt(void *arg) for (t = types; *t; t++) { for (p = planes; p->width; p++) { - struct i915_ggtt_view view = { + struct i915_gtt_view view = { .type = *t, .rotated.plane[0] = *p, }; @@ -994,7 +1017,7 @@ static int igt_vma_remapped_gtt(void *arg) goto out; if (!plane_info[0].dst_stride) - plane_info[0].dst_stride = *t == I915_GGTT_VIEW_ROTATED ? + plane_info[0].dst_stride = *t == I915_GTT_VIEW_ROTATED ? p->height : p->width; vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, PIN_MAPPABLE); @@ -1003,7 +1026,7 @@ static int igt_vma_remapped_gtt(void *arg) goto out; } - GEM_BUG_ON(vma->ggtt_view.type != *t); + GEM_BUG_ON(vma->gtt_view.type != *t); map = i915_vma_pin_iomap(vma); i915_vma_unpin(vma); @@ -1017,7 +1040,7 @@ static int igt_vma_remapped_gtt(void *arg) unsigned int offset; u32 val = y << 16 | x; - if (*t == I915_GGTT_VIEW_ROTATED) + if (*t == I915_GTT_VIEW_ROTATED) offset = (x * plane_info[0].dst_stride + y) * PAGE_SIZE; else offset = (y * plane_info[0].dst_stride + x) * PAGE_SIZE; @@ -1034,7 +1057,7 @@ static int igt_vma_remapped_gtt(void *arg) goto out; } - GEM_BUG_ON(vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL); + GEM_BUG_ON(vma->gtt_view.type != I915_GTT_VIEW_NORMAL); map = i915_vma_pin_iomap(vma); i915_vma_unpin(vma); @@ -1049,7 +1072,7 @@ static int igt_vma_remapped_gtt(void *arg) u32 exp = y << 16 | x; u32 val; - if (*t == I915_GGTT_VIEW_ROTATED) + if (*t == I915_GTT_VIEW_ROTATED) src_idx = rotated_index(&view.rotated, 0, x, y); else src_idx = remapped_index(&view.remapped, 0, x, y); @@ -1058,7 +1081,7 @@ static int igt_vma_remapped_gtt(void *arg) val = ioread32(&map[offset / sizeof(*map)]); if (val != exp) { pr_err("%s VMA write test failed, expected 0x%x, found 0x%x\n", - *t == I915_GGTT_VIEW_ROTATED ? "Rotated" : "Remapped", + *t == I915_GTT_VIEW_ROTATED ? "Rotated" : "Remapped", exp, val); i915_vma_unpin_iomap(vma); err = -EINVAL; @@ -1085,5 +1108,5 @@ int i915_vma_live_selftests(struct drm_i915_private *i915) SUBTEST(igt_vma_remapped_gtt), }; - return i915_subtests(tests, i915); + return i915_live_subtests(tests, i915); } diff --git a/drivers/gpu/drm/i915/selftests/igt_flush_test.c b/drivers/gpu/drm/i915/selftests/igt_flush_test.c index 7b0939e3f007..c383d31d46b0 100644 --- a/drivers/gpu/drm/i915/selftests/igt_flush_test.c +++ b/drivers/gpu/drm/i915/selftests/igt_flush_test.c @@ -14,21 +14,37 @@ int igt_flush_test(struct drm_i915_private *i915) { - struct intel_gt *gt = &i915->gt; - int ret = intel_gt_is_wedged(gt) ? -EIO : 0; + struct intel_gt *gt; + unsigned int i; + int ret = 0; - cond_resched(); + for_each_gt(gt, i915, i) { + struct intel_engine_cs *engine; + unsigned long timeout_ms = 0; + unsigned int id; - if (intel_gt_wait_for_idle(gt, HZ / 5) == -ETIME) { - pr_err("%pS timed out, cancelling all further testing.\n", - __builtin_return_address(0)); + if (intel_gt_is_wedged(gt)) + ret = -EIO; - GEM_TRACE("%pS timed out.\n", - __builtin_return_address(0)); - GEM_TRACE_DUMP(); + for_each_engine(engine, gt, id) { + if (engine->props.preempt_timeout_ms > timeout_ms) + timeout_ms = engine->props.preempt_timeout_ms; + } - intel_gt_set_wedged(gt); - ret = -EIO; + cond_resched(); + + /* 2x longest preempt timeout, experimentally determined */ + if (intel_gt_wait_for_idle(gt, HZ * timeout_ms / 500) == -ETIME) { + pr_err("%pS timed out, cancelling all further testing.\n", + __builtin_return_address(0)); + + GEM_TRACE("%pS timed out.\n", + __builtin_return_address(0)); + GEM_TRACE_DUMP(); + + intel_gt_set_wedged(gt); + ret = -EIO; + } } return ret; diff --git a/drivers/gpu/drm/i915/selftests/igt_live_test.c b/drivers/gpu/drm/i915/selftests/igt_live_test.c index c130010a7033..7d41874a49c5 100644 --- a/drivers/gpu/drm/i915/selftests/igt_live_test.c +++ b/drivers/gpu/drm/i915/selftests/igt_live_test.c @@ -5,7 +5,8 @@ */ #include "i915_drv.h" -#include "gt/intel_gt_requests.h" +#include "gt/intel_gt.h" +#include "gt/intel_gt_print.h" #include "../i915_selftest.h" #include "igt_flush_test.h" @@ -16,28 +17,33 @@ int igt_live_test_begin(struct igt_live_test *t, const char *func, const char *name) { - struct intel_gt *gt = &i915->gt; struct intel_engine_cs *engine; enum intel_engine_id id; + struct intel_gt *gt; + unsigned int i; int err; t->i915 = i915; t->func = func; t->name = name; - err = intel_gt_wait_for_idle(gt, MAX_SCHEDULE_TIMEOUT); - if (err) { - pr_err("%s(%s): failed to idle before, with err=%d!", - func, name, err); - return err; + for_each_gt(gt, i915, i) { + + err = intel_gt_wait_for_idle(gt, MAX_SCHEDULE_TIMEOUT); + if (err) { + gt_err(gt, "%s(%s): GT failed to idle before, with err=%d!", + func, name, err); + return err; + } + + for_each_engine(engine, gt, id) + t->reset_engine[i][id] = + i915_reset_engine_count(&i915->gpu_error, + engine); } t->reset_global = i915_reset_count(&i915->gpu_error); - for_each_engine(engine, gt, id) - t->reset_engine[id] = - i915_reset_engine_count(&i915->gpu_error, engine); - return 0; } @@ -46,6 +52,8 @@ int igt_live_test_end(struct igt_live_test *t) struct drm_i915_private *i915 = t->i915; struct intel_engine_cs *engine; enum intel_engine_id id; + struct intel_gt *gt; + unsigned int i; if (igt_flush_test(i915)) return -EIO; @@ -57,16 +65,18 @@ int igt_live_test_end(struct igt_live_test *t) return -EIO; } - for_each_engine(engine, &i915->gt, id) { - if (t->reset_engine[id] == - i915_reset_engine_count(&i915->gpu_error, engine)) - continue; + for_each_gt(gt, i915, i) { + for_each_engine(engine, gt, id) { + if (t->reset_engine[i][id] == + i915_reset_engine_count(&i915->gpu_error, engine)) + continue; - pr_err("%s(%s): engine '%s' was reset %d times!\n", - t->func, t->name, engine->name, - i915_reset_engine_count(&i915->gpu_error, engine) - - t->reset_engine[id]); - return -EIO; + gt_err(gt, "%s(%s): engine '%s' was reset %d times!\n", + t->func, t->name, engine->name, + i915_reset_engine_count(&i915->gpu_error, engine) - + t->reset_engine[i][id]); + return -EIO; + } } return 0; diff --git a/drivers/gpu/drm/i915/selftests/igt_live_test.h b/drivers/gpu/drm/i915/selftests/igt_live_test.h index 36ed42736c52..83e3ad430922 100644 --- a/drivers/gpu/drm/i915/selftests/igt_live_test.h +++ b/drivers/gpu/drm/i915/selftests/igt_live_test.h @@ -7,6 +7,7 @@ #ifndef IGT_LIVE_TEST_H #define IGT_LIVE_TEST_H +#include "gt/intel_gt_defines.h" /* for I915_MAX_GT */ #include "gt/intel_engine.h" /* for I915_NUM_ENGINES */ struct drm_i915_private; @@ -17,7 +18,7 @@ struct igt_live_test { const char *name; unsigned int reset_global; - unsigned int reset_engine[I915_NUM_ENGINES]; + unsigned int reset_engine[I915_MAX_GT][I915_NUM_ENGINES]; }; /* diff --git a/drivers/gpu/drm/i915/selftests/igt_mmap.c b/drivers/gpu/drm/i915/selftests/igt_mmap.c index 583a4ff8b8c9..e920a461bd36 100644 --- a/drivers/gpu/drm/i915/selftests/igt_mmap.c +++ b/drivers/gpu/drm/i915/selftests/igt_mmap.c @@ -9,15 +9,28 @@ #include "i915_drv.h" #include "igt_mmap.h" -unsigned long igt_mmap_node(struct drm_i915_private *i915, - struct drm_vma_offset_node *node, - unsigned long addr, - unsigned long prot, - unsigned long flags) +unsigned long igt_mmap_offset(struct drm_i915_private *i915, + u64 offset, + unsigned long size, + unsigned long prot, + unsigned long flags) { + struct drm_vma_offset_node *node; struct file *file; + unsigned long addr; int err; + /* no need to refcount, we own this object */ + drm_vma_offset_lock_lookup(i915->drm.vma_offset_manager); + node = drm_vma_offset_exact_lookup_locked(i915->drm.vma_offset_manager, + offset / PAGE_SIZE, size / PAGE_SIZE); + drm_vma_offset_unlock_lookup(i915->drm.vma_offset_manager); + + if (GEM_WARN_ON(!node)) { + pr_info("Failed to lookup %llx\n", offset); + return -ENOENT; + } + /* Pretend to open("/dev/dri/card0") */ file = mock_drm_getfile(i915->drm.primary, O_RDWR); if (IS_ERR(file)) @@ -29,7 +42,7 @@ unsigned long igt_mmap_node(struct drm_i915_private *i915, goto out_file; } - addr = vm_mmap(file, addr, drm_vma_node_size(node) << PAGE_SHIFT, + addr = vm_mmap(file, 0, drm_vma_node_size(node) << PAGE_SHIFT, prot, flags, drm_vma_node_offset_addr(node)); drm_vma_node_revoke(node, file->private_data); diff --git a/drivers/gpu/drm/i915/selftests/igt_mmap.h b/drivers/gpu/drm/i915/selftests/igt_mmap.h index 6e716cb59d7e..acbe34d81a6d 100644 --- a/drivers/gpu/drm/i915/selftests/igt_mmap.h +++ b/drivers/gpu/drm/i915/selftests/igt_mmap.h @@ -7,13 +7,15 @@ #ifndef IGT_MMAP_H #define IGT_MMAP_H +#include <linux/types.h> + struct drm_i915_private; struct drm_vma_offset_node; -unsigned long igt_mmap_node(struct drm_i915_private *i915, - struct drm_vma_offset_node *node, - unsigned long addr, - unsigned long prot, - unsigned long flags); +unsigned long igt_mmap_offset(struct drm_i915_private *i915, + u64 offset, + unsigned long size, + unsigned long prot, + unsigned long flags); #endif /* IGT_MMAP_H */ diff --git a/drivers/gpu/drm/i915/selftests/igt_reset.c b/drivers/gpu/drm/i915/selftests/igt_reset.c index 9f8590b868a9..a2838c65f8a5 100644 --- a/drivers/gpu/drm/i915/selftests/igt_reset.c +++ b/drivers/gpu/drm/i915/selftests/igt_reset.c @@ -36,7 +36,7 @@ void igt_global_reset_unlock(struct intel_gt *gt) enum intel_engine_id id; for_each_engine(engine, gt, id) - clear_bit(I915_RESET_ENGINE + id, >->reset.flags); + clear_and_wake_up_bit(I915_RESET_ENGINE + id, >->reset.flags); clear_bit(I915_RESET_BACKOFF, >->reset.flags); wake_up_all(>->reset.queue); diff --git a/drivers/gpu/drm/i915/selftests/igt_spinner.c b/drivers/gpu/drm/i915/selftests/igt_spinner.c index 24d87d0fc747..820364171ebe 100644 --- a/drivers/gpu/drm/i915/selftests/igt_spinner.c +++ b/drivers/gpu/drm/i915/selftests/igt_spinner.c @@ -3,11 +3,13 @@ * * Copyright © 2018 Intel Corporation */ -#include "gt/intel_gpu_commands.h" -#include "gt/intel_gt.h" +#include "gem/i915_gem_internal.h" #include "gem/selftests/igt_gem_utils.h" +#include "gt/intel_gpu_commands.h" +#include "gt/intel_gt.h" +#include "i915_wait_util.h" #include "igt_spinner.h" int igt_spinner_init(struct igt_spinner *spin, struct intel_gt *gt) @@ -96,7 +98,7 @@ int igt_spinner_pin(struct igt_spinner *spin, if (!spin->batch) { unsigned int mode; - mode = i915_coherent_map_type(spin->gt->i915, spin->obj, false); + mode = intel_gt_coherent_map_type(spin->gt, spin->obj, false); vaddr = igt_spinner_pin_obj(ce, ww, spin->obj, mode, &spin->batch_vma); if (IS_ERR(vaddr)) return PTR_ERR(vaddr); @@ -115,23 +117,7 @@ static unsigned int seqno_offset(u64 fence) static u64 hws_address(const struct i915_vma *hws, const struct i915_request *rq) { - return hws->node.start + seqno_offset(rq->fence.context); -} - -static int move_to_active(struct i915_vma *vma, - struct i915_request *rq, - unsigned int flags) -{ - int err; - - i915_vma_lock(vma); - err = i915_request_await_object(rq, vma->obj, - flags & EXEC_OBJECT_WRITE); - if (err == 0) - err = i915_vma_move_to_active(vma, rq, flags); - i915_vma_unlock(vma); - - return err; + return i915_vma_offset(hws) + seqno_offset(rq->fence.context); } struct i915_request * @@ -164,25 +150,25 @@ igt_spinner_create_request(struct igt_spinner *spin, if (IS_ERR(rq)) return ERR_CAST(rq); - err = move_to_active(vma, rq, 0); + err = igt_vma_move_to_active_unlocked(vma, rq, 0); if (err) goto cancel_rq; - err = move_to_active(hws, rq, 0); + err = igt_vma_move_to_active_unlocked(hws, rq, 0); if (err) goto cancel_rq; batch = spin->batch; - if (GRAPHICS_VER(rq->engine->i915) >= 8) { + if (GRAPHICS_VER(rq->i915) >= 8) { *batch++ = MI_STORE_DWORD_IMM_GEN4; *batch++ = lower_32_bits(hws_address(hws, rq)); *batch++ = upper_32_bits(hws_address(hws, rq)); - } else if (GRAPHICS_VER(rq->engine->i915) >= 6) { + } else if (GRAPHICS_VER(rq->i915) >= 6) { *batch++ = MI_STORE_DWORD_IMM_GEN4; *batch++ = 0; *batch++ = hws_address(hws, rq); - } else if (GRAPHICS_VER(rq->engine->i915) >= 4) { + } else if (GRAPHICS_VER(rq->i915) >= 4) { *batch++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; *batch++ = 0; *batch++ = hws_address(hws, rq); @@ -194,16 +180,19 @@ igt_spinner_create_request(struct igt_spinner *spin, *batch++ = arbitration_command; - if (GRAPHICS_VER(rq->engine->i915) >= 8) + memset32(batch, MI_NOOP, 128); + batch += 128; + + if (GRAPHICS_VER(rq->i915) >= 8) *batch++ = MI_BATCH_BUFFER_START | BIT(8) | 1; - else if (IS_HASWELL(rq->engine->i915)) + else if (IS_HASWELL(rq->i915)) *batch++ = MI_BATCH_BUFFER_START | MI_BATCH_PPGTT_HSW; - else if (GRAPHICS_VER(rq->engine->i915) >= 6) + else if (GRAPHICS_VER(rq->i915) >= 6) *batch++ = MI_BATCH_BUFFER_START; else *batch++ = MI_BATCH_BUFFER_START | MI_BATCH_GTT; - *batch++ = lower_32_bits(vma->node.start); - *batch++ = upper_32_bits(vma->node.start); + *batch++ = lower_32_bits(i915_vma_offset(vma)); + *batch++ = upper_32_bits(i915_vma_offset(vma)); *batch++ = MI_BATCH_BUFFER_END; /* not reached */ @@ -216,9 +205,9 @@ igt_spinner_create_request(struct igt_spinner *spin, } flags = 0; - if (GRAPHICS_VER(rq->engine->i915) <= 5) + if (GRAPHICS_VER(rq->i915) <= 5) flags |= I915_DISPATCH_SECURE; - err = engine->emit_bb_start(rq, vma->node.start, PAGE_SIZE, flags); + err = engine->emit_bb_start(rq, i915_vma_offset(vma), PAGE_SIZE, flags); cancel_rq: if (err) { diff --git a/drivers/gpu/drm/i915/selftests/intel_memory_region.c b/drivers/gpu/drm/i915/selftests/intel_memory_region.c index c85d516b85cd..7b856b5090f9 100644 --- a/drivers/gpu/drm/i915/selftests/intel_memory_region.c +++ b/drivers/gpu/drm/i915/selftests/intel_memory_region.c @@ -6,6 +6,8 @@ #include <linux/prime_numbers.h> #include <linux/sort.h> +#include <drm/drm_buddy.h> + #include "../i915_selftest.h" #include "mock_drm.h" @@ -15,12 +17,15 @@ #include "gem/i915_gem_context.h" #include "gem/i915_gem_lmem.h" #include "gem/i915_gem_region.h" -#include "gem/i915_gem_object_blt.h" +#include "gem/i915_gem_ttm.h" #include "gem/selftests/igt_gem_utils.h" #include "gem/selftests/mock_context.h" +#include "gt/intel_engine_pm.h" #include "gt/intel_engine_user.h" #include "gt/intel_gt.h" +#include "gt/intel_migrate.h" #include "i915_memcpy.h" +#include "i915_ttm_buddy_manager.h" #include "selftests/igt_flush_test.h" #include "selftests/i915_random.h" @@ -57,16 +62,15 @@ static int igt_mock_fill(void *arg) LIST_HEAD(objects); int err = 0; - page_size = mem->chunk_size; + page_size = PAGE_SIZE; + max_pages = div64_u64(total, page_size); rem = total; -retry: - max_pages = div64_u64(rem, page_size); for_each_prime_number_from(page_num, 1, max_pages) { resource_size_t size = page_num * page_size; struct drm_i915_gem_object *obj; - obj = i915_gem_object_create_region(mem, size, 0); + obj = i915_gem_object_create_region(mem, size, 0, 0); if (IS_ERR(obj)) { err = PTR_ERR(obj); break; @@ -86,11 +90,6 @@ retry: err = 0; if (err == -ENXIO) { if (page_num * page_size <= rem) { - if (mem->is_range_manager && max_pages > 1) { - max_pages >>= 1; - goto retry; - } - pr_err("%s failed, space still left in region\n", __func__); err = -EINVAL; @@ -113,7 +112,7 @@ igt_object_create(struct intel_memory_region *mem, struct drm_i915_gem_object *obj; int err; - obj = i915_gem_object_create_region(mem, size, flags); + obj = i915_gem_object_create_region(mem, size, 0, flags); if (IS_ERR(obj)) return obj; @@ -157,6 +156,7 @@ static bool is_contiguous(struct drm_i915_gem_object *obj) static int igt_mock_reserve(void *arg) { struct intel_memory_region *mem = arg; + struct drm_i915_private *i915 = mem->i915; resource_size_t avail = resource_size(&mem->region); struct drm_i915_gem_object *obj; const u32 chunk_size = SZ_32M; @@ -166,16 +166,18 @@ static int igt_mock_reserve(void *arg) LIST_HEAD(objects); int err = 0; - if (!list_empty(&mem->reserved)) { - pr_err("%s region reserved list is not empty\n", __func__); - return -EINVAL; - } - count = avail / chunk_size; order = i915_random_order(count, &prng); if (!order) return 0; + mem = mock_region_create(i915, 0, SZ_2G, I915_GTT_PAGE_SIZE_4K, 0, 0); + if (IS_ERR(mem)) { + pr_err("failed to create memory region\n"); + err = PTR_ERR(mem); + goto out_free_order; + } + /* Reserve a bunch of ranges within the region */ for (i = 0; i < count; ++i) { u64 start = order[i] * chunk_size; @@ -205,18 +207,12 @@ static int igt_mock_reserve(void *arg) do { u32 size = i915_prandom_u32_max_state(cur_avail, &prng); -retry: size = max_t(u32, round_up(size, PAGE_SIZE), PAGE_SIZE); obj = igt_object_create(mem, &objects, size, 0); if (IS_ERR(obj)) { - if (PTR_ERR(obj) == -ENXIO) { - if (mem->is_range_manager && - size > mem->chunk_size) { - size >>= 1; - goto retry; - } + if (PTR_ERR(obj) == -ENXIO) break; - } + err = PTR_ERR(obj); goto out_close; } @@ -230,9 +226,10 @@ retry: } out_close: - kfree(order); close_objects(mem, &objects); - intel_memory_region_unreserve(mem); + intel_memory_region_destroy(mem); +out_free_order: + kfree(order); return err; } @@ -252,7 +249,7 @@ static int igt_mock_contiguous(void *arg) total = resource_size(&mem->region); /* Min size */ - obj = igt_object_create(mem, &objects, mem->chunk_size, + obj = igt_object_create(mem, &objects, PAGE_SIZE, I915_BO_ALLOC_CONTIGUOUS); if (IS_ERR(obj)) return PTR_ERR(obj); @@ -333,17 +330,15 @@ static int igt_mock_contiguous(void *arg) min = target; target = total >> 1; - if (!mem->is_range_manager) { - /* Make sure we can still allocate all the fragmented space */ - obj = igt_object_create(mem, &objects, target, 0); - if (IS_ERR(obj)) { - err = PTR_ERR(obj); - goto err_close_objects; - } - - igt_object_release(obj); + /* Make sure we can still allocate all the fragmented space */ + obj = igt_object_create(mem, &objects, target, 0); + if (IS_ERR(obj)) { + err = PTR_ERR(obj); + goto err_close_objects; } + igt_object_release(obj); + /* * Even though we have enough free space, we don't have a big enough * contiguous block. Make sure that holds true. @@ -362,7 +357,7 @@ static int igt_mock_contiguous(void *arg) } target >>= 1; - } while (target >= mem->chunk_size); + } while (target >= PAGE_SIZE); err_close_objects: list_splice_tail(&holes, &objects); @@ -374,7 +369,9 @@ static int igt_mock_splintered_region(void *arg) { struct intel_memory_region *mem = arg; struct drm_i915_private *i915 = mem->i915; + struct i915_ttm_buddy_resource *res; struct drm_i915_gem_object *obj; + struct drm_buddy *mm; unsigned int expected_order; LIST_HEAD(objects); u64 size; @@ -382,63 +379,287 @@ static int igt_mock_splintered_region(void *arg) /* * Sanity check we can still allocate everything even if the - * max_order != mm.size. i.e our starting address space size is not a + * mm.max_order != mm.size. i.e our starting address space size is not a * power-of-two. */ size = (SZ_4G - 1) & PAGE_MASK; - mem = mock_region_create(i915, 0, size, PAGE_SIZE, 0); + mem = mock_region_create(i915, 0, size, PAGE_SIZE, 0, 0); if (IS_ERR(mem)) return PTR_ERR(mem); + obj = igt_object_create(mem, &objects, size, 0); + if (IS_ERR(obj)) { + err = PTR_ERR(obj); + goto out_close; + } + + res = to_ttm_buddy_resource(obj->mm.res); + mm = res->mm; + if (mm->size != size) { + pr_err("%s size mismatch(%llu != %llu)\n", + __func__, mm->size, size); + err = -EINVAL; + goto out_put; + } + expected_order = get_order(rounddown_pow_of_two(size)); - if (mem->max_order != expected_order) { + if (mm->max_order != expected_order) { pr_err("%s order mismatch(%u != %u)\n", - __func__, mem->max_order, expected_order); + __func__, mm->max_order, expected_order); err = -EINVAL; goto out_put; } - obj = igt_object_create(mem, &objects, size, 0); + close_objects(mem, &objects); + + obj = igt_object_create(mem, &objects, roundup_pow_of_two(size), + I915_BO_ALLOC_CONTIGUOUS); + if (!IS_ERR(obj)) { + pr_err("%s too large contiguous allocation was not rejected\n", + __func__); + err = -EINVAL; + goto out_close; + } + + obj = igt_object_create(mem, &objects, size, I915_BO_ALLOC_CONTIGUOUS); if (IS_ERR(obj)) { + pr_err("%s largest possible contiguous allocation failed\n", + __func__); err = PTR_ERR(obj); goto out_close; } +out_close: close_objects(mem, &objects); +out_put: + intel_memory_region_destroy(mem); + return err; +} + +#ifndef SZ_8G +#define SZ_8G BIT_ULL(33) +#endif + +static int igt_mock_max_segment(void *arg) +{ + struct intel_memory_region *mem = arg; + struct drm_i915_private *i915 = mem->i915; + struct i915_ttm_buddy_resource *res; + struct drm_i915_gem_object *obj; + struct drm_buddy_block *block; + struct drm_buddy *mm; + struct list_head *blocks; + struct scatterlist *sg; + I915_RND_STATE(prng); + LIST_HEAD(objects); + unsigned int max_segment; + unsigned int ps; + u64 size; + int err = 0; /* - * While we should be able allocate everything without any flag - * restrictions, if we consider I915_BO_ALLOC_CONTIGUOUS then we are - * actually limited to the largest power-of-two for the region size i.e - * max_order, due to the inner workings of the buddy allocator. So make - * sure that does indeed hold true. + * While we may create very large contiguous blocks, we may need + * to break those down for consumption elsewhere. In particular, + * dma-mapping with scatterlist elements have an implicit limit of + * UINT_MAX on each element. */ - if (!mem->is_range_manager) { - obj = igt_object_create(mem, &objects, size, - I915_BO_ALLOC_CONTIGUOUS); - if (!IS_ERR(obj)) { - pr_err("%s too large contiguous allocation was not rejected\n", - __func__); + size = SZ_8G; + ps = PAGE_SIZE; + if (i915_prandom_u64_state(&prng) & 1) + ps = SZ_64K; /* For something like DG2 */ + + max_segment = round_down(UINT_MAX, ps); + + mem = mock_region_create(i915, 0, size, ps, 0, 0); + if (IS_ERR(mem)) + return PTR_ERR(mem); + + obj = igt_object_create(mem, &objects, size, 0); + if (IS_ERR(obj)) { + err = PTR_ERR(obj); + goto out_put; + } + + res = to_ttm_buddy_resource(obj->mm.res); + blocks = &res->blocks; + mm = res->mm; + size = 0; + list_for_each_entry(block, blocks, link) { + if (drm_buddy_block_size(mm, block) > size) + size = drm_buddy_block_size(mm, block); + } + if (size < max_segment) { + pr_err("%s: Failed to create a huge contiguous block [> %u], largest block %lld\n", + __func__, max_segment, size); + err = -EINVAL; + goto out_close; + } + + for (sg = obj->mm.pages->sgl; sg; sg = sg_next(sg)) { + dma_addr_t daddr = sg_dma_address(sg); + + if (sg->length > max_segment) { + pr_err("%s: Created an oversized scatterlist entry, %u > %u\n", + __func__, sg->length, max_segment); + err = -EINVAL; + goto out_close; + } + + if (!IS_ALIGNED(daddr, ps)) { + pr_err("%s: Created an unaligned scatterlist entry, addr=%pa, ps=%u\n", + __func__, &daddr, ps); err = -EINVAL; goto out_close; } } - obj = igt_object_create(mem, &objects, rounddown_pow_of_two(size), - I915_BO_ALLOC_CONTIGUOUS); +out_close: + close_objects(mem, &objects); +out_put: + intel_memory_region_destroy(mem); + return err; +} + +static u64 igt_object_mappable_total(struct drm_i915_gem_object *obj) +{ + struct intel_memory_region *mr = obj->mm.region; + struct i915_ttm_buddy_resource *bman_res = + to_ttm_buddy_resource(obj->mm.res); + struct drm_buddy *mm = bman_res->mm; + struct drm_buddy_block *block; + u64 total; + + total = 0; + list_for_each_entry(block, &bman_res->blocks, link) { + u64 start = drm_buddy_block_offset(block); + u64 end = start + drm_buddy_block_size(mm, block); + + if (start < resource_size(&mr->io)) + total += min_t(u64, end, resource_size(&mr->io)) - start; + } + + return total; +} + +static int igt_mock_io_size(void *arg) +{ + struct intel_memory_region *mr = arg; + struct drm_i915_private *i915 = mr->i915; + struct drm_i915_gem_object *obj; + u64 mappable_theft_total; + u64 io_size; + u64 total; + u64 ps; + u64 rem; + u64 size; + I915_RND_STATE(prng); + LIST_HEAD(objects); + int err = 0; + + ps = SZ_4K; + if (i915_prandom_u64_state(&prng) & 1) + ps = SZ_64K; /* For something like DG2 */ + + div64_u64_rem(i915_prandom_u64_state(&prng), SZ_8G, &total); + total = round_down(total, ps); + total = max_t(u64, total, SZ_1G); + + div64_u64_rem(i915_prandom_u64_state(&prng), total - ps, &io_size); + io_size = round_down(io_size, ps); + io_size = max_t(u64, io_size, SZ_256M); /* 256M seems to be the common lower limit */ + + pr_info("%s with ps=%llx, io_size=%llx, total=%llx\n", + __func__, ps, io_size, total); + + mr = mock_region_create(i915, 0, total, ps, 0, io_size); + if (IS_ERR(mr)) { + err = PTR_ERR(mr); + goto out_err; + } + + mappable_theft_total = 0; + rem = total - io_size; + do { + div64_u64_rem(i915_prandom_u64_state(&prng), rem, &size); + size = round_down(size, ps); + size = max(size, ps); + + obj = igt_object_create(mr, &objects, size, + I915_BO_ALLOC_GPU_ONLY); + if (IS_ERR(obj)) { + pr_err("%s TOPDOWN failed with rem=%llx, size=%llx\n", + __func__, rem, size); + err = PTR_ERR(obj); + goto out_close; + } + + mappable_theft_total += igt_object_mappable_total(obj); + rem -= size; + } while (rem); + + pr_info("%s mappable theft=(%lluMiB/%lluMiB), total=%lluMiB\n", + __func__, + (u64)mappable_theft_total >> 20, + (u64)io_size >> 20, + (u64)total >> 20); + + /* + * Even if we allocate all of the non-mappable portion, we should still + * be able to dip into the mappable portion. + */ + obj = igt_object_create(mr, &objects, io_size, + I915_BO_ALLOC_GPU_ONLY); if (IS_ERR(obj)) { - pr_err("%s largest possible contiguous allocation failed\n", - __func__); + pr_err("%s allocation unexpectedly failed\n", __func__); err = PTR_ERR(obj); goto out_close; } + close_objects(mr, &objects); + + rem = io_size; + do { + div64_u64_rem(i915_prandom_u64_state(&prng), rem, &size); + size = round_down(size, ps); + size = max(size, ps); + + obj = igt_object_create(mr, &objects, size, 0); + if (IS_ERR(obj)) { + pr_err("%s MAPPABLE failed with rem=%llx, size=%llx\n", + __func__, rem, size); + err = PTR_ERR(obj); + goto out_close; + } + + if (igt_object_mappable_total(obj) != size) { + pr_err("%s allocation is not mappable(size=%llx)\n", + __func__, size); + err = -EINVAL; + goto out_close; + } + rem -= size; + } while (rem); + + /* + * We assume CPU access is required by default, which should result in a + * failure here, even though the non-mappable portion is free. + */ + obj = igt_object_create(mr, &objects, ps, 0); + if (!IS_ERR(obj)) { + pr_err("%s allocation unexpectedly succeeded\n", __func__); + err = -EINVAL; + goto out_close; + } + out_close: - close_objects(mem, &objects); -out_put: - intel_memory_region_put(mem); + close_objects(mr, &objects); + intel_memory_region_destroy(mr); +out_err: + if (err == -ENOMEM) + err = 0; + return err; } @@ -579,6 +800,68 @@ out_put: return err; } +static int igt_lmem_create_with_ps(void *arg) +{ + struct drm_i915_private *i915 = arg; + int err = 0; + u32 ps; + + for (ps = PAGE_SIZE; ps <= SZ_1G; ps <<= 1) { + struct drm_i915_gem_object *obj; + dma_addr_t daddr; + + obj = __i915_gem_object_create_lmem_with_ps(i915, ps, ps, 0); + if (IS_ERR(obj)) { + err = PTR_ERR(obj); + if (err == -ENXIO || err == -E2BIG) { + pr_info("%s not enough lmem for ps(%u) err=%d\n", + __func__, ps, err); + err = 0; + } + + break; + } + + if (obj->base.size != ps) { + pr_err("%s size(%zu) != ps(%u)\n", + __func__, obj->base.size, ps); + err = -EINVAL; + goto out_put; + } + + i915_gem_object_lock(obj, NULL); + err = i915_gem_object_pin_pages(obj); + if (err) { + if (err == -ENXIO || err == -E2BIG || err == -ENOMEM) { + pr_info("%s not enough lmem for ps(%u) err=%d\n", + __func__, ps, err); + err = 0; + } + goto out_put; + } + + daddr = i915_gem_object_get_dma_address(obj, 0); + if (!IS_ALIGNED(daddr, ps)) { + pr_err("%s daddr(%pa) not aligned with ps(%u)\n", + __func__, &daddr, ps); + err = -EINVAL; + goto out_unpin; + } + +out_unpin: + i915_gem_object_unpin_pages(obj); + __i915_gem_object_put_pages(obj); +out_put: + i915_gem_object_unlock(obj); + i915_gem_object_put(obj); + + if (err) + break; + } + + return err; +} + static int igt_lmem_create_cleared_cpu(void *arg) { struct drm_i915_private *i915 = arg; @@ -741,6 +1024,7 @@ static int igt_lmem_write_cpu(void *arg) PAGE_SIZE - 64, }; struct intel_engine_cs *engine; + struct i915_request *rq; u32 *vaddr; u32 sz; u32 i; @@ -767,15 +1051,30 @@ static int igt_lmem_write_cpu(void *arg) goto out_put; } + i915_gem_object_lock(obj, NULL); + + err = dma_resv_reserve_fences(obj->base.resv, 1); + if (err) { + i915_gem_object_unlock(obj); + goto out_put; + } + /* Put the pages into a known state -- from the gpu for added fun */ intel_engine_pm_get(engine); - err = i915_gem_object_fill_blt(obj, engine->kernel_context, 0xdeadbeaf); - intel_engine_pm_put(engine); - if (err) - goto out_unpin; + err = intel_context_migrate_clear(engine->gt->migrate.context, NULL, + obj->mm.pages->sgl, + i915_gem_get_pat_index(i915, + I915_CACHE_NONE), + true, 0xdeadbeaf, &rq); + if (rq) { + dma_resv_add_fence(obj->base.resv, &rq->fence, + DMA_RESV_USAGE_WRITE); + i915_request_put(rq); + } - i915_gem_object_lock(obj, NULL); - err = i915_gem_object_set_to_wc_domain(obj, true); + intel_engine_pm_put(engine); + if (!err) + err = i915_gem_object_set_to_wc_domain(obj, true); i915_gem_object_unlock(obj); if (err) goto out_unpin; @@ -858,7 +1157,7 @@ create_region_for_mapping(struct intel_memory_region *mr, u64 size, u32 type, struct drm_i915_gem_object *obj; void *addr; - obj = i915_gem_object_create_region(mr, size, 0); + obj = i915_gem_object_create_region(mr, size, 0, 0); if (IS_ERR(obj)) { if (PTR_ERR(obj) == -ENOSPC) /* Stolen memory */ return ERR_PTR(-ENODEV); @@ -1046,6 +1345,8 @@ int intel_memory_region_mock_selftests(void) SUBTEST(igt_mock_fill), SUBTEST(igt_mock_contiguous), SUBTEST(igt_mock_splintered_region), + SUBTEST(igt_mock_max_segment), + SUBTEST(igt_mock_io_size), }; struct intel_memory_region *mem; struct drm_i915_private *i915; @@ -1055,7 +1356,7 @@ int intel_memory_region_mock_selftests(void) if (!i915) return -ENOMEM; - mem = mock_region_create(i915, 0, SZ_2G, I915_GTT_PAGE_SIZE_4K, 0); + mem = mock_region_create(i915, 0, SZ_2G, I915_GTT_PAGE_SIZE_4K, 0, 0); if (IS_ERR(mem)) { pr_err("failed to create memory region\n"); err = PTR_ERR(mem); @@ -1064,7 +1365,7 @@ int intel_memory_region_mock_selftests(void) err = i915_subtests(tests, mem); - intel_memory_region_put(mem); + intel_memory_region_destroy(mem); out_unref: mock_destroy_device(i915); return err; @@ -1074,6 +1375,7 @@ int intel_memory_region_live_selftests(struct drm_i915_private *i915) { static const struct i915_subtest tests[] = { SUBTEST(igt_lmem_create), + SUBTEST(igt_lmem_create_with_ps), SUBTEST(igt_lmem_create_cleared_cpu), SUBTEST(igt_lmem_write_cpu), SUBTEST(igt_lmem_write_gpu), @@ -1084,7 +1386,7 @@ int intel_memory_region_live_selftests(struct drm_i915_private *i915) return 0; } - if (intel_gt_is_wedged(&i915->gt)) + if (intel_gt_is_wedged(to_gt(i915))) return 0; return i915_live_subtests(tests, i915); @@ -1096,7 +1398,7 @@ int intel_memory_region_perf_selftests(struct drm_i915_private *i915) SUBTEST(perf_memcpy), }; - if (intel_gt_is_wedged(&i915->gt)) + if (intel_gt_is_wedged(to_gt(i915))) return 0; return i915_live_subtests(tests, i915); diff --git a/drivers/gpu/drm/i915/selftests/intel_scheduler_helpers.c b/drivers/gpu/drm/i915/selftests/intel_scheduler_helpers.c new file mode 100644 index 000000000000..e14ac0ab1314 --- /dev/null +++ b/drivers/gpu/drm/i915/selftests/intel_scheduler_helpers.c @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021 Intel Corporation + */ + +#include <linux/jiffies.h> + +//#include "gt/intel_engine_user.h" +#include "gt/intel_gt.h" +#include "i915_drv.h" +#include "i915_selftest.h" + +#include "selftests/intel_scheduler_helpers.h" + +#define REDUCED_TIMESLICE 5 +#define REDUCED_PREEMPT 10 +#define WAIT_FOR_RESET_TIME_MS 10000 + +struct intel_engine_cs *intel_selftest_find_any_engine(struct intel_gt *gt) +{ + struct intel_engine_cs *engine; + enum intel_engine_id id; + + for_each_engine(engine, gt, id) + return engine; + + pr_err("No valid engine found!\n"); + return NULL; +} + +int intel_selftest_modify_policy(struct intel_engine_cs *engine, + struct intel_selftest_saved_policy *saved, + enum selftest_scheduler_modify modify_type) +{ + int err; + + saved->reset = engine->i915->params.reset; + saved->flags = engine->flags; + saved->timeslice = engine->props.timeslice_duration_ms; + saved->preempt_timeout = engine->props.preempt_timeout_ms; + + switch (modify_type) { + case SELFTEST_SCHEDULER_MODIFY_FAST_RESET: + /* + * Enable force pre-emption on time slice expiration + * together with engine reset on pre-emption timeout. + * This is required to make the GuC notice and reset + * the single hanging context. + * Also, reduce the preemption timeout to something + * small to speed the test up. + */ + engine->i915->params.reset = 2; + engine->flags |= I915_ENGINE_WANT_FORCED_PREEMPTION; + engine->props.timeslice_duration_ms = REDUCED_TIMESLICE; + engine->props.preempt_timeout_ms = REDUCED_PREEMPT; + break; + + case SELFTEST_SCHEDULER_MODIFY_NO_HANGCHECK: + engine->props.preempt_timeout_ms = 0; + break; + + default: + pr_err("Invalid scheduler policy modification type: %d!\n", modify_type); + return -EINVAL; + } + + if (!intel_engine_uses_guc(engine)) + return 0; + + err = intel_guc_global_policies_update(&engine->gt->uc.guc); + if (err) + intel_selftest_restore_policy(engine, saved); + + return err; +} + +int intel_selftest_restore_policy(struct intel_engine_cs *engine, + struct intel_selftest_saved_policy *saved) +{ + /* Restore the original policies */ + engine->i915->params.reset = saved->reset; + engine->flags = saved->flags; + engine->props.timeslice_duration_ms = saved->timeslice; + engine->props.preempt_timeout_ms = saved->preempt_timeout; + + if (!intel_engine_uses_guc(engine)) + return 0; + + return intel_guc_global_policies_update(&engine->gt->uc.guc); +} + +int intel_selftest_wait_for_rq(struct i915_request *rq) +{ + long ret; + + ret = i915_request_wait(rq, 0, msecs_to_jiffies(WAIT_FOR_RESET_TIME_MS)); + if (ret < 0) + return ret; + + return 0; +} diff --git a/drivers/gpu/drm/i915/selftests/intel_scheduler_helpers.h b/drivers/gpu/drm/i915/selftests/intel_scheduler_helpers.h new file mode 100644 index 000000000000..ae60bb507f45 --- /dev/null +++ b/drivers/gpu/drm/i915/selftests/intel_scheduler_helpers.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef _INTEL_SELFTEST_SCHEDULER_HELPERS_H_ +#define _INTEL_SELFTEST_SCHEDULER_HELPERS_H_ + +#include <linux/types.h> + +struct i915_request; +struct intel_engine_cs; +struct intel_gt; + +struct intel_selftest_saved_policy { + u32 flags; + u32 reset; + u64 timeslice; + u64 preempt_timeout; +}; + +enum selftest_scheduler_modify { + SELFTEST_SCHEDULER_MODIFY_NO_HANGCHECK = 0, + SELFTEST_SCHEDULER_MODIFY_FAST_RESET, +}; + +struct intel_engine_cs *intel_selftest_find_any_engine(struct intel_gt *gt); +int intel_selftest_modify_policy(struct intel_engine_cs *engine, + struct intel_selftest_saved_policy *saved, + enum selftest_scheduler_modify modify_type); +int intel_selftest_restore_policy(struct intel_engine_cs *engine, + struct intel_selftest_saved_policy *saved); +int intel_selftest_wait_for_rq(struct i915_request *rq); + +#endif diff --git a/drivers/gpu/drm/i915/selftests/intel_uncore.c b/drivers/gpu/drm/i915/selftests/intel_uncore.c index 8ef9e6a4ad05..507bf42a1aaf 100644 --- a/drivers/gpu/drm/i915/selftests/intel_uncore.c +++ b/drivers/gpu/drm/i915/selftests/intel_uncore.c @@ -24,6 +24,8 @@ #include "../i915_selftest.h" +#include "gt/intel_gt.h" + static int intel_fw_table_check(const struct intel_forcewake_range *ranges, unsigned int num_ranges, bool is_watertight) @@ -62,29 +64,42 @@ static int intel_fw_table_check(const struct intel_forcewake_range *ranges, static int intel_shadow_table_check(void) { struct { - const i915_reg_t *regs; + const struct i915_mmio_range *regs; unsigned int size; - } reg_lists[] = { + } range_lists[] = { { gen8_shadowed_regs, ARRAY_SIZE(gen8_shadowed_regs) }, { gen11_shadowed_regs, ARRAY_SIZE(gen11_shadowed_regs) }, { gen12_shadowed_regs, ARRAY_SIZE(gen12_shadowed_regs) }, + { dg2_shadowed_regs, ARRAY_SIZE(dg2_shadowed_regs) }, + { mtl_shadowed_regs, ARRAY_SIZE(mtl_shadowed_regs) }, + { xelpmp_shadowed_regs, ARRAY_SIZE(xelpmp_shadowed_regs) }, }; - const i915_reg_t *reg; + const struct i915_mmio_range *range; unsigned int i, j; s32 prev; - for (j = 0; j < ARRAY_SIZE(reg_lists); ++j) { - reg = reg_lists[j].regs; - for (i = 0, prev = -1; i < reg_lists[j].size; i++, reg++) { - u32 offset = i915_mmio_reg_offset(*reg); + for (j = 0; j < ARRAY_SIZE(range_lists); ++j) { + range = range_lists[j].regs; + for (i = 0, prev = -1; i < range_lists[j].size; i++, range++) { + if (range->end < range->start) { + pr_err("%s: range[%d]:(%06x-%06x) has end before start\n", + __func__, i, range->start, range->end); + return -EINVAL; + } + + if (prev >= (s32)range->start) { + pr_err("%s: range[%d]:(%06x-%06x) is before end of previous (%06x)\n", + __func__, i, range->start, range->end, prev); + return -EINVAL; + } - if (prev >= (s32)offset) { - pr_err("%s: entry[%d]:(%x) is before previous (%x)\n", - __func__, i, offset, prev); + if (range->start % 4) { + pr_err("%s: range[%d]:(%06x-%06x) has non-dword-aligned start\n", + __func__, i, range->start, range->end); return -EINVAL; } - prev = offset; + prev = range->end; } } @@ -103,6 +118,8 @@ int intel_uncore_mock_selftests(void) { __gen9_fw_ranges, ARRAY_SIZE(__gen9_fw_ranges), true }, { __gen11_fw_ranges, ARRAY_SIZE(__gen11_fw_ranges), true }, { __gen12_fw_ranges, ARRAY_SIZE(__gen12_fw_ranges), true }, + { __mtl_fw_ranges, ARRAY_SIZE(__mtl_fw_ranges), true }, + { __xelpmp_fw_ranges, ARRAY_SIZE(__xelpmp_fw_ranges), true }, }; int err, i; @@ -192,7 +209,7 @@ static int live_forcewake_ops(void *arg) for_each_engine(engine, gt, id) { i915_reg_t mmio = _MMIO(engine->mmio_base + r->offset); - u32 __iomem *reg = uncore->regs + engine->mmio_base + r->offset; + u32 __iomem *reg = intel_uncore_regs(uncore) + engine->mmio_base + r->offset; enum forcewake_domains fw_domains; u32 val; @@ -260,13 +277,15 @@ static int live_forcewake_domains(void *arg) #define FW_RANGE 0x40000 struct intel_gt *gt = arg; struct intel_uncore *uncore = gt->uncore; + struct drm_i915_private *i915 = gt->i915; + struct intel_display *display = i915->display; unsigned long *valid; u32 offset; int err; - if (!HAS_FPGA_DBG_UNCLAIMED(gt->i915) && - !IS_VALLEYVIEW(gt->i915) && - !IS_CHERRYVIEW(gt->i915)) + if (!HAS_FPGA_DBG_UNCLAIMED(display) && + !IS_VALLEYVIEW(i915) && + !IS_CHERRYVIEW(i915)) return 0; /* @@ -332,5 +351,5 @@ int intel_uncore_live_selftests(struct drm_i915_private *i915) SUBTEST(live_forcewake_domains), }; - return intel_gt_live_subtests(tests, &i915->gt); + return intel_gt_live_subtests(tests, to_gt(i915)); } diff --git a/drivers/gpu/drm/i915/selftests/lib_sw_fence.c b/drivers/gpu/drm/i915/selftests/lib_sw_fence.c index 080b90b63d16..d79e4defb71d 100644 --- a/drivers/gpu/drm/i915/selftests/lib_sw_fence.c +++ b/drivers/gpu/drm/i915/selftests/lib_sw_fence.c @@ -26,7 +26,7 @@ /* Small library of different fence types useful for writing tests */ -static int __i915_sw_fence_call +static int nop_fence_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state) { return NOTIFY_DONE; @@ -41,12 +41,12 @@ void __onstack_fence_init(struct i915_sw_fence *fence, __init_waitqueue_head(&fence->wait, name, key); atomic_set(&fence->pending, 1); fence->error = 0; - fence->flags = (unsigned long)nop_fence_notify; + fence->fn = nop_fence_notify; } void onstack_fence_fini(struct i915_sw_fence *fence) { - if (!fence->flags) + if (!fence->fn) return; i915_sw_fence_commit(fence); @@ -55,7 +55,7 @@ void onstack_fence_fini(struct i915_sw_fence *fence) static void timed_fence_wake(struct timer_list *t) { - struct timed_fence *tf = from_timer(tf, t, timer); + struct timed_fence *tf = timer_container_of(tf, t, timer); i915_sw_fence_commit(&tf->fence); } @@ -74,10 +74,10 @@ void timed_fence_init(struct timed_fence *tf, unsigned long expires) void timed_fence_fini(struct timed_fence *tf) { - if (del_timer_sync(&tf->timer)) + if (timer_delete_sync(&tf->timer)) i915_sw_fence_commit(&tf->fence); - destroy_timer_on_stack(&tf->timer); + timer_destroy_on_stack(&tf->timer); i915_sw_fence_fini(&tf->fence); } @@ -89,7 +89,7 @@ struct heap_fence { }; }; -static int __i915_sw_fence_call +static int heap_fence_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state) { struct heap_fence *h = container_of(fence, typeof(*h), fence); diff --git a/drivers/gpu/drm/i915/selftests/librapl.c b/drivers/gpu/drm/i915/selftests/librapl.c index eb03b5b28bad..25b8726b9dff 100644 --- a/drivers/gpu/drm/i915/selftests/librapl.c +++ b/drivers/gpu/drm/i915/selftests/librapl.c @@ -22,12 +22,12 @@ u64 librapl_energy_uJ(void) unsigned long long power; u32 units; - if (rdmsrl_safe(MSR_RAPL_POWER_UNIT, &power)) + if (rdmsrq_safe(MSR_RAPL_POWER_UNIT, &power)) return 0; units = (power & 0x1f00) >> 8; - if (rdmsrl_safe(MSR_PP1_ENERGY_STATUS, &power)) + if (rdmsrq_safe(MSR_PP1_ENERGY_STATUS, &power)) return 0; return (1000000 * power) >> units; /* convert to uJ */ diff --git a/drivers/gpu/drm/i915/selftests/mock_gem_device.c b/drivers/gpu/drm/i915/selftests/mock_gem_device.c index d189c4bd4bef..b59626c4994c 100644 --- a/drivers/gpu/drm/i915/selftests/mock_gem_device.c +++ b/drivers/gpu/drm/i915/selftests/mock_gem_device.c @@ -28,9 +28,12 @@ #include <drm/drm_managed.h> +#include "display/intel_display_device.h" + #include "gt/intel_gt.h" #include "gt/intel_gt_requests.h" #include "gt/mock_engine.h" +#include "i915_driver.h" #include "intel_memory_region.h" #include "intel_region_ttm.h" @@ -45,14 +48,15 @@ void mock_device_flush(struct drm_i915_private *i915) { - struct intel_gt *gt = &i915->gt; + struct intel_gt *gt = to_gt(i915); struct intel_engine_cs *engine; enum intel_engine_id id; do { for_each_engine(engine, gt, id) mock_engine_flush(engine); - } while (intel_gt_retire_requests_timeout(gt, MAX_SCHEDULE_TIMEOUT)); + } while (intel_gt_retire_requests_timeout(gt, MAX_SCHEDULE_TIMEOUT, + NULL)); } static void mock_device_release(struct drm_device *dev) @@ -63,16 +67,16 @@ static void mock_device_release(struct drm_device *dev) goto out; mock_device_flush(i915); - intel_gt_driver_remove(&i915->gt); + intel_gt_driver_remove(to_gt(i915)); i915_gem_drain_workqueue(i915); - i915_gem_drain_freed_objects(i915); - mock_fini_ggtt(&i915->ggtt); + mock_fini_ggtt(to_gt(i915)->ggtt); + destroy_workqueue(i915->unordered_wq); destroy_workqueue(i915->wq); intel_region_ttm_device_fini(i915); - intel_gt_driver_late_release(&i915->gt); + intel_gt_driver_late_release_all(i915); intel_memory_regions_driver_release(i915); drm_mode_config_cleanup(&i915->drm); @@ -111,12 +115,36 @@ static struct dev_pm_domain pm_domain = { }, }; +static void mock_gt_probe(struct drm_i915_private *i915) +{ + i915->gt[0]->name = "Mock GT"; +} + +static const struct intel_device_info mock_info = { + .__runtime.graphics.ip.ver = -1, + .__runtime.page_sizes = (I915_GTT_PAGE_SIZE_4K | + I915_GTT_PAGE_SIZE_64K | + I915_GTT_PAGE_SIZE_2M), + .memory_regions = BIT(INTEL_REGION_SMEM), + .platform_engine_mask = BIT(0), + + /* simply use legacy cache level for mock device */ + .max_pat_index = 3, + .cachelevel_to_pat = { + [I915_CACHE_NONE] = 0, + [I915_CACHE_LLC] = 1, + [I915_CACHE_L3_LLC] = 2, + [I915_CACHE_WT] = 3, + }, +}; + struct drm_i915_private *mock_gem_device(void) { -#if IS_ENABLED(CONFIG_IOMMU_API) && defined(CONFIG_INTEL_IOMMU) +#if IS_ENABLED(CONFIG_IOMMU_API) && IS_ENABLED(CONFIG_INTEL_IOMMU) static struct dev_iommu fake_iommu = { .priv = (void *)-1 }; #endif struct drm_i915_private *i915; + struct intel_display *display; struct pci_dev *pdev; int ret; @@ -129,7 +157,7 @@ struct drm_i915_private *mock_gem_device(void) dev_set_name(&pdev->dev, "mock"); dma_coerce_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); -#if IS_ENABLED(CONFIG_IOMMU_API) && defined(CONFIG_INTEL_IOMMU) +#if IS_ENABLED(CONFIG_IOMMU_API) && IS_ENABLED(CONFIG_INTEL_IOMMU) /* HACK to disable iommu for the fake device; force identity mapping */ pdev->dev.iommu = &fake_iommu; #endif @@ -148,7 +176,20 @@ struct drm_i915_private *mock_gem_device(void) return NULL; } - pci_set_drvdata(pdev, i915); + pci_set_drvdata(pdev, &i915->drm); + + /* Device parameters start as a copy of module parameters. */ + i915_params_copy(&i915->params, &i915_modparams); + + /* Set up device info and initial runtime info. */ + intel_device_info_driver_create(i915, pdev->device, &mock_info); + + /* FIXME: Can we run selftests using a mock device without display? */ + display = intel_display_device_probe(pdev, i915_driver_parent_interface()); + if (IS_ERR(display)) + goto err_device; + + i915->display = display; dev_pm_domain_set(&pdev->dev, &pm_domain); pm_runtime_enable(&pdev->dev); @@ -156,30 +197,23 @@ struct drm_i915_private *mock_gem_device(void) if (pm_runtime_enabled(&pdev->dev)) WARN_ON(pm_runtime_get_sync(&pdev->dev)); - - i915_params_copy(&i915->params, &i915_modparams); - intel_runtime_pm_init_early(&i915->runtime_pm); + /* wakeref tracking has significant overhead */ + i915->runtime_pm.no_wakeref_tracking = true; /* Using the global GTT may ask questions about KMS users, so prepare */ drm_mode_config_init(&i915->drm); - mkwrite_device_info(i915)->graphics_ver = -1; - - mkwrite_device_info(i915)->page_sizes = - I915_GTT_PAGE_SIZE_4K | - I915_GTT_PAGE_SIZE_64K | - I915_GTT_PAGE_SIZE_2M; - - mkwrite_device_info(i915)->memory_regions = REGION_SMEM; intel_memory_regions_hw_probe(i915); - mock_uncore_init(&i915->uncore, i915); + spin_lock_init(&i915->gpu_error.lock); i915_gem_init__mm(i915); - intel_gt_init_early(&i915->gt, i915); - atomic_inc(&i915->gt.wakeref.count); /* disable; no hw support */ - i915->gt.awake = -ENODEV; + intel_root_gt_init_early(i915); + mock_uncore_init(&i915->uncore, i915); + atomic_inc(&to_gt(i915)->wakeref.count); /* disable; no hw support */ + to_gt(i915)->awake = INTEL_WAKEREF_MOCK_GT; + mock_gt_probe(i915); ret = intel_region_ttm_device_init(i915); if (ret) @@ -189,22 +223,30 @@ struct drm_i915_private *mock_gem_device(void) if (!i915->wq) goto err_drv; + i915->unordered_wq = alloc_workqueue("mock-unordered", 0, 0); + if (!i915->unordered_wq) + goto err_wq; + mock_init_contexts(i915); - mock_init_ggtt(i915, &i915->ggtt); - i915->gt.vm = i915_vm_get(&i915->ggtt.vm); + /* allocate the ggtt */ + ret = intel_gt_assign_ggtt(to_gt(i915)); + if (ret) + goto err_unlock; + + mock_init_ggtt(to_gt(i915)); + to_gt(i915)->vm = i915_vm_get(&to_gt(i915)->ggtt->vm); - mkwrite_device_info(i915)->platform_engine_mask = BIT(0); - i915->gt.info.engine_mask = BIT(0); + to_gt(i915)->info.engine_mask = BIT(0); - i915->gt.engine[RCS0] = mock_engine(i915, "mock", RCS0); - if (!i915->gt.engine[RCS0]) + to_gt(i915)->engine[RCS0] = mock_engine(i915, "mock", RCS0); + if (!to_gt(i915)->engine[RCS0]) goto err_unlock; - if (mock_engine_init(i915->gt.engine[RCS0])) + if (mock_engine_init(to_gt(i915)->engine[RCS0])) goto err_context; - __clear_bit(I915_WEDGED, &i915->gt.reset.flags); + __clear_bit(I915_WEDGED, &to_gt(i915)->reset.flags); intel_engines_driver_register(i915); i915->do_release = true; @@ -213,15 +255,18 @@ struct drm_i915_private *mock_gem_device(void) return i915; err_context: - intel_gt_driver_remove(&i915->gt); + intel_gt_driver_remove(to_gt(i915)); err_unlock: + destroy_workqueue(i915->unordered_wq); +err_wq: destroy_workqueue(i915->wq); err_drv: intel_region_ttm_device_fini(i915); err_ttm: - intel_gt_driver_late_release(&i915->gt); + intel_gt_driver_late_release_all(i915); intel_memory_regions_driver_release(i915); drm_mode_config_cleanup(&i915->drm); +err_device: mock_destroy_device(i915); return NULL; @@ -231,6 +276,8 @@ void mock_destroy_device(struct drm_i915_private *i915) { struct device *dev = i915->drm.dev; + intel_display_device_remove(i915->display); + devres_release_group(dev, NULL); put_device(dev); } diff --git a/drivers/gpu/drm/i915/selftests/mock_gtt.c b/drivers/gpu/drm/i915/selftests/mock_gtt.c index 5c7ae40bba63..a516c0aa88fd 100644 --- a/drivers/gpu/drm/i915/selftests/mock_gtt.c +++ b/drivers/gpu/drm/i915/selftests/mock_gtt.c @@ -27,29 +27,29 @@ static void mock_insert_page(struct i915_address_space *vm, dma_addr_t addr, u64 offset, - enum i915_cache_level level, + unsigned int pat_index, u32 flags) { } static void mock_insert_entries(struct i915_address_space *vm, - struct i915_vma *vma, - enum i915_cache_level level, u32 flags) + struct i915_vma_resource *vma_res, + unsigned int pat_index, u32 flags) { } static void mock_bind_ppgtt(struct i915_address_space *vm, struct i915_vm_pt_stash *stash, - struct i915_vma *vma, - enum i915_cache_level cache_level, + struct i915_vma_resource *vma_res, + unsigned int pat_index, u32 flags) { GEM_BUG_ON(flags & I915_VMA_GLOBAL_BIND); - set_bit(I915_VMA_LOCAL_BIND_BIT, __i915_vma_flags(vma)); + vma_res->bound_flags |= flags; } static void mock_unbind_ppgtt(struct i915_address_space *vm, - struct i915_vma *vma) + struct i915_vma_resource *vma_res) { } @@ -70,15 +70,15 @@ struct i915_ppgtt *mock_ppgtt(struct drm_i915_private *i915, const char *name) if (!ppgtt) return NULL; - ppgtt->vm.gt = &i915->gt; + ppgtt->vm.gt = to_gt(i915); ppgtt->vm.i915 = i915; ppgtt->vm.total = round_down(U64_MAX, PAGE_SIZE); - ppgtt->vm.file = ERR_PTR(-ENODEV); ppgtt->vm.dma = i915->drm.dev; i915_address_space_init(&ppgtt->vm, VM_CLASS_PPGTT); ppgtt->vm.alloc_pt_dma = alloc_pt_dma; + ppgtt->vm.alloc_scratch_dma = alloc_pt_dma; ppgtt->vm.clear_range = mock_clear_range; ppgtt->vm.insert_page = mock_insert_page; @@ -87,38 +87,37 @@ struct i915_ppgtt *mock_ppgtt(struct drm_i915_private *i915, const char *name) ppgtt->vm.vma_ops.bind_vma = mock_bind_ppgtt; ppgtt->vm.vma_ops.unbind_vma = mock_unbind_ppgtt; - ppgtt->vm.vma_ops.set_pages = ppgtt_set_pages; - ppgtt->vm.vma_ops.clear_pages = clear_pages; return ppgtt; } static void mock_bind_ggtt(struct i915_address_space *vm, struct i915_vm_pt_stash *stash, - struct i915_vma *vma, - enum i915_cache_level cache_level, + struct i915_vma_resource *vma_res, + unsigned int pat_index, u32 flags) { } static void mock_unbind_ggtt(struct i915_address_space *vm, - struct i915_vma *vma) + struct i915_vma_resource *vma_res) { } -void mock_init_ggtt(struct drm_i915_private *i915, struct i915_ggtt *ggtt) +void mock_init_ggtt(struct intel_gt *gt) { - memset(ggtt, 0, sizeof(*ggtt)); + struct i915_ggtt *ggtt = gt->ggtt; - ggtt->vm.gt = &i915->gt; - ggtt->vm.i915 = i915; + ggtt->vm.gt = gt; + ggtt->vm.i915 = gt->i915; ggtt->vm.is_ggtt = true; - ggtt->gmadr = (struct resource) DEFINE_RES_MEM(0, 2048 * PAGE_SIZE); + ggtt->gmadr = DEFINE_RES_MEM(0, 2048 * PAGE_SIZE); ggtt->mappable_end = resource_size(&ggtt->gmadr); ggtt->vm.total = 4096 * PAGE_SIZE; ggtt->vm.alloc_pt_dma = alloc_pt_dma; + ggtt->vm.alloc_scratch_dma = alloc_pt_dma; ggtt->vm.clear_range = mock_clear_range; ggtt->vm.insert_page = mock_insert_page; @@ -127,11 +126,8 @@ void mock_init_ggtt(struct drm_i915_private *i915, struct i915_ggtt *ggtt) ggtt->vm.vma_ops.bind_vma = mock_bind_ggtt; ggtt->vm.vma_ops.unbind_vma = mock_unbind_ggtt; - ggtt->vm.vma_ops.set_pages = ggtt_set_pages; - ggtt->vm.vma_ops.clear_pages = clear_pages; i915_address_space_init(&ggtt->vm, VM_CLASS_GGTT); - i915->gt.ggtt = ggtt; } void mock_fini_ggtt(struct i915_ggtt *ggtt) diff --git a/drivers/gpu/drm/i915/selftests/mock_gtt.h b/drivers/gpu/drm/i915/selftests/mock_gtt.h index e3f224f43beb..d6eb90bd7f3f 100644 --- a/drivers/gpu/drm/i915/selftests/mock_gtt.h +++ b/drivers/gpu/drm/i915/selftests/mock_gtt.h @@ -27,8 +27,9 @@ struct drm_i915_private; struct i915_ggtt; +struct intel_gt; -void mock_init_ggtt(struct drm_i915_private *i915, struct i915_ggtt *ggtt); +void mock_init_ggtt(struct intel_gt *gt); void mock_fini_ggtt(struct i915_ggtt *ggtt); struct i915_ppgtt *mock_ppgtt(struct drm_i915_private *i915, const char *name); diff --git a/drivers/gpu/drm/i915/selftests/mock_region.c b/drivers/gpu/drm/i915/selftests/mock_region.c index eafc5a04975c..6324eb32d4dd 100644 --- a/drivers/gpu/drm/i915/selftests/mock_region.c +++ b/drivers/gpu/drm/i915/selftests/mock_region.c @@ -3,9 +3,8 @@ * Copyright © 2019-2021 Intel Corporation */ -#include <linux/scatterlist.h> - #include <drm/ttm/ttm_placement.h> +#include <linux/scatterlist.h> #include "gem/i915_gem_region.h" #include "intel_memory_region.h" @@ -16,35 +15,39 @@ static void mock_region_put_pages(struct drm_i915_gem_object *obj, struct sg_table *pages) { - intel_region_ttm_node_free(obj->mm.region, obj->mm.st_mm_node); - sg_free_table(pages); - kfree(pages); + i915_refct_sgt_put(obj->mm.rsgt); + obj->mm.rsgt = NULL; + intel_region_ttm_resource_free(obj->mm.region, obj->mm.res); } static int mock_region_get_pages(struct drm_i915_gem_object *obj) { - unsigned int flags; struct sg_table *pages; - - flags = I915_ALLOC_MIN_PAGE_SIZE; - if (obj->flags & I915_BO_ALLOC_CONTIGUOUS) - flags |= I915_ALLOC_CONTIGUOUS; - - obj->mm.st_mm_node = intel_region_ttm_node_alloc(obj->mm.region, - obj->base.size, - flags); - if (IS_ERR(obj->mm.st_mm_node)) - return PTR_ERR(obj->mm.st_mm_node); - - pages = intel_region_ttm_node_to_st(obj->mm.region, obj->mm.st_mm_node); - if (IS_ERR(pages)) { - intel_region_ttm_node_free(obj->mm.region, obj->mm.st_mm_node); - return PTR_ERR(pages); + int err; + + obj->mm.res = intel_region_ttm_resource_alloc(obj->mm.region, + obj->bo_offset, + obj->base.size, + obj->flags); + if (IS_ERR(obj->mm.res)) + return PTR_ERR(obj->mm.res); + + obj->mm.rsgt = intel_region_ttm_resource_to_rsgt(obj->mm.region, + obj->mm.res, + obj->mm.region->min_page_size); + if (IS_ERR(obj->mm.rsgt)) { + err = PTR_ERR(obj->mm.rsgt); + goto err_free_resource; } - __i915_gem_object_set_pages(obj, pages, i915_sg_dma_sizes(pages->sgl)); + pages = &obj->mm.rsgt->table; + __i915_gem_object_set_pages(obj, pages); return 0; + +err_free_resource: + intel_region_ttm_resource_free(obj->mm.region, obj->mm.res); + return err; } static const struct drm_i915_gem_object_ops mock_region_obj_ops = { @@ -56,7 +59,9 @@ static const struct drm_i915_gem_object_ops mock_region_obj_ops = { static int mock_object_init(struct intel_memory_region *mem, struct drm_i915_gem_object *obj, + resource_size_t offset, resource_size_t size, + resource_size_t page_size, unsigned int flags) { static struct lock_class_key lock_class; @@ -68,6 +73,8 @@ static int mock_object_init(struct intel_memory_region *mem, drm_gem_private_object_init(&i915->drm, &obj->base, size); i915_gem_object_init(obj, &mock_region_obj_ops, &lock_class, flags); + obj->bo_offset = offset; + obj->read_domains = I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT; i915_gem_object_set_cache_coherency(obj, I915_CACHE_NONE); @@ -77,13 +84,16 @@ static int mock_object_init(struct intel_memory_region *mem, return 0; } -static void mock_region_fini(struct intel_memory_region *mem) +static int mock_region_fini(struct intel_memory_region *mem) { struct drm_i915_private *i915 = mem->i915; int instance = mem->instance; + int ret; - intel_region_ttm_fini(mem); + ret = intel_region_ttm_fini(mem); ida_free(&i915->selftest.mock_region_instances, instance); + + return ret; } static const struct intel_memory_region_ops mock_region_ops = { @@ -97,7 +107,8 @@ mock_region_create(struct drm_i915_private *i915, resource_size_t start, resource_size_t size, resource_size_t min_page_size, - resource_size_t io_start) + resource_size_t io_start, + resource_size_t io_size) { int instance = ida_alloc_max(&i915->selftest.mock_region_instances, TTM_NUM_MEM_TYPES - TTM_PL_PRIV - 1, @@ -107,6 +118,7 @@ mock_region_create(struct drm_i915_private *i915, return ERR_PTR(instance); return intel_memory_region_create(i915, start, size, min_page_size, - io_start, INTEL_MEMORY_MOCK, instance, + io_start, io_size, + INTEL_MEMORY_MOCK, instance, &mock_region_ops); } diff --git a/drivers/gpu/drm/i915/selftests/mock_region.h b/drivers/gpu/drm/i915/selftests/mock_region.h index 329bf74dfaca..e36c3a433551 100644 --- a/drivers/gpu/drm/i915/selftests/mock_region.h +++ b/drivers/gpu/drm/i915/selftests/mock_region.h @@ -16,6 +16,7 @@ mock_region_create(struct drm_i915_private *i915, resource_size_t start, resource_size_t size, resource_size_t min_page_size, - resource_size_t io_start); + resource_size_t io_start, + resource_size_t io_size); #endif /* !__MOCK_REGION_H */ diff --git a/drivers/gpu/drm/i915/selftests/mock_request.c b/drivers/gpu/drm/i915/selftests/mock_request.c index 09f747228dff..1b0cf073e964 100644 --- a/drivers/gpu/drm/i915/selftests/mock_request.c +++ b/drivers/gpu/drm/i915/selftests/mock_request.c @@ -35,7 +35,7 @@ mock_request(struct intel_context *ce, unsigned long delay) /* NB the i915->requests slab cache is enlarged to fit mock_request */ request = intel_context_create_request(ce); if (IS_ERR(request)) - return NULL; + return request; request->mock.delay = delay; return request; diff --git a/drivers/gpu/drm/i915/selftests/mock_uncore.c b/drivers/gpu/drm/i915/selftests/mock_uncore.c index ca57e4008701..f2d6be5e1230 100644 --- a/drivers/gpu/drm/i915/selftests/mock_uncore.c +++ b/drivers/gpu/drm/i915/selftests/mock_uncore.c @@ -42,7 +42,7 @@ __nop_read(64) void mock_uncore_init(struct intel_uncore *uncore, struct drm_i915_private *i915) { - intel_uncore_init_early(uncore, i915); + intel_uncore_init_early(uncore, to_gt(i915)); ASSIGN_RAW_WRITE_MMIO_VFUNCS(uncore, nop); ASSIGN_RAW_READ_MMIO_VFUNCS(uncore, nop); diff --git a/drivers/gpu/drm/i915/selftests/scatterlist.c b/drivers/gpu/drm/i915/selftests/scatterlist.c index d599186d5b71..7e59591bbed6 100644 --- a/drivers/gpu/drm/i915/selftests/scatterlist.c +++ b/drivers/gpu/drm/i915/selftests/scatterlist.c @@ -22,7 +22,7 @@ */ #include <linux/prime_numbers.h> -#include <linux/random.h> +#include <linux/prandom.h> #include "i915_selftest.h" #include "i915_utils.h" @@ -220,6 +220,10 @@ static int alloc_table(struct pfn_table *pt, struct scatterlist *sg; unsigned long n, pfn; + /* restricted by sg_alloc_table */ + if (overflows_type(max, unsigned int)) + return -E2BIG; + if (sg_alloc_table(&pt->st, max, GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN)) return alloc_error; |