diff options
Diffstat (limited to 'drivers/gpu/drm/i915/gt')
77 files changed, 2684 insertions, 779 deletions
diff --git a/drivers/gpu/drm/i915/gt/gen6_ppgtt.c b/drivers/gpu/drm/i915/gt/gen6_ppgtt.c index 5aaacc53fa4c..c2bdc133c89a 100644 --- a/drivers/gpu/drm/i915/gt/gen6_ppgtt.c +++ b/drivers/gpu/drm/i915/gt/gen6_ppgtt.c @@ -109,7 +109,7 @@ static void gen6_ppgtt_clear_range(struct i915_address_space *vm, static void gen6_ppgtt_insert_entries(struct i915_address_space *vm, struct i915_vma_resource *vma_res, - enum i915_cache_level cache_level, + unsigned int pat_index, u32 flags) { struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); @@ -117,7 +117,7 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm, unsigned int first_entry = vma_res->start / I915_GTT_PAGE_SIZE; unsigned int act_pt = first_entry / GEN6_PTES; unsigned int act_pte = first_entry % GEN6_PTES; - const u32 pte_encode = vm->pte_encode(0, cache_level, flags); + const u32 pte_encode = vm->pte_encode(0, pat_index, flags); struct sgt_dma iter = sgt_dma(vma_res); gen6_pte_t *vaddr; @@ -227,7 +227,9 @@ static int gen6_ppgtt_init_scratch(struct gen6_ppgtt *ppgtt) vm->scratch[0]->encode = vm->pte_encode(px_dma(vm->scratch[0]), - I915_CACHE_NONE, PTE_READ_ONLY); + i915_gem_get_pat_index(vm->i915, + I915_CACHE_NONE), + PTE_READ_ONLY); vm->scratch[1] = vm->alloc_pt_dma(vm, I915_GTT_PAGE_SIZE_4K); if (IS_ERR(vm->scratch[1])) { @@ -278,7 +280,7 @@ static void gen6_ppgtt_cleanup(struct i915_address_space *vm) static void pd_vma_bind(struct i915_address_space *vm, struct i915_vm_pt_stash *stash, struct i915_vma_resource *vma_res, - enum i915_cache_level cache_level, + unsigned int pat_index, u32 unused) { struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); diff --git a/drivers/gpu/drm/i915/gt/gen8_engine_cs.c b/drivers/gpu/drm/i915/gt/gen8_engine_cs.c index e1c76e5bfa82..23857cc08eca 100644 --- a/drivers/gpu/drm/i915/gt/gen8_engine_cs.c +++ b/drivers/gpu/drm/i915/gt/gen8_engine_cs.c @@ -177,14 +177,40 @@ u32 *gen12_emit_aux_table_inv(struct intel_gt *gt, u32 *cs, const i915_reg_t inv return cs; } +static int mtl_dummy_pipe_control(struct i915_request *rq) +{ + /* Wa_14016712196 */ + if (IS_MTL_GRAPHICS_STEP(rq->engine->i915, M, STEP_A0, STEP_B0) || + IS_MTL_GRAPHICS_STEP(rq->engine->i915, P, STEP_A0, STEP_B0)) { + u32 *cs; + + /* dummy PIPE_CONTROL + depth flush */ + cs = intel_ring_begin(rq, 6); + if (IS_ERR(cs)) + return PTR_ERR(cs); + cs = gen12_emit_pipe_control(cs, + 0, + PIPE_CONTROL_DEPTH_CACHE_FLUSH, + LRC_PPHWSP_SCRATCH_ADDR); + intel_ring_advance(rq, cs); + } + + return 0; +} + int gen12_emit_flush_rcs(struct i915_request *rq, u32 mode) { struct intel_engine_cs *engine = rq->engine; if (mode & EMIT_FLUSH) { u32 flags = 0; + int err; u32 *cs; + err = mtl_dummy_pipe_control(rq); + if (err) + return err; + flags |= PIPE_CONTROL_TILE_CACHE_FLUSH; flags |= PIPE_CONTROL_FLUSH_L3; flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH; @@ -217,6 +243,11 @@ int gen12_emit_flush_rcs(struct i915_request *rq, u32 mode) if (mode & EMIT_INVALIDATE) { u32 flags = 0; u32 *cs, count; + int err; + + err = mtl_dummy_pipe_control(rq); + if (err) + return err; flags |= PIPE_CONTROL_COMMAND_CACHE_INVALIDATE; flags |= PIPE_CONTROL_TLB_INVALIDATE; @@ -733,6 +764,13 @@ u32 *gen12_emit_fini_breadcrumb_rcs(struct i915_request *rq, u32 *cs) PIPE_CONTROL_DC_FLUSH_ENABLE | PIPE_CONTROL_FLUSH_ENABLE); + /* Wa_14016712196 */ + if (IS_MTL_GRAPHICS_STEP(i915, M, STEP_A0, STEP_B0) || + IS_MTL_GRAPHICS_STEP(i915, P, STEP_A0, STEP_B0)) + /* dummy PIPE_CONTROL + depth flush */ + cs = gen12_emit_pipe_control(cs, 0, + PIPE_CONTROL_DEPTH_CACHE_FLUSH, 0); + if (GRAPHICS_VER(i915) == 12 && GRAPHICS_VER_FULL(i915) < IP_VER(12, 50)) /* Wa_1409600907 */ flags |= PIPE_CONTROL_DEPTH_STALL; diff --git a/drivers/gpu/drm/i915/gt/gen8_ppgtt.c b/drivers/gpu/drm/i915/gt/gen8_ppgtt.c index 4daaa6f55668..f948d33e5ec5 100644 --- a/drivers/gpu/drm/i915/gt/gen8_ppgtt.c +++ b/drivers/gpu/drm/i915/gt/gen8_ppgtt.c @@ -29,7 +29,7 @@ static u64 gen8_pde_encode(const dma_addr_t addr, } static u64 gen8_pte_encode(dma_addr_t addr, - enum i915_cache_level level, + unsigned int pat_index, u32 flags) { gen8_pte_t pte = addr | GEN8_PAGE_PRESENT | GEN8_PAGE_RW; @@ -40,7 +40,12 @@ static u64 gen8_pte_encode(dma_addr_t addr, if (flags & PTE_LM) pte |= GEN12_PPGTT_PTE_LM; - switch (level) { + /* + * For pre-gen12 platforms pat_index is the same as enum + * i915_cache_level, so the switch-case here is still valid. + * See translation table defined by LEGACY_CACHELEVEL. + */ + switch (pat_index) { case I915_CACHE_NONE: pte |= PPAT_UNCACHED; break; @@ -55,6 +60,33 @@ static u64 gen8_pte_encode(dma_addr_t addr, return pte; } +static u64 gen12_pte_encode(dma_addr_t addr, + unsigned int pat_index, + u32 flags) +{ + gen8_pte_t pte = addr | GEN8_PAGE_PRESENT | GEN8_PAGE_RW; + + if (unlikely(flags & PTE_READ_ONLY)) + pte &= ~GEN8_PAGE_RW; + + if (flags & PTE_LM) + pte |= GEN12_PPGTT_PTE_LM; + + if (pat_index & BIT(0)) + pte |= GEN12_PPGTT_PTE_PAT0; + + if (pat_index & BIT(1)) + pte |= GEN12_PPGTT_PTE_PAT1; + + if (pat_index & BIT(2)) + pte |= GEN12_PPGTT_PTE_PAT2; + + if (pat_index & BIT(3)) + pte |= MTL_PPGTT_PTE_PAT3; + + return pte; +} + static void gen8_ppgtt_notify_vgt(struct i915_ppgtt *ppgtt, bool create) { struct drm_i915_private *i915 = ppgtt->vm.i915; @@ -423,11 +455,11 @@ gen8_ppgtt_insert_pte(struct i915_ppgtt *ppgtt, struct i915_page_directory *pdp, struct sgt_dma *iter, u64 idx, - enum i915_cache_level cache_level, + unsigned int pat_index, u32 flags) { struct i915_page_directory *pd; - const gen8_pte_t pte_encode = gen8_pte_encode(0, cache_level, flags); + const gen8_pte_t pte_encode = ppgtt->vm.pte_encode(0, pat_index, flags); gen8_pte_t *vaddr; pd = i915_pd_entry(pdp, gen8_pd_index(idx, 2)); @@ -470,10 +502,10 @@ static void xehpsdv_ppgtt_insert_huge(struct i915_address_space *vm, struct i915_vma_resource *vma_res, struct sgt_dma *iter, - enum i915_cache_level cache_level, + unsigned int pat_index, u32 flags) { - const gen8_pte_t pte_encode = vm->pte_encode(0, cache_level, flags); + const gen8_pte_t pte_encode = vm->pte_encode(0, pat_index, flags); unsigned int rem = sg_dma_len(iter->sg); u64 start = vma_res->start; u64 end = start + vma_res->vma_size; @@ -570,6 +602,7 @@ xehpsdv_ppgtt_insert_huge(struct i915_address_space *vm, } } while (rem >= page_size && index < max); + drm_clflush_virt_range(vaddr, PAGE_SIZE); vma_res->page_sizes_gtt |= page_size; } while (iter->sg && sg_dma_len(iter->sg)); } @@ -577,10 +610,10 @@ xehpsdv_ppgtt_insert_huge(struct i915_address_space *vm, static void gen8_ppgtt_insert_huge(struct i915_address_space *vm, struct i915_vma_resource *vma_res, struct sgt_dma *iter, - enum i915_cache_level cache_level, + unsigned int pat_index, u32 flags) { - const gen8_pte_t pte_encode = gen8_pte_encode(0, cache_level, flags); + const gen8_pte_t pte_encode = vm->pte_encode(0, pat_index, flags); unsigned int rem = sg_dma_len(iter->sg); u64 start = vma_res->start; @@ -700,17 +733,17 @@ static void gen8_ppgtt_insert_huge(struct i915_address_space *vm, static void gen8_ppgtt_insert(struct i915_address_space *vm, struct i915_vma_resource *vma_res, - enum i915_cache_level cache_level, + unsigned int pat_index, u32 flags) { struct i915_ppgtt * const ppgtt = i915_vm_to_ppgtt(vm); struct sgt_dma iter = sgt_dma(vma_res); if (vma_res->bi.page_sizes.sg > I915_GTT_PAGE_SIZE) { - if (HAS_64K_PAGES(vm->i915)) - xehpsdv_ppgtt_insert_huge(vm, vma_res, &iter, cache_level, flags); + if (GRAPHICS_VER_FULL(vm->i915) >= IP_VER(12, 50)) + xehpsdv_ppgtt_insert_huge(vm, vma_res, &iter, pat_index, flags); else - gen8_ppgtt_insert_huge(vm, vma_res, &iter, cache_level, flags); + gen8_ppgtt_insert_huge(vm, vma_res, &iter, pat_index, flags); } else { u64 idx = vma_res->start >> GEN8_PTE_SHIFT; @@ -719,7 +752,7 @@ static void gen8_ppgtt_insert(struct i915_address_space *vm, gen8_pdp_for_page_index(vm, idx); idx = gen8_ppgtt_insert_pte(ppgtt, pdp, &iter, idx, - cache_level, flags); + pat_index, flags); } while (idx); vma_res->page_sizes_gtt = I915_GTT_PAGE_SIZE; @@ -729,7 +762,7 @@ static void gen8_ppgtt_insert(struct i915_address_space *vm, static void gen8_ppgtt_insert_entry(struct i915_address_space *vm, dma_addr_t addr, u64 offset, - enum i915_cache_level level, + unsigned int pat_index, u32 flags) { u64 idx = offset >> GEN8_PTE_SHIFT; @@ -743,14 +776,14 @@ static void gen8_ppgtt_insert_entry(struct i915_address_space *vm, GEM_BUG_ON(pt->is_compact); vaddr = px_vaddr(pt); - vaddr[gen8_pd_index(idx, 0)] = gen8_pte_encode(addr, level, flags); + vaddr[gen8_pd_index(idx, 0)] = vm->pte_encode(addr, pat_index, flags); drm_clflush_virt_range(&vaddr[gen8_pd_index(idx, 0)], sizeof(*vaddr)); } static void __xehpsdv_ppgtt_insert_entry_lm(struct i915_address_space *vm, dma_addr_t addr, u64 offset, - enum i915_cache_level level, + unsigned int pat_index, u32 flags) { u64 idx = offset >> GEN8_PTE_SHIFT; @@ -773,20 +806,20 @@ static void __xehpsdv_ppgtt_insert_entry_lm(struct i915_address_space *vm, } vaddr = px_vaddr(pt); - vaddr[gen8_pd_index(idx, 0) / 16] = gen8_pte_encode(addr, level, flags); + vaddr[gen8_pd_index(idx, 0) / 16] = vm->pte_encode(addr, pat_index, flags); } static void xehpsdv_ppgtt_insert_entry(struct i915_address_space *vm, dma_addr_t addr, u64 offset, - enum i915_cache_level level, + unsigned int pat_index, u32 flags) { if (flags & PTE_LM) return __xehpsdv_ppgtt_insert_entry_lm(vm, addr, offset, - level, flags); + pat_index, flags); - return gen8_ppgtt_insert_entry(vm, addr, offset, level, flags); + return gen8_ppgtt_insert_entry(vm, addr, offset, pat_index, flags); } static int gen8_init_scratch(struct i915_address_space *vm) @@ -820,8 +853,10 @@ static int gen8_init_scratch(struct i915_address_space *vm) pte_flags |= PTE_LM; vm->scratch[0]->encode = - gen8_pte_encode(px_dma(vm->scratch[0]), - I915_CACHE_NONE, pte_flags); + vm->pte_encode(px_dma(vm->scratch[0]), + i915_gem_get_pat_index(vm->i915, + I915_CACHE_NONE), + pte_flags); for (i = 1; i <= vm->top; i++) { struct drm_i915_gem_object *obj; @@ -963,7 +998,10 @@ struct i915_ppgtt *gen8_ppgtt_create(struct intel_gt *gt, */ ppgtt->vm.alloc_scratch_dma = alloc_pt_dma; - ppgtt->vm.pte_encode = gen8_pte_encode; + if (GRAPHICS_VER(gt->i915) >= 12) + ppgtt->vm.pte_encode = gen12_pte_encode; + else + ppgtt->vm.pte_encode = gen8_pte_encode; ppgtt->vm.bind_async_flags = I915_VMA_LOCAL_BIND; ppgtt->vm.insert_entries = gen8_ppgtt_insert; diff --git a/drivers/gpu/drm/i915/gt/gen8_ppgtt.h b/drivers/gpu/drm/i915/gt/gen8_ppgtt.h index f541d19264b4..19c635441642 100644 --- a/drivers/gpu/drm/i915/gt/gen8_ppgtt.h +++ b/drivers/gpu/drm/i915/gt/gen8_ppgtt.h @@ -10,13 +10,12 @@ struct i915_address_space; struct intel_gt; -enum i915_cache_level; struct i915_ppgtt *gen8_ppgtt_create(struct intel_gt *gt, unsigned long lmem_pt_obj_flags); u64 gen8_ggtt_pte_encode(dma_addr_t addr, - enum i915_cache_level level, + unsigned int pat_index, u32 flags); #endif diff --git a/drivers/gpu/drm/i915/gt/intel_context.c b/drivers/gpu/drm/i915/gt/intel_context.c index 2aa63ec521b8..a53b26178f0a 100644 --- a/drivers/gpu/drm/i915/gt/intel_context.c +++ b/drivers/gpu/drm/i915/gt/intel_context.c @@ -578,10 +578,13 @@ void intel_context_bind_parent_child(struct intel_context *parent, child->parallel.parent = parent; } -u64 intel_context_get_total_runtime_ns(const struct intel_context *ce) +u64 intel_context_get_total_runtime_ns(struct intel_context *ce) { u64 total, active; + if (ce->ops->update_stats) + ce->ops->update_stats(ce); + total = ce->stats.runtime.total; if (ce->ops->flags & COPS_RUNTIME_CYCLES) total *= ce->engine->gt->clock_period_ns; diff --git a/drivers/gpu/drm/i915/gt/intel_context.h b/drivers/gpu/drm/i915/gt/intel_context.h index 48f888c3da08..a80e3b7c24ff 100644 --- a/drivers/gpu/drm/i915/gt/intel_context.h +++ b/drivers/gpu/drm/i915/gt/intel_context.h @@ -97,7 +97,7 @@ void intel_context_bind_parent_child(struct intel_context *parent, /** * intel_context_lock_pinned - Stablises the 'pinned' status of the HW context - * @ce - the context + * @ce: the context * * Acquire a lock on the pinned status of the HW context, such that the context * can neither be bound to the GPU or unbound whilst the lock is held, i.e. @@ -111,7 +111,7 @@ static inline int intel_context_lock_pinned(struct intel_context *ce) /** * intel_context_is_pinned - Reports the 'pinned' status - * @ce - the context + * @ce: the context * * While in use by the GPU, the context, along with its ring and page * tables is pinned into memory and the GTT. @@ -133,7 +133,7 @@ static inline void intel_context_cancel_request(struct intel_context *ce, /** * intel_context_unlock_pinned - Releases the earlier locking of 'pinned' status - * @ce - the context + * @ce: the context * * Releases the lock earlier acquired by intel_context_unlock_pinned(). */ @@ -375,7 +375,7 @@ intel_context_clear_nopreempt(struct intel_context *ce) clear_bit(CONTEXT_NOPREEMPT, &ce->flags); } -u64 intel_context_get_total_runtime_ns(const struct intel_context *ce); +u64 intel_context_get_total_runtime_ns(struct intel_context *ce); u64 intel_context_get_avg_runtime_ns(struct intel_context *ce); static inline u64 intel_context_clock(void) diff --git a/drivers/gpu/drm/i915/gt/intel_context_types.h b/drivers/gpu/drm/i915/gt/intel_context_types.h index e36670f2e626..aceaac28a33e 100644 --- a/drivers/gpu/drm/i915/gt/intel_context_types.h +++ b/drivers/gpu/drm/i915/gt/intel_context_types.h @@ -58,6 +58,8 @@ struct intel_context_ops { void (*sched_disable)(struct intel_context *ce); + void (*update_stats)(struct intel_context *ce); + void (*reset)(struct intel_context *ce); void (*destroy)(struct kref *kref); diff --git a/drivers/gpu/drm/i915/gt/intel_engine_cs.c b/drivers/gpu/drm/i915/gt/intel_engine_cs.c index 5c6c9a6d469c..0aff5bb13c53 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_cs.c +++ b/drivers/gpu/drm/i915/gt/intel_engine_cs.c @@ -1515,7 +1515,7 @@ int intel_engines_init(struct intel_gt *gt) } /** - * intel_engines_cleanup_common - cleans up the engine state created by + * intel_engine_cleanup_common - cleans up the engine state created by * the common initiailizers. * @engine: Engine to cleanup. * diff --git a/drivers/gpu/drm/i915/gt/intel_engine_pm.c b/drivers/gpu/drm/i915/gt/intel_engine_pm.c index ee531a5c142c..21af0ec52223 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_pm.c +++ b/drivers/gpu/drm/i915/gt/intel_engine_pm.c @@ -296,9 +296,7 @@ static const struct intel_wakeref_ops wf_ops = { void intel_engine_init__pm(struct intel_engine_cs *engine) { - struct intel_runtime_pm *rpm = engine->uncore->rpm; - - intel_wakeref_init(&engine->wakeref, rpm, &wf_ops); + intel_wakeref_init(&engine->wakeref, engine->i915, &wf_ops); intel_engine_init_heartbeat(engine); intel_gsc_idle_msg_enable(engine); diff --git a/drivers/gpu/drm/i915/gt/intel_engine_types.h b/drivers/gpu/drm/i915/gt/intel_engine_types.h index 960291f88fd6..e99a6fa03d45 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_types.h +++ b/drivers/gpu/drm/i915/gt/intel_engine_types.h @@ -289,6 +289,7 @@ struct intel_engine_execlists { */ u8 csb_head; + /* private: selftest */ I915_SELFTEST_DECLARE(struct st_preempt_hang preempt_hang;) }; diff --git a/drivers/gpu/drm/i915/gt/intel_engine_user.c b/drivers/gpu/drm/i915/gt/intel_engine_user.c index cd4f1b126f75..dcedff41a825 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_user.c +++ b/drivers/gpu/drm/i915/gt/intel_engine_user.c @@ -117,7 +117,7 @@ static void set_scheduler_caps(struct drm_i915_private *i915) disabled |= (I915_SCHEDULER_CAP_ENABLED | I915_SCHEDULER_CAP_PRIORITY); - if (intel_uc_uses_guc_submission(&to_gt(i915)->uc)) + if (intel_uc_uses_guc_submission(&engine->gt->uc)) enabled |= I915_SCHEDULER_CAP_STATIC_PRIORITY_MAP; for (i = 0; i < ARRAY_SIZE(map); i++) { diff --git a/drivers/gpu/drm/i915/gt/intel_execlists_submission.c b/drivers/gpu/drm/i915/gt/intel_execlists_submission.c index 750326434677..2ebd937f3b4c 100644 --- a/drivers/gpu/drm/i915/gt/intel_execlists_submission.c +++ b/drivers/gpu/drm/i915/gt/intel_execlists_submission.c @@ -2327,6 +2327,7 @@ static u32 active_ccid(struct intel_engine_cs *engine) static void execlists_capture(struct intel_engine_cs *engine) { + struct drm_i915_private *i915 = engine->i915; struct execlists_capture *cap; if (!IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR)) @@ -2375,7 +2376,7 @@ static void execlists_capture(struct intel_engine_cs *engine) goto err_rq; INIT_WORK(&cap->work, execlists_capture_work); - schedule_work(&cap->work); + queue_work(i915->unordered_wq, &cap->work); return; err_rq: @@ -3680,7 +3681,7 @@ static void virtual_context_destroy(struct kref *kref) * lock, we can delegate the free of the engine to an RCU worker. */ INIT_RCU_WORK(&ve->rcu, rcu_virtual_context_destroy); - queue_rcu_work(system_wq, &ve->rcu); + queue_rcu_work(ve->context.engine->i915->unordered_wq, &ve->rcu); } static void virtual_engine_initial_hint(struct virtual_engine *ve) diff --git a/drivers/gpu/drm/i915/gt/intel_ggtt.c b/drivers/gpu/drm/i915/gt/intel_ggtt.c index 3c7f1ed92f5b..dd0ed941441a 100644 --- a/drivers/gpu/drm/i915/gt/intel_ggtt.c +++ b/drivers/gpu/drm/i915/gt/intel_ggtt.c @@ -220,8 +220,28 @@ static void guc_ggtt_invalidate(struct i915_ggtt *ggtt) } } +static u64 mtl_ggtt_pte_encode(dma_addr_t addr, + unsigned int pat_index, + u32 flags) +{ + gen8_pte_t pte = addr | GEN8_PAGE_PRESENT; + + WARN_ON_ONCE(addr & ~GEN12_GGTT_PTE_ADDR_MASK); + + if (flags & PTE_LM) + pte |= GEN12_GGTT_PTE_LM; + + if (pat_index & BIT(0)) + pte |= MTL_GGTT_PTE_PAT0; + + if (pat_index & BIT(1)) + pte |= MTL_GGTT_PTE_PAT1; + + return pte; +} + u64 gen8_ggtt_pte_encode(dma_addr_t addr, - enum i915_cache_level level, + unsigned int pat_index, u32 flags) { gen8_pte_t pte = addr | GEN8_PAGE_PRESENT; @@ -240,25 +260,25 @@ static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte) static void gen8_ggtt_insert_page(struct i915_address_space *vm, dma_addr_t addr, u64 offset, - enum i915_cache_level level, + unsigned int pat_index, u32 flags) { struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); gen8_pte_t __iomem *pte = (gen8_pte_t __iomem *)ggtt->gsm + offset / I915_GTT_PAGE_SIZE; - gen8_set_pte(pte, gen8_ggtt_pte_encode(addr, level, flags)); + gen8_set_pte(pte, ggtt->vm.pte_encode(addr, pat_index, flags)); ggtt->invalidate(ggtt); } static void gen8_ggtt_insert_entries(struct i915_address_space *vm, struct i915_vma_resource *vma_res, - enum i915_cache_level level, + unsigned int pat_index, u32 flags) { - const gen8_pte_t pte_encode = gen8_ggtt_pte_encode(0, level, flags); struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); + const gen8_pte_t pte_encode = ggtt->vm.pte_encode(0, pat_index, flags); gen8_pte_t __iomem *gte; gen8_pte_t __iomem *end; struct sgt_iter iter; @@ -315,14 +335,14 @@ static void gen8_ggtt_clear_range(struct i915_address_space *vm, static void gen6_ggtt_insert_page(struct i915_address_space *vm, dma_addr_t addr, u64 offset, - enum i915_cache_level level, + unsigned int pat_index, u32 flags) { struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); gen6_pte_t __iomem *pte = (gen6_pte_t __iomem *)ggtt->gsm + offset / I915_GTT_PAGE_SIZE; - iowrite32(vm->pte_encode(addr, level, flags), pte); + iowrite32(vm->pte_encode(addr, pat_index, flags), pte); ggtt->invalidate(ggtt); } @@ -335,7 +355,7 @@ static void gen6_ggtt_insert_page(struct i915_address_space *vm, */ static void gen6_ggtt_insert_entries(struct i915_address_space *vm, struct i915_vma_resource *vma_res, - enum i915_cache_level level, + unsigned int pat_index, u32 flags) { struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); @@ -352,7 +372,7 @@ static void gen6_ggtt_insert_entries(struct i915_address_space *vm, iowrite32(vm->scratch[0]->encode, gte++); end += (vma_res->node_size + vma_res->guard) / I915_GTT_PAGE_SIZE; for_each_sgt_daddr(addr, iter, vma_res->bi.pages) - iowrite32(vm->pte_encode(addr, level, flags), gte++); + iowrite32(vm->pte_encode(addr, pat_index, flags), gte++); GEM_BUG_ON(gte > end); /* Fill the allocated but "unused" space beyond the end of the buffer */ @@ -387,14 +407,15 @@ struct insert_page { struct i915_address_space *vm; dma_addr_t addr; u64 offset; - enum i915_cache_level level; + unsigned int pat_index; }; static int bxt_vtd_ggtt_insert_page__cb(void *_arg) { struct insert_page *arg = _arg; - gen8_ggtt_insert_page(arg->vm, arg->addr, arg->offset, arg->level, 0); + gen8_ggtt_insert_page(arg->vm, arg->addr, arg->offset, + arg->pat_index, 0); bxt_vtd_ggtt_wa(arg->vm); return 0; @@ -403,10 +424,10 @@ static int bxt_vtd_ggtt_insert_page__cb(void *_arg) static void bxt_vtd_ggtt_insert_page__BKL(struct i915_address_space *vm, dma_addr_t addr, u64 offset, - enum i915_cache_level level, + unsigned int pat_index, u32 unused) { - struct insert_page arg = { vm, addr, offset, level }; + struct insert_page arg = { vm, addr, offset, pat_index }; stop_machine(bxt_vtd_ggtt_insert_page__cb, &arg, NULL); } @@ -414,7 +435,7 @@ static void bxt_vtd_ggtt_insert_page__BKL(struct i915_address_space *vm, struct insert_entries { struct i915_address_space *vm; struct i915_vma_resource *vma_res; - enum i915_cache_level level; + unsigned int pat_index; u32 flags; }; @@ -422,7 +443,8 @@ static int bxt_vtd_ggtt_insert_entries__cb(void *_arg) { struct insert_entries *arg = _arg; - gen8_ggtt_insert_entries(arg->vm, arg->vma_res, arg->level, arg->flags); + gen8_ggtt_insert_entries(arg->vm, arg->vma_res, + arg->pat_index, arg->flags); bxt_vtd_ggtt_wa(arg->vm); return 0; @@ -430,10 +452,10 @@ static int bxt_vtd_ggtt_insert_entries__cb(void *_arg) static void bxt_vtd_ggtt_insert_entries__BKL(struct i915_address_space *vm, struct i915_vma_resource *vma_res, - enum i915_cache_level level, + unsigned int pat_index, u32 flags) { - struct insert_entries arg = { vm, vma_res, level, flags }; + struct insert_entries arg = { vm, vma_res, pat_index, flags }; stop_machine(bxt_vtd_ggtt_insert_entries__cb, &arg, NULL); } @@ -462,7 +484,7 @@ static void gen6_ggtt_clear_range(struct i915_address_space *vm, void intel_ggtt_bind_vma(struct i915_address_space *vm, struct i915_vm_pt_stash *stash, struct i915_vma_resource *vma_res, - enum i915_cache_level cache_level, + unsigned int pat_index, u32 flags) { u32 pte_flags; @@ -479,7 +501,7 @@ void intel_ggtt_bind_vma(struct i915_address_space *vm, if (vma_res->bi.lmem) pte_flags |= PTE_LM; - vm->insert_entries(vm, vma_res, cache_level, pte_flags); + vm->insert_entries(vm, vma_res, pat_index, pte_flags); vma_res->page_sizes_gtt = I915_GTT_PAGE_SIZE; } @@ -628,7 +650,7 @@ err: static void aliasing_gtt_bind_vma(struct i915_address_space *vm, struct i915_vm_pt_stash *stash, struct i915_vma_resource *vma_res, - enum i915_cache_level cache_level, + unsigned int pat_index, u32 flags) { u32 pte_flags; @@ -640,10 +662,10 @@ static void aliasing_gtt_bind_vma(struct i915_address_space *vm, if (flags & I915_VMA_LOCAL_BIND) ppgtt_bind_vma(&i915_vm_to_ggtt(vm)->alias->vm, - stash, vma_res, cache_level, flags); + stash, vma_res, pat_index, flags); if (flags & I915_VMA_GLOBAL_BIND) - vm->insert_entries(vm, vma_res, cache_level, pte_flags); + vm->insert_entries(vm, vma_res, pat_index, pte_flags); vma_res->bound_flags |= flags; } @@ -900,7 +922,9 @@ static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size) ggtt->vm.scratch[0]->encode = ggtt->vm.pte_encode(px_dma(ggtt->vm.scratch[0]), - I915_CACHE_NONE, pte_flags); + i915_gem_get_pat_index(i915, + I915_CACHE_NONE), + pte_flags); return 0; } @@ -981,18 +1005,26 @@ static int gen8_gmch_probe(struct i915_ggtt *ggtt) ggtt->vm.vma_ops.bind_vma = intel_ggtt_bind_vma; ggtt->vm.vma_ops.unbind_vma = intel_ggtt_unbind_vma; - ggtt->vm.pte_encode = gen8_ggtt_pte_encode; + if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70)) + ggtt->vm.pte_encode = mtl_ggtt_pte_encode; + else + ggtt->vm.pte_encode = gen8_ggtt_pte_encode; return ggtt_probe_common(ggtt, size); } +/* + * For pre-gen8 platforms pat_index is the same as enum i915_cache_level, + * so the switch-case statements in these PTE encode functions are still valid. + * See translation table LEGACY_CACHELEVEL. + */ static u64 snb_pte_encode(dma_addr_t addr, - enum i915_cache_level level, + unsigned int pat_index, u32 flags) { gen6_pte_t pte = GEN6_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID; - switch (level) { + switch (pat_index) { case I915_CACHE_L3_LLC: case I915_CACHE_LLC: pte |= GEN6_PTE_CACHE_LLC; @@ -1001,19 +1033,19 @@ static u64 snb_pte_encode(dma_addr_t addr, pte |= GEN6_PTE_UNCACHED; break; default: - MISSING_CASE(level); + MISSING_CASE(pat_index); } return pte; } static u64 ivb_pte_encode(dma_addr_t addr, - enum i915_cache_level level, + unsigned int pat_index, u32 flags) { gen6_pte_t pte = GEN6_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID; - switch (level) { + switch (pat_index) { case I915_CACHE_L3_LLC: pte |= GEN7_PTE_CACHE_L3_LLC; break; @@ -1024,14 +1056,14 @@ static u64 ivb_pte_encode(dma_addr_t addr, pte |= GEN6_PTE_UNCACHED; break; default: - MISSING_CASE(level); + MISSING_CASE(pat_index); } return pte; } static u64 byt_pte_encode(dma_addr_t addr, - enum i915_cache_level level, + unsigned int pat_index, u32 flags) { gen6_pte_t pte = GEN6_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID; @@ -1039,31 +1071,31 @@ static u64 byt_pte_encode(dma_addr_t addr, if (!(flags & PTE_READ_ONLY)) pte |= BYT_PTE_WRITEABLE; - if (level != I915_CACHE_NONE) + if (pat_index != I915_CACHE_NONE) pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES; return pte; } static u64 hsw_pte_encode(dma_addr_t addr, - enum i915_cache_level level, + unsigned int pat_index, u32 flags) { gen6_pte_t pte = HSW_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID; - if (level != I915_CACHE_NONE) + if (pat_index != I915_CACHE_NONE) pte |= HSW_WB_LLC_AGE3; return pte; } static u64 iris_pte_encode(dma_addr_t addr, - enum i915_cache_level level, + unsigned int pat_index, u32 flags) { gen6_pte_t pte = HSW_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID; - switch (level) { + switch (pat_index) { case I915_CACHE_NONE: break; case I915_CACHE_WT: @@ -1266,7 +1298,9 @@ bool i915_ggtt_resume_vm(struct i915_address_space *vm) */ vma->resource->bound_flags = 0; vma->ops->bind_vma(vm, NULL, vma->resource, - obj ? obj->cache_level : 0, + obj ? obj->pat_index : + i915_gem_get_pat_index(vm->i915, + I915_CACHE_NONE), was_bound); if (obj) { /* only used during resume => exclusive access */ @@ -1292,6 +1326,9 @@ void i915_ggtt_resume(struct i915_ggtt *ggtt) ggtt->vm.scratch_range(&ggtt->vm, ggtt->error_capture.start, ggtt->error_capture.size); + list_for_each_entry(gt, &ggtt->gt_list, ggtt_link) + intel_uc_resume_mappings(>->uc); + ggtt->invalidate(ggtt); if (flush) diff --git a/drivers/gpu/drm/i915/gt/intel_ggtt_gmch.c b/drivers/gpu/drm/i915/gt/intel_ggtt_gmch.c index d6a74ae2527b..866c416afb73 100644 --- a/drivers/gpu/drm/i915/gt/intel_ggtt_gmch.c +++ b/drivers/gpu/drm/i915/gt/intel_ggtt_gmch.c @@ -18,10 +18,10 @@ static void gmch_ggtt_insert_page(struct i915_address_space *vm, dma_addr_t addr, u64 offset, - enum i915_cache_level cache_level, + unsigned int pat_index, u32 unused) { - unsigned int flags = (cache_level == I915_CACHE_NONE) ? + unsigned int flags = (pat_index == I915_CACHE_NONE) ? AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY; intel_gmch_gtt_insert_page(addr, offset >> PAGE_SHIFT, flags); @@ -29,10 +29,10 @@ static void gmch_ggtt_insert_page(struct i915_address_space *vm, static void gmch_ggtt_insert_entries(struct i915_address_space *vm, struct i915_vma_resource *vma_res, - enum i915_cache_level cache_level, + unsigned int pat_index, u32 unused) { - unsigned int flags = (cache_level == I915_CACHE_NONE) ? + unsigned int flags = (pat_index == I915_CACHE_NONE) ? AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY; intel_gmch_gtt_insert_sg_entries(vma_res->bi.pages, vma_res->start >> PAGE_SHIFT, diff --git a/drivers/gpu/drm/i915/gt/intel_gt_buffer_pool.c b/drivers/gpu/drm/i915/gt/intel_gt_buffer_pool.c index cadfd85785b1..86b5a9ba323d 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_buffer_pool.c +++ b/drivers/gpu/drm/i915/gt/intel_gt_buffer_pool.c @@ -88,10 +88,11 @@ static void pool_free_work(struct work_struct *wrk) { struct intel_gt_buffer_pool *pool = container_of(wrk, typeof(*pool), work.work); + struct intel_gt *gt = container_of(pool, struct intel_gt, buffer_pool); if (pool_free_older_than(pool, HZ)) - schedule_delayed_work(&pool->work, - round_jiffies_up_relative(HZ)); + queue_delayed_work(gt->i915->unordered_wq, &pool->work, + round_jiffies_up_relative(HZ)); } static void pool_retire(struct i915_active *ref) @@ -99,6 +100,7 @@ static void pool_retire(struct i915_active *ref) struct intel_gt_buffer_pool_node *node = container_of(ref, typeof(*node), active); struct intel_gt_buffer_pool *pool = node->pool; + struct intel_gt *gt = container_of(pool, struct intel_gt, buffer_pool); struct list_head *list = bucket_for_size(pool, node->obj->base.size); unsigned long flags; @@ -116,8 +118,8 @@ static void pool_retire(struct i915_active *ref) WRITE_ONCE(node->age, jiffies ?: 1); /* 0 reserved for active nodes */ spin_unlock_irqrestore(&pool->lock, flags); - schedule_delayed_work(&pool->work, - round_jiffies_up_relative(HZ)); + queue_delayed_work(gt->i915->unordered_wq, &pool->work, + round_jiffies_up_relative(HZ)); } void intel_gt_buffer_pool_mark_used(struct intel_gt_buffer_pool_node *node) diff --git a/drivers/gpu/drm/i915/gt/intel_gt_irq.c b/drivers/gpu/drm/i915/gt/intel_gt_irq.c index 1b25a6039152..62fd00c9e519 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_irq.c +++ b/drivers/gpu/drm/i915/gt/intel_gt_irq.c @@ -7,6 +7,7 @@ #include "i915_drv.h" #include "i915_irq.h" +#include "i915_reg.h" #include "intel_breadcrumbs.h" #include "intel_gt.h" #include "intel_gt_irq.h" @@ -15,6 +16,7 @@ #include "intel_uncore.h" #include "intel_rps.h" #include "pxp/intel_pxp_irq.h" +#include "uc/intel_gsc_proxy.h" static void guc_irq_handler(struct intel_guc *guc, u16 iir) { @@ -81,6 +83,9 @@ gen11_other_irq_handler(struct intel_gt *gt, const u8 instance, if (instance == OTHER_GSC_INSTANCE) return intel_gsc_irq_handler(gt, iir); + if (instance == OTHER_GSC_HECI_2_INSTANCE) + return intel_gsc_proxy_irq_handler(>->uc.gsc, iir); + WARN_ONCE(1, "unhandled other interrupt instance=0x%x, iir=0x%x\n", instance, iir); } @@ -100,7 +105,10 @@ static struct intel_gt *pick_gt(struct intel_gt *gt, u8 class, u8 instance) case VIDEO_ENHANCEMENT_CLASS: return media_gt; case OTHER_CLASS: - if (instance == OTHER_GSC_INSTANCE && HAS_ENGINE(media_gt, GSC0)) + if (instance == OTHER_GSC_HECI_2_INSTANCE) + return media_gt; + if ((instance == OTHER_GSC_INSTANCE || instance == OTHER_KCR_INSTANCE) && + HAS_ENGINE(media_gt, GSC0)) return media_gt; fallthrough; default: @@ -256,6 +264,7 @@ void gen11_gt_irq_postinstall(struct intel_gt *gt) u32 irqs = GT_RENDER_USER_INTERRUPT; u32 guc_mask = intel_uc_wants_guc(>->uc) ? GUC_INTR_GUC2HOST : 0; u32 gsc_mask = 0; + u32 heci_mask = 0; u32 dmask; u32 smask; @@ -267,10 +276,16 @@ void gen11_gt_irq_postinstall(struct intel_gt *gt) dmask = irqs << 16 | irqs; smask = irqs << 16; - if (HAS_ENGINE(gt, GSC0)) + if (HAS_ENGINE(gt, GSC0)) { + /* + * the heci2 interrupt is enabled via the same register as the + * GSC interrupt, but it has its own mask register. + */ gsc_mask = irqs; - else if (HAS_HECI_GSC(gt->i915)) + heci_mask = GSC_IRQ_INTF(1); /* HECI2 IRQ for SW Proxy*/ + } else if (HAS_HECI_GSC(gt->i915)) { gsc_mask = GSC_IRQ_INTF(0) | GSC_IRQ_INTF(1); + } BUILD_BUG_ON(irqs & 0xffff0000); @@ -280,7 +295,7 @@ void gen11_gt_irq_postinstall(struct intel_gt *gt) if (CCS_MASK(gt)) intel_uncore_write(uncore, GEN12_CCS_RSVD_INTR_ENABLE, smask); if (gsc_mask) - intel_uncore_write(uncore, GEN11_GUNIT_CSME_INTR_ENABLE, gsc_mask); + intel_uncore_write(uncore, GEN11_GUNIT_CSME_INTR_ENABLE, gsc_mask | heci_mask); /* Unmask irqs on RCS, BCS, VCS and VECS engines. */ intel_uncore_write(uncore, GEN11_RCS0_RSVD_INTR_MASK, ~smask); @@ -308,6 +323,9 @@ void gen11_gt_irq_postinstall(struct intel_gt *gt) intel_uncore_write(uncore, GEN12_CCS2_CCS3_INTR_MASK, ~dmask); if (gsc_mask) intel_uncore_write(uncore, GEN11_GUNIT_CSME_INTR_MASK, ~gsc_mask); + if (heci_mask) + intel_uncore_write(uncore, GEN12_HECI2_RSVD_INTR_MASK, + ~REG_FIELD_PREP(ENGINE1_MASK, heci_mask)); if (guc_mask) { /* the enable bit is common for both GTs but the masks are separate */ @@ -358,7 +376,7 @@ static void gen7_parity_error_irq_handler(struct intel_gt *gt, u32 iir) if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT) gt->i915->l3_parity.which_slice |= 1 << 0; - schedule_work(>->i915->l3_parity.error_work); + queue_work(gt->i915->unordered_wq, >->i915->l3_parity.error_work); } void gen6_gt_irq_handler(struct intel_gt *gt, u32 gt_iir) diff --git a/drivers/gpu/drm/i915/gt/intel_gt_pm.c b/drivers/gpu/drm/i915/gt/intel_gt_pm.c index e02cb90723ae..5a942af0a14e 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_pm.c +++ b/drivers/gpu/drm/i915/gt/intel_gt_pm.c @@ -87,7 +87,7 @@ static int __gt_unpark(struct intel_wakeref *wf) intel_rc6_unpark(>->rc6); intel_rps_unpark(>->rps); - i915_pmu_gt_unparked(i915); + i915_pmu_gt_unparked(gt); intel_guc_busyness_unpark(gt); intel_gt_unpark_requests(gt); @@ -109,7 +109,7 @@ static int __gt_park(struct intel_wakeref *wf) intel_guc_busyness_park(gt); i915_vma_parked(gt); - i915_pmu_gt_parked(i915); + i915_pmu_gt_parked(gt); intel_rps_park(>->rps); intel_rc6_park(>->rc6); @@ -137,7 +137,7 @@ void intel_gt_pm_init_early(struct intel_gt *gt) * runtime_pm is per-device rather than per-tile, so this is still the * correct structure. */ - intel_wakeref_init(>->wakeref, >->i915->runtime_pm, &wf_ops); + intel_wakeref_init(>->wakeref, gt->i915, &wf_ops); seqcount_mutex_init(>->stats.lock, >->wakeref.mutex); } diff --git a/drivers/gpu/drm/i915/gt/intel_gt_pm_debugfs.c b/drivers/gpu/drm/i915/gt/intel_gt_pm_debugfs.c index 80dbbef86b1d..357e2f865727 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_pm_debugfs.c +++ b/drivers/gpu/drm/i915/gt/intel_gt_pm_debugfs.c @@ -539,7 +539,10 @@ static bool rps_eval(void *data) { struct intel_gt *gt = data; - return HAS_RPS(gt->i915); + if (intel_guc_slpc_is_used(>->uc.guc)) + return false; + else + return HAS_RPS(gt->i915); } DEFINE_INTEL_GT_DEBUGFS_ATTRIBUTE(rps_boost); diff --git a/drivers/gpu/drm/i915/gt/intel_gt_regs.h b/drivers/gpu/drm/i915/gt/intel_gt_regs.h index fd1f9cd35e9d..718cb2c80f79 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_regs.h +++ b/drivers/gpu/drm/i915/gt/intel_gt_regs.h @@ -7,7 +7,8 @@ #define __INTEL_GT_REGS__ #include "i915_reg_defs.h" -#include "display/intel_display_reg_defs.h" /* VLV_DISPLAY_BASE */ + +#define VLV_GUNIT_BASE 0x180000 /* * The perf control registers are technically multicast registers, but the @@ -356,7 +357,11 @@ #define GEN7_TLB_RD_ADDR _MMIO(0x4700) #define GEN12_PAT_INDEX(index) _MMIO(0x4800 + (index) * 4) -#define XEHP_PAT_INDEX(index) MCR_REG(0x4800 + (index) * 4) +#define _PAT_INDEX(index) _PICK_EVEN_2RANGES(index, 8, \ + 0x4800, 0x4804, \ + 0x4848, 0x484c) +#define XEHP_PAT_INDEX(index) MCR_REG(_PAT_INDEX(index)) +#define XELPMP_PAT_INDEX(index) _MMIO(_PAT_INDEX(index)) #define XEHP_TILE0_ADDR_RANGE MCR_REG(0x4900) #define XEHP_TILE_LMEM_RANGE_SHIFT 8 @@ -525,6 +530,11 @@ #define GEN8_RC6_CTX_INFO _MMIO(0x8504) +#define GEN12_SQCNT1 _MMIO(0x8718) +#define GEN12_SQCNT1_PMON_ENABLE REG_BIT(30) +#define GEN12_SQCNT1_OABPC REG_BIT(29) +#define GEN12_STRICT_RAR_ENABLE REG_BIT(23) + #define XEHP_SQCM MCR_REG(0x8724) #define EN_32B_ACCESS REG_BIT(30) @@ -1460,7 +1470,7 @@ #define GEN12_RCU_MODE _MMIO(0x14800) #define GEN12_RCU_MODE_CCS_ENABLE REG_BIT(0) -#define CHV_FUSE_GT _MMIO(VLV_DISPLAY_BASE + 0x2168) +#define CHV_FUSE_GT _MMIO(VLV_GUNIT_BASE + 0x2168) #define CHV_FGT_DISABLE_SS0 (1 << 10) #define CHV_FGT_DISABLE_SS1 (1 << 11) #define CHV_FGT_EU_DIS_SS0_R0_SHIFT 16 @@ -1587,6 +1597,7 @@ #define GEN11_GT_INTR_DW(x) _MMIO(0x190018 + ((x) * 4)) #define GEN11_CSME (31) +#define GEN12_HECI_2 (30) #define GEN11_GUNIT (28) #define GEN11_GUC (25) #define MTL_MGUC (24) @@ -1628,6 +1639,7 @@ /* irq instances for OTHER_CLASS */ #define OTHER_GUC_INSTANCE 0 #define OTHER_GTPM_INSTANCE 1 +#define OTHER_GSC_HECI_2_INSTANCE 3 #define OTHER_KCR_INSTANCE 4 #define OTHER_GSC_INSTANCE 6 #define OTHER_MEDIA_GUC_INSTANCE 16 @@ -1643,6 +1655,7 @@ #define GEN12_VCS6_VCS7_INTR_MASK _MMIO(0x1900b4) #define GEN11_VECS0_VECS1_INTR_MASK _MMIO(0x1900d0) #define GEN12_VECS2_VECS3_INTR_MASK _MMIO(0x1900d4) +#define GEN12_HECI2_RSVD_INTR_MASK _MMIO(0x1900e4) #define GEN11_GUC_SG_INTR_MASK _MMIO(0x1900e8) #define MTL_GUC_MGUC_INTR_MASK _MMIO(0x1900e8) /* MTL+ */ #define GEN11_GPM_WGBOXPERF_INTR_MASK _MMIO(0x1900ec) diff --git a/drivers/gpu/drm/i915/gt/intel_gt_requests.c b/drivers/gpu/drm/i915/gt/intel_gt_requests.c index 1dfd01668c79..d1a382dfaa1d 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_requests.c +++ b/drivers/gpu/drm/i915/gt/intel_gt_requests.c @@ -116,7 +116,7 @@ void intel_engine_add_retire(struct intel_engine_cs *engine, GEM_BUG_ON(intel_engine_is_virtual(engine)); if (add_retire(engine, tl)) - schedule_work(&engine->retire_work); + queue_work(engine->i915->unordered_wq, &engine->retire_work); } void intel_engine_init_retire(struct intel_engine_cs *engine) @@ -207,8 +207,8 @@ static void retire_work_handler(struct work_struct *work) struct intel_gt *gt = container_of(work, typeof(*gt), requests.retire_work.work); - schedule_delayed_work(>->requests.retire_work, - round_jiffies_up_relative(HZ)); + queue_delayed_work(gt->i915->unordered_wq, >->requests.retire_work, + round_jiffies_up_relative(HZ)); intel_gt_retire_requests(gt); } @@ -224,8 +224,8 @@ void intel_gt_park_requests(struct intel_gt *gt) void intel_gt_unpark_requests(struct intel_gt *gt) { - schedule_delayed_work(>->requests.retire_work, - round_jiffies_up_relative(HZ)); + queue_delayed_work(gt->i915->unordered_wq, >->requests.retire_work, + round_jiffies_up_relative(HZ)); } void intel_gt_fini_requests(struct intel_gt *gt) diff --git a/drivers/gpu/drm/i915/gt/intel_gt_sysfs_pm.c b/drivers/gpu/drm/i915/gt/intel_gt_sysfs_pm.c index 28f27091cd3b..ee2b44f896a2 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_sysfs_pm.c +++ b/drivers/gpu/drm/i915/gt/intel_gt_sysfs_pm.c @@ -451,6 +451,33 @@ static ssize_t punit_req_freq_mhz_show(struct kobject *kobj, return sysfs_emit(buff, "%u\n", preq); } +static ssize_t slpc_ignore_eff_freq_show(struct kobject *kobj, + struct kobj_attribute *attr, + char *buff) +{ + struct intel_gt *gt = intel_gt_sysfs_get_drvdata(kobj, attr->attr.name); + struct intel_guc_slpc *slpc = >->uc.guc.slpc; + + return sysfs_emit(buff, "%u\n", slpc->ignore_eff_freq); +} + +static ssize_t slpc_ignore_eff_freq_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buff, size_t count) +{ + struct intel_gt *gt = intel_gt_sysfs_get_drvdata(kobj, attr->attr.name); + struct intel_guc_slpc *slpc = >->uc.guc.slpc; + int err; + u32 val; + + err = kstrtou32(buff, 0, &val); + if (err) + return err; + + err = intel_guc_slpc_set_ignore_eff_freq(slpc, val); + return err ?: count; +} + struct intel_gt_bool_throttle_attr { struct attribute attr; ssize_t (*show)(struct kobject *kobj, struct kobj_attribute *attr, @@ -663,6 +690,8 @@ static struct kobj_attribute attr_media_freq_factor_scale = INTEL_GT_ATTR_RO(media_RP0_freq_mhz); INTEL_GT_ATTR_RO(media_RPn_freq_mhz); +INTEL_GT_ATTR_RW(slpc_ignore_eff_freq); + static const struct attribute *media_perf_power_attrs[] = { &attr_media_freq_factor.attr, &attr_media_freq_factor_scale.attr, @@ -744,6 +773,12 @@ void intel_gt_sysfs_pm_init(struct intel_gt *gt, struct kobject *kobj) if (ret) gt_warn(gt, "failed to create punit_req_freq_mhz sysfs (%pe)", ERR_PTR(ret)); + if (intel_uc_uses_guc_slpc(>->uc)) { + ret = sysfs_create_file(kobj, &attr_slpc_ignore_eff_freq.attr); + if (ret) + gt_warn(gt, "failed to create ignore_eff_freq sysfs (%pe)", ERR_PTR(ret)); + } + if (i915_mmio_reg_valid(intel_gt_perf_limit_reasons_reg(gt))) { ret = sysfs_create_files(kobj, throttle_reason_attrs); if (ret) diff --git a/drivers/gpu/drm/i915/gt/intel_gtt.c b/drivers/gpu/drm/i915/gt/intel_gtt.c index 4f436ba7a3c8..2f6a9be0ffe6 100644 --- a/drivers/gpu/drm/i915/gt/intel_gtt.c +++ b/drivers/gpu/drm/i915/gt/intel_gtt.c @@ -468,6 +468,44 @@ void gtt_write_workarounds(struct intel_gt *gt) } } +static void xelpmp_setup_private_ppat(struct intel_uncore *uncore) +{ + intel_uncore_write(uncore, XELPMP_PAT_INDEX(0), + MTL_PPAT_L4_0_WB); + intel_uncore_write(uncore, XELPMP_PAT_INDEX(1), + MTL_PPAT_L4_1_WT); + intel_uncore_write(uncore, XELPMP_PAT_INDEX(2), + MTL_PPAT_L4_3_UC); + intel_uncore_write(uncore, XELPMP_PAT_INDEX(3), + MTL_PPAT_L4_0_WB | MTL_2_COH_1W); + intel_uncore_write(uncore, XELPMP_PAT_INDEX(4), + MTL_PPAT_L4_0_WB | MTL_3_COH_2W); + + /* + * Remaining PAT entries are left at the hardware-default + * fully-cached setting + */ +} + +static void xelpg_setup_private_ppat(struct intel_gt *gt) +{ + intel_gt_mcr_multicast_write(gt, XEHP_PAT_INDEX(0), + MTL_PPAT_L4_0_WB); + intel_gt_mcr_multicast_write(gt, XEHP_PAT_INDEX(1), + MTL_PPAT_L4_1_WT); + intel_gt_mcr_multicast_write(gt, XEHP_PAT_INDEX(2), + MTL_PPAT_L4_3_UC); + intel_gt_mcr_multicast_write(gt, XEHP_PAT_INDEX(3), + MTL_PPAT_L4_0_WB | MTL_2_COH_1W); + intel_gt_mcr_multicast_write(gt, XEHP_PAT_INDEX(4), + MTL_PPAT_L4_0_WB | MTL_3_COH_2W); + + /* + * Remaining PAT entries are left at the hardware-default + * fully-cached setting + */ +} + static void tgl_setup_private_ppat(struct intel_uncore *uncore) { /* TGL doesn't support LLC or AGE settings */ @@ -603,7 +641,14 @@ void setup_private_pat(struct intel_gt *gt) GEM_BUG_ON(GRAPHICS_VER(i915) < 8); - if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50)) + if (gt->type == GT_MEDIA) { + xelpmp_setup_private_ppat(gt->uncore); + return; + } + + if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70)) + xelpg_setup_private_ppat(gt); + else if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50)) xehp_setup_private_ppat(gt); else if (GRAPHICS_VER(i915) >= 12) tgl_setup_private_ppat(uncore); diff --git a/drivers/gpu/drm/i915/gt/intel_gtt.h b/drivers/gpu/drm/i915/gt/intel_gtt.h index 69ce55f517f5..4d6296cdbcfd 100644 --- a/drivers/gpu/drm/i915/gt/intel_gtt.h +++ b/drivers/gpu/drm/i915/gt/intel_gtt.h @@ -88,9 +88,17 @@ typedef u64 gen8_pte_t; #define BYT_PTE_SNOOPED_BY_CPU_CACHES REG_BIT(2) #define BYT_PTE_WRITEABLE REG_BIT(1) +#define MTL_PPGTT_PTE_PAT3 BIT_ULL(62) #define GEN12_PPGTT_PTE_LM BIT_ULL(11) +#define GEN12_PPGTT_PTE_PAT2 BIT_ULL(7) +#define GEN12_PPGTT_PTE_PAT1 BIT_ULL(4) +#define GEN12_PPGTT_PTE_PAT0 BIT_ULL(3) -#define GEN12_GGTT_PTE_LM BIT_ULL(1) +#define GEN12_GGTT_PTE_LM BIT_ULL(1) +#define MTL_GGTT_PTE_PAT0 BIT_ULL(52) +#define MTL_GGTT_PTE_PAT1 BIT_ULL(53) +#define GEN12_GGTT_PTE_ADDR_MASK GENMASK_ULL(45, 12) +#define MTL_GGTT_PTE_PAT_MASK GENMASK_ULL(53, 52) #define GEN12_PDE_64K BIT(6) #define GEN12_PTE_PS64 BIT(8) @@ -147,7 +155,13 @@ typedef u64 gen8_pte_t; #define GEN8_PDE_IPS_64K BIT(11) #define GEN8_PDE_PS_2M BIT(7) -enum i915_cache_level; +#define MTL_PPAT_L4_CACHE_POLICY_MASK REG_GENMASK(3, 2) +#define MTL_PAT_INDEX_COH_MODE_MASK REG_GENMASK(1, 0) +#define MTL_PPAT_L4_3_UC REG_FIELD_PREP(MTL_PPAT_L4_CACHE_POLICY_MASK, 3) +#define MTL_PPAT_L4_1_WT REG_FIELD_PREP(MTL_PPAT_L4_CACHE_POLICY_MASK, 1) +#define MTL_PPAT_L4_0_WB REG_FIELD_PREP(MTL_PPAT_L4_CACHE_POLICY_MASK, 0) +#define MTL_3_COH_2W REG_FIELD_PREP(MTL_PAT_INDEX_COH_MODE_MASK, 3) +#define MTL_2_COH_1W REG_FIELD_PREP(MTL_PAT_INDEX_COH_MODE_MASK, 2) struct drm_i915_gem_object; struct i915_fence_reg; @@ -216,7 +230,7 @@ struct i915_vma_ops { void (*bind_vma)(struct i915_address_space *vm, struct i915_vm_pt_stash *stash, struct i915_vma_resource *vma_res, - enum i915_cache_level cache_level, + unsigned int pat_index, u32 flags); /* * Unmap an object from an address space. This usually consists of @@ -288,7 +302,7 @@ struct i915_address_space { (*alloc_scratch_dma)(struct i915_address_space *vm, int sz); u64 (*pte_encode)(dma_addr_t addr, - enum i915_cache_level level, + unsigned int pat_index, u32 flags); /* Create a valid PTE */ #define PTE_READ_ONLY BIT(0) #define PTE_LM BIT(1) @@ -303,20 +317,20 @@ struct i915_address_space { void (*insert_page)(struct i915_address_space *vm, dma_addr_t addr, u64 offset, - enum i915_cache_level cache_level, + unsigned int pat_index, u32 flags); void (*insert_entries)(struct i915_address_space *vm, struct i915_vma_resource *vma_res, - enum i915_cache_level cache_level, + unsigned int pat_index, u32 flags); void (*raw_insert_page)(struct i915_address_space *vm, dma_addr_t addr, u64 offset, - enum i915_cache_level cache_level, + unsigned int pat_index, u32 flags); void (*raw_insert_entries)(struct i915_address_space *vm, struct i915_vma_resource *vma_res, - enum i915_cache_level cache_level, + unsigned int pat_index, u32 flags); void (*cleanup)(struct i915_address_space *vm); @@ -493,7 +507,7 @@ static inline void i915_vm_put(struct i915_address_space *vm) /** * i915_vm_resv_put - Release a reference on the vm's reservation lock - * @resv: Pointer to a reservation lock obtained from i915_vm_resv_get() + * @vm: The vm whose reservation lock reference we want to release */ static inline void i915_vm_resv_put(struct i915_address_space *vm) { @@ -563,7 +577,7 @@ void ppgtt_init(struct i915_ppgtt *ppgtt, struct intel_gt *gt, void intel_ggtt_bind_vma(struct i915_address_space *vm, struct i915_vm_pt_stash *stash, struct i915_vma_resource *vma_res, - enum i915_cache_level cache_level, + unsigned int pat_index, u32 flags); void intel_ggtt_unbind_vma(struct i915_address_space *vm, struct i915_vma_resource *vma_res); @@ -641,7 +655,7 @@ void gen6_ggtt_invalidate(struct i915_ggtt *ggtt); void ppgtt_bind_vma(struct i915_address_space *vm, struct i915_vm_pt_stash *stash, struct i915_vma_resource *vma_res, - enum i915_cache_level cache_level, + unsigned int pat_index, u32 flags); void ppgtt_unbind_vma(struct i915_address_space *vm, struct i915_vma_resource *vma_res); diff --git a/drivers/gpu/drm/i915/gt/intel_lrc.c b/drivers/gpu/drm/i915/gt/intel_lrc.c index 81a96c52a92b..a4ec20aaafe2 100644 --- a/drivers/gpu/drm/i915/gt/intel_lrc.c +++ b/drivers/gpu/drm/i915/gt/intel_lrc.c @@ -1370,7 +1370,9 @@ gen12_emit_indirect_ctx_rcs(const struct intel_context *ce, u32 *cs) cs, GEN12_GFX_CCS_AUX_NV); /* Wa_16014892111 */ - if (IS_DG2(ce->engine->i915)) + if (IS_MTL_GRAPHICS_STEP(ce->engine->i915, M, STEP_A0, STEP_B0) || + IS_MTL_GRAPHICS_STEP(ce->engine->i915, P, STEP_A0, STEP_B0) || + IS_DG2(ce->engine->i915)) cs = dg2_emit_draw_watermark_setting(cs); return cs; diff --git a/drivers/gpu/drm/i915/gt/intel_migrate.c b/drivers/gpu/drm/i915/gt/intel_migrate.c index 3f638f198796..6023288b0e2d 100644 --- a/drivers/gpu/drm/i915/gt/intel_migrate.c +++ b/drivers/gpu/drm/i915/gt/intel_migrate.c @@ -45,7 +45,9 @@ static void xehpsdv_toggle_pdes(struct i915_address_space *vm, * Insert a dummy PTE into every PT that will map to LMEM to ensure * we have a correctly setup PDE structure for later use. */ - vm->insert_page(vm, 0, d->offset, I915_CACHE_NONE, PTE_LM); + vm->insert_page(vm, 0, d->offset, + i915_gem_get_pat_index(vm->i915, I915_CACHE_NONE), + PTE_LM); GEM_BUG_ON(!pt->is_compact); d->offset += SZ_2M; } @@ -63,7 +65,9 @@ static void xehpsdv_insert_pte(struct i915_address_space *vm, * alignment is 64K underneath for the pt, and we are careful * not to access the space in the void. */ - vm->insert_page(vm, px_dma(pt), d->offset, I915_CACHE_NONE, PTE_LM); + vm->insert_page(vm, px_dma(pt), d->offset, + i915_gem_get_pat_index(vm->i915, I915_CACHE_NONE), + PTE_LM); d->offset += SZ_64K; } @@ -73,7 +77,8 @@ static void insert_pte(struct i915_address_space *vm, { struct insert_pte_data *d = data; - vm->insert_page(vm, px_dma(pt), d->offset, I915_CACHE_NONE, + vm->insert_page(vm, px_dma(pt), d->offset, + i915_gem_get_pat_index(vm->i915, I915_CACHE_NONE), i915_gem_object_is_lmem(pt->base) ? PTE_LM : 0); d->offset += PAGE_SIZE; } @@ -356,13 +361,13 @@ static int max_pte_pkt_size(struct i915_request *rq, int pkt) static int emit_pte(struct i915_request *rq, struct sgt_dma *it, - enum i915_cache_level cache_level, + unsigned int pat_index, bool is_lmem, u64 offset, int length) { bool has_64K_pages = HAS_64K_PAGES(rq->engine->i915); - const u64 encode = rq->context->vm->pte_encode(0, cache_level, + const u64 encode = rq->context->vm->pte_encode(0, pat_index, is_lmem ? PTE_LM : 0); struct intel_ring *ring = rq->ring; int pkt, dword_length; @@ -673,17 +678,17 @@ int intel_context_migrate_copy(struct intel_context *ce, const struct i915_deps *deps, struct scatterlist *src, - enum i915_cache_level src_cache_level, + unsigned int src_pat_index, bool src_is_lmem, struct scatterlist *dst, - enum i915_cache_level dst_cache_level, + unsigned int dst_pat_index, bool dst_is_lmem, struct i915_request **out) { struct sgt_dma it_src = sg_sgt(src), it_dst = sg_sgt(dst), it_ccs; struct drm_i915_private *i915 = ce->engine->i915; u64 ccs_bytes_to_cpy = 0, bytes_to_cpy; - enum i915_cache_level ccs_cache_level; + unsigned int ccs_pat_index; u32 src_offset, dst_offset; u8 src_access, dst_access; struct i915_request *rq; @@ -707,12 +712,12 @@ intel_context_migrate_copy(struct intel_context *ce, dst_sz = scatter_list_length(dst); if (src_is_lmem) { it_ccs = it_dst; - ccs_cache_level = dst_cache_level; + ccs_pat_index = dst_pat_index; ccs_is_src = false; } else if (dst_is_lmem) { bytes_to_cpy = dst_sz; it_ccs = it_src; - ccs_cache_level = src_cache_level; + ccs_pat_index = src_pat_index; ccs_is_src = true; } @@ -773,7 +778,7 @@ intel_context_migrate_copy(struct intel_context *ce, src_sz = calculate_chunk_sz(i915, src_is_lmem, bytes_to_cpy, ccs_bytes_to_cpy); - len = emit_pte(rq, &it_src, src_cache_level, src_is_lmem, + len = emit_pte(rq, &it_src, src_pat_index, src_is_lmem, src_offset, src_sz); if (!len) { err = -EINVAL; @@ -784,7 +789,7 @@ intel_context_migrate_copy(struct intel_context *ce, goto out_rq; } - err = emit_pte(rq, &it_dst, dst_cache_level, dst_is_lmem, + err = emit_pte(rq, &it_dst, dst_pat_index, dst_is_lmem, dst_offset, len); if (err < 0) goto out_rq; @@ -811,7 +816,7 @@ intel_context_migrate_copy(struct intel_context *ce, goto out_rq; ccs_sz = GET_CCS_BYTES(i915, len); - err = emit_pte(rq, &it_ccs, ccs_cache_level, false, + err = emit_pte(rq, &it_ccs, ccs_pat_index, false, ccs_is_src ? src_offset : dst_offset, ccs_sz); if (err < 0) @@ -920,7 +925,7 @@ static int emit_clear(struct i915_request *rq, u32 offset, int size, GEM_BUG_ON(size >> PAGE_SHIFT > S16_MAX); - if (HAS_FLAT_CCS(i915) && ver >= 12) + if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50)) ring_sz = XY_FAST_COLOR_BLT_DW; else if (ver >= 8) ring_sz = 8; @@ -931,7 +936,7 @@ static int emit_clear(struct i915_request *rq, u32 offset, int size, if (IS_ERR(cs)) return PTR_ERR(cs); - if (HAS_FLAT_CCS(i915) && ver >= 12) { + if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50)) { *cs++ = XY_FAST_COLOR_BLT_CMD | XY_FAST_COLOR_BLT_DEPTH_32 | (XY_FAST_COLOR_BLT_DW - 2); *cs++ = FIELD_PREP(XY_FAST_COLOR_BLT_MOCS_MASK, mocs) | @@ -979,7 +984,7 @@ int intel_context_migrate_clear(struct intel_context *ce, const struct i915_deps *deps, struct scatterlist *sg, - enum i915_cache_level cache_level, + unsigned int pat_index, bool is_lmem, u32 value, struct i915_request **out) @@ -1027,7 +1032,7 @@ intel_context_migrate_clear(struct intel_context *ce, if (err) goto out_rq; - len = emit_pte(rq, &it, cache_level, is_lmem, offset, CHUNK_SZ); + len = emit_pte(rq, &it, pat_index, is_lmem, offset, CHUNK_SZ); if (len <= 0) { err = len; goto out_rq; @@ -1074,10 +1079,10 @@ int intel_migrate_copy(struct intel_migrate *m, struct i915_gem_ww_ctx *ww, const struct i915_deps *deps, struct scatterlist *src, - enum i915_cache_level src_cache_level, + unsigned int src_pat_index, bool src_is_lmem, struct scatterlist *dst, - enum i915_cache_level dst_cache_level, + unsigned int dst_pat_index, bool dst_is_lmem, struct i915_request **out) { @@ -1098,8 +1103,8 @@ int intel_migrate_copy(struct intel_migrate *m, goto out; err = intel_context_migrate_copy(ce, deps, - src, src_cache_level, src_is_lmem, - dst, dst_cache_level, dst_is_lmem, + src, src_pat_index, src_is_lmem, + dst, dst_pat_index, dst_is_lmem, out); intel_context_unpin(ce); @@ -1113,7 +1118,7 @@ intel_migrate_clear(struct intel_migrate *m, struct i915_gem_ww_ctx *ww, const struct i915_deps *deps, struct scatterlist *sg, - enum i915_cache_level cache_level, + unsigned int pat_index, bool is_lmem, u32 value, struct i915_request **out) @@ -1134,7 +1139,7 @@ intel_migrate_clear(struct intel_migrate *m, if (err) goto out; - err = intel_context_migrate_clear(ce, deps, sg, cache_level, + err = intel_context_migrate_clear(ce, deps, sg, pat_index, is_lmem, value, out); intel_context_unpin(ce); diff --git a/drivers/gpu/drm/i915/gt/intel_migrate.h b/drivers/gpu/drm/i915/gt/intel_migrate.h index ccc677ec4aa3..11fc09a00c4b 100644 --- a/drivers/gpu/drm/i915/gt/intel_migrate.h +++ b/drivers/gpu/drm/i915/gt/intel_migrate.h @@ -16,7 +16,6 @@ struct i915_request; struct i915_gem_ww_ctx; struct intel_gt; struct scatterlist; -enum i915_cache_level; int intel_migrate_init(struct intel_migrate *m, struct intel_gt *gt); @@ -26,20 +25,20 @@ int intel_migrate_copy(struct intel_migrate *m, struct i915_gem_ww_ctx *ww, const struct i915_deps *deps, struct scatterlist *src, - enum i915_cache_level src_cache_level, + unsigned int src_pat_index, bool src_is_lmem, struct scatterlist *dst, - enum i915_cache_level dst_cache_level, + unsigned int dst_pat_index, bool dst_is_lmem, struct i915_request **out); int intel_context_migrate_copy(struct intel_context *ce, const struct i915_deps *deps, struct scatterlist *src, - enum i915_cache_level src_cache_level, + unsigned int src_pat_index, bool src_is_lmem, struct scatterlist *dst, - enum i915_cache_level dst_cache_level, + unsigned int dst_pat_index, bool dst_is_lmem, struct i915_request **out); @@ -48,7 +47,7 @@ intel_migrate_clear(struct intel_migrate *m, struct i915_gem_ww_ctx *ww, const struct i915_deps *deps, struct scatterlist *sg, - enum i915_cache_level cache_level, + unsigned int pat_index, bool is_lmem, u32 value, struct i915_request **out); @@ -56,7 +55,7 @@ int intel_context_migrate_clear(struct intel_context *ce, const struct i915_deps *deps, struct scatterlist *sg, - enum i915_cache_level cache_level, + unsigned int pat_index, bool is_lmem, u32 value, struct i915_request **out); diff --git a/drivers/gpu/drm/i915/gt/intel_mocs.c b/drivers/gpu/drm/i915/gt/intel_mocs.c index 69b489e8dfed..2c014407225c 100644 --- a/drivers/gpu/drm/i915/gt/intel_mocs.c +++ b/drivers/gpu/drm/i915/gt/intel_mocs.c @@ -40,6 +40,10 @@ struct drm_i915_mocs_table { #define LE_COS(value) ((value) << 15) #define LE_SSE(value) ((value) << 17) +/* Defines for the tables (GLOB_MOCS_0 - GLOB_MOCS_16) */ +#define _L4_CACHEABILITY(value) ((value) << 2) +#define IG_PAT(value) ((value) << 8) + /* Defines for the tables (LNCFMOCS0 - LNCFMOCS31) - two entries per word */ #define L3_ESC(value) ((value) << 0) #define L3_SCC(value) ((value) << 1) @@ -50,6 +54,7 @@ struct drm_i915_mocs_table { /* Helper defines */ #define GEN9_NUM_MOCS_ENTRIES 64 /* 63-64 are reserved, but configured. */ #define PVC_NUM_MOCS_ENTRIES 3 +#define MTL_NUM_MOCS_ENTRIES 16 /* (e)LLC caching options */ /* @@ -73,6 +78,12 @@ struct drm_i915_mocs_table { #define L3_2_RESERVED _L3_CACHEABILITY(2) #define L3_3_WB _L3_CACHEABILITY(3) +/* L4 caching options */ +#define L4_0_WB _L4_CACHEABILITY(0) +#define L4_1_WT _L4_CACHEABILITY(1) +#define L4_2_RESERVED _L4_CACHEABILITY(2) +#define L4_3_UC _L4_CACHEABILITY(3) + #define MOCS_ENTRY(__idx, __control_value, __l3cc_value) \ [__idx] = { \ .control_value = __control_value, \ @@ -416,6 +427,57 @@ static const struct drm_i915_mocs_entry pvc_mocs_table[] = { MOCS_ENTRY(2, 0, L3_3_WB), }; +static const struct drm_i915_mocs_entry mtl_mocs_table[] = { + /* Error - Reserved for Non-Use */ + MOCS_ENTRY(0, + IG_PAT(0), + L3_LKUP(1) | L3_3_WB), + /* Cached - L3 + L4 */ + MOCS_ENTRY(1, + IG_PAT(1), + L3_LKUP(1) | L3_3_WB), + /* L4 - GO:L3 */ + MOCS_ENTRY(2, + IG_PAT(1), + L3_LKUP(1) | L3_1_UC), + /* Uncached - GO:L3 */ + MOCS_ENTRY(3, + IG_PAT(1) | L4_3_UC, + L3_LKUP(1) | L3_1_UC), + /* L4 - GO:Mem */ + MOCS_ENTRY(4, + IG_PAT(1), + L3_LKUP(1) | L3_GLBGO(1) | L3_1_UC), + /* Uncached - GO:Mem */ + MOCS_ENTRY(5, + IG_PAT(1) | L4_3_UC, + L3_LKUP(1) | L3_GLBGO(1) | L3_1_UC), + /* L4 - L3:NoLKUP; GO:L3 */ + MOCS_ENTRY(6, + IG_PAT(1), + L3_1_UC), + /* Uncached - L3:NoLKUP; GO:L3 */ + MOCS_ENTRY(7, + IG_PAT(1) | L4_3_UC, + L3_1_UC), + /* L4 - L3:NoLKUP; GO:Mem */ + MOCS_ENTRY(8, + IG_PAT(1), + L3_GLBGO(1) | L3_1_UC), + /* Uncached - L3:NoLKUP; GO:Mem */ + MOCS_ENTRY(9, + IG_PAT(1) | L4_3_UC, + L3_GLBGO(1) | L3_1_UC), + /* Display - L3; L4:WT */ + MOCS_ENTRY(14, + IG_PAT(1) | L4_1_WT, + L3_LKUP(1) | L3_3_WB), + /* CCS - Non-Displayable */ + MOCS_ENTRY(15, + IG_PAT(1), + L3_GLBGO(1) | L3_1_UC), +}; + enum { HAS_GLOBAL_MOCS = BIT(0), HAS_ENGINE_MOCS = BIT(1), @@ -445,7 +507,13 @@ static unsigned int get_mocs_settings(const struct drm_i915_private *i915, memset(table, 0, sizeof(struct drm_i915_mocs_table)); table->unused_entries_index = I915_MOCS_PTE; - if (IS_PONTEVECCHIO(i915)) { + if (IS_METEORLAKE(i915)) { + table->size = ARRAY_SIZE(mtl_mocs_table); + table->table = mtl_mocs_table; + table->n_entries = MTL_NUM_MOCS_ENTRIES; + table->uc_index = 9; + table->unused_entries_index = 1; + } else if (IS_PONTEVECCHIO(i915)) { table->size = ARRAY_SIZE(pvc_mocs_table); table->table = pvc_mocs_table; table->n_entries = PVC_NUM_MOCS_ENTRIES; diff --git a/drivers/gpu/drm/i915/gt/intel_ppgtt.c b/drivers/gpu/drm/i915/gt/intel_ppgtt.c index 7ecfa672f738..436756bfbb1a 100644 --- a/drivers/gpu/drm/i915/gt/intel_ppgtt.c +++ b/drivers/gpu/drm/i915/gt/intel_ppgtt.c @@ -181,7 +181,7 @@ struct i915_ppgtt *i915_ppgtt_create(struct intel_gt *gt, void ppgtt_bind_vma(struct i915_address_space *vm, struct i915_vm_pt_stash *stash, struct i915_vma_resource *vma_res, - enum i915_cache_level cache_level, + unsigned int pat_index, u32 flags) { u32 pte_flags; @@ -199,7 +199,7 @@ void ppgtt_bind_vma(struct i915_address_space *vm, if (vma_res->bi.lmem) pte_flags |= PTE_LM; - vm->insert_entries(vm, vma_res, cache_level, pte_flags); + vm->insert_entries(vm, vma_res, pat_index, pte_flags); wmb(); } diff --git a/drivers/gpu/drm/i915/gt/intel_rc6.c b/drivers/gpu/drm/i915/gt/intel_rc6.c index 8f3cd68d14f8..58bb1c55294c 100644 --- a/drivers/gpu/drm/i915/gt/intel_rc6.c +++ b/drivers/gpu/drm/i915/gt/intel_rc6.c @@ -53,11 +53,6 @@ static struct drm_i915_private *rc6_to_i915(struct intel_rc6 *rc) return rc6_to_gt(rc)->i915; } -static void set(struct intel_uncore *uncore, i915_reg_t reg, u32 val) -{ - intel_uncore_write_fw(uncore, reg, val); -} - static void gen11_rc6_enable(struct intel_rc6 *rc6) { struct intel_gt *gt = rc6_to_gt(rc6); @@ -72,19 +67,19 @@ static void gen11_rc6_enable(struct intel_rc6 *rc6) */ if (!intel_uc_uses_guc_rc(>->uc)) { /* 2b: Program RC6 thresholds.*/ - set(uncore, GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16 | 85); - set(uncore, GEN10_MEDIA_WAKE_RATE_LIMIT, 150); + intel_uncore_write_fw(uncore, GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16 | 85); + intel_uncore_write_fw(uncore, GEN10_MEDIA_WAKE_RATE_LIMIT, 150); - set(uncore, GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */ - set(uncore, GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */ + intel_uncore_write_fw(uncore, GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */ + intel_uncore_write_fw(uncore, GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */ for_each_engine(engine, rc6_to_gt(rc6), id) - set(uncore, RING_MAX_IDLE(engine->mmio_base), 10); + intel_uncore_write_fw(uncore, RING_MAX_IDLE(engine->mmio_base), 10); - set(uncore, GUC_MAX_IDLE_COUNT, 0xA); + intel_uncore_write_fw(uncore, GUC_MAX_IDLE_COUNT, 0xA); - set(uncore, GEN6_RC_SLEEP, 0); + intel_uncore_write_fw(uncore, GEN6_RC_SLEEP, 0); - set(uncore, GEN6_RC6_THRESHOLD, 50000); /* 50/125ms per EI */ + intel_uncore_write_fw(uncore, GEN6_RC6_THRESHOLD, 50000); /* 50/125ms per EI */ } /* @@ -105,8 +100,8 @@ static void gen11_rc6_enable(struct intel_rc6 *rc6) * Broadwell+, To be conservative, we want to factor in a context * switch on top (due to ksoftirqd). */ - set(uncore, GEN9_MEDIA_PG_IDLE_HYSTERESIS, 60); - set(uncore, GEN9_RENDER_PG_IDLE_HYSTERESIS, 60); + intel_uncore_write_fw(uncore, GEN9_MEDIA_PG_IDLE_HYSTERESIS, 60); + intel_uncore_write_fw(uncore, GEN9_RENDER_PG_IDLE_HYSTERESIS, 60); /* 3a: Enable RC6 * @@ -122,8 +117,14 @@ static void gen11_rc6_enable(struct intel_rc6 *rc6) GEN6_RC_CTL_RC6_ENABLE | GEN6_RC_CTL_EI_MODE(1); - /* Wa_16011777198 - Render powergating must remain disabled */ - if (IS_DG2_GRAPHICS_STEP(gt->i915, G10, STEP_A0, STEP_C0) || + /* + * Wa_16011777198 and BSpec 52698 - Render powergating must be off. + * FIXME BSpec is outdated, disabling powergating for MTL is just + * temporary wa and should be removed after fixing real cause + * of forcewake timeouts. + */ + if (IS_METEORLAKE(gt->i915) || + IS_DG2_GRAPHICS_STEP(gt->i915, G10, STEP_A0, STEP_C0) || IS_DG2_GRAPHICS_STEP(gt->i915, G11, STEP_A0, STEP_B0)) pg_enable = GEN9_MEDIA_PG_ENABLE | @@ -141,7 +142,7 @@ static void gen11_rc6_enable(struct intel_rc6 *rc6) VDN_MFX_POWERGATE_ENABLE(i)); } - set(uncore, GEN9_PG_ENABLE, pg_enable); + intel_uncore_write_fw(uncore, GEN9_PG_ENABLE, pg_enable); } static void gen9_rc6_enable(struct intel_rc6 *rc6) @@ -152,26 +153,26 @@ static void gen9_rc6_enable(struct intel_rc6 *rc6) /* 2b: Program RC6 thresholds.*/ if (GRAPHICS_VER(rc6_to_i915(rc6)) >= 11) { - set(uncore, GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16 | 85); - set(uncore, GEN10_MEDIA_WAKE_RATE_LIMIT, 150); + intel_uncore_write_fw(uncore, GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16 | 85); + intel_uncore_write_fw(uncore, GEN10_MEDIA_WAKE_RATE_LIMIT, 150); } else if (IS_SKYLAKE(rc6_to_i915(rc6))) { /* * WaRsDoubleRc6WrlWithCoarsePowerGating:skl Doubling WRL only * when CPG is enabled */ - set(uncore, GEN6_RC6_WAKE_RATE_LIMIT, 108 << 16); + intel_uncore_write_fw(uncore, GEN6_RC6_WAKE_RATE_LIMIT, 108 << 16); } else { - set(uncore, GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16); + intel_uncore_write_fw(uncore, GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16); } - set(uncore, GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */ - set(uncore, GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */ + intel_uncore_write_fw(uncore, GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */ + intel_uncore_write_fw(uncore, GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */ for_each_engine(engine, rc6_to_gt(rc6), id) - set(uncore, RING_MAX_IDLE(engine->mmio_base), 10); + intel_uncore_write_fw(uncore, RING_MAX_IDLE(engine->mmio_base), 10); - set(uncore, GUC_MAX_IDLE_COUNT, 0xA); + intel_uncore_write_fw(uncore, GUC_MAX_IDLE_COUNT, 0xA); - set(uncore, GEN6_RC_SLEEP, 0); + intel_uncore_write_fw(uncore, GEN6_RC_SLEEP, 0); /* * 2c: Program Coarse Power Gating Policies. @@ -194,11 +195,11 @@ static void gen9_rc6_enable(struct intel_rc6 *rc6) * conservative, we have to factor in a context switch on top (due * to ksoftirqd). */ - set(uncore, GEN9_MEDIA_PG_IDLE_HYSTERESIS, 250); - set(uncore, GEN9_RENDER_PG_IDLE_HYSTERESIS, 250); + intel_uncore_write_fw(uncore, GEN9_MEDIA_PG_IDLE_HYSTERESIS, 250); + intel_uncore_write_fw(uncore, GEN9_RENDER_PG_IDLE_HYSTERESIS, 250); /* 3a: Enable RC6 */ - set(uncore, GEN6_RC6_THRESHOLD, 37500); /* 37.5/125ms per EI */ + intel_uncore_write_fw(uncore, GEN6_RC6_THRESHOLD, 37500); /* 37.5/125ms per EI */ rc6->ctl_enable = GEN6_RC_CTL_HW_ENABLE | @@ -210,8 +211,8 @@ static void gen9_rc6_enable(struct intel_rc6 *rc6) * - Render/Media PG need to be disabled with RC6. */ if (!NEEDS_WaRsDisableCoarsePowerGating(rc6_to_i915(rc6))) - set(uncore, GEN9_PG_ENABLE, - GEN9_RENDER_PG_ENABLE | GEN9_MEDIA_PG_ENABLE); + intel_uncore_write_fw(uncore, GEN9_PG_ENABLE, + GEN9_RENDER_PG_ENABLE | GEN9_MEDIA_PG_ENABLE); } static void gen8_rc6_enable(struct intel_rc6 *rc6) @@ -221,13 +222,13 @@ static void gen8_rc6_enable(struct intel_rc6 *rc6) enum intel_engine_id id; /* 2b: Program RC6 thresholds.*/ - set(uncore, GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16); - set(uncore, GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */ - set(uncore, GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */ + intel_uncore_write_fw(uncore, GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16); + intel_uncore_write_fw(uncore, GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */ + intel_uncore_write_fw(uncore, GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */ for_each_engine(engine, rc6_to_gt(rc6), id) - set(uncore, RING_MAX_IDLE(engine->mmio_base), 10); - set(uncore, GEN6_RC_SLEEP, 0); - set(uncore, GEN6_RC6_THRESHOLD, 625); /* 800us/1.28 for TO */ + intel_uncore_write_fw(uncore, RING_MAX_IDLE(engine->mmio_base), 10); + intel_uncore_write_fw(uncore, GEN6_RC_SLEEP, 0); + intel_uncore_write_fw(uncore, GEN6_RC6_THRESHOLD, 625); /* 800us/1.28 for TO */ /* 3: Enable RC6 */ rc6->ctl_enable = @@ -245,20 +246,20 @@ static void gen6_rc6_enable(struct intel_rc6 *rc6) u32 rc6vids, rc6_mask; int ret; - set(uncore, GEN6_RC1_WAKE_RATE_LIMIT, 1000 << 16); - set(uncore, GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16 | 30); - set(uncore, GEN6_RC6pp_WAKE_RATE_LIMIT, 30); - set(uncore, GEN6_RC_EVALUATION_INTERVAL, 125000); - set(uncore, GEN6_RC_IDLE_HYSTERSIS, 25); + intel_uncore_write_fw(uncore, GEN6_RC1_WAKE_RATE_LIMIT, 1000 << 16); + intel_uncore_write_fw(uncore, GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16 | 30); + intel_uncore_write_fw(uncore, GEN6_RC6pp_WAKE_RATE_LIMIT, 30); + intel_uncore_write_fw(uncore, GEN6_RC_EVALUATION_INTERVAL, 125000); + intel_uncore_write_fw(uncore, GEN6_RC_IDLE_HYSTERSIS, 25); for_each_engine(engine, rc6_to_gt(rc6), id) - set(uncore, RING_MAX_IDLE(engine->mmio_base), 10); + intel_uncore_write_fw(uncore, RING_MAX_IDLE(engine->mmio_base), 10); - set(uncore, GEN6_RC_SLEEP, 0); - set(uncore, GEN6_RC1e_THRESHOLD, 1000); - set(uncore, GEN6_RC6_THRESHOLD, 50000); - set(uncore, GEN6_RC6p_THRESHOLD, 150000); - set(uncore, GEN6_RC6pp_THRESHOLD, 64000); /* unused */ + intel_uncore_write_fw(uncore, GEN6_RC_SLEEP, 0); + intel_uncore_write_fw(uncore, GEN6_RC1e_THRESHOLD, 1000); + intel_uncore_write_fw(uncore, GEN6_RC6_THRESHOLD, 50000); + intel_uncore_write_fw(uncore, GEN6_RC6p_THRESHOLD, 150000); + intel_uncore_write_fw(uncore, GEN6_RC6pp_THRESHOLD, 64000); /* unused */ /* We don't use those on Haswell */ rc6_mask = GEN6_RC_CTL_RC6_ENABLE; @@ -372,22 +373,22 @@ static void chv_rc6_enable(struct intel_rc6 *rc6) enum intel_engine_id id; /* 2a: Program RC6 thresholds.*/ - set(uncore, GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16); - set(uncore, GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */ - set(uncore, GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */ + intel_uncore_write_fw(uncore, GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16); + intel_uncore_write_fw(uncore, GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */ + intel_uncore_write_fw(uncore, GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */ for_each_engine(engine, rc6_to_gt(rc6), id) - set(uncore, RING_MAX_IDLE(engine->mmio_base), 10); - set(uncore, GEN6_RC_SLEEP, 0); + intel_uncore_write_fw(uncore, RING_MAX_IDLE(engine->mmio_base), 10); + intel_uncore_write_fw(uncore, GEN6_RC_SLEEP, 0); /* TO threshold set to 500 us (0x186 * 1.28 us) */ - set(uncore, GEN6_RC6_THRESHOLD, 0x186); + intel_uncore_write_fw(uncore, GEN6_RC6_THRESHOLD, 0x186); /* Allows RC6 residency counter to work */ - set(uncore, VLV_COUNTER_CONTROL, - _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH | - VLV_MEDIA_RC6_COUNT_EN | - VLV_RENDER_RC6_COUNT_EN)); + intel_uncore_write_fw(uncore, VLV_COUNTER_CONTROL, + _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH | + VLV_MEDIA_RC6_COUNT_EN | + VLV_RENDER_RC6_COUNT_EN)); /* 3: Enable RC6 */ rc6->ctl_enable = GEN7_RC_CTL_TO_MODE; @@ -399,22 +400,22 @@ static void vlv_rc6_enable(struct intel_rc6 *rc6) struct intel_engine_cs *engine; enum intel_engine_id id; - set(uncore, GEN6_RC6_WAKE_RATE_LIMIT, 0x00280000); - set(uncore, GEN6_RC_EVALUATION_INTERVAL, 125000); - set(uncore, GEN6_RC_IDLE_HYSTERSIS, 25); + intel_uncore_write_fw(uncore, GEN6_RC6_WAKE_RATE_LIMIT, 0x00280000); + intel_uncore_write_fw(uncore, GEN6_RC_EVALUATION_INTERVAL, 125000); + intel_uncore_write_fw(uncore, GEN6_RC_IDLE_HYSTERSIS, 25); for_each_engine(engine, rc6_to_gt(rc6), id) - set(uncore, RING_MAX_IDLE(engine->mmio_base), 10); + intel_uncore_write_fw(uncore, RING_MAX_IDLE(engine->mmio_base), 10); - set(uncore, GEN6_RC6_THRESHOLD, 0x557); + intel_uncore_write_fw(uncore, GEN6_RC6_THRESHOLD, 0x557); /* Allows RC6 residency counter to work */ - set(uncore, VLV_COUNTER_CONTROL, - _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH | - VLV_MEDIA_RC0_COUNT_EN | - VLV_RENDER_RC0_COUNT_EN | - VLV_MEDIA_RC6_COUNT_EN | - VLV_RENDER_RC6_COUNT_EN)); + intel_uncore_write_fw(uncore, VLV_COUNTER_CONTROL, + _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH | + VLV_MEDIA_RC0_COUNT_EN | + VLV_RENDER_RC0_COUNT_EN | + VLV_MEDIA_RC6_COUNT_EN | + VLV_RENDER_RC6_COUNT_EN)); rc6->ctl_enable = GEN7_RC_CTL_TO_MODE | VLV_RC_CTL_CTX_RST_PARALLEL; @@ -575,9 +576,9 @@ static void __intel_rc6_disable(struct intel_rc6 *rc6) intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL); if (GRAPHICS_VER(i915) >= 9) - set(uncore, GEN9_PG_ENABLE, 0); - set(uncore, GEN6_RC_CONTROL, 0); - set(uncore, GEN6_RC_STATE, 0); + intel_uncore_write_fw(uncore, GEN9_PG_ENABLE, 0); + intel_uncore_write_fw(uncore, GEN6_RC_CONTROL, 0); + intel_uncore_write_fw(uncore, GEN6_RC_STATE, 0); intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL); } @@ -684,7 +685,7 @@ void intel_rc6_unpark(struct intel_rc6 *rc6) return; /* Restore HW timers for automatic RC6 entry while busy */ - set(uncore, GEN6_RC_CONTROL, rc6->ctl_enable); + intel_uncore_write_fw(uncore, GEN6_RC_CONTROL, rc6->ctl_enable); } void intel_rc6_park(struct intel_rc6 *rc6) @@ -704,7 +705,7 @@ void intel_rc6_park(struct intel_rc6 *rc6) return; /* Turn off the HW timers and go directly to rc6 */ - set(uncore, GEN6_RC_CONTROL, GEN6_RC_CTL_RC6_ENABLE); + intel_uncore_write_fw(uncore, GEN6_RC_CONTROL, GEN6_RC_CTL_RC6_ENABLE); if (HAS_RC6pp(rc6_to_i915(rc6))) target = 0x6; /* deepest rc6 */ @@ -712,7 +713,7 @@ void intel_rc6_park(struct intel_rc6 *rc6) target = 0x5; /* deep rc6 */ else target = 0x4; /* normal rc6 */ - set(uncore, GEN6_RC_STATE, target << RC_SW_TARGET_STATE_SHIFT); + intel_uncore_write_fw(uncore, GEN6_RC_STATE, target << RC_SW_TARGET_STATE_SHIFT); } void intel_rc6_disable(struct intel_rc6 *rc6) @@ -735,7 +736,7 @@ void intel_rc6_fini(struct intel_rc6 *rc6) /* We want the BIOS C6 state preserved across loads for MTL */ if (IS_METEORLAKE(rc6_to_i915(rc6)) && rc6->bios_state_captured) - set(uncore, GEN6_RC_STATE, rc6->bios_rc_state); + intel_uncore_write_fw(uncore, GEN6_RC_STATE, rc6->bios_rc_state); pctx = fetch_and_zero(&rc6->pctx); if (pctx) @@ -766,18 +767,18 @@ static u64 vlv_residency_raw(struct intel_uncore *uncore, const i915_reg_t reg) * before we have set the default VLV_COUNTER_CONTROL value. So always * set the high bit to be safe. */ - set(uncore, VLV_COUNTER_CONTROL, - _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH)); + intel_uncore_write_fw(uncore, VLV_COUNTER_CONTROL, + _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH)); upper = intel_uncore_read_fw(uncore, reg); do { tmp = upper; - set(uncore, VLV_COUNTER_CONTROL, - _MASKED_BIT_DISABLE(VLV_COUNT_RANGE_HIGH)); + intel_uncore_write_fw(uncore, VLV_COUNTER_CONTROL, + _MASKED_BIT_DISABLE(VLV_COUNT_RANGE_HIGH)); lower = intel_uncore_read_fw(uncore, reg); - set(uncore, VLV_COUNTER_CONTROL, - _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH)); + intel_uncore_write_fw(uncore, VLV_COUNTER_CONTROL, + _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH)); upper = intel_uncore_read_fw(uncore, reg); } while (upper != tmp && --loop); diff --git a/drivers/gpu/drm/i915/gt/intel_reset.c b/drivers/gpu/drm/i915/gt/intel_reset.c index 797ea8340467..e2152f75ba2e 100644 --- a/drivers/gpu/drm/i915/gt/intel_reset.c +++ b/drivers/gpu/drm/i915/gt/intel_reset.c @@ -7,7 +7,7 @@ #include <linux/stop_machine.h> #include <linux/string_helpers.h> -#include "display/intel_display.h" +#include "display/intel_display_reset.h" #include "display/intel_overlay.h" #include "gem/i915_gem_context.h" @@ -20,6 +20,7 @@ #include "i915_file_private.h" #include "i915_gpu_error.h" #include "i915_irq.h" +#include "i915_reg.h" #include "intel_breadcrumbs.h" #include "intel_engine_pm.h" #include "intel_engine_regs.h" @@ -1370,11 +1371,11 @@ static void intel_gt_reset_global(struct intel_gt *gt, /* Use a watchdog to ensure that our reset completes */ intel_wedge_on_timeout(&w, gt, 60 * HZ) { - intel_display_prepare_reset(gt->i915); + intel_display_reset_prepare(gt->i915); intel_gt_reset(gt, engine_mask, reason); - intel_display_finish_reset(gt->i915); + intel_display_reset_finish(gt->i915); } if (!test_bit(I915_WEDGED, >->reset.flags)) @@ -1624,7 +1625,7 @@ void __intel_init_wedge(struct intel_wedge_me *w, w->name = name; INIT_DELAYED_WORK_ONSTACK(&w->work, intel_wedge_me); - schedule_delayed_work(&w->work, timeout); + queue_delayed_work(gt->i915->unordered_wq, &w->work, timeout); } void __intel_fini_wedge(struct intel_wedge_me *w) diff --git a/drivers/gpu/drm/i915/gt/intel_rps.c b/drivers/gpu/drm/i915/gt/intel_rps.c index b2671ac59dc0..e92e626d4994 100644 --- a/drivers/gpu/drm/i915/gt/intel_rps.c +++ b/drivers/gpu/drm/i915/gt/intel_rps.c @@ -8,8 +8,10 @@ #include <drm/i915_drm.h> #include "display/intel_display.h" +#include "display/intel_display_irq.h" #include "i915_drv.h" #include "i915_irq.h" +#include "i915_reg.h" #include "intel_breadcrumbs.h" #include "intel_gt.h" #include "intel_gt_clock_utils.h" @@ -71,13 +73,14 @@ static void set(struct intel_uncore *uncore, i915_reg_t reg, u32 val) static void rps_timer(struct timer_list *t) { struct intel_rps *rps = from_timer(rps, t, timer); + struct intel_gt *gt = rps_to_gt(rps); struct intel_engine_cs *engine; ktime_t dt, last, timestamp; enum intel_engine_id id; s64 max_busy[3] = {}; timestamp = 0; - for_each_engine(engine, rps_to_gt(rps), id) { + for_each_engine(engine, gt, id) { s64 busy; int i; @@ -121,7 +124,7 @@ static void rps_timer(struct timer_list *t) busy += div_u64(max_busy[i], 1 << i); } - GT_TRACE(rps_to_gt(rps), + GT_TRACE(gt, "busy:%lld [%d%%], max:[%lld, %lld, %lld], interval:%d\n", busy, (int)div64_u64(100 * busy, dt), max_busy[0], max_busy[1], max_busy[2], @@ -131,12 +134,12 @@ static void rps_timer(struct timer_list *t) rps->cur_freq < rps->max_freq_softlimit) { rps->pm_iir |= GEN6_PM_RP_UP_THRESHOLD; rps->pm_interval = 1; - schedule_work(&rps->work); + queue_work(gt->i915->unordered_wq, &rps->work); } else if (100 * busy < rps->power.down_threshold * dt && rps->cur_freq > rps->min_freq_softlimit) { rps->pm_iir |= GEN6_PM_RP_DOWN_THRESHOLD; rps->pm_interval = 1; - schedule_work(&rps->work); + queue_work(gt->i915->unordered_wq, &rps->work); } else { rps->last_adj = 0; } @@ -971,7 +974,7 @@ static int rps_set_boost_freq(struct intel_rps *rps, u32 val) } mutex_unlock(&rps->lock); if (boost) - schedule_work(&rps->work); + queue_work(rps_to_gt(rps)->i915->unordered_wq, &rps->work); return 0; } @@ -1023,7 +1026,8 @@ void intel_rps_boost(struct i915_request *rq) if (!atomic_fetch_inc(&slpc->num_waiters)) { GT_TRACE(rps_to_gt(rps), "boost fence:%llx:%llx\n", rq->fence.context, rq->fence.seqno); - schedule_work(&slpc->boost_work); + queue_work(rps_to_gt(rps)->i915->unordered_wq, + &slpc->boost_work); } return; @@ -1039,7 +1043,7 @@ void intel_rps_boost(struct i915_request *rq) rq->fence.context, rq->fence.seqno); if (READ_ONCE(rps->cur_freq) < rps->boost_freq) - schedule_work(&rps->work); + queue_work(rps_to_gt(rps)->i915->unordered_wq, &rps->work); WRITE_ONCE(rps->boosts, rps->boosts + 1); /* debug only */ } @@ -1898,7 +1902,7 @@ void gen11_rps_irq_handler(struct intel_rps *rps, u32 pm_iir) gen6_gt_pm_mask_irq(gt, events); rps->pm_iir |= events; - schedule_work(&rps->work); + queue_work(gt->i915->unordered_wq, &rps->work); } void gen6_rps_irq_handler(struct intel_rps *rps, u32 pm_iir) @@ -1915,7 +1919,7 @@ void gen6_rps_irq_handler(struct intel_rps *rps, u32 pm_iir) gen6_gt_pm_mask_irq(gt, events); rps->pm_iir |= events; - schedule_work(&rps->work); + queue_work(gt->i915->unordered_wq, &rps->work); spin_unlock(gt->irq_lock); } diff --git a/drivers/gpu/drm/i915/gt/intel_workarounds.c b/drivers/gpu/drm/i915/gt/intel_workarounds.c index b925ef47304b..4d2dece96011 100644 --- a/drivers/gpu/drm/i915/gt/intel_workarounds.c +++ b/drivers/gpu/drm/i915/gt/intel_workarounds.c @@ -812,11 +812,25 @@ static void dg2_ctx_workarounds_init(struct intel_engine_cs *engine, wa_masked_en(wal, CACHE_MODE_1, MSAA_OPTIMIZATION_REDUC_DISABLE); } +static void mtl_ctx_gt_tuning_init(struct intel_engine_cs *engine, + struct i915_wa_list *wal) +{ + struct drm_i915_private *i915 = engine->i915; + + dg2_ctx_gt_tuning_init(engine, wal); + + if (IS_MTL_GRAPHICS_STEP(i915, M, STEP_B0, STEP_FOREVER) || + IS_MTL_GRAPHICS_STEP(i915, P, STEP_B0, STEP_FOREVER)) + wa_add(wal, DRAW_WATERMARK, VERT_WM_VAL, 0x3FF, 0, false); +} + static void mtl_ctx_workarounds_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) { struct drm_i915_private *i915 = engine->i915; + mtl_ctx_gt_tuning_init(engine, wal); + if (IS_MTL_GRAPHICS_STEP(i915, M, STEP_A0, STEP_B0) || IS_MTL_GRAPHICS_STEP(i915, P, STEP_A0, STEP_B0)) { /* Wa_14014947963 */ @@ -1695,14 +1709,20 @@ pvc_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) static void xelpg_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) { + /* Wa_14018778641 / Wa_18018781329 */ + wa_mcr_write_or(wal, RENDER_MOD_CTRL, FORCE_MISS_FTLB); + wa_mcr_write_or(wal, COMP_MOD_CTRL, FORCE_MISS_FTLB); + + /* Wa_22016670082 */ + wa_write_or(wal, GEN12_SQCNT1, GEN12_STRICT_RAR_ENABLE); + if (IS_MTL_GRAPHICS_STEP(gt->i915, M, STEP_A0, STEP_B0) || IS_MTL_GRAPHICS_STEP(gt->i915, P, STEP_A0, STEP_B0)) { /* Wa_14014830051 */ wa_mcr_write_clr(wal, SARB_CHICKEN1, COMP_CKN_IN); - /* Wa_18018781329 */ - wa_mcr_write_or(wal, RENDER_MOD_CTRL, FORCE_MISS_FTLB); - wa_mcr_write_or(wal, COMP_MOD_CTRL, FORCE_MISS_FTLB); + /* Wa_14015795083 */ + wa_write_clr(wal, GEN7_MISCCPCTL, GEN12_DOP_CLOCK_GATE_RENDER_ENABLE); } /* @@ -1715,17 +1735,16 @@ xelpg_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) static void xelpmp_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) { - if (IS_MTL_MEDIA_STEP(gt->i915, STEP_A0, STEP_B0)) { - /* - * Wa_18018781329 - * - * Note that although these registers are MCR on the primary - * GT, the media GT's versions are regular singleton registers. - */ - wa_write_or(wal, XELPMP_GSC_MOD_CTRL, FORCE_MISS_FTLB); - wa_write_or(wal, XELPMP_VDBX_MOD_CTRL, FORCE_MISS_FTLB); - wa_write_or(wal, XELPMP_VEBX_MOD_CTRL, FORCE_MISS_FTLB); - } + /* + * Wa_14018778641 + * Wa_18018781329 + * + * Note that although these registers are MCR on the primary + * GT, the media GT's versions are regular singleton registers. + */ + wa_write_or(wal, XELPMP_GSC_MOD_CTRL, FORCE_MISS_FTLB); + wa_write_or(wal, XELPMP_VDBX_MOD_CTRL, FORCE_MISS_FTLB); + wa_write_or(wal, XELPMP_VEBX_MOD_CTRL, FORCE_MISS_FTLB); debug_dump_steering(gt); } @@ -1743,6 +1762,13 @@ xelpmp_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) */ static void gt_tuning_settings(struct intel_gt *gt, struct i915_wa_list *wal) { + if (IS_METEORLAKE(gt->i915)) { + if (gt->type != GT_MEDIA) + wa_mcr_write_or(wal, XEHP_L3SCQREG7, BLEND_FILL_CACHING_OPT_DIS); + + wa_mcr_write_or(wal, XEHP_SQCM, EN_32B_ACCESS); + } + if (IS_PONTEVECCHIO(gt->i915)) { wa_mcr_write(wal, XEHPC_L3SCRUB, SCRUB_CL_DWNGRADE_SHARED | SCRUB_RATE_4B_PER_CLK); @@ -2939,7 +2965,7 @@ static void add_render_compute_tuning_settings(struct drm_i915_private *i915, struct i915_wa_list *wal) { - if (IS_DG2(i915)) + if (IS_METEORLAKE(i915) || IS_DG2(i915)) wa_mcr_write_clr_set(wal, RT_CTRL, STACKID_CTRL, STACKID_CTRL_512); /* diff --git a/drivers/gpu/drm/i915/gt/selftest_engine_cs.c b/drivers/gpu/drm/i915/gt/selftest_engine_cs.c index 542ce6d2de19..78cdfc6f315f 100644 --- a/drivers/gpu/drm/i915/gt/selftest_engine_cs.c +++ b/drivers/gpu/drm/i915/gt/selftest_engine_cs.c @@ -27,7 +27,7 @@ static void perf_begin(struct intel_gt *gt) /* Boost gpufreq to max [waitboost] and keep it fixed */ atomic_inc(>->rps.num_waiters); - schedule_work(>->rps.work); + queue_work(gt->i915->unordered_wq, >->rps.work); flush_work(>->rps.work); } diff --git a/drivers/gpu/drm/i915/gt/selftest_engine_pm.c b/drivers/gpu/drm/i915/gt/selftest_engine_pm.c index 87c94314cf67..10e556a7eac4 100644 --- a/drivers/gpu/drm/i915/gt/selftest_engine_pm.c +++ b/drivers/gpu/drm/i915/gt/selftest_engine_pm.c @@ -5,6 +5,7 @@ #include <linux/sort.h> +#include "gt/intel_gt_print.h" #include "i915_selftest.h" #include "intel_engine_regs.h" #include "intel_gpu_commands.h" @@ -402,7 +403,7 @@ static int live_engine_pm(void *arg) /* gt wakeref is async (deferred to workqueue) */ if (intel_gt_pm_wait_for_idle(gt)) { - pr_err("GT failed to idle\n"); + gt_err(gt, "GT failed to idle\n"); return -EINVAL; } } diff --git a/drivers/gpu/drm/i915/gt/selftest_migrate.c b/drivers/gpu/drm/i915/gt/selftest_migrate.c index e677f2da093d..3def5ca72dec 100644 --- a/drivers/gpu/drm/i915/gt/selftest_migrate.c +++ b/drivers/gpu/drm/i915/gt/selftest_migrate.c @@ -137,7 +137,7 @@ err_free_src: static int intel_context_copy_ccs(struct intel_context *ce, const struct i915_deps *deps, struct scatterlist *sg, - enum i915_cache_level cache_level, + unsigned int pat_index, bool write_to_ccs, struct i915_request **out) { @@ -185,7 +185,7 @@ static int intel_context_copy_ccs(struct intel_context *ce, if (err) goto out_rq; - len = emit_pte(rq, &it, cache_level, true, offset, CHUNK_SZ); + len = emit_pte(rq, &it, pat_index, true, offset, CHUNK_SZ); if (len <= 0) { err = len; goto out_rq; @@ -223,7 +223,7 @@ intel_migrate_ccs_copy(struct intel_migrate *m, struct i915_gem_ww_ctx *ww, const struct i915_deps *deps, struct scatterlist *sg, - enum i915_cache_level cache_level, + unsigned int pat_index, bool write_to_ccs, struct i915_request **out) { @@ -243,7 +243,7 @@ intel_migrate_ccs_copy(struct intel_migrate *m, if (err) goto out; - err = intel_context_copy_ccs(ce, deps, sg, cache_level, + err = intel_context_copy_ccs(ce, deps, sg, pat_index, write_to_ccs, out); intel_context_unpin(ce); @@ -300,7 +300,7 @@ static int clear(struct intel_migrate *migrate, /* Write the obj data into ccs surface */ err = intel_migrate_ccs_copy(migrate, &ww, NULL, obj->mm.pages->sgl, - obj->cache_level, + obj->pat_index, true, &rq); if (rq && !err) { if (i915_request_wait(rq, 0, HZ) < 0) { @@ -351,7 +351,7 @@ static int clear(struct intel_migrate *migrate, err = intel_migrate_ccs_copy(migrate, &ww, NULL, obj->mm.pages->sgl, - obj->cache_level, + obj->pat_index, false, &rq); if (rq && !err) { if (i915_request_wait(rq, 0, HZ) < 0) { @@ -414,9 +414,9 @@ static int __migrate_copy(struct intel_migrate *migrate, struct i915_request **out) { return intel_migrate_copy(migrate, ww, NULL, - src->mm.pages->sgl, src->cache_level, + src->mm.pages->sgl, src->pat_index, i915_gem_object_is_lmem(src), - dst->mm.pages->sgl, dst->cache_level, + dst->mm.pages->sgl, dst->pat_index, i915_gem_object_is_lmem(dst), out); } @@ -428,9 +428,9 @@ static int __global_copy(struct intel_migrate *migrate, struct i915_request **out) { return intel_context_migrate_copy(migrate->context, NULL, - src->mm.pages->sgl, src->cache_level, + src->mm.pages->sgl, src->pat_index, i915_gem_object_is_lmem(src), - dst->mm.pages->sgl, dst->cache_level, + dst->mm.pages->sgl, dst->pat_index, i915_gem_object_is_lmem(dst), out); } @@ -455,7 +455,7 @@ static int __migrate_clear(struct intel_migrate *migrate, { return intel_migrate_clear(migrate, ww, NULL, obj->mm.pages->sgl, - obj->cache_level, + obj->pat_index, i915_gem_object_is_lmem(obj), value, out); } @@ -468,7 +468,7 @@ static int __global_clear(struct intel_migrate *migrate, { return intel_context_migrate_clear(migrate->context, NULL, obj->mm.pages->sgl, - obj->cache_level, + obj->pat_index, i915_gem_object_is_lmem(obj), value, out); } @@ -648,7 +648,7 @@ static int live_emit_pte_full_ring(void *arg) */ pr_info("%s emite_pte ring space=%u\n", __func__, rq->ring->space); it = sg_sgt(obj->mm.pages->sgl); - len = emit_pte(rq, &it, obj->cache_level, false, 0, CHUNK_SZ); + len = emit_pte(rq, &it, obj->pat_index, false, 0, CHUNK_SZ); if (!len) { err = -EINVAL; goto out_rq; @@ -844,7 +844,7 @@ static int wrap_ktime_compare(const void *A, const void *B) static int __perf_clear_blt(struct intel_context *ce, struct scatterlist *sg, - enum i915_cache_level cache_level, + unsigned int pat_index, bool is_lmem, size_t sz) { @@ -858,7 +858,7 @@ static int __perf_clear_blt(struct intel_context *ce, t0 = ktime_get(); - err = intel_context_migrate_clear(ce, NULL, sg, cache_level, + err = intel_context_migrate_clear(ce, NULL, sg, pat_index, is_lmem, 0, &rq); if (rq) { if (i915_request_wait(rq, 0, MAX_SCHEDULE_TIMEOUT) < 0) @@ -904,7 +904,8 @@ static int perf_clear_blt(void *arg) err = __perf_clear_blt(gt->migrate.context, dst->mm.pages->sgl, - I915_CACHE_NONE, + i915_gem_get_pat_index(gt->i915, + I915_CACHE_NONE), i915_gem_object_is_lmem(dst), sizes[i]); @@ -919,10 +920,10 @@ static int perf_clear_blt(void *arg) static int __perf_copy_blt(struct intel_context *ce, struct scatterlist *src, - enum i915_cache_level src_cache_level, + unsigned int src_pat_index, bool src_is_lmem, struct scatterlist *dst, - enum i915_cache_level dst_cache_level, + unsigned int dst_pat_index, bool dst_is_lmem, size_t sz) { @@ -937,9 +938,9 @@ static int __perf_copy_blt(struct intel_context *ce, t0 = ktime_get(); err = intel_context_migrate_copy(ce, NULL, - src, src_cache_level, + src, src_pat_index, src_is_lmem, - dst, dst_cache_level, + dst, dst_pat_index, dst_is_lmem, &rq); if (rq) { @@ -994,10 +995,12 @@ static int perf_copy_blt(void *arg) err = __perf_copy_blt(gt->migrate.context, src->mm.pages->sgl, - I915_CACHE_NONE, + i915_gem_get_pat_index(gt->i915, + I915_CACHE_NONE), i915_gem_object_is_lmem(src), dst->mm.pages->sgl, - I915_CACHE_NONE, + i915_gem_get_pat_index(gt->i915, + I915_CACHE_NONE), i915_gem_object_is_lmem(dst), sz); diff --git a/drivers/gpu/drm/i915/gt/selftest_mocs.c b/drivers/gpu/drm/i915/gt/selftest_mocs.c index ca009a6a13bd..a8446ab82501 100644 --- a/drivers/gpu/drm/i915/gt/selftest_mocs.c +++ b/drivers/gpu/drm/i915/gt/selftest_mocs.c @@ -131,13 +131,14 @@ static int read_mocs_table(struct i915_request *rq, const struct drm_i915_mocs_table *table, u32 *offset) { + struct intel_gt *gt = rq->engine->gt; u32 addr; if (!table) return 0; if (HAS_GLOBAL_MOCS_REGISTERS(rq->engine->i915)) - addr = global_mocs_offset(); + addr = global_mocs_offset() + gt->uncore->gsi_offset; else addr = mocs_offset(rq->engine); diff --git a/drivers/gpu/drm/i915/gt/selftest_reset.c b/drivers/gpu/drm/i915/gt/selftest_reset.c index a9e0a91bc0e0..79aa6ac66ad2 100644 --- a/drivers/gpu/drm/i915/gt/selftest_reset.c +++ b/drivers/gpu/drm/i915/gt/selftest_reset.c @@ -86,7 +86,9 @@ __igt_reset_stolen(struct intel_gt *gt, ggtt->vm.insert_page(&ggtt->vm, dma, ggtt->error_capture.start, - I915_CACHE_NONE, 0); + i915_gem_get_pat_index(gt->i915, + I915_CACHE_NONE), + 0); mb(); s = io_mapping_map_wc(&ggtt->iomap, @@ -127,7 +129,9 @@ __igt_reset_stolen(struct intel_gt *gt, ggtt->vm.insert_page(&ggtt->vm, dma, ggtt->error_capture.start, - I915_CACHE_NONE, 0); + i915_gem_get_pat_index(gt->i915, + I915_CACHE_NONE), + 0); mb(); s = io_mapping_map_wc(&ggtt->iomap, diff --git a/drivers/gpu/drm/i915/gt/selftest_rps.c b/drivers/gpu/drm/i915/gt/selftest_rps.c index 84e77e8dbba1..fb30f733b036 100644 --- a/drivers/gpu/drm/i915/gt/selftest_rps.c +++ b/drivers/gpu/drm/i915/gt/selftest_rps.c @@ -8,6 +8,7 @@ #include "gem/i915_gem_internal.h" +#include "i915_reg.h" #include "intel_engine_heartbeat.h" #include "intel_engine_pm.h" #include "intel_engine_regs.h" diff --git a/drivers/gpu/drm/i915/gt/selftest_slpc.c b/drivers/gpu/drm/i915/gt/selftest_slpc.c index bd44ce73a504..952c8d52d68a 100644 --- a/drivers/gpu/drm/i915/gt/selftest_slpc.c +++ b/drivers/gpu/drm/i915/gt/selftest_slpc.c @@ -70,6 +70,31 @@ static int slpc_set_freq(struct intel_gt *gt, u32 freq) return err; } +static int slpc_restore_freq(struct intel_guc_slpc *slpc, u32 min, u32 max) +{ + int err; + + err = slpc_set_max_freq(slpc, max); + if (err) { + pr_err("Unable to restore max freq"); + return err; + } + + err = slpc_set_min_freq(slpc, min); + if (err) { + pr_err("Unable to restore min freq"); + return err; + } + + err = intel_guc_slpc_set_ignore_eff_freq(slpc, false); + if (err) { + pr_err("Unable to restore efficient freq"); + return err; + } + + return 0; +} + static u64 measure_power_at_freq(struct intel_gt *gt, int *freq, u64 *power) { int err = 0; @@ -268,8 +293,7 @@ static int run_test(struct intel_gt *gt, int test_type) /* * Set min frequency to RPn so that we can test the whole - * range of RPn-RP0. This also turns off efficient freq - * usage and makes results more predictable. + * range of RPn-RP0. */ err = slpc_set_min_freq(slpc, slpc->min_freq); if (err) { @@ -277,6 +301,15 @@ static int run_test(struct intel_gt *gt, int test_type) return err; } + /* + * Turn off efficient frequency so RPn/RP0 ranges are obeyed. + */ + err = intel_guc_slpc_set_ignore_eff_freq(slpc, true); + if (err) { + pr_err("Unable to turn off efficient freq!"); + return err; + } + intel_gt_pm_wait_for_idle(gt); intel_gt_pm_get(gt); for_each_engine(engine, gt, id) { @@ -358,9 +391,8 @@ static int run_test(struct intel_gt *gt, int test_type) break; } - /* Restore min/max frequencies */ - slpc_set_max_freq(slpc, slpc_max_freq); - slpc_set_min_freq(slpc, slpc_min_freq); + /* Restore min/max/efficient frequencies */ + err = slpc_restore_freq(slpc, slpc_min_freq, slpc_max_freq); if (igt_flush_test(gt->i915)) err = -EIO; diff --git a/drivers/gpu/drm/i915/gt/selftest_timeline.c b/drivers/gpu/drm/i915/gt/selftest_timeline.c index 9f536c251179..39c3ec12df1a 100644 --- a/drivers/gpu/drm/i915/gt/selftest_timeline.c +++ b/drivers/gpu/drm/i915/gt/selftest_timeline.c @@ -836,7 +836,7 @@ static int setup_watcher(struct hwsp_watcher *w, struct intel_gt *gt, return PTR_ERR(obj); /* keep the same cache settings as timeline */ - i915_gem_object_set_cache_coherency(obj, tl->hwsp_ggtt->obj->cache_level); + i915_gem_object_set_pat_index(obj, tl->hwsp_ggtt->obj->pat_index); w->map = i915_gem_object_pin_map_unlocked(obj, page_unmask_bits(tl->hwsp_ggtt->obj->mm.mapping)); if (IS_ERR(w->map)) { diff --git a/drivers/gpu/drm/i915/gt/selftest_tlb.c b/drivers/gpu/drm/i915/gt/selftest_tlb.c index e6cac1f15d6e..3bd6b540257b 100644 --- a/drivers/gpu/drm/i915/gt/selftest_tlb.c +++ b/drivers/gpu/drm/i915/gt/selftest_tlb.c @@ -36,6 +36,8 @@ pte_tlbinv(struct intel_context *ce, u64 length, struct rnd_state *prng) { + const unsigned int pat_index = + i915_gem_get_pat_index(ce->vm->i915, I915_CACHE_NONE); struct drm_i915_gem_object *batch; struct drm_mm_node vb_node; struct i915_request *rq; @@ -155,7 +157,7 @@ pte_tlbinv(struct intel_context *ce, /* Flip the PTE between A and B */ if (i915_gem_object_is_lmem(vb->obj)) pte_flags |= PTE_LM; - ce->vm->insert_entries(ce->vm, &vb_res, 0, pte_flags); + ce->vm->insert_entries(ce->vm, &vb_res, pat_index, pte_flags); /* Flush the PTE update to concurrent HW */ tlbinv(ce->vm, addr & -length, length); @@ -188,11 +190,18 @@ out: static struct drm_i915_gem_object *create_lmem(struct intel_gt *gt) { + struct intel_memory_region *mr = gt->i915->mm.regions[INTEL_REGION_LMEM_0]; + resource_size_t size = SZ_1G; + /* * Allocation of largest possible page size allows to test all types - * of pages. + * of pages. To succeed with both allocations, especially in case of Small + * BAR, try to allocate no more than quarter of mappable memory. */ - return i915_gem_object_create_lmem(gt->i915, SZ_1G, I915_BO_ALLOC_CONTIGUOUS); + if (mr && size > mr->io_size / 4) + size = mr->io_size / 4; + + return i915_gem_object_create_lmem(gt->i915, size, I915_BO_ALLOC_CONTIGUOUS); } static struct drm_i915_gem_object *create_smem(struct intel_gt *gt) diff --git a/drivers/gpu/drm/i915/gt/uc/abi/guc_communication_ctb_abi.h b/drivers/gpu/drm/i915/gt/uc/abi/guc_communication_ctb_abi.h index 28b8387f97b7..f7d70db16d76 100644 --- a/drivers/gpu/drm/i915/gt/uc/abi/guc_communication_ctb_abi.h +++ b/drivers/gpu/drm/i915/gt/uc/abi/guc_communication_ctb_abi.h @@ -167,25 +167,4 @@ static_assert(sizeof(struct guc_ct_buffer_desc) == 64); * - **flags**, holds various bits to control message handling */ -/* - * Definition of the command transport message header (DW0) - * - * bit[4..0] message len (in dwords) - * bit[7..5] reserved - * bit[8] response (G2H only) - * bit[8] write fence to desc (H2G only) - * bit[9] write status to H2G buff (H2G only) - * bit[10] send status back via G2H (H2G only) - * bit[15..11] reserved - * bit[31..16] action code - */ -#define GUC_CT_MSG_LEN_SHIFT 0 -#define GUC_CT_MSG_LEN_MASK 0x1F -#define GUC_CT_MSG_IS_RESPONSE (1 << 8) -#define GUC_CT_MSG_WRITE_FENCE_TO_DESC (1 << 8) -#define GUC_CT_MSG_WRITE_STATUS_TO_BUFF (1 << 9) -#define GUC_CT_MSG_SEND_STATUS (1 << 10) -#define GUC_CT_MSG_ACTION_SHIFT 16 -#define GUC_CT_MSG_ACTION_MASK 0xFFFF - #endif /* _ABI_GUC_COMMUNICATION_CTB_ABI_H */ diff --git a/drivers/gpu/drm/i915/gt/uc/abi/guc_errors_abi.h b/drivers/gpu/drm/i915/gt/uc/abi/guc_errors_abi.h index bcb1129b3610..dabeaf4f245f 100644 --- a/drivers/gpu/drm/i915/gt/uc/abi/guc_errors_abi.h +++ b/drivers/gpu/drm/i915/gt/uc/abi/guc_errors_abi.h @@ -44,6 +44,7 @@ enum intel_guc_load_status { enum intel_bootrom_load_status { INTEL_BOOTROM_STATUS_NO_KEY_FOUND = 0x13, INTEL_BOOTROM_STATUS_AES_PROD_KEY_FOUND = 0x1A, + INTEL_BOOTROM_STATUS_PROD_KEY_CHECK_FAILURE = 0x2B, INTEL_BOOTROM_STATUS_RSA_FAILED = 0x50, INTEL_BOOTROM_STATUS_PAVPC_FAILED = 0x73, INTEL_BOOTROM_STATUS_WOPCM_FAILED = 0x74, diff --git a/drivers/gpu/drm/i915/gt/uc/abi/guc_messages_abi.h b/drivers/gpu/drm/i915/gt/uc/abi/guc_messages_abi.h index 7d5ba4d97d70..98eb4f46572b 100644 --- a/drivers/gpu/drm/i915/gt/uc/abi/guc_messages_abi.h +++ b/drivers/gpu/drm/i915/gt/uc/abi/guc_messages_abi.h @@ -24,6 +24,7 @@ * | | 30:28 | **TYPE** - message type | * | | | - _`GUC_HXG_TYPE_REQUEST` = 0 | * | | | - _`GUC_HXG_TYPE_EVENT` = 1 | + * | | | - _`GUC_HXG_TYPE_FAST_REQUEST` = 2 | * | | | - _`GUC_HXG_TYPE_NO_RESPONSE_BUSY` = 3 | * | | | - _`GUC_HXG_TYPE_NO_RESPONSE_RETRY` = 5 | * | | | - _`GUC_HXG_TYPE_RESPONSE_FAILURE` = 6 | @@ -46,6 +47,7 @@ #define GUC_HXG_MSG_0_TYPE (0x7 << 28) #define GUC_HXG_TYPE_REQUEST 0u #define GUC_HXG_TYPE_EVENT 1u +#define GUC_HXG_TYPE_FAST_REQUEST 2u #define GUC_HXG_TYPE_NO_RESPONSE_BUSY 3u #define GUC_HXG_TYPE_NO_RESPONSE_RETRY 5u #define GUC_HXG_TYPE_RESPONSE_FAILURE 6u @@ -90,6 +92,34 @@ #define GUC_HXG_REQUEST_MSG_n_DATAn GUC_HXG_MSG_n_PAYLOAD /** + * DOC: HXG Fast Request + * + * The `HXG Request`_ message should be used to initiate asynchronous activity + * for which confirmation or return data is not expected. + * + * If confirmation is required then `HXG Request`_ shall be used instead. + * + * The recipient of this message may only use `HXG Failure`_ message if it was + * unable to accept this request (like invalid data). + * + * Format of `HXG Fast Request`_ message is same as `HXG Request`_ except @TYPE. + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN - see `HXG Message`_ | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = `GUC_HXG_TYPE_FAST_REQUEST`_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:16 | DATA0 - see `HXG Request`_ | + * | +-------+--------------------------------------------------------------+ + * | | 15:0 | ACTION - see `HXG Request`_ | + * +---+-------+--------------------------------------------------------------+ + * |...| | DATAn - see `HXG Request`_ | + * +---+-------+--------------------------------------------------------------+ + */ + +/** * DOC: HXG Event * * The `HXG Event`_ message should be used to initiate asynchronous activity diff --git a/drivers/gpu/drm/i915/gt/uc/guc_capture_fwif.h b/drivers/gpu/drm/i915/gt/uc/guc_capture_fwif.h index 9d589c28f40f..1fc0c17b1230 100644 --- a/drivers/gpu/drm/i915/gt/uc/guc_capture_fwif.h +++ b/drivers/gpu/drm/i915/gt/uc/guc_capture_fwif.h @@ -12,7 +12,7 @@ struct intel_guc; struct file; -/** +/* * struct __guc_capture_bufstate * * Book-keeping structure used to track read and write pointers @@ -26,7 +26,7 @@ struct __guc_capture_bufstate { u32 wr; }; -/** +/* * struct __guc_capture_parsed_output - extracted error capture node * * A single unit of extracted error-capture output data grouped together @@ -58,7 +58,7 @@ struct __guc_capture_parsed_output { #define GCAP_PARSED_REGLIST_INDEX_ENGINST BIT(GUC_CAPTURE_LIST_TYPE_ENGINE_INSTANCE) }; -/** +/* * struct guc_debug_capture_list_header / struct guc_debug_capture_list * * As part of ADS registration, these header structures (followed by @@ -76,7 +76,7 @@ struct guc_debug_capture_list { struct guc_mmio_reg regs[]; } __packed; -/** +/* * struct __guc_mmio_reg_descr / struct __guc_mmio_reg_descr_group * * intel_guc_capture module uses these structures to maintain static @@ -101,7 +101,7 @@ struct __guc_mmio_reg_descr_group { struct __guc_mmio_reg_descr *extlist; /* only used for steered registers */ }; -/** +/* * struct guc_state_capture_header_t / struct guc_state_capture_t / * guc_state_capture_group_header_t / guc_state_capture_group_t * @@ -148,7 +148,7 @@ struct guc_state_capture_group_t { struct guc_state_capture_t capture_entries[]; } __packed; -/** +/* * struct __guc_capture_ads_cache * * A structure to cache register lists that were populated and registered @@ -187,6 +187,10 @@ struct intel_guc_state_capture { struct __guc_capture_ads_cache ads_cache[GUC_CAPTURE_LIST_INDEX_MAX] [GUC_CAPTURE_LIST_TYPE_MAX] [GUC_MAX_ENGINE_CLASSES]; + + /** + * @ads_null_cache: ADS null cache. + */ void *ads_null_cache; /** @@ -202,6 +206,10 @@ struct intel_guc_state_capture { struct list_head cachelist; #define PREALLOC_NODES_MAX_COUNT (3 * GUC_MAX_ENGINE_CLASSES * GUC_MAX_INSTANCES_PER_CLASS) #define PREALLOC_NODES_DEFAULT_NUMREGS 64 + + /** + * @max_mmio_per_node: Max MMIO per node. + */ int max_mmio_per_node; /** diff --git a/drivers/gpu/drm/i915/gt/uc/intel_gsc_binary_headers.h b/drivers/gpu/drm/i915/gt/uc/intel_gsc_binary_headers.h new file mode 100644 index 000000000000..714f0c256118 --- /dev/null +++ b/drivers/gpu/drm/i915/gt/uc/intel_gsc_binary_headers.h @@ -0,0 +1,74 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _INTEL_GSC_BINARY_HEADERS_H_ +#define _INTEL_GSC_BINARY_HEADERS_H_ + +#include <linux/types.h> + +/* Code partition directory (CPD) structures */ +struct intel_gsc_cpd_header_v2 { + u32 header_marker; +#define INTEL_GSC_CPD_HEADER_MARKER 0x44504324 + + u32 num_of_entries; + u8 header_version; + u8 entry_version; + u8 header_length; /* in bytes */ + u8 flags; + u32 partition_name; + u32 crc32; +} __packed; + +struct intel_gsc_cpd_entry { + u8 name[12]; + + /* + * Bits 0-24: offset from the beginning of the code partition + * Bit 25: huffman compressed + * Bits 26-31: reserved + */ + u32 offset; +#define INTEL_GSC_CPD_ENTRY_OFFSET_MASK GENMASK(24, 0) +#define INTEL_GSC_CPD_ENTRY_HUFFMAN_COMP BIT(25) + + /* + * Module/Item length, in bytes. For Huffman-compressed modules, this + * refers to the uncompressed size. For software-compressed modules, + * this refers to the compressed size. + */ + u32 length; + + u8 reserved[4]; +} __packed; + +struct intel_gsc_version { + u16 major; + u16 minor; + u16 hotfix; + u16 build; +} __packed; + +struct intel_gsc_manifest_header { + u32 header_type; /* 0x4 for manifest type */ + u32 header_length; /* in dwords */ + u32 header_version; + u32 flags; + u32 vendor; + u32 date; + u32 size; /* In dwords, size of entire manifest (header + extensions) */ + u32 header_id; + u32 internal_data; + struct intel_gsc_version fw_version; + u32 security_version; + struct intel_gsc_version meu_kit_version; + u32 meu_manifest_version; + u8 general_data[4]; + u8 reserved3[56]; + u32 modulus_size; /* in dwords */ + u32 exponent_size; /* in dwords */ +} __packed; + +#endif diff --git a/drivers/gpu/drm/i915/gt/uc/intel_gsc_fw.c b/drivers/gpu/drm/i915/gt/uc/intel_gsc_fw.c index 1d9fdfb11268..60e9c6c9e775 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_gsc_fw.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_gsc_fw.c @@ -13,6 +13,7 @@ #define GSC_FW_STATUS_REG _MMIO(0x116C40) #define GSC_FW_CURRENT_STATE REG_GENMASK(3, 0) #define GSC_FW_CURRENT_STATE_RESET 0 +#define GSC_FW_PROXY_STATE_NORMAL 5 #define GSC_FW_INIT_COMPLETE_BIT REG_BIT(9) static bool gsc_is_in_reset(struct intel_uncore *uncore) @@ -23,12 +24,27 @@ static bool gsc_is_in_reset(struct intel_uncore *uncore) GSC_FW_CURRENT_STATE_RESET; } -bool intel_gsc_uc_fw_init_done(struct intel_gsc_uc *gsc) +static u32 gsc_uc_get_fw_status(struct intel_uncore *uncore) { - struct intel_uncore *uncore = gsc_uc_to_gt(gsc)->uncore; - u32 fw_status = intel_uncore_read(uncore, GSC_FW_STATUS_REG); + intel_wakeref_t wakeref; + u32 fw_status = 0; + + with_intel_runtime_pm(uncore->rpm, wakeref) + fw_status = intel_uncore_read(uncore, GSC_FW_STATUS_REG); + + return fw_status; +} + +bool intel_gsc_uc_fw_proxy_init_done(struct intel_gsc_uc *gsc) +{ + return REG_FIELD_GET(GSC_FW_CURRENT_STATE, + gsc_uc_get_fw_status(gsc_uc_to_gt(gsc)->uncore)) == + GSC_FW_PROXY_STATE_NORMAL; +} - return fw_status & GSC_FW_INIT_COMPLETE_BIT; +bool intel_gsc_uc_fw_init_done(struct intel_gsc_uc *gsc) +{ + return gsc_uc_get_fw_status(gsc_uc_to_gt(gsc)->uncore) & GSC_FW_INIT_COMPLETE_BIT; } static int emit_gsc_fw_load(struct i915_request *rq, struct intel_gsc_uc *gsc) @@ -110,6 +126,13 @@ static int gsc_fw_load_prepare(struct intel_gsc_uc *gsc) if (obj->base.size < gsc->fw.size) return -ENOSPC; + /* + * Wa_22016122933: For MTL the shared memory needs to be mapped + * as WC on CPU side and UC (PAT index 2) on GPU side + */ + if (IS_METEORLAKE(i915)) + i915_gem_object_set_cache_coherency(obj, I915_CACHE_NONE); + dst = i915_gem_object_pin_map_unlocked(obj, i915_coherent_map_type(i915, obj, true)); if (IS_ERR(dst)) @@ -125,6 +148,12 @@ static int gsc_fw_load_prepare(struct intel_gsc_uc *gsc) memset(dst, 0, obj->base.size); memcpy(dst, src, gsc->fw.size); + /* + * Wa_22016122933: Making sure the data in dst is + * visible to GSC right away + */ + intel_guc_write_barrier(>->uc.guc); + i915_gem_object_unpin_map(gsc->fw.obj); i915_gem_object_unpin_map(obj); diff --git a/drivers/gpu/drm/i915/gt/uc/intel_gsc_fw.h b/drivers/gpu/drm/i915/gt/uc/intel_gsc_fw.h index f4c1106bb2a9..fff8928218df 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_gsc_fw.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_gsc_fw.h @@ -13,5 +13,6 @@ struct intel_uncore; int intel_gsc_uc_fw_upload(struct intel_gsc_uc *gsc); bool intel_gsc_uc_fw_init_done(struct intel_gsc_uc *gsc); +bool intel_gsc_uc_fw_proxy_init_done(struct intel_gsc_uc *gsc); #endif diff --git a/drivers/gpu/drm/i915/gt/uc/intel_gsc_proxy.c b/drivers/gpu/drm/i915/gt/uc/intel_gsc_proxy.c new file mode 100644 index 000000000000..5f138de3c14f --- /dev/null +++ b/drivers/gpu/drm/i915/gt/uc/intel_gsc_proxy.c @@ -0,0 +1,424 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2023 Intel Corporation + */ + +#include <linux/component.h> + +#include <drm/i915_component.h> +#include <drm/i915_gsc_proxy_mei_interface.h> + +#include "gt/intel_gt.h" +#include "gt/intel_gt_print.h" +#include "intel_gsc_proxy.h" +#include "intel_gsc_uc.h" +#include "intel_gsc_uc_heci_cmd_submit.h" +#include "i915_drv.h" +#include "i915_reg.h" + +/* + * GSC proxy: + * The GSC uC needs to communicate with the CSME to perform certain operations. + * Since the GSC can't perform this communication directly on platforms where it + * is integrated in GT, i915 needs to transfer the messages from GSC to CSME + * and back. i915 must manually start the proxy flow after the GSC is loaded to + * signal to GSC that we're ready to handle its messages and allow it to query + * its init data from CSME; GSC will then trigger an HECI2 interrupt if it needs + * to send messages to CSME again. + * The proxy flow is as follow: + * 1 - i915 submits a request to GSC asking for the message to CSME + * 2 - GSC replies with the proxy header + payload for CSME + * 3 - i915 sends the reply from GSC as-is to CSME via the mei proxy component + * 4 - CSME replies with the proxy header + payload for GSC + * 5 - i915 submits a request to GSC with the reply from CSME + * 6 - GSC replies either with a new header + payload (same as step 2, so we + * restart from there) or with an end message. + */ + +/* + * The component should load quite quickly in most cases, but it could take + * a bit. Using a very big timeout just to cover the worst case scenario + */ +#define GSC_PROXY_INIT_TIMEOUT_MS 20000 + +/* the protocol supports up to 32K in each direction */ +#define GSC_PROXY_BUFFER_SIZE SZ_32K +#define GSC_PROXY_CHANNEL_SIZE (GSC_PROXY_BUFFER_SIZE * 2) +#define GSC_PROXY_MAX_MSG_SIZE (GSC_PROXY_BUFFER_SIZE - sizeof(struct intel_gsc_mtl_header)) + +/* FW-defined proxy header */ +struct intel_gsc_proxy_header { + /* + * hdr: + * Bits 0-7: type of the proxy message (see enum intel_gsc_proxy_type) + * Bits 8-15: rsvd + * Bits 16-31: length in bytes of the payload following the proxy header + */ + u32 hdr; +#define GSC_PROXY_TYPE GENMASK(7, 0) +#define GSC_PROXY_PAYLOAD_LENGTH GENMASK(31, 16) + + u32 source; /* Source of the Proxy message */ + u32 destination; /* Destination of the Proxy message */ +#define GSC_PROXY_ADDRESSING_KMD 0x10000 +#define GSC_PROXY_ADDRESSING_GSC 0x20000 +#define GSC_PROXY_ADDRESSING_CSME 0x30000 + + u32 status; /* Command status */ +} __packed; + +/* FW-defined proxy types */ +enum intel_gsc_proxy_type { + GSC_PROXY_MSG_TYPE_PROXY_INVALID = 0, + GSC_PROXY_MSG_TYPE_PROXY_QUERY = 1, + GSC_PROXY_MSG_TYPE_PROXY_PAYLOAD = 2, + GSC_PROXY_MSG_TYPE_PROXY_END = 3, + GSC_PROXY_MSG_TYPE_PROXY_NOTIFICATION = 4, +}; + +struct gsc_proxy_msg { + struct intel_gsc_mtl_header header; + struct intel_gsc_proxy_header proxy_header; +} __packed; + +static int proxy_send_to_csme(struct intel_gsc_uc *gsc) +{ + struct intel_gt *gt = gsc_uc_to_gt(gsc); + struct i915_gsc_proxy_component *comp = gsc->proxy.component; + struct intel_gsc_mtl_header *hdr; + void *in = gsc->proxy.to_csme; + void *out = gsc->proxy.to_gsc; + u32 in_size; + int ret; + + /* CSME msg only includes the proxy */ + hdr = in; + in += sizeof(struct intel_gsc_mtl_header); + out += sizeof(struct intel_gsc_mtl_header); + + in_size = hdr->message_size - sizeof(struct intel_gsc_mtl_header); + + /* the message must contain at least the proxy header */ + if (in_size < sizeof(struct intel_gsc_proxy_header) || + in_size > GSC_PROXY_MAX_MSG_SIZE) { + gt_err(gt, "Invalid CSME message size: %u\n", in_size); + return -EINVAL; + } + + ret = comp->ops->send(comp->mei_dev, in, in_size); + if (ret < 0) { + gt_err(gt, "Failed to send CSME message\n"); + return ret; + } + + ret = comp->ops->recv(comp->mei_dev, out, GSC_PROXY_MAX_MSG_SIZE); + if (ret < 0) { + gt_err(gt, "Failed to receive CSME message\n"); + return ret; + } + + return ret; +} + +static int proxy_send_to_gsc(struct intel_gsc_uc *gsc) +{ + struct intel_gt *gt = gsc_uc_to_gt(gsc); + u32 *marker = gsc->proxy.to_csme; /* first dw of the reply header */ + u64 addr_in = i915_ggtt_offset(gsc->proxy.vma); + u64 addr_out = addr_in + GSC_PROXY_BUFFER_SIZE; + u32 size = ((struct gsc_proxy_msg *)gsc->proxy.to_gsc)->header.message_size; + int err; + + /* the message must contain at least the gsc and proxy headers */ + if (size < sizeof(struct gsc_proxy_msg) || size > GSC_PROXY_BUFFER_SIZE) { + gt_err(gt, "Invalid GSC proxy message size: %u\n", size); + return -EINVAL; + } + + /* clear the message marker */ + *marker = 0; + + /* make sure the marker write is flushed */ + wmb(); + + /* send the request */ + err = intel_gsc_uc_heci_cmd_submit_packet(gsc, addr_in, size, + addr_out, GSC_PROXY_BUFFER_SIZE); + + if (!err) { + /* wait for the reply to show up */ + err = wait_for(*marker != 0, 300); + if (err) + gt_err(gt, "Failed to get a proxy reply from gsc\n"); + } + + return err; +} + +static int validate_proxy_header(struct intel_gsc_proxy_header *header, + u32 source, u32 dest) +{ + u32 type = FIELD_GET(GSC_PROXY_TYPE, header->hdr); + u32 length = FIELD_GET(GSC_PROXY_PAYLOAD_LENGTH, header->hdr); + int ret = 0; + + if (header->destination != dest || header->source != source) { + ret = -ENOEXEC; + goto fail; + } + + switch (type) { + case GSC_PROXY_MSG_TYPE_PROXY_PAYLOAD: + if (length > 0) + break; + fallthrough; + case GSC_PROXY_MSG_TYPE_PROXY_INVALID: + ret = -EIO; + goto fail; + default: + break; + } + +fail: + return ret; +} + +static int proxy_query(struct intel_gsc_uc *gsc) +{ + struct intel_gt *gt = gsc_uc_to_gt(gsc); + struct gsc_proxy_msg *to_gsc = gsc->proxy.to_gsc; + struct gsc_proxy_msg *to_csme = gsc->proxy.to_csme; + int ret; + + intel_gsc_uc_heci_cmd_emit_mtl_header(&to_gsc->header, + HECI_MEADDRESS_PROXY, + sizeof(struct gsc_proxy_msg), + 0); + + to_gsc->proxy_header.hdr = + FIELD_PREP(GSC_PROXY_TYPE, GSC_PROXY_MSG_TYPE_PROXY_QUERY) | + FIELD_PREP(GSC_PROXY_PAYLOAD_LENGTH, 0); + + to_gsc->proxy_header.source = GSC_PROXY_ADDRESSING_KMD; + to_gsc->proxy_header.destination = GSC_PROXY_ADDRESSING_GSC; + to_gsc->proxy_header.status = 0; + + while (1) { + /* clear the GSC response header space */ + memset(gsc->proxy.to_csme, 0, sizeof(struct gsc_proxy_msg)); + + /* send proxy message to GSC */ + ret = proxy_send_to_gsc(gsc); + if (ret) { + gt_err(gt, "failed to send proxy message to GSC! %d\n", ret); + goto proxy_error; + } + + /* stop if this was the last message */ + if (FIELD_GET(GSC_PROXY_TYPE, to_csme->proxy_header.hdr) == + GSC_PROXY_MSG_TYPE_PROXY_END) + break; + + /* make sure the GSC-to-CSME proxy header is sane */ + ret = validate_proxy_header(&to_csme->proxy_header, + GSC_PROXY_ADDRESSING_GSC, + GSC_PROXY_ADDRESSING_CSME); + if (ret) { + gt_err(gt, "invalid GSC to CSME proxy header! %d\n", ret); + goto proxy_error; + } + + /* send the GSC message to the CSME */ + ret = proxy_send_to_csme(gsc); + if (ret < 0) { + gt_err(gt, "failed to send proxy message to CSME! %d\n", ret); + goto proxy_error; + } + + /* update the GSC message size with the returned value from CSME */ + to_gsc->header.message_size = ret + sizeof(struct intel_gsc_mtl_header); + + /* make sure the CSME-to-GSC proxy header is sane */ + ret = validate_proxy_header(&to_gsc->proxy_header, + GSC_PROXY_ADDRESSING_CSME, + GSC_PROXY_ADDRESSING_GSC); + if (ret) { + gt_err(gt, "invalid CSME to GSC proxy header! %d\n", ret); + goto proxy_error; + } + } + +proxy_error: + return ret < 0 ? ret : 0; +} + +int intel_gsc_proxy_request_handler(struct intel_gsc_uc *gsc) +{ + struct intel_gt *gt = gsc_uc_to_gt(gsc); + int err; + + if (!gsc->proxy.component_added) + return -ENODEV; + + assert_rpm_wakelock_held(gt->uncore->rpm); + + /* when GSC is loaded, we can queue this before the component is bound */ + err = wait_for(gsc->proxy.component, GSC_PROXY_INIT_TIMEOUT_MS); + if (err) { + gt_err(gt, "GSC proxy component didn't bind within the expected timeout\n"); + return -EIO; + } + + mutex_lock(&gsc->proxy.mutex); + if (!gsc->proxy.component) { + gt_err(gt, "GSC proxy worker called without the component being bound!\n"); + err = -EIO; + } else { + /* + * write the status bit to clear it and allow new proxy + * interrupts to be generated while we handle the current + * request, but be sure not to write the reset bit + */ + intel_uncore_rmw(gt->uncore, HECI_H_CSR(MTL_GSC_HECI2_BASE), + HECI_H_CSR_RST, HECI_H_CSR_IS); + err = proxy_query(gsc); + } + mutex_unlock(&gsc->proxy.mutex); + return err; +} + +void intel_gsc_proxy_irq_handler(struct intel_gsc_uc *gsc, u32 iir) +{ + struct intel_gt *gt = gsc_uc_to_gt(gsc); + + if (unlikely(!iir)) + return; + + lockdep_assert_held(gt->irq_lock); + + if (!gsc->proxy.component) { + gt_err(gt, "GSC proxy irq received without the component being bound!\n"); + return; + } + + gsc->gsc_work_actions |= GSC_ACTION_SW_PROXY; + queue_work(gsc->wq, &gsc->work); +} + +static int i915_gsc_proxy_component_bind(struct device *i915_kdev, + struct device *mei_kdev, void *data) +{ + struct drm_i915_private *i915 = kdev_to_i915(i915_kdev); + struct intel_gt *gt = i915->media_gt; + struct intel_gsc_uc *gsc = >->uc.gsc; + intel_wakeref_t wakeref; + + /* enable HECI2 IRQs */ + with_intel_runtime_pm(&i915->runtime_pm, wakeref) + intel_uncore_rmw(gt->uncore, HECI_H_CSR(MTL_GSC_HECI2_BASE), + HECI_H_CSR_RST, HECI_H_CSR_IE); + + mutex_lock(&gsc->proxy.mutex); + gsc->proxy.component = data; + gsc->proxy.component->mei_dev = mei_kdev; + mutex_unlock(&gsc->proxy.mutex); + + return 0; +} + +static void i915_gsc_proxy_component_unbind(struct device *i915_kdev, + struct device *mei_kdev, void *data) +{ + struct drm_i915_private *i915 = kdev_to_i915(i915_kdev); + struct intel_gt *gt = i915->media_gt; + struct intel_gsc_uc *gsc = >->uc.gsc; + intel_wakeref_t wakeref; + + mutex_lock(&gsc->proxy.mutex); + gsc->proxy.component = NULL; + mutex_unlock(&gsc->proxy.mutex); + + /* disable HECI2 IRQs */ + with_intel_runtime_pm(&i915->runtime_pm, wakeref) + intel_uncore_rmw(gt->uncore, HECI_H_CSR(MTL_GSC_HECI2_BASE), + HECI_H_CSR_IE | HECI_H_CSR_RST, 0); +} + +static const struct component_ops i915_gsc_proxy_component_ops = { + .bind = i915_gsc_proxy_component_bind, + .unbind = i915_gsc_proxy_component_unbind, +}; + +static int proxy_channel_alloc(struct intel_gsc_uc *gsc) +{ + struct intel_gt *gt = gsc_uc_to_gt(gsc); + struct i915_vma *vma; + void *vaddr; + int err; + + err = intel_guc_allocate_and_map_vma(>->uc.guc, GSC_PROXY_CHANNEL_SIZE, + &vma, &vaddr); + if (err) + return err; + + gsc->proxy.vma = vma; + gsc->proxy.to_gsc = vaddr; + gsc->proxy.to_csme = vaddr + GSC_PROXY_BUFFER_SIZE; + + return 0; +} + +static void proxy_channel_free(struct intel_gsc_uc *gsc) +{ + if (!gsc->proxy.vma) + return; + + gsc->proxy.to_gsc = NULL; + gsc->proxy.to_csme = NULL; + i915_vma_unpin_and_release(&gsc->proxy.vma, I915_VMA_RELEASE_MAP); +} + +void intel_gsc_proxy_fini(struct intel_gsc_uc *gsc) +{ + struct intel_gt *gt = gsc_uc_to_gt(gsc); + struct drm_i915_private *i915 = gt->i915; + + if (fetch_and_zero(&gsc->proxy.component_added)) + component_del(i915->drm.dev, &i915_gsc_proxy_component_ops); + + proxy_channel_free(gsc); +} + +int intel_gsc_proxy_init(struct intel_gsc_uc *gsc) +{ + int err; + struct intel_gt *gt = gsc_uc_to_gt(gsc); + struct drm_i915_private *i915 = gt->i915; + + mutex_init(&gsc->proxy.mutex); + + if (!IS_ENABLED(CONFIG_INTEL_MEI_GSC_PROXY)) { + gt_info(gt, "can't init GSC proxy due to missing mei component\n"); + return -ENODEV; + } + + err = proxy_channel_alloc(gsc); + if (err) + return err; + + err = component_add_typed(i915->drm.dev, &i915_gsc_proxy_component_ops, + I915_COMPONENT_GSC_PROXY); + if (err < 0) { + gt_err(gt, "Failed to add GSC_PROXY component (%d)\n", err); + goto out_free; + } + + gsc->proxy.component_added = true; + + return 0; + +out_free: + proxy_channel_free(gsc); + return err; +} + diff --git a/drivers/gpu/drm/i915/gt/uc/intel_gsc_proxy.h b/drivers/gpu/drm/i915/gt/uc/intel_gsc_proxy.h new file mode 100644 index 000000000000..fc5aef10bfb4 --- /dev/null +++ b/drivers/gpu/drm/i915/gt/uc/intel_gsc_proxy.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _INTEL_GSC_PROXY_H_ +#define _INTEL_GSC_PROXY_H_ + +#include <linux/types.h> + +struct intel_gsc_uc; + +int intel_gsc_proxy_init(struct intel_gsc_uc *gsc); +void intel_gsc_proxy_fini(struct intel_gsc_uc *gsc); +int intel_gsc_proxy_request_handler(struct intel_gsc_uc *gsc); +void intel_gsc_proxy_irq_handler(struct intel_gsc_uc *gsc, u32 iir); + +#endif diff --git a/drivers/gpu/drm/i915/gt/uc/intel_gsc_uc.c b/drivers/gpu/drm/i915/gt/uc/intel_gsc_uc.c index 2d5b70b3384c..c659cc01f32f 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_gsc_uc.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_gsc_uc.c @@ -10,15 +10,79 @@ #include "intel_gsc_uc.h" #include "intel_gsc_fw.h" #include "i915_drv.h" +#include "intel_gsc_proxy.h" static void gsc_work(struct work_struct *work) { struct intel_gsc_uc *gsc = container_of(work, typeof(*gsc), work); struct intel_gt *gt = gsc_uc_to_gt(gsc); intel_wakeref_t wakeref; + u32 actions; + int ret; + + wakeref = intel_runtime_pm_get(gt->uncore->rpm); + + spin_lock_irq(gt->irq_lock); + actions = gsc->gsc_work_actions; + gsc->gsc_work_actions = 0; + spin_unlock_irq(gt->irq_lock); + + if (actions & GSC_ACTION_FW_LOAD) { + ret = intel_gsc_uc_fw_upload(gsc); + if (!ret) + /* setup proxy on a new load */ + actions |= GSC_ACTION_SW_PROXY; + else if (ret != -EEXIST) + goto out_put; + + /* + * The HuC auth can be done both before or after the proxy init; + * if done after, a proxy request will be issued and must be + * serviced before the authentication can complete. + * Since this worker also handles proxy requests, we can't + * perform an action that requires the proxy from within it and + * then stall waiting for it, because we'd be blocking the + * service path. Therefore, it is easier for us to load HuC + * first and do proxy later. The GSC will ack the HuC auth and + * then send the HuC proxy request as part of the proxy init + * flow. + * Note that we can only do the GSC auth if the GuC auth was + * successful. + */ + if (intel_uc_uses_huc(>->uc) && + intel_huc_is_authenticated(>->uc.huc, INTEL_HUC_AUTH_BY_GUC)) + intel_huc_auth(>->uc.huc, INTEL_HUC_AUTH_BY_GSC); + } + + if (actions & GSC_ACTION_SW_PROXY) { + if (!intel_gsc_uc_fw_init_done(gsc)) { + gt_err(gt, "Proxy request received with GSC not loaded!\n"); + goto out_put; + } + + ret = intel_gsc_proxy_request_handler(gsc); + if (ret) + goto out_put; + + /* mark the GSC FW init as done the first time we run this */ + if (actions & GSC_ACTION_FW_LOAD) { + /* + * If there is a proxy establishment error, the GSC might still + * complete the request handling cleanly, so we need to check the + * status register to check if the proxy init was actually successful + */ + if (intel_gsc_uc_fw_proxy_init_done(gsc)) { + drm_dbg(>->i915->drm, "GSC Proxy initialized\n"); + intel_uc_fw_change_status(&gsc->fw, INTEL_UC_FIRMWARE_RUNNING); + } else { + drm_err(>->i915->drm, + "GSC status reports proxy init not complete\n"); + } + } + } - with_intel_runtime_pm(gt->uncore->rpm, wakeref) - intel_gsc_uc_fw_upload(gsc); +out_put: + intel_runtime_pm_put(gt->uncore->rpm, wakeref); } static bool gsc_engine_supported(struct intel_gt *gt) @@ -43,17 +107,30 @@ static bool gsc_engine_supported(struct intel_gt *gt) void intel_gsc_uc_init_early(struct intel_gsc_uc *gsc) { - intel_uc_fw_init_early(&gsc->fw, INTEL_UC_FW_TYPE_GSC); + struct intel_gt *gt = gsc_uc_to_gt(gsc); + + /* + * GSC FW needs to be copied to a dedicated memory allocations for + * loading (see gsc->local), so we don't need to GGTT map the FW image + * itself into GGTT. + */ + intel_uc_fw_init_early(&gsc->fw, INTEL_UC_FW_TYPE_GSC, false); INIT_WORK(&gsc->work, gsc_work); /* we can arrive here from i915_driver_early_probe for primary * GT with it being not fully setup hence check device info's * engine mask */ - if (!gsc_engine_supported(gsc_uc_to_gt(gsc))) { + if (!gsc_engine_supported(gt)) { intel_uc_fw_change_status(&gsc->fw, INTEL_UC_FIRMWARE_NOT_SUPPORTED); return; } + + gsc->wq = alloc_ordered_workqueue("i915_gsc", 0); + if (!gsc->wq) { + gt_err(gt, "failed to allocate WQ for GSC, disabling FW\n"); + intel_uc_fw_change_status(&gsc->fw, INTEL_UC_FIRMWARE_NOT_SUPPORTED); + } } int intel_gsc_uc_init(struct intel_gsc_uc *gsc) @@ -88,6 +165,9 @@ int intel_gsc_uc_init(struct intel_gsc_uc *gsc) gsc->ce = ce; + /* if we fail to init proxy we still want to load GSC for PM */ + intel_gsc_proxy_init(gsc); + intel_uc_fw_change_status(&gsc->fw, INTEL_UC_FIRMWARE_LOADABLE); return 0; @@ -107,6 +187,12 @@ void intel_gsc_uc_fini(struct intel_gsc_uc *gsc) return; flush_work(&gsc->work); + if (gsc->wq) { + destroy_workqueue(gsc->wq); + gsc->wq = NULL; + } + + intel_gsc_proxy_fini(gsc); if (gsc->ce) intel_engine_destroy_pinned_context(fetch_and_zero(&gsc->ce)); @@ -145,11 +231,17 @@ void intel_gsc_uc_resume(struct intel_gsc_uc *gsc) void intel_gsc_uc_load_start(struct intel_gsc_uc *gsc) { + struct intel_gt *gt = gsc_uc_to_gt(gsc); + if (!intel_uc_fw_is_loadable(&gsc->fw)) return; if (intel_gsc_uc_fw_init_done(gsc)) return; - queue_work(system_unbound_wq, &gsc->work); + spin_lock_irq(gt->irq_lock); + gsc->gsc_work_actions |= GSC_ACTION_FW_LOAD; + spin_unlock_irq(gt->irq_lock); + + queue_work(gsc->wq, &gsc->work); } diff --git a/drivers/gpu/drm/i915/gt/uc/intel_gsc_uc.h b/drivers/gpu/drm/i915/gt/uc/intel_gsc_uc.h index 5f50fa1ff8b9..a2a0813b8a76 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_gsc_uc.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_gsc_uc.h @@ -10,6 +10,7 @@ struct i915_vma; struct intel_context; +struct i915_gsc_proxy_component; struct intel_gsc_uc { /* Generic uC firmware management */ @@ -19,7 +20,21 @@ struct intel_gsc_uc { struct i915_vma *local; /* private memory for GSC usage */ struct intel_context *ce; /* for submission to GSC FW via GSC engine */ - struct work_struct work; /* for delayed load */ + /* for delayed load and proxy handling */ + struct workqueue_struct *wq; + struct work_struct work; + u32 gsc_work_actions; /* protected by gt->irq_lock */ +#define GSC_ACTION_FW_LOAD BIT(0) +#define GSC_ACTION_SW_PROXY BIT(1) + + struct { + struct i915_gsc_proxy_component *component; + bool component_added; + struct i915_vma *vma; + void *to_gsc; + void *to_csme; + struct mutex mutex; /* protects the tee channel binding */ + } proxy; }; void intel_gsc_uc_init_early(struct intel_gsc_uc *gsc); diff --git a/drivers/gpu/drm/i915/gt/uc/intel_gsc_uc_heci_cmd_submit.c b/drivers/gpu/drm/i915/gt/uc/intel_gsc_uc_heci_cmd_submit.c index ea0da06e2f39..89ed5ee9cded 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_gsc_uc_heci_cmd_submit.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_gsc_uc_heci_cmd_submit.c @@ -3,6 +3,7 @@ * Copyright © 2023 Intel Corporation */ +#include "gt/intel_context.h" #include "gt/intel_engine_pm.h" #include "gt/intel_gpu_commands.h" #include "gt/intel_gt.h" @@ -98,7 +99,7 @@ void intel_gsc_uc_heci_cmd_emit_mtl_header(struct intel_gsc_mtl_header *header, u64 host_session_id) { host_session_id &= ~HOST_SESSION_MASK; - if (heci_client_id == HECI_MEADDRESS_PXP) + if (host_session_id && heci_client_id == HECI_MEADDRESS_PXP) host_session_id |= HOST_SESSION_PXP_SINGLE; header->validity_marker = GSC_HECI_VALIDITY_MARKER; @@ -107,3 +108,104 @@ void intel_gsc_uc_heci_cmd_emit_mtl_header(struct intel_gsc_mtl_header *header, header->header_version = MTL_GSC_HEADER_VERSION; header->message_size = message_size; } + +static void +emit_gsc_heci_pkt_nonpriv(u32 *cmd, struct intel_gsc_heci_non_priv_pkt *pkt) +{ + *cmd++ = GSC_HECI_CMD_PKT; + *cmd++ = lower_32_bits(pkt->addr_in); + *cmd++ = upper_32_bits(pkt->addr_in); + *cmd++ = pkt->size_in; + *cmd++ = lower_32_bits(pkt->addr_out); + *cmd++ = upper_32_bits(pkt->addr_out); + *cmd++ = pkt->size_out; + *cmd++ = 0; + *cmd++ = MI_BATCH_BUFFER_END; +} + +int +intel_gsc_uc_heci_cmd_submit_nonpriv(struct intel_gsc_uc *gsc, + struct intel_context *ce, + struct intel_gsc_heci_non_priv_pkt *pkt, + u32 *cmd, int timeout_ms) +{ + struct intel_engine_cs *engine; + struct i915_gem_ww_ctx ww; + struct i915_request *rq; + int err, trials = 0; + + i915_gem_ww_ctx_init(&ww, false); +retry: + err = i915_gem_object_lock(pkt->bb_vma->obj, &ww); + if (err) + goto out_ww; + err = i915_gem_object_lock(pkt->heci_pkt_vma->obj, &ww); + if (err) + goto out_ww; + err = intel_context_pin_ww(ce, &ww); + if (err) + goto out_ww; + + rq = i915_request_create(ce); + if (IS_ERR(rq)) { + err = PTR_ERR(rq); + goto out_unpin_ce; + } + + emit_gsc_heci_pkt_nonpriv(cmd, pkt); + + err = i915_vma_move_to_active(pkt->bb_vma, rq, 0); + if (err) + goto out_rq; + err = i915_vma_move_to_active(pkt->heci_pkt_vma, rq, EXEC_OBJECT_WRITE); + if (err) + goto out_rq; + + engine = rq->context->engine; + if (engine->emit_init_breadcrumb) { + err = engine->emit_init_breadcrumb(rq); + if (err) + goto out_rq; + } + + err = engine->emit_bb_start(rq, i915_vma_offset(pkt->bb_vma), PAGE_SIZE, 0); + if (err) + goto out_rq; + + err = ce->engine->emit_flush(rq, 0); + if (err) + drm_err(&gsc_uc_to_gt(gsc)->i915->drm, + "Failed emit-flush for gsc-heci-non-priv-pkterr=%d\n", err); + +out_rq: + i915_request_get(rq); + + if (unlikely(err)) + i915_request_set_error_once(rq, err); + + i915_request_add(rq); + + if (!err) { + if (i915_request_wait(rq, I915_WAIT_INTERRUPTIBLE, + msecs_to_jiffies(timeout_ms)) < 0) + err = -ETIME; + } + + i915_request_put(rq); + +out_unpin_ce: + intel_context_unpin(ce); +out_ww: + if (err == -EDEADLK) { + err = i915_gem_ww_ctx_backoff(&ww); + if (!err) { + if (++trials < 10) + goto retry; + else + err = -EAGAIN; + } + } + i915_gem_ww_ctx_fini(&ww); + + return err; +} diff --git a/drivers/gpu/drm/i915/gt/uc/intel_gsc_uc_heci_cmd_submit.h b/drivers/gpu/drm/i915/gt/uc/intel_gsc_uc_heci_cmd_submit.h index 3d56ae501991..ef70e304904a 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_gsc_uc_heci_cmd_submit.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_gsc_uc_heci_cmd_submit.h @@ -8,12 +8,16 @@ #include <linux/types.h> +struct i915_vma; +struct intel_context; struct intel_gsc_uc; + struct intel_gsc_mtl_header { u32 validity_marker; #define GSC_HECI_VALIDITY_MARKER 0xA578875A u8 heci_client_id; +#define HECI_MEADDRESS_PROXY 10 #define HECI_MEADDRESS_PXP 17 #define HECI_MEADDRESS_HDCP 18 @@ -47,7 +51,8 @@ struct intel_gsc_mtl_header { * we distinguish the flags using OUTFLAG or INFLAG */ u32 flags; -#define GSC_OUTFLAG_MSG_PENDING 1 +#define GSC_OUTFLAG_MSG_PENDING BIT(0) +#define GSC_INFLAG_MSG_CLEANUP BIT(1) u32 status; } __packed; @@ -58,4 +63,24 @@ int intel_gsc_uc_heci_cmd_submit_packet(struct intel_gsc_uc *gsc, void intel_gsc_uc_heci_cmd_emit_mtl_header(struct intel_gsc_mtl_header *header, u8 heci_client_id, u32 message_size, u64 host_session_id); + +struct intel_gsc_heci_non_priv_pkt { + u64 addr_in; + u32 size_in; + u64 addr_out; + u32 size_out; + struct i915_vma *heci_pkt_vma; + struct i915_vma *bb_vma; +}; + +void +intel_gsc_uc_heci_cmd_emit_mtl_header(struct intel_gsc_mtl_header *header, + u8 heci_client_id, u32 msg_size, + u64 host_session_id); + +int +intel_gsc_uc_heci_cmd_submit_nonpriv(struct intel_gsc_uc *gsc, + struct intel_context *ce, + struct intel_gsc_heci_non_priv_pkt *pkt, + u32 *cs, int timeout_ms); #endif diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc.c b/drivers/gpu/drm/i915/gt/uc/intel_guc.c index d76508fa3af7..2eb891b270ae 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc.c @@ -16,6 +16,7 @@ #include "intel_guc_submission.h" #include "i915_drv.h" #include "i915_irq.h" +#include "i915_reg.h" /** * DOC: GuC @@ -163,7 +164,7 @@ void intel_guc_init_early(struct intel_guc *guc) struct intel_gt *gt = guc_to_gt(guc); struct drm_i915_private *i915 = gt->i915; - intel_uc_fw_init_early(&guc->fw, INTEL_UC_FW_TYPE_GUC); + intel_uc_fw_init_early(&guc->fw, INTEL_UC_FW_TYPE_GUC, true); intel_guc_ct_init_early(&guc->ct); intel_guc_log_init_early(&guc->log); intel_guc_submission_init_early(guc); @@ -743,6 +744,13 @@ struct i915_vma *intel_guc_allocate_vma(struct intel_guc *guc, u32 size) if (IS_ERR(obj)) return ERR_CAST(obj); + /* + * Wa_22016122933: For MTL the shared memory needs to be mapped + * as WC on CPU side and UC (PAT index 2) on GPU side + */ + if (IS_METEORLAKE(gt->i915)) + i915_gem_object_set_cache_coherency(obj, I915_CACHE_NONE); + vma = i915_vma_instance(obj, >->ggtt->vm, NULL); if (IS_ERR(vma)) goto err; diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc.h b/drivers/gpu/drm/i915/gt/uc/intel_guc.h index e46aac1a41e6..8dc291ff0093 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc.h @@ -42,6 +42,7 @@ struct intel_guc { /** @capture: the error-state-capture module's data and objects */ struct intel_guc_state_capture *capture; + /** @dbgfs_node: debugfs node */ struct dentry *dbgfs_node; /** @sched_engine: Global engine used to submit requests to GuC */ diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c index 69ce06faf8cd..63724e17829a 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c @@ -643,6 +643,39 @@ static void guc_init_golden_context(struct intel_guc *guc) GEM_BUG_ON(guc->ads_golden_ctxt_size != total_size); } +static u32 guc_get_capture_engine_mask(struct iosys_map *info_map, u32 capture_class) +{ + u32 mask; + + switch (capture_class) { + case GUC_CAPTURE_LIST_CLASS_RENDER_COMPUTE: + mask = info_map_read(info_map, engine_enabled_masks[GUC_RENDER_CLASS]); + mask |= info_map_read(info_map, engine_enabled_masks[GUC_COMPUTE_CLASS]); + break; + + case GUC_CAPTURE_LIST_CLASS_VIDEO: + mask = info_map_read(info_map, engine_enabled_masks[GUC_VIDEO_CLASS]); + break; + + case GUC_CAPTURE_LIST_CLASS_VIDEOENHANCE: + mask = info_map_read(info_map, engine_enabled_masks[GUC_VIDEOENHANCE_CLASS]); + break; + + case GUC_CAPTURE_LIST_CLASS_BLITTER: + mask = info_map_read(info_map, engine_enabled_masks[GUC_BLITTER_CLASS]); + break; + + case GUC_CAPTURE_LIST_CLASS_GSC_OTHER: + mask = info_map_read(info_map, engine_enabled_masks[GUC_GSC_OTHER_CLASS]); + break; + + default: + mask = 0; + } + + return mask; +} + static int guc_capture_prep_lists(struct intel_guc *guc) { @@ -678,9 +711,10 @@ guc_capture_prep_lists(struct intel_guc *guc) for (i = 0; i < GUC_CAPTURE_LIST_INDEX_MAX; i++) { for (j = 0; j < GUC_MAX_ENGINE_CLASSES; j++) { + u32 engine_mask = guc_get_capture_engine_mask(&info_map, j); /* null list if we dont have said engine or list */ - if (!info_map_read(&info_map, engine_enabled_masks[j])) { + if (!engine_mask) { if (ads_is_mapped) { ads_blob_write(guc, ads.capture_class[i][j], null_ggtt); ads_blob_write(guc, ads.capture_instance[i][j], null_ggtt); diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_capture.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_capture.c index e0e793167d61..331cec07c125 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_capture.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_capture.c @@ -30,12 +30,12 @@ #define COMMON_BASE_GLOBAL \ { FORCEWAKE_MT, 0, 0, "FORCEWAKE" } -#define COMMON_GEN9BASE_GLOBAL \ +#define COMMON_GEN8BASE_GLOBAL \ { ERROR_GEN6, 0, 0, "ERROR_GEN6" }, \ { DONE_REG, 0, 0, "DONE_REG" }, \ { HSW_GTT_CACHE_EN, 0, 0, "HSW_GTT_CACHE_EN" } -#define GEN9_GLOBAL \ +#define GEN8_GLOBAL \ { GEN8_FAULT_TLB_DATA0, 0, 0, "GEN8_FAULT_TLB_DATA0" }, \ { GEN8_FAULT_TLB_DATA1, 0, 0, "GEN8_FAULT_TLB_DATA1" } @@ -96,67 +96,65 @@ { GEN12_SFC_DONE(2), 0, 0, "SFC_DONE[2]" }, \ { GEN12_SFC_DONE(3), 0, 0, "SFC_DONE[3]" } -/* XE_LPD - Global */ -static const struct __guc_mmio_reg_descr xe_lpd_global_regs[] = { +/* XE_LP Global */ +static const struct __guc_mmio_reg_descr xe_lp_global_regs[] = { COMMON_BASE_GLOBAL, - COMMON_GEN9BASE_GLOBAL, + COMMON_GEN8BASE_GLOBAL, COMMON_GEN12BASE_GLOBAL, }; -/* XE_LPD - Render / Compute Per-Class */ -static const struct __guc_mmio_reg_descr xe_lpd_rc_class_regs[] = { +/* XE_LP Render / Compute Per-Class */ +static const struct __guc_mmio_reg_descr xe_lp_rc_class_regs[] = { COMMON_BASE_HAS_EU, COMMON_BASE_RENDER, COMMON_GEN12BASE_RENDER, }; -/* GEN9/XE_LPD - Render / Compute Per-Engine-Instance */ -static const struct __guc_mmio_reg_descr xe_lpd_rc_inst_regs[] = { +/* GEN8+ Render / Compute Per-Engine-Instance */ +static const struct __guc_mmio_reg_descr gen8_rc_inst_regs[] = { COMMON_BASE_ENGINE_INSTANCE, }; -/* GEN9/XE_LPD - Media Decode/Encode Per-Engine-Instance */ -static const struct __guc_mmio_reg_descr xe_lpd_vd_inst_regs[] = { +/* GEN8+ Media Decode/Encode Per-Engine-Instance */ +static const struct __guc_mmio_reg_descr gen8_vd_inst_regs[] = { COMMON_BASE_ENGINE_INSTANCE, }; -/* XE_LPD - Video Enhancement Per-Class */ -static const struct __guc_mmio_reg_descr xe_lpd_vec_class_regs[] = { +/* XE_LP Video Enhancement Per-Class */ +static const struct __guc_mmio_reg_descr xe_lp_vec_class_regs[] = { COMMON_GEN12BASE_VEC, }; -/* GEN9/XE_LPD - Video Enhancement Per-Engine-Instance */ -static const struct __guc_mmio_reg_descr xe_lpd_vec_inst_regs[] = { +/* GEN8+ Video Enhancement Per-Engine-Instance */ +static const struct __guc_mmio_reg_descr gen8_vec_inst_regs[] = { COMMON_BASE_ENGINE_INSTANCE, }; -/* GEN9/XE_LPD - Blitter Per-Engine-Instance */ -static const struct __guc_mmio_reg_descr xe_lpd_blt_inst_regs[] = { +/* GEN8+ Blitter Per-Engine-Instance */ +static const struct __guc_mmio_reg_descr gen8_blt_inst_regs[] = { COMMON_BASE_ENGINE_INSTANCE, }; -/* XE_LPD - GSC Per-Engine-Instance */ -static const struct __guc_mmio_reg_descr xe_lpd_gsc_inst_regs[] = { +/* XE_LP - GSC Per-Engine-Instance */ +static const struct __guc_mmio_reg_descr xe_lp_gsc_inst_regs[] = { COMMON_BASE_ENGINE_INSTANCE, }; -/* GEN9 - Global */ -static const struct __guc_mmio_reg_descr default_global_regs[] = { +/* GEN8 - Global */ +static const struct __guc_mmio_reg_descr gen8_global_regs[] = { COMMON_BASE_GLOBAL, - COMMON_GEN9BASE_GLOBAL, - GEN9_GLOBAL, + COMMON_GEN8BASE_GLOBAL, + GEN8_GLOBAL, }; -static const struct __guc_mmio_reg_descr default_rc_class_regs[] = { +static const struct __guc_mmio_reg_descr gen8_rc_class_regs[] = { COMMON_BASE_HAS_EU, COMMON_BASE_RENDER, }; /* - * Empty lists: - * GEN9/XE_LPD - Blitter Per-Class - * GEN9/XE_LPD - Media Decode/Encode Per-Class - * GEN9 - VEC Class + * Empty list to prevent warnings about unknown class/instance types + * as not all class/instanace types have entries on all platforms. */ static const struct __guc_mmio_reg_descr empty_regs_list[] = { }; @@ -174,37 +172,33 @@ static const struct __guc_mmio_reg_descr empty_regs_list[] = { } /* List of lists */ -static const struct __guc_mmio_reg_descr_group default_lists[] = { - MAKE_REGLIST(default_global_regs, PF, GLOBAL, 0), - MAKE_REGLIST(default_rc_class_regs, PF, ENGINE_CLASS, GUC_RENDER_CLASS), - MAKE_REGLIST(xe_lpd_rc_inst_regs, PF, ENGINE_INSTANCE, GUC_RENDER_CLASS), - MAKE_REGLIST(default_rc_class_regs, PF, ENGINE_CLASS, GUC_COMPUTE_CLASS), - MAKE_REGLIST(xe_lpd_rc_inst_regs, PF, ENGINE_INSTANCE, GUC_COMPUTE_CLASS), - MAKE_REGLIST(empty_regs_list, PF, ENGINE_CLASS, GUC_VIDEO_CLASS), - MAKE_REGLIST(xe_lpd_vd_inst_regs, PF, ENGINE_INSTANCE, GUC_VIDEO_CLASS), - MAKE_REGLIST(empty_regs_list, PF, ENGINE_CLASS, GUC_VIDEOENHANCE_CLASS), - MAKE_REGLIST(xe_lpd_vec_inst_regs, PF, ENGINE_INSTANCE, GUC_VIDEOENHANCE_CLASS), - MAKE_REGLIST(empty_regs_list, PF, ENGINE_CLASS, GUC_BLITTER_CLASS), - MAKE_REGLIST(xe_lpd_blt_inst_regs, PF, ENGINE_INSTANCE, GUC_BLITTER_CLASS), - MAKE_REGLIST(empty_regs_list, PF, ENGINE_CLASS, GUC_GSC_OTHER_CLASS), - MAKE_REGLIST(xe_lpd_gsc_inst_regs, PF, ENGINE_INSTANCE, GUC_GSC_OTHER_CLASS), +static const struct __guc_mmio_reg_descr_group gen8_lists[] = { + MAKE_REGLIST(gen8_global_regs, PF, GLOBAL, 0), + MAKE_REGLIST(gen8_rc_class_regs, PF, ENGINE_CLASS, GUC_CAPTURE_LIST_CLASS_RENDER_COMPUTE), + MAKE_REGLIST(gen8_rc_inst_regs, PF, ENGINE_INSTANCE, GUC_CAPTURE_LIST_CLASS_RENDER_COMPUTE), + MAKE_REGLIST(empty_regs_list, PF, ENGINE_CLASS, GUC_CAPTURE_LIST_CLASS_VIDEO), + MAKE_REGLIST(gen8_vd_inst_regs, PF, ENGINE_INSTANCE, GUC_CAPTURE_LIST_CLASS_VIDEO), + MAKE_REGLIST(empty_regs_list, PF, ENGINE_CLASS, GUC_CAPTURE_LIST_CLASS_VIDEOENHANCE), + MAKE_REGLIST(gen8_vec_inst_regs, PF, ENGINE_INSTANCE, GUC_CAPTURE_LIST_CLASS_VIDEOENHANCE), + MAKE_REGLIST(empty_regs_list, PF, ENGINE_CLASS, GUC_CAPTURE_LIST_CLASS_BLITTER), + MAKE_REGLIST(gen8_blt_inst_regs, PF, ENGINE_INSTANCE, GUC_CAPTURE_LIST_CLASS_BLITTER), + MAKE_REGLIST(empty_regs_list, PF, ENGINE_CLASS, GUC_CAPTURE_LIST_CLASS_GSC_OTHER), + MAKE_REGLIST(empty_regs_list, PF, ENGINE_INSTANCE, GUC_CAPTURE_LIST_CLASS_GSC_OTHER), {} }; -static const struct __guc_mmio_reg_descr_group xe_lpd_lists[] = { - MAKE_REGLIST(xe_lpd_global_regs, PF, GLOBAL, 0), - MAKE_REGLIST(xe_lpd_rc_class_regs, PF, ENGINE_CLASS, GUC_RENDER_CLASS), - MAKE_REGLIST(xe_lpd_rc_inst_regs, PF, ENGINE_INSTANCE, GUC_RENDER_CLASS), - MAKE_REGLIST(xe_lpd_rc_class_regs, PF, ENGINE_CLASS, GUC_COMPUTE_CLASS), - MAKE_REGLIST(xe_lpd_rc_inst_regs, PF, ENGINE_INSTANCE, GUC_COMPUTE_CLASS), - MAKE_REGLIST(empty_regs_list, PF, ENGINE_CLASS, GUC_VIDEO_CLASS), - MAKE_REGLIST(xe_lpd_vd_inst_regs, PF, ENGINE_INSTANCE, GUC_VIDEO_CLASS), - MAKE_REGLIST(xe_lpd_vec_class_regs, PF, ENGINE_CLASS, GUC_VIDEOENHANCE_CLASS), - MAKE_REGLIST(xe_lpd_vec_inst_regs, PF, ENGINE_INSTANCE, GUC_VIDEOENHANCE_CLASS), - MAKE_REGLIST(empty_regs_list, PF, ENGINE_CLASS, GUC_BLITTER_CLASS), - MAKE_REGLIST(xe_lpd_blt_inst_regs, PF, ENGINE_INSTANCE, GUC_BLITTER_CLASS), - MAKE_REGLIST(empty_regs_list, PF, ENGINE_CLASS, GUC_GSC_OTHER_CLASS), - MAKE_REGLIST(xe_lpd_gsc_inst_regs, PF, ENGINE_INSTANCE, GUC_GSC_OTHER_CLASS), +static const struct __guc_mmio_reg_descr_group xe_lp_lists[] = { + MAKE_REGLIST(xe_lp_global_regs, PF, GLOBAL, 0), + MAKE_REGLIST(xe_lp_rc_class_regs, PF, ENGINE_CLASS, GUC_CAPTURE_LIST_CLASS_RENDER_COMPUTE), + MAKE_REGLIST(gen8_rc_inst_regs, PF, ENGINE_INSTANCE, GUC_CAPTURE_LIST_CLASS_RENDER_COMPUTE), + MAKE_REGLIST(empty_regs_list, PF, ENGINE_CLASS, GUC_CAPTURE_LIST_CLASS_VIDEO), + MAKE_REGLIST(gen8_vd_inst_regs, PF, ENGINE_INSTANCE, GUC_CAPTURE_LIST_CLASS_VIDEO), + MAKE_REGLIST(xe_lp_vec_class_regs, PF, ENGINE_CLASS, GUC_CAPTURE_LIST_CLASS_VIDEOENHANCE), + MAKE_REGLIST(gen8_vec_inst_regs, PF, ENGINE_INSTANCE, GUC_CAPTURE_LIST_CLASS_VIDEOENHANCE), + MAKE_REGLIST(empty_regs_list, PF, ENGINE_CLASS, GUC_CAPTURE_LIST_CLASS_BLITTER), + MAKE_REGLIST(gen8_blt_inst_regs, PF, ENGINE_INSTANCE, GUC_CAPTURE_LIST_CLASS_BLITTER), + MAKE_REGLIST(empty_regs_list, PF, ENGINE_CLASS, GUC_CAPTURE_LIST_CLASS_GSC_OTHER), + MAKE_REGLIST(xe_lp_gsc_inst_regs, PF, ENGINE_INSTANCE, GUC_CAPTURE_LIST_CLASS_GSC_OTHER), {} }; @@ -260,11 +254,15 @@ struct __ext_steer_reg { i915_mcr_reg_t reg; }; -static const struct __ext_steer_reg xe_extregs[] = { +static const struct __ext_steer_reg gen8_extregs[] = { {"GEN8_SAMPLER_INSTDONE", GEN8_SAMPLER_INSTDONE}, {"GEN8_ROW_INSTDONE", GEN8_ROW_INSTDONE} }; +static const struct __ext_steer_reg xehpg_extregs[] = { + {"XEHPG_INSTDONE_GEOM_SVG", XEHPG_INSTDONE_GEOM_SVG} +}; + static void __fill_ext_reg(struct __guc_mmio_reg_descr *ext, const struct __ext_steer_reg *extlist, int slice_id, int subslice_id) @@ -295,86 +293,30 @@ __alloc_ext_regs(struct __guc_mmio_reg_descr_group *newlist, } static void -guc_capture_alloc_steered_lists_xe_lpd(struct intel_guc *guc, - const struct __guc_mmio_reg_descr_group *lists) +guc_capture_alloc_steered_lists(struct intel_guc *guc, + const struct __guc_mmio_reg_descr_group *lists) { struct intel_gt *gt = guc_to_gt(guc); int slice, subslice, iter, i, num_steer_regs, num_tot_regs = 0; const struct __guc_mmio_reg_descr_group *list; struct __guc_mmio_reg_descr_group *extlists; struct __guc_mmio_reg_descr *extarray; - struct sseu_dev_info *sseu; + bool has_xehpg_extregs; - /* In XE_LPD we only have steered registers for the render-class */ + /* steered registers currently only exist for the render-class */ list = guc_capture_get_one_list(lists, GUC_CAPTURE_LIST_INDEX_PF, - GUC_CAPTURE_LIST_TYPE_ENGINE_CLASS, GUC_RENDER_CLASS); + GUC_CAPTURE_LIST_TYPE_ENGINE_CLASS, + GUC_CAPTURE_LIST_CLASS_RENDER_COMPUTE); /* skip if extlists was previously allocated */ if (!list || guc->capture->extlists) return; - num_steer_regs = ARRAY_SIZE(xe_extregs); - - sseu = >->info.sseu; - for_each_ss_steering(iter, gt, slice, subslice) - num_tot_regs += num_steer_regs; - - if (!num_tot_regs) - return; - - /* allocate an extra for an end marker */ - extlists = kcalloc(2, sizeof(struct __guc_mmio_reg_descr_group), GFP_KERNEL); - if (!extlists) - return; - - if (__alloc_ext_regs(&extlists[0], list, num_tot_regs)) { - kfree(extlists); - return; - } - - extarray = extlists[0].extlist; - for_each_ss_steering(iter, gt, slice, subslice) { - for (i = 0; i < num_steer_regs; ++i) { - __fill_ext_reg(extarray, &xe_extregs[i], slice, subslice); - ++extarray; - } - } - - guc->capture->extlists = extlists; -} - -static const struct __ext_steer_reg xehpg_extregs[] = { - {"XEHPG_INSTDONE_GEOM_SVG", XEHPG_INSTDONE_GEOM_SVG} -}; - -static bool __has_xehpg_extregs(u32 ipver) -{ - return (ipver >= IP_VER(12, 55)); -} - -static void -guc_capture_alloc_steered_lists_xe_hpg(struct intel_guc *guc, - const struct __guc_mmio_reg_descr_group *lists, - u32 ipver) -{ - struct intel_gt *gt = guc_to_gt(guc); - struct sseu_dev_info *sseu; - int slice, subslice, i, iter, num_steer_regs, num_tot_regs = 0; - const struct __guc_mmio_reg_descr_group *list; - struct __guc_mmio_reg_descr_group *extlists; - struct __guc_mmio_reg_descr *extarray; - - /* In XE_LP / HPG we only have render-class steering registers during error-capture */ - list = guc_capture_get_one_list(lists, GUC_CAPTURE_LIST_INDEX_PF, - GUC_CAPTURE_LIST_TYPE_ENGINE_CLASS, GUC_RENDER_CLASS); - /* skip if extlists was previously allocated */ - if (!list || guc->capture->extlists) - return; + has_xehpg_extregs = GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 55); - num_steer_regs = ARRAY_SIZE(xe_extregs); - if (__has_xehpg_extregs(ipver)) + num_steer_regs = ARRAY_SIZE(gen8_extregs); + if (has_xehpg_extregs) num_steer_regs += ARRAY_SIZE(xehpg_extregs); - sseu = >->info.sseu; for_each_ss_steering(iter, gt, slice, subslice) num_tot_regs += num_steer_regs; @@ -393,11 +335,12 @@ guc_capture_alloc_steered_lists_xe_hpg(struct intel_guc *guc, extarray = extlists[0].extlist; for_each_ss_steering(iter, gt, slice, subslice) { - for (i = 0; i < ARRAY_SIZE(xe_extregs); ++i) { - __fill_ext_reg(extarray, &xe_extregs[i], slice, subslice); + for (i = 0; i < ARRAY_SIZE(gen8_extregs); ++i) { + __fill_ext_reg(extarray, &gen8_extregs[i], slice, subslice); ++extarray; } - if (__has_xehpg_extregs(ipver)) { + + if (has_xehpg_extregs) { for (i = 0; i < ARRAY_SIZE(xehpg_extregs); ++i) { __fill_ext_reg(extarray, &xehpg_extregs[i], slice, subslice); ++extarray; @@ -413,26 +356,22 @@ static const struct __guc_mmio_reg_descr_group * guc_capture_get_device_reglist(struct intel_guc *guc) { struct drm_i915_private *i915 = guc_to_gt(guc)->i915; + const struct __guc_mmio_reg_descr_group *lists; - if (GRAPHICS_VER(i915) > 11) { - /* - * For certain engine classes, there are slice and subslice - * level registers requiring steering. We allocate and populate - * these at init time based on hw config add it as an extension - * list at the end of the pre-populated render list. - */ - if (IS_DG2(i915)) - guc_capture_alloc_steered_lists_xe_hpg(guc, xe_lpd_lists, IP_VER(12, 55)); - else if (IS_XEHPSDV(i915)) - guc_capture_alloc_steered_lists_xe_hpg(guc, xe_lpd_lists, IP_VER(12, 50)); - else - guc_capture_alloc_steered_lists_xe_lpd(guc, xe_lpd_lists); + if (GRAPHICS_VER(i915) >= 12) + lists = xe_lp_lists; + else + lists = gen8_lists; - return xe_lpd_lists; - } + /* + * For certain engine classes, there are slice and subslice + * level registers requiring steering. We allocate and populate + * these at init time based on hw config add it as an extension + * list at the end of the pre-populated render list. + */ + guc_capture_alloc_steered_lists(guc, lists); - /* if GuC submission is enabled on a non-POR platform, just use a common baseline */ - return default_lists; + return lists; } static const char * @@ -456,17 +395,15 @@ static const char * __stringify_engclass(u32 class) { switch (class) { - case GUC_RENDER_CLASS: - return "Render"; - case GUC_VIDEO_CLASS: + case GUC_CAPTURE_LIST_CLASS_RENDER_COMPUTE: + return "Render/Compute"; + case GUC_CAPTURE_LIST_CLASS_VIDEO: return "Video"; - case GUC_VIDEOENHANCE_CLASS: + case GUC_CAPTURE_LIST_CLASS_VIDEOENHANCE: return "VideoEnhance"; - case GUC_BLITTER_CLASS: + case GUC_CAPTURE_LIST_CLASS_BLITTER: return "Blitter"; - case GUC_COMPUTE_CLASS: - return "Compute"; - case GUC_GSC_OTHER_CLASS: + case GUC_CAPTURE_LIST_CLASS_GSC_OTHER: return "GSC-Other"; default: break; @@ -1596,6 +1533,36 @@ void intel_guc_capture_free_node(struct intel_engine_coredump *ee) ee->guc_capture_node = NULL; } +bool intel_guc_capture_is_matching_engine(struct intel_gt *gt, + struct intel_context *ce, + struct intel_engine_cs *engine) +{ + struct __guc_capture_parsed_output *n; + struct intel_guc *guc; + + if (!gt || !ce || !engine) + return false; + + guc = >->uc.guc; + if (!guc->capture) + return false; + + /* + * Look for a matching GuC reported error capture node from + * the internal output link-list based on lrca, guc-id and engine + * identification. + */ + list_for_each_entry(n, &guc->capture->outlist, link) { + if (n->eng_inst == GUC_ID_TO_ENGINE_INSTANCE(engine->guc_id) && + n->eng_class == GUC_ID_TO_ENGINE_CLASS(engine->guc_id) && + n->guc_id == ce->guc_id.id && + (n->lrca & CTX_GTT_ADDRESS_MASK) == (ce->lrc.lrca & CTX_GTT_ADDRESS_MASK)) + return true; + } + + return false; +} + void intel_guc_capture_get_matching_node(struct intel_gt *gt, struct intel_engine_coredump *ee, struct intel_context *ce) @@ -1611,6 +1578,7 @@ void intel_guc_capture_get_matching_node(struct intel_gt *gt, return; GEM_BUG_ON(ee->guc_capture_node); + /* * Look for a matching GuC reported error capture node from * the internal output link-list based on lrca, guc-id and engine diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_capture.h b/drivers/gpu/drm/i915/gt/uc/intel_guc_capture.h index fbd3713c7832..302256d45431 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_capture.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_capture.h @@ -11,6 +11,7 @@ struct drm_i915_error_state_buf; struct guc_gt_system_info; struct intel_engine_coredump; +struct intel_engine_cs; struct intel_context; struct intel_gt; struct intel_guc; @@ -20,6 +21,8 @@ int intel_guc_capture_print_engine_node(struct drm_i915_error_state_buf *m, const struct intel_engine_coredump *ee); void intel_guc_capture_get_matching_node(struct intel_gt *gt, struct intel_engine_coredump *ee, struct intel_context *ce); +bool intel_guc_capture_is_matching_engine(struct intel_gt *gt, struct intel_context *ce, + struct intel_engine_cs *engine); void intel_guc_capture_process(struct intel_guc *guc); int intel_guc_capture_getlist(struct intel_guc *guc, u32 owner, u32 type, u32 classid, void **outptr); diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c index 1803a633ed64..f28a3a83742d 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c @@ -13,6 +13,30 @@ #include "intel_guc_ct.h" #include "intel_guc_print.h" +#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GUC) +enum { + CT_DEAD_ALIVE = 0, + CT_DEAD_SETUP, + CT_DEAD_WRITE, + CT_DEAD_DEADLOCK, + CT_DEAD_H2G_HAS_ROOM, + CT_DEAD_READ, + CT_DEAD_PROCESS_FAILED, +}; + +static void ct_dead_ct_worker_func(struct work_struct *w); + +#define CT_DEAD(ct, reason) \ + do { \ + if (!(ct)->dead_ct_reported) { \ + (ct)->dead_ct_reason |= 1 << CT_DEAD_##reason; \ + queue_work(system_unbound_wq, &(ct)->dead_ct_worker); \ + } \ + } while (0) +#else +#define CT_DEAD(ct, reason) do { } while (0) +#endif + static inline struct intel_guc *ct_to_guc(struct intel_guc_ct *ct) { return container_of(ct, struct intel_guc, ct); @@ -93,6 +117,9 @@ void intel_guc_ct_init_early(struct intel_guc_ct *ct) spin_lock_init(&ct->requests.lock); INIT_LIST_HEAD(&ct->requests.pending); INIT_LIST_HEAD(&ct->requests.incoming); +#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GUC) + INIT_WORK(&ct->dead_ct_worker, ct_dead_ct_worker_func); +#endif INIT_WORK(&ct->requests.worker, ct_incoming_request_worker_func); tasklet_setup(&ct->receive_tasklet, ct_receive_tasklet_func); init_waitqueue_head(&ct->wq); @@ -319,11 +346,16 @@ int intel_guc_ct_enable(struct intel_guc_ct *ct) ct->enabled = true; ct->stall_time = KTIME_MAX; +#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GUC) + ct->dead_ct_reported = false; + ct->dead_ct_reason = CT_DEAD_ALIVE; +#endif return 0; err_out: CT_PROBE_ERROR(ct, "Failed to enable CTB (%pe)\n", ERR_PTR(err)); + CT_DEAD(ct, SETUP); return err; } @@ -344,6 +376,24 @@ void intel_guc_ct_disable(struct intel_guc_ct *ct) } } +#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM) +static void ct_track_lost_and_found(struct intel_guc_ct *ct, u32 fence, u32 action) +{ + unsigned int lost = fence % ARRAY_SIZE(ct->requests.lost_and_found); +#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GUC) + unsigned long entries[SZ_32]; + unsigned int n; + + n = stack_trace_save(entries, ARRAY_SIZE(entries), 1); + + /* May be called under spinlock, so avoid sleeping */ + ct->requests.lost_and_found[lost].stack = stack_depot_save(entries, n, GFP_NOWAIT); +#endif + ct->requests.lost_and_found[lost].fence = fence; + ct->requests.lost_and_found[lost].action = action; +} +#endif + static u32 ct_get_next_fence(struct intel_guc_ct *ct) { /* For now it's trivial */ @@ -394,11 +444,11 @@ static int ct_write(struct intel_guc_ct *ct, FIELD_PREP(GUC_CTB_MSG_0_NUM_DWORDS, len) | FIELD_PREP(GUC_CTB_MSG_0_FENCE, fence); - type = (flags & INTEL_GUC_CT_SEND_NB) ? GUC_HXG_TYPE_EVENT : + type = (flags & INTEL_GUC_CT_SEND_NB) ? GUC_HXG_TYPE_FAST_REQUEST : GUC_HXG_TYPE_REQUEST; hxg = FIELD_PREP(GUC_HXG_MSG_0_TYPE, type) | - FIELD_PREP(GUC_HXG_EVENT_MSG_0_ACTION | - GUC_HXG_EVENT_MSG_0_DATA0, action[0]); + FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION | + GUC_HXG_REQUEST_MSG_0_DATA0, action[0]); CT_DEBUG(ct, "writing (tail %u) %*ph %*ph %*ph\n", tail, 4, &header, 4, &hxg, 4 * (len - 1), &action[1]); @@ -415,6 +465,11 @@ static int ct_write(struct intel_guc_ct *ct, } GEM_BUG_ON(tail > size); +#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM) + ct_track_lost_and_found(ct, fence, + FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, action[0])); +#endif + /* * make sure H2G buffer update and LRC tail update (if this triggering a * submission) are visible before updating the descriptor tail @@ -434,6 +489,7 @@ static int ct_write(struct intel_guc_ct *ct, corrupted: CT_ERROR(ct, "Corrupted descriptor head=%u tail=%u status=%#x\n", desc->head, desc->tail, desc->status); + CT_DEAD(ct, WRITE); ctb->broken = true; return -EPIPE; } @@ -504,6 +560,7 @@ static inline bool ct_deadlocked(struct intel_guc_ct *ct) CT_ERROR(ct, "Head: %u\n (Dwords)", ct->ctbs.recv.desc->head); CT_ERROR(ct, "Tail: %u\n (Dwords)", ct->ctbs.recv.desc->tail); + CT_DEAD(ct, DEADLOCK); ct->ctbs.send.broken = true; } @@ -552,6 +609,7 @@ static inline bool h2g_has_room(struct intel_guc_ct *ct, u32 len_dw) head, ctb->size); desc->status |= GUC_CTB_STATUS_OVERFLOW; ctb->broken = true; + CT_DEAD(ct, H2G_HAS_ROOM); return false; } @@ -640,7 +698,7 @@ static int ct_send(struct intel_guc_ct *ct, GEM_BUG_ON(!ct->enabled); GEM_BUG_ON(!len); - GEM_BUG_ON(len & ~GUC_CT_MSG_LEN_MASK); + GEM_BUG_ON(len > GUC_CTB_HXG_MSG_MAX_LEN - GUC_CTB_HDR_LEN); GEM_BUG_ON(!response_buf && response_buf_size); might_sleep(); @@ -902,15 +960,59 @@ static int ct_read(struct intel_guc_ct *ct, struct ct_incoming_msg **msg) /* now update descriptor */ WRITE_ONCE(desc->head, head); + /* + * Wa_22016122933: Making sure the head update is + * visible to GuC right away + */ + intel_guc_write_barrier(ct_to_guc(ct)); + return available - len; corrupted: CT_ERROR(ct, "Corrupted descriptor head=%u tail=%u status=%#x\n", desc->head, desc->tail, desc->status); ctb->broken = true; + CT_DEAD(ct, READ); return -EPIPE; } +#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM) +static bool ct_check_lost_and_found(struct intel_guc_ct *ct, u32 fence) +{ + unsigned int n; + char *buf = NULL; + bool found = false; + + lockdep_assert_held(&ct->requests.lock); + + for (n = 0; n < ARRAY_SIZE(ct->requests.lost_and_found); n++) { + if (ct->requests.lost_and_found[n].fence != fence) + continue; + found = true; + +#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GUC) + buf = kmalloc(SZ_4K, GFP_NOWAIT); + if (buf && stack_depot_snprint(ct->requests.lost_and_found[n].stack, + buf, SZ_4K, 0)) { + CT_ERROR(ct, "Fence %u was used by action %#04x sent at\n%s", + fence, ct->requests.lost_and_found[n].action, buf); + break; + } +#endif + CT_ERROR(ct, "Fence %u was used by action %#04x\n", + fence, ct->requests.lost_and_found[n].action); + break; + } + kfree(buf); + return found; +} +#else +static bool ct_check_lost_and_found(struct intel_guc_ct *ct, u32 fence) +{ + return false; +} +#endif + static int ct_handle_response(struct intel_guc_ct *ct, struct ct_incoming_msg *response) { u32 len = FIELD_GET(GUC_CTB_MSG_0_NUM_DWORDS, response->msg[0]); @@ -952,12 +1054,13 @@ static int ct_handle_response(struct intel_guc_ct *ct, struct ct_incoming_msg *r break; } if (!found) { - CT_ERROR(ct, "Unsolicited response (fence %u)\n", fence); - CT_ERROR(ct, "Could not find fence=%u, last_fence=%u\n", fence, - ct->requests.last_fence); - list_for_each_entry(req, &ct->requests.pending, link) - CT_ERROR(ct, "request %u awaits response\n", - req->fence); + CT_ERROR(ct, "Unsolicited response message: len %u, data %#x (fence %u, last %u)\n", + len, hxg[0], fence, ct->requests.last_fence); + if (!ct_check_lost_and_found(ct, fence)) { + list_for_each_entry(req, &ct->requests.pending, link) + CT_ERROR(ct, "request %u awaits response\n", + req->fence); + } err = -ENOKEY; } spin_unlock_irqrestore(&ct->requests.lock, flags); @@ -1057,6 +1160,7 @@ static bool ct_process_incoming_requests(struct intel_guc_ct *ct) if (unlikely(err)) { CT_ERROR(ct, "Failed to process CT message (%pe) %*ph\n", ERR_PTR(err), 4 * request->size, request->msg); + CT_DEAD(ct, PROCESS_FAILED); ct_free_msg(request); } @@ -1233,3 +1337,19 @@ void intel_guc_ct_print_info(struct intel_guc_ct *ct, drm_printf(p, "Tail: %u\n", ct->ctbs.recv.desc->tail); } + +#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GUC) +static void ct_dead_ct_worker_func(struct work_struct *w) +{ + struct intel_guc_ct *ct = container_of(w, struct intel_guc_ct, dead_ct_worker); + struct intel_guc *guc = ct_to_guc(ct); + + if (ct->dead_ct_reported) + return; + + ct->dead_ct_reported = true; + + guc_info(guc, "CTB is dead - reason=0x%X\n", ct->dead_ct_reason); + intel_klog_error_capture(guc_to_gt(guc), (intel_engine_mask_t)~0U); +} +#endif diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.h b/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.h index f709a19c7e21..58e42901ff49 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.h @@ -8,6 +8,7 @@ #include <linux/interrupt.h> #include <linux/spinlock.h> +#include <linux/stackdepot.h> #include <linux/workqueue.h> #include <linux/ktime.h> #include <linux/wait.h> @@ -81,10 +82,26 @@ struct intel_guc_ct { struct list_head incoming; /* incoming requests */ struct work_struct worker; /* handler for incoming requests */ + +#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM) + struct { + u16 fence; + u16 action; +#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GUC) + depot_stack_handle_t stack; +#endif + } lost_and_found[SZ_16]; +#endif } requests; /** @stall_time: time of first time a CTB submission is stalled */ ktime_t stall_time; + +#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GUC) + int dead_ct_reason; + bool dead_ct_reported; + struct work_struct dead_ct_worker; +#endif }; void intel_guc_ct_init_early(struct intel_guc_ct *ct); diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_fw.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_fw.c index 6fda3aec5c66..364d0d546ec8 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_fw.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_fw.c @@ -129,6 +129,7 @@ static inline bool guc_load_done(struct intel_uncore *uncore, u32 *status, bool case INTEL_BOOTROM_STATUS_RC6CTXCONFIG_FAILED: case INTEL_BOOTROM_STATUS_MPUMAP_INCORRECT: case INTEL_BOOTROM_STATUS_EXCEPTION: + case INTEL_BOOTROM_STATUS_PROD_KEY_CHECK_FAILURE: *success = false; return true; } @@ -190,8 +191,10 @@ static int guc_wait_ucode(struct intel_guc *guc) if (!ret || !success) break; - guc_dbg(guc, "load still in progress, count = %d, freq = %dMHz\n", - count, intel_rps_read_actual_frequency(&uncore->gt->rps)); + guc_dbg(guc, "load still in progress, count = %d, freq = %dMHz, status = 0x%08X [0x%02X/%02X]\n", + count, intel_rps_read_actual_frequency(&uncore->gt->rps), status, + REG_FIELD_GET(GS_BOOTROM_MASK, status), + REG_FIELD_GET(GS_UKERNEL_MASK, status)); } after = ktime_get(); delta = ktime_sub(after, before); @@ -219,6 +222,11 @@ static int guc_wait_ucode(struct intel_guc *guc) guc_info(guc, "firmware signature verification failed\n"); ret = -ENOEXEC; break; + + case INTEL_BOOTROM_STATUS_PROD_KEY_CHECK_FAILURE: + guc_info(guc, "firmware production part check failure\n"); + ret = -ENOEXEC; + break; } switch (ukernel) { diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_fwif.h b/drivers/gpu/drm/i915/gt/uc/intel_guc_fwif.h index 4ae5fc2f6002..b4d56eccfb1f 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_fwif.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_fwif.h @@ -35,13 +35,6 @@ #define GUC_MAX_CONTEXT_ID 65535 #define GUC_INVALID_CONTEXT_ID GUC_MAX_CONTEXT_ID -#define GUC_RENDER_ENGINE 0 -#define GUC_VIDEO_ENGINE 1 -#define GUC_BLITTER_ENGINE 2 -#define GUC_VIDEOENHANCE_ENGINE 3 -#define GUC_VIDEO_ENGINE2 4 -#define GUC_MAX_ENGINES_NUM (GUC_VIDEO_ENGINE2 + 1) - #define GUC_RENDER_CLASS 0 #define GUC_VIDEO_CLASS 1 #define GUC_VIDEOENHANCE_CLASS 2 @@ -411,6 +404,15 @@ enum guc_capture_type { GUC_CAPTURE_LIST_TYPE_MAX, }; +/* Class indecies for capture_class and capture_instance arrays */ +enum { + GUC_CAPTURE_LIST_CLASS_RENDER_COMPUTE = 0, + GUC_CAPTURE_LIST_CLASS_VIDEO = 1, + GUC_CAPTURE_LIST_CLASS_VIDEOENHANCE = 2, + GUC_CAPTURE_LIST_CLASS_BLITTER = 3, + GUC_CAPTURE_LIST_CLASS_GSC_OTHER = 4, +}; + /* GuC Additional Data Struct */ struct guc_ads { struct guc_mmio_reg_set reg_state_list[GUC_MAX_ENGINE_CLASSES][GUC_MAX_INSTANCES_PER_CLASS]; @@ -451,7 +453,7 @@ enum guc_log_buffer_type { GUC_MAX_LOG_BUFFER }; -/** +/* * struct guc_log_buffer_state - GuC log buffer state * * Below state structure is used for coordination of retrieval of GuC firmware @@ -490,32 +492,6 @@ struct guc_log_buffer_state { u32 version; } __packed; -struct guc_ctx_report { - u32 report_return_status; - u32 reserved1[64]; - u32 affected_count; - u32 reserved2[2]; -} __packed; - -/* GuC Shared Context Data Struct */ -struct guc_shared_ctx_data { - u32 addr_of_last_preempted_data_low; - u32 addr_of_last_preempted_data_high; - u32 addr_of_last_preempted_data_high_tmp; - u32 padding; - u32 is_mapped_to_proxy; - u32 proxy_ctx_id; - u32 engine_reset_ctx_id; - u32 media_reset_count; - u32 reserved1[8]; - u32 uk_last_ctx_switch_reason; - u32 was_reset; - u32 lrca_gpu_addr; - u64 execlist_ctx; - u32 reserved2[66]; - struct guc_ctx_report preempt_ctx_report[GUC_MAX_ENGINES_NUM]; -} __packed; - /* This action will be programmed in C1BC - SOFT_SCRATCH_15_REG */ enum intel_guc_recv_message { INTEL_GUC_RECV_MSG_CRASH_DUMP_POSTED = BIT(1), diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_slpc.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_slpc.c index 026d73855f36..01b75529311c 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_slpc.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_slpc.c @@ -277,6 +277,7 @@ int intel_guc_slpc_init(struct intel_guc_slpc *slpc) slpc->max_freq_softlimit = 0; slpc->min_freq_softlimit = 0; + slpc->ignore_eff_freq = false; slpc->min_is_rpmax = false; slpc->boost_freq = 0; @@ -457,6 +458,29 @@ int intel_guc_slpc_get_max_freq(struct intel_guc_slpc *slpc, u32 *val) return ret; } +int intel_guc_slpc_set_ignore_eff_freq(struct intel_guc_slpc *slpc, bool val) +{ + struct drm_i915_private *i915 = slpc_to_i915(slpc); + intel_wakeref_t wakeref; + int ret; + + mutex_lock(&slpc->lock); + wakeref = intel_runtime_pm_get(&i915->runtime_pm); + + ret = slpc_set_param(slpc, + SLPC_PARAM_IGNORE_EFFICIENT_FREQUENCY, + val); + if (ret) + guc_probe_error(slpc_to_guc(slpc), "Failed to set efficient freq(%d): %pe\n", + val, ERR_PTR(ret)); + else + slpc->ignore_eff_freq = val; + + intel_runtime_pm_put(&i915->runtime_pm, wakeref); + mutex_unlock(&slpc->lock); + return ret; +} + /** * intel_guc_slpc_set_min_freq() - Set min frequency limit for SLPC. * @slpc: pointer to intel_guc_slpc. @@ -482,16 +506,6 @@ int intel_guc_slpc_set_min_freq(struct intel_guc_slpc *slpc, u32 val) mutex_lock(&slpc->lock); wakeref = intel_runtime_pm_get(&i915->runtime_pm); - /* Ignore efficient freq if lower min freq is requested */ - ret = slpc_set_param(slpc, - SLPC_PARAM_IGNORE_EFFICIENT_FREQUENCY, - val < slpc->rp1_freq); - if (ret) { - guc_probe_error(slpc_to_guc(slpc), "Failed to toggle efficient freq: %pe\n", - ERR_PTR(ret)); - goto out; - } - ret = slpc_set_param(slpc, SLPC_PARAM_GLOBAL_MIN_GT_UNSLICE_FREQ_MHZ, val); @@ -499,7 +513,6 @@ int intel_guc_slpc_set_min_freq(struct intel_guc_slpc *slpc, u32 val) if (!ret) slpc->min_freq_softlimit = val; -out: intel_runtime_pm_put(&i915->runtime_pm, wakeref); mutex_unlock(&slpc->lock); @@ -752,6 +765,9 @@ int intel_guc_slpc_enable(struct intel_guc_slpc *slpc) /* Set cached media freq ratio mode */ intel_guc_slpc_set_media_ratio_mode(slpc, slpc->media_ratio_mode); + /* Set cached value of ignore efficient freq */ + intel_guc_slpc_set_ignore_eff_freq(slpc, slpc->ignore_eff_freq); + return 0; } @@ -821,6 +837,8 @@ int intel_guc_slpc_print_info(struct intel_guc_slpc *slpc, struct drm_printer *p slpc_decode_min_freq(slpc)); drm_printf(p, "\twaitboosts: %u\n", slpc->num_boosts); + drm_printf(p, "\tBoosts outstanding: %u\n", + atomic_read(&slpc->num_waiters)); } } diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_slpc.h b/drivers/gpu/drm/i915/gt/uc/intel_guc_slpc.h index 17ed515f6a85..597eb5413ddf 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_slpc.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_slpc.h @@ -46,5 +46,6 @@ void intel_guc_slpc_boost(struct intel_guc_slpc *slpc); void intel_guc_slpc_dec_waiters(struct intel_guc_slpc *slpc); int intel_guc_slpc_unset_gucrc_mode(struct intel_guc_slpc *slpc); int intel_guc_slpc_override_gucrc_mode(struct intel_guc_slpc *slpc, u32 mode); +int intel_guc_slpc_set_ignore_eff_freq(struct intel_guc_slpc *slpc, bool val); #endif diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_slpc_types.h b/drivers/gpu/drm/i915/gt/uc/intel_guc_slpc_types.h index a6ef53b04e04..a88651331497 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_slpc_types.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_slpc_types.h @@ -31,6 +31,7 @@ struct intel_guc_slpc { /* frequency softlimits */ u32 min_freq_softlimit; u32 max_freq_softlimit; + bool ignore_eff_freq; /* cached media ratio mode */ u32 media_ratio_mode; diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c index 88e881b100cf..a0e3ef1c65d2 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c @@ -1402,13 +1402,34 @@ static void __update_guc_busyness_stats(struct intel_guc *guc) spin_unlock_irqrestore(&guc->timestamp.lock, flags); } +static void __guc_context_update_stats(struct intel_context *ce) +{ + struct intel_guc *guc = ce_to_guc(ce); + unsigned long flags; + + spin_lock_irqsave(&guc->timestamp.lock, flags); + lrc_update_runtime(ce); + spin_unlock_irqrestore(&guc->timestamp.lock, flags); +} + +static void guc_context_update_stats(struct intel_context *ce) +{ + if (!intel_context_pin_if_active(ce)) + return; + + __guc_context_update_stats(ce); + intel_context_unpin(ce); +} + static void guc_timestamp_ping(struct work_struct *wrk) { struct intel_guc *guc = container_of(wrk, typeof(*guc), timestamp.work.work); struct intel_uc *uc = container_of(guc, typeof(*uc), guc); struct intel_gt *gt = guc_to_gt(guc); + struct intel_context *ce; intel_wakeref_t wakeref; + unsigned long index; int srcu, ret; /* @@ -1424,6 +1445,10 @@ static void guc_timestamp_ping(struct work_struct *wrk) with_intel_runtime_pm(>->i915->runtime_pm, wakeref) __update_guc_busyness_stats(guc); + /* adjust context stats for overflow */ + xa_for_each(&guc->context_lookup, index, ce) + guc_context_update_stats(ce); + intel_gt_reset_unlock(gt, srcu); guc_enable_busyness_worker(guc); @@ -1629,16 +1654,16 @@ static void guc_reset_state(struct intel_context *ce, u32 head, bool scrub) static void guc_engine_reset_prepare(struct intel_engine_cs *engine) { - if (!IS_GRAPHICS_VER(engine->i915, 11, 12)) - return; - - intel_engine_stop_cs(engine); - /* * Wa_22011802037: In addition to stopping the cs, we need * to wait for any pending mi force wakeups */ - intel_engine_wait_for_pending_mi_fw(engine); + if (IS_MTL_GRAPHICS_STEP(engine->i915, M, STEP_A0, STEP_B0) || + (GRAPHICS_VER(engine->i915) >= 11 && + GRAPHICS_VER_FULL(engine->i915) < IP_VER(12, 70))) { + intel_engine_stop_cs(engine); + intel_engine_wait_for_pending_mi_fw(engine); + } } static void guc_reset_nop(struct intel_engine_cs *engine) @@ -2774,6 +2799,7 @@ static void guc_context_unpin(struct intel_context *ce) { struct intel_guc *guc = ce_to_guc(ce); + __guc_context_update_stats(ce); unpin_guc_id(guc, ce); lrc_unpin(ce); @@ -3455,6 +3481,7 @@ static void remove_from_context(struct i915_request *rq) } static const struct intel_context_ops guc_context_ops = { + .flags = COPS_RUNTIME_CYCLES, .alloc = guc_context_alloc, .close = guc_context_close, @@ -3473,6 +3500,8 @@ static const struct intel_context_ops guc_context_ops = { .sched_disable = guc_context_sched_disable, + .update_stats = guc_context_update_stats, + .reset = lrc_reset, .destroy = guc_context_destroy, @@ -3728,6 +3757,7 @@ static int guc_virtual_context_alloc(struct intel_context *ce) } static const struct intel_context_ops virtual_guc_context_ops = { + .flags = COPS_RUNTIME_CYCLES, .alloc = guc_virtual_context_alloc, .close = guc_context_close, @@ -3745,6 +3775,7 @@ static const struct intel_context_ops virtual_guc_context_ops = { .exit = guc_virtual_context_exit, .sched_disable = guc_context_sched_disable, + .update_stats = guc_context_update_stats, .destroy = guc_context_destroy, @@ -4697,13 +4728,37 @@ static void capture_error_state(struct intel_guc *guc, { struct intel_gt *gt = guc_to_gt(guc); struct drm_i915_private *i915 = gt->i915; - struct intel_engine_cs *engine = __context_to_physical_engine(ce); intel_wakeref_t wakeref; + intel_engine_mask_t engine_mask; + + if (intel_engine_is_virtual(ce->engine)) { + struct intel_engine_cs *e; + intel_engine_mask_t tmp, virtual_mask = ce->engine->mask; + + engine_mask = 0; + for_each_engine_masked(e, ce->engine->gt, virtual_mask, tmp) { + bool match = intel_guc_capture_is_matching_engine(gt, ce, e); + + if (match) { + intel_engine_set_hung_context(e, ce); + engine_mask |= e->mask; + atomic_inc(&i915->gpu_error.reset_engine_count[e->uabi_class]); + } + } + + if (!engine_mask) { + guc_warn(guc, "No matching physical engine capture for virtual engine context 0x%04X / %s", + ce->guc_id.id, ce->engine->name); + engine_mask = ~0U; + } + } else { + intel_engine_set_hung_context(ce->engine, ce); + engine_mask = ce->engine->mask; + atomic_inc(&i915->gpu_error.reset_engine_count[ce->engine->uabi_class]); + } - intel_engine_set_hung_context(engine, ce); with_intel_runtime_pm(&i915->runtime_pm, wakeref) - i915_capture_error_state(gt, engine->mask, CORE_DUMP_FLAG_IS_GUC_CAPTURE); - atomic_inc(&i915->gpu_error.reset_engine_count[engine->uabi_class]); + i915_capture_error_state(gt, engine_mask, CORE_DUMP_FLAG_IS_GUC_CAPTURE); } static void guc_context_replay(struct intel_context *ce) diff --git a/drivers/gpu/drm/i915/gt/uc/intel_huc.c b/drivers/gpu/drm/i915/gt/uc/intel_huc.c index 04724ff56ded..ddd146265beb 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_huc.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_huc.c @@ -6,23 +6,16 @@ #include <linux/types.h> #include "gt/intel_gt.h" -#include "gt/intel_gt_print.h" #include "intel_guc_reg.h" #include "intel_huc.h" +#include "intel_huc_print.h" #include "i915_drv.h" +#include "i915_reg.h" +#include "pxp/intel_pxp_cmd_interface_43.h" #include <linux/device/bus.h> #include <linux/mei_aux.h> -#define huc_printk(_huc, _level, _fmt, ...) \ - gt_##_level(huc_to_gt(_huc), "HuC: " _fmt, ##__VA_ARGS__) -#define huc_err(_huc, _fmt, ...) huc_printk((_huc), err, _fmt, ##__VA_ARGS__) -#define huc_warn(_huc, _fmt, ...) huc_printk((_huc), warn, _fmt, ##__VA_ARGS__) -#define huc_notice(_huc, _fmt, ...) huc_printk((_huc), notice, _fmt, ##__VA_ARGS__) -#define huc_info(_huc, _fmt, ...) huc_printk((_huc), info, _fmt, ##__VA_ARGS__) -#define huc_dbg(_huc, _fmt, ...) huc_printk((_huc), dbg, _fmt, ##__VA_ARGS__) -#define huc_probe_error(_huc, _fmt, ...) huc_printk((_huc), probe_error, _fmt, ##__VA_ARGS__) - /** * DOC: HuC * @@ -31,15 +24,23 @@ * capabilities by adding HuC specific commands to batch buffers. * * The kernel driver is only responsible for loading the HuC firmware and - * triggering its security authentication, which is performed by the GuC on - * older platforms and by the GSC on newer ones. For the GuC to correctly - * perform the authentication, the HuC binary must be loaded before the GuC one. + * triggering its security authentication. This is done differently depending + * on the platform: + * - older platforms (from Gen9 to most Gen12s): the load is performed via DMA + * and the authentication via GuC + * - DG2: load and authentication are both performed via GSC. + * - MTL and newer platforms: the load is performed via DMA (same as with + * not-DG2 older platforms), while the authentication is done in 2-steps, + * a first auth for clear-media workloads via GuC and a second one for all + * workloads via GSC. + * On platforms where the GuC does the authentication, to correctly do so the + * HuC binary must be loaded before the GuC one. * Loading the HuC is optional; however, not using the HuC might negatively * impact power usage and/or performance of media workloads, depending on the * use-cases. * HuC must be reloaded on events that cause the WOPCM to lose its contents - * (S3/S4, FLR); GuC-authenticated HuC must also be reloaded on GuC/GT reset, - * while GSC-managed HuC will survive that. + * (S3/S4, FLR); on older platforms the HuC must also be reloaded on GuC/GT + * reset, while on newer ones it will survive that. * * See https://github.com/intel/media-driver for the latest details on HuC * functionality. @@ -115,7 +116,7 @@ static enum hrtimer_restart huc_delayed_load_timer_callback(struct hrtimer *hrti { struct intel_huc *huc = container_of(hrtimer, struct intel_huc, delayed_load.timer); - if (!intel_huc_is_authenticated(huc)) { + if (!intel_huc_is_authenticated(huc, INTEL_HUC_AUTH_BY_GSC)) { if (huc->delayed_load.status == INTEL_HUC_WAITING_ON_GSC) huc_notice(huc, "timed out waiting for MEI GSC\n"); else if (huc->delayed_load.status == INTEL_HUC_WAITING_ON_PXP) @@ -133,7 +134,7 @@ static void huc_delayed_load_start(struct intel_huc *huc) { ktime_t delay; - GEM_BUG_ON(intel_huc_is_authenticated(huc)); + GEM_BUG_ON(intel_huc_is_authenticated(huc, INTEL_HUC_AUTH_BY_GSC)); /* * On resume we don't have to wait for MEI-GSC to be re-probed, but we @@ -276,7 +277,7 @@ void intel_huc_init_early(struct intel_huc *huc) struct drm_i915_private *i915 = huc_to_gt(huc)->i915; struct intel_gt *gt = huc_to_gt(huc); - intel_uc_fw_init_early(&huc->fw, INTEL_UC_FW_TYPE_HUC); + intel_uc_fw_init_early(&huc->fw, INTEL_UC_FW_TYPE_HUC, true); /* * we always init the fence as already completed, even if HuC is not @@ -293,13 +294,23 @@ void intel_huc_init_early(struct intel_huc *huc) } if (GRAPHICS_VER(i915) >= 11) { - huc->status.reg = GEN11_HUC_KERNEL_LOAD_INFO; - huc->status.mask = HUC_LOAD_SUCCESSFUL; - huc->status.value = HUC_LOAD_SUCCESSFUL; + huc->status[INTEL_HUC_AUTH_BY_GUC].reg = GEN11_HUC_KERNEL_LOAD_INFO; + huc->status[INTEL_HUC_AUTH_BY_GUC].mask = HUC_LOAD_SUCCESSFUL; + huc->status[INTEL_HUC_AUTH_BY_GUC].value = HUC_LOAD_SUCCESSFUL; + } else { + huc->status[INTEL_HUC_AUTH_BY_GUC].reg = HUC_STATUS2; + huc->status[INTEL_HUC_AUTH_BY_GUC].mask = HUC_FW_VERIFIED; + huc->status[INTEL_HUC_AUTH_BY_GUC].value = HUC_FW_VERIFIED; + } + + if (IS_DG2(i915)) { + huc->status[INTEL_HUC_AUTH_BY_GSC].reg = GEN11_HUC_KERNEL_LOAD_INFO; + huc->status[INTEL_HUC_AUTH_BY_GSC].mask = HUC_LOAD_SUCCESSFUL; + huc->status[INTEL_HUC_AUTH_BY_GSC].value = HUC_LOAD_SUCCESSFUL; } else { - huc->status.reg = HUC_STATUS2; - huc->status.mask = HUC_FW_VERIFIED; - huc->status.value = HUC_FW_VERIFIED; + huc->status[INTEL_HUC_AUTH_BY_GSC].reg = HECI_FWSTS5(MTL_GSC_HECI1_BASE); + huc->status[INTEL_HUC_AUTH_BY_GSC].mask = HECI_FWSTS5_HUC_AUTH_DONE; + huc->status[INTEL_HUC_AUTH_BY_GSC].value = HECI_FWSTS5_HUC_AUTH_DONE; } } @@ -307,51 +318,91 @@ void intel_huc_init_early(struct intel_huc *huc) static int check_huc_loading_mode(struct intel_huc *huc) { struct intel_gt *gt = huc_to_gt(huc); - bool fw_needs_gsc = intel_huc_is_loaded_by_gsc(huc); - bool hw_uses_gsc = false; + bool gsc_enabled = huc->fw.has_gsc_headers; /* * The fuse for HuC load via GSC is only valid on platforms that have * GuC deprivilege. */ if (HAS_GUC_DEPRIVILEGE(gt->i915)) - hw_uses_gsc = intel_uncore_read(gt->uncore, GUC_SHIM_CONTROL2) & - GSC_LOADS_HUC; + huc->loaded_via_gsc = intel_uncore_read(gt->uncore, GUC_SHIM_CONTROL2) & + GSC_LOADS_HUC; - if (fw_needs_gsc != hw_uses_gsc) { - huc_err(huc, "mismatch between FW (%s) and HW (%s) load modes\n", - HUC_LOAD_MODE_STRING(fw_needs_gsc), HUC_LOAD_MODE_STRING(hw_uses_gsc)); + if (huc->loaded_via_gsc && !gsc_enabled) { + huc_err(huc, "HW requires a GSC-enabled blob, but we found a legacy one\n"); return -ENOEXEC; } - /* make sure we can access the GSC via the mei driver if we need it */ - if (!(IS_ENABLED(CONFIG_INTEL_MEI_PXP) && IS_ENABLED(CONFIG_INTEL_MEI_GSC)) && - fw_needs_gsc) { - huc_info(huc, "can't load due to missing MEI modules\n"); - return -EIO; + /* + * On newer platforms we have GSC-enabled binaries but we load the HuC + * via DMA. To do so we need to find the location of the legacy-style + * binary inside the GSC-enabled one, which we do at fetch time. Make + * sure that we were able to do so if the fuse says we need to load via + * DMA and the binary is GSC-enabled. + */ + if (!huc->loaded_via_gsc && gsc_enabled && !huc->fw.dma_start_offset) { + huc_err(huc, "HW in DMA mode, but we have an incompatible GSC-enabled blob\n"); + return -ENOEXEC; + } + + /* + * If the HuC is loaded via GSC, we need to be able to access the GSC. + * On DG2 this is done via the mei components, while on newer platforms + * it is done via the GSCCS, + */ + if (huc->loaded_via_gsc) { + if (IS_DG2(gt->i915)) { + if (!IS_ENABLED(CONFIG_INTEL_MEI_PXP) || + !IS_ENABLED(CONFIG_INTEL_MEI_GSC)) { + huc_info(huc, "can't load due to missing mei modules\n"); + return -EIO; + } + } else { + if (!HAS_ENGINE(gt, GSC0)) { + huc_info(huc, "can't load due to missing GSCCS\n"); + return -EIO; + } + } } - huc_dbg(huc, "loaded by GSC = %s\n", str_yes_no(fw_needs_gsc)); + huc_dbg(huc, "loaded by GSC = %s\n", str_yes_no(huc->loaded_via_gsc)); return 0; } int intel_huc_init(struct intel_huc *huc) { + struct intel_gt *gt = huc_to_gt(huc); int err; err = check_huc_loading_mode(huc); if (err) goto out; + if (HAS_ENGINE(gt, GSC0)) { + struct i915_vma *vma; + + vma = intel_guc_allocate_vma(>->uc.guc, PXP43_HUC_AUTH_INOUT_SIZE * 2); + if (IS_ERR(vma)) { + err = PTR_ERR(vma); + huc_info(huc, "Failed to allocate heci pkt\n"); + goto out; + } + + huc->heci_pkt = vma; + } + err = intel_uc_fw_init(&huc->fw); if (err) - goto out; + goto out_pkt; intel_uc_fw_change_status(&huc->fw, INTEL_UC_FIRMWARE_LOADABLE); return 0; +out_pkt: + if (huc->heci_pkt) + i915_vma_unpin_and_release(&huc->heci_pkt, 0); out: intel_uc_fw_change_status(&huc->fw, INTEL_UC_FIRMWARE_INIT_FAIL); huc_info(huc, "initialization failed %pe\n", ERR_PTR(err)); @@ -366,6 +417,9 @@ void intel_huc_fini(struct intel_huc *huc) */ delayed_huc_load_fini(huc); + if (huc->heci_pkt) + i915_vma_unpin_and_release(&huc->heci_pkt, 0); + if (intel_uc_fw_is_loadable(&huc->fw)) intel_uc_fw_fini(&huc->fw); } @@ -383,34 +437,45 @@ void intel_huc_suspend(struct intel_huc *huc) delayed_huc_load_complete(huc); } -int intel_huc_wait_for_auth_complete(struct intel_huc *huc) +static const char *auth_mode_string(struct intel_huc *huc, + enum intel_huc_authentication_type type) +{ + bool partial = huc->fw.has_gsc_headers && type == INTEL_HUC_AUTH_BY_GUC; + + return partial ? "clear media" : "all workloads"; +} + +int intel_huc_wait_for_auth_complete(struct intel_huc *huc, + enum intel_huc_authentication_type type) { struct intel_gt *gt = huc_to_gt(huc); int ret; ret = __intel_wait_for_register(gt->uncore, - huc->status.reg, - huc->status.mask, - huc->status.value, + huc->status[type].reg, + huc->status[type].mask, + huc->status[type].value, 2, 50, NULL); /* mark the load process as complete even if the wait failed */ delayed_huc_load_complete(huc); if (ret) { - huc_err(huc, "firmware not verified %pe\n", ERR_PTR(ret)); + huc_err(huc, "firmware not verified for %s: %pe\n", + auth_mode_string(huc, type), ERR_PTR(ret)); intel_uc_fw_change_status(&huc->fw, INTEL_UC_FIRMWARE_LOAD_FAIL); return ret; } intel_uc_fw_change_status(&huc->fw, INTEL_UC_FIRMWARE_RUNNING); - huc_info(huc, "authenticated!\n"); + huc_info(huc, "authenticated for %s\n", auth_mode_string(huc, type)); return 0; } /** * intel_huc_auth() - Authenticate HuC uCode * @huc: intel_huc structure + * @type: authentication type (via GuC or via GSC) * * Called after HuC and GuC firmware loading during intel_uc_init_hw(). * @@ -418,7 +483,7 @@ int intel_huc_wait_for_auth_complete(struct intel_huc *huc) * passing the offset of the RSA signature to intel_guc_auth_huc(). It then * waits for up to 50ms for firmware verification ACK. */ -int intel_huc_auth(struct intel_huc *huc) +int intel_huc_auth(struct intel_huc *huc, enum intel_huc_authentication_type type) { struct intel_gt *gt = huc_to_gt(huc); struct intel_guc *guc = >->uc.guc; @@ -427,44 +492,67 @@ int intel_huc_auth(struct intel_huc *huc) if (!intel_uc_fw_is_loaded(&huc->fw)) return -ENOEXEC; - /* GSC will do the auth */ + /* GSC will do the auth with the load */ if (intel_huc_is_loaded_by_gsc(huc)) return -ENODEV; + if (intel_huc_is_authenticated(huc, type)) + return -EEXIST; + ret = i915_inject_probe_error(gt->i915, -ENXIO); if (ret) goto fail; - GEM_BUG_ON(intel_uc_fw_is_running(&huc->fw)); - - ret = intel_guc_auth_huc(guc, intel_guc_ggtt_offset(guc, huc->fw.rsa_data)); - if (ret) { - huc_err(huc, "authentication by GuC failed %pe\n", ERR_PTR(ret)); - goto fail; + switch (type) { + case INTEL_HUC_AUTH_BY_GUC: + ret = intel_guc_auth_huc(guc, intel_guc_ggtt_offset(guc, huc->fw.rsa_data)); + break; + case INTEL_HUC_AUTH_BY_GSC: + ret = intel_huc_fw_auth_via_gsccs(huc); + break; + default: + MISSING_CASE(type); + ret = -EINVAL; } + if (ret) + goto fail; /* Check authentication status, it should be done by now */ - ret = intel_huc_wait_for_auth_complete(huc); + ret = intel_huc_wait_for_auth_complete(huc, type); if (ret) goto fail; return 0; fail: - huc_probe_error(huc, "authentication failed %pe\n", ERR_PTR(ret)); + huc_probe_error(huc, "%s authentication failed %pe\n", + auth_mode_string(huc, type), ERR_PTR(ret)); return ret; } -bool intel_huc_is_authenticated(struct intel_huc *huc) +bool intel_huc_is_authenticated(struct intel_huc *huc, + enum intel_huc_authentication_type type) { struct intel_gt *gt = huc_to_gt(huc); intel_wakeref_t wakeref; u32 status = 0; with_intel_runtime_pm(gt->uncore->rpm, wakeref) - status = intel_uncore_read(gt->uncore, huc->status.reg); + status = intel_uncore_read(gt->uncore, huc->status[type].reg); + + return (status & huc->status[type].mask) == huc->status[type].value; +} + +static bool huc_is_fully_authenticated(struct intel_huc *huc) +{ + struct intel_uc_fw *huc_fw = &huc->fw; - return (status & huc->status.mask) == huc->status.value; + if (!huc_fw->has_gsc_headers) + return intel_huc_is_authenticated(huc, INTEL_HUC_AUTH_BY_GUC); + else if (intel_huc_is_loaded_by_gsc(huc) || HAS_ENGINE(huc_to_gt(huc), GSC0)) + return intel_huc_is_authenticated(huc, INTEL_HUC_AUTH_BY_GSC); + else + return false; } /** @@ -479,7 +567,9 @@ bool intel_huc_is_authenticated(struct intel_huc *huc) */ int intel_huc_check_status(struct intel_huc *huc) { - switch (__intel_uc_fw_status(&huc->fw)) { + struct intel_uc_fw *huc_fw = &huc->fw; + + switch (__intel_uc_fw_status(huc_fw)) { case INTEL_UC_FIRMWARE_NOT_SUPPORTED: return -ENODEV; case INTEL_UC_FIRMWARE_DISABLED: @@ -496,7 +586,17 @@ int intel_huc_check_status(struct intel_huc *huc) break; } - return intel_huc_is_authenticated(huc); + /* + * GSC-enabled binaries loaded via DMA are first partially + * authenticated by GuC and then fully authenticated by GSC + */ + if (huc_is_fully_authenticated(huc)) + return 1; /* full auth */ + else if (huc_fw->has_gsc_headers && !intel_huc_is_loaded_by_gsc(huc) && + intel_huc_is_authenticated(huc, INTEL_HUC_AUTH_BY_GUC)) + return 2; /* clear media only */ + else + return 0; } static bool huc_has_delayed_load(struct intel_huc *huc) @@ -510,7 +610,10 @@ void intel_huc_update_auth_status(struct intel_huc *huc) if (!intel_uc_fw_is_loadable(&huc->fw)) return; - if (intel_huc_is_authenticated(huc)) + if (!huc->fw.has_gsc_headers) + return; + + if (huc_is_fully_authenticated(huc)) intel_uc_fw_change_status(&huc->fw, INTEL_UC_FIRMWARE_RUNNING); else if (huc_has_delayed_load(huc)) @@ -543,5 +646,5 @@ void intel_huc_load_status(struct intel_huc *huc, struct drm_printer *p) with_intel_runtime_pm(gt->uncore->rpm, wakeref) drm_printf(p, "HuC status: 0x%08x\n", - intel_uncore_read(gt->uncore, huc->status.reg)); + intel_uncore_read(gt->uncore, huc->status[INTEL_HUC_AUTH_BY_GUC].reg)); } diff --git a/drivers/gpu/drm/i915/gt/uc/intel_huc.h b/drivers/gpu/drm/i915/gt/uc/intel_huc.h index 0789184d81a2..ba5cb08e9e7b 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_huc.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_huc.h @@ -15,6 +15,7 @@ #include <linux/hrtimer.h> struct bus_type; +struct i915_vma; enum intel_huc_delayed_load_status { INTEL_HUC_WAITING_ON_GSC = 0, @@ -22,6 +23,12 @@ enum intel_huc_delayed_load_status { INTEL_HUC_DELAYED_LOAD_ERROR, }; +enum intel_huc_authentication_type { + INTEL_HUC_AUTH_BY_GUC = 0, + INTEL_HUC_AUTH_BY_GSC, + INTEL_HUC_AUTH_MAX_MODES +}; + struct intel_huc { /* Generic uC firmware management */ struct intel_uc_fw fw; @@ -31,7 +38,7 @@ struct intel_huc { i915_reg_t reg; u32 mask; u32 value; - } status; + } status[INTEL_HUC_AUTH_MAX_MODES]; struct { struct i915_sw_fence fence; @@ -39,6 +46,11 @@ struct intel_huc { struct notifier_block nb; enum intel_huc_delayed_load_status status; } delayed_load; + + /* for load via GSCCS */ + struct i915_vma *heci_pkt; + + bool loaded_via_gsc; }; int intel_huc_sanitize(struct intel_huc *huc); @@ -46,11 +58,13 @@ void intel_huc_init_early(struct intel_huc *huc); int intel_huc_init(struct intel_huc *huc); void intel_huc_fini(struct intel_huc *huc); void intel_huc_suspend(struct intel_huc *huc); -int intel_huc_auth(struct intel_huc *huc); -int intel_huc_wait_for_auth_complete(struct intel_huc *huc); +int intel_huc_auth(struct intel_huc *huc, enum intel_huc_authentication_type type); +int intel_huc_wait_for_auth_complete(struct intel_huc *huc, + enum intel_huc_authentication_type type); +bool intel_huc_is_authenticated(struct intel_huc *huc, + enum intel_huc_authentication_type type); int intel_huc_check_status(struct intel_huc *huc); void intel_huc_update_auth_status(struct intel_huc *huc); -bool intel_huc_is_authenticated(struct intel_huc *huc); void intel_huc_register_gsc_notifier(struct intel_huc *huc, const struct bus_type *bus); void intel_huc_unregister_gsc_notifier(struct intel_huc *huc, const struct bus_type *bus); @@ -73,13 +87,13 @@ static inline bool intel_huc_is_used(struct intel_huc *huc) static inline bool intel_huc_is_loaded_by_gsc(const struct intel_huc *huc) { - return huc->fw.loaded_via_gsc; + return huc->loaded_via_gsc; } static inline bool intel_huc_wait_required(struct intel_huc *huc) { return intel_huc_is_used(huc) && intel_huc_is_loaded_by_gsc(huc) && - !intel_huc_is_authenticated(huc); + !intel_huc_is_authenticated(huc, INTEL_HUC_AUTH_BY_GSC); } void intel_huc_load_status(struct intel_huc *huc, struct drm_printer *p); diff --git a/drivers/gpu/drm/i915/gt/uc/intel_huc_fw.c b/drivers/gpu/drm/i915/gt/uc/intel_huc_fw.c index 534b0aa43316..e608152fecfc 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_huc_fw.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_huc_fw.c @@ -5,10 +5,241 @@ #include "gt/intel_gsc.h" #include "gt/intel_gt.h" +#include "intel_gsc_binary_headers.h" +#include "intel_gsc_uc_heci_cmd_submit.h" #include "intel_huc.h" #include "intel_huc_fw.h" +#include "intel_huc_print.h" #include "i915_drv.h" #include "pxp/intel_pxp_huc.h" +#include "pxp/intel_pxp_cmd_interface_43.h" + +struct mtl_huc_auth_msg_in { + struct intel_gsc_mtl_header header; + struct pxp43_new_huc_auth_in huc_in; +} __packed; + +struct mtl_huc_auth_msg_out { + struct intel_gsc_mtl_header header; + struct pxp43_huc_auth_out huc_out; +} __packed; + +int intel_huc_fw_auth_via_gsccs(struct intel_huc *huc) +{ + struct intel_gt *gt = huc_to_gt(huc); + struct drm_i915_private *i915 = gt->i915; + struct drm_i915_gem_object *obj; + struct mtl_huc_auth_msg_in *msg_in; + struct mtl_huc_auth_msg_out *msg_out; + void *pkt_vaddr; + u64 pkt_offset; + int retry = 5; + int err = 0; + + if (!huc->heci_pkt) + return -ENODEV; + + obj = huc->heci_pkt->obj; + pkt_offset = i915_ggtt_offset(huc->heci_pkt); + + pkt_vaddr = i915_gem_object_pin_map_unlocked(obj, + i915_coherent_map_type(i915, obj, true)); + if (IS_ERR(pkt_vaddr)) + return PTR_ERR(pkt_vaddr); + + msg_in = pkt_vaddr; + msg_out = pkt_vaddr + PXP43_HUC_AUTH_INOUT_SIZE; + + intel_gsc_uc_heci_cmd_emit_mtl_header(&msg_in->header, + HECI_MEADDRESS_PXP, + sizeof(*msg_in), 0); + + msg_in->huc_in.header.api_version = PXP_APIVER(4, 3); + msg_in->huc_in.header.command_id = PXP43_CMDID_NEW_HUC_AUTH; + msg_in->huc_in.header.status = 0; + msg_in->huc_in.header.buffer_len = sizeof(msg_in->huc_in) - + sizeof(msg_in->huc_in.header); + msg_in->huc_in.huc_base_address = huc->fw.vma_res.start; + msg_in->huc_in.huc_size = huc->fw.obj->base.size; + + do { + err = intel_gsc_uc_heci_cmd_submit_packet(>->uc.gsc, + pkt_offset, sizeof(*msg_in), + pkt_offset + PXP43_HUC_AUTH_INOUT_SIZE, + PXP43_HUC_AUTH_INOUT_SIZE); + if (err) { + huc_err(huc, "failed to submit GSC request to auth: %d\n", err); + goto out_unpin; + } + + if (msg_out->header.flags & GSC_OUTFLAG_MSG_PENDING) { + msg_in->header.gsc_message_handle = msg_out->header.gsc_message_handle; + err = -EBUSY; + msleep(50); + } + } while (--retry && err == -EBUSY); + + if (err) + goto out_unpin; + + if (msg_out->header.message_size != sizeof(*msg_out)) { + huc_err(huc, "invalid GSC reply length %u [expected %zu]\n", + msg_out->header.message_size, sizeof(*msg_out)); + err = -EPROTO; + goto out_unpin; + } + + /* + * The GSC will return PXP_STATUS_OP_NOT_PERMITTED if the HuC is already + * loaded. If the same error is ever returned with HuC not loaded we'll + * still catch it when we check the authentication bit later. + */ + if (msg_out->huc_out.header.status != PXP_STATUS_SUCCESS && + msg_out->huc_out.header.status != PXP_STATUS_OP_NOT_PERMITTED) { + huc_err(huc, "auth failed with GSC error = 0x%x\n", + msg_out->huc_out.header.status); + err = -EIO; + goto out_unpin; + } + +out_unpin: + i915_gem_object_unpin_map(obj); + return err; +} + +static void get_version_from_gsc_manifest(struct intel_uc_fw_ver *ver, const void *data) +{ + const struct intel_gsc_manifest_header *manifest = data; + + ver->major = manifest->fw_version.major; + ver->minor = manifest->fw_version.minor; + ver->patch = manifest->fw_version.hotfix; +} + +static bool css_valid(const void *data, size_t size) +{ + const struct uc_css_header *css = data; + + if (unlikely(size < sizeof(struct uc_css_header))) + return false; + + if (css->module_type != 0x6) + return false; + + if (css->module_vendor != PCI_VENDOR_ID_INTEL) + return false; + + return true; +} + +static inline u32 entry_offset(const struct intel_gsc_cpd_entry *entry) +{ + return entry->offset & INTEL_GSC_CPD_ENTRY_OFFSET_MASK; +} + +int intel_huc_fw_get_binary_info(struct intel_uc_fw *huc_fw, const void *data, size_t size) +{ + struct intel_huc *huc = container_of(huc_fw, struct intel_huc, fw); + const struct intel_gsc_cpd_header_v2 *header = data; + const struct intel_gsc_cpd_entry *entry; + size_t min_size = sizeof(*header); + int i; + + if (!huc_fw->has_gsc_headers) { + huc_err(huc, "Invalid FW type for GSC header parsing!\n"); + return -EINVAL; + } + + if (size < sizeof(*header)) { + huc_err(huc, "FW too small! %zu < %zu\n", size, min_size); + return -ENODATA; + } + + /* + * The GSC-enabled HuC binary starts with a directory header, followed + * by a series of entries. Each entry is identified by a name and + * points to a specific section of the binary containing the relevant + * data. The entries we're interested in are: + * - "HUCP.man": points to the GSC manifest header for the HuC, which + * contains the version info. + * - "huc_fw": points to the legacy-style binary that can be used for + * load via the DMA. This entry only contains a valid CSS + * on binaries for platforms that support 2-step HuC load + * via dma and auth via GSC (like MTL). + * + * -------------------------------------------------- + * [ intel_gsc_cpd_header_v2 ] + * -------------------------------------------------- + * [ intel_gsc_cpd_entry[] ] + * [ entry1 ] + * [ ... ] + * [ entryX ] + * [ "HUCP.man" ] + * [ ... ] + * [ offset >----------------------------]------o + * [ ... ] | + * [ entryY ] | + * [ "huc_fw" ] | + * [ ... ] | + * [ offset >----------------------------]----------o + * -------------------------------------------------- | | + * | | + * -------------------------------------------------- | | + * [ intel_gsc_manifest_header ]<-----o | + * [ ... ] | + * [ intel_gsc_version fw_version ] | + * [ ... ] | + * -------------------------------------------------- | + * | + * -------------------------------------------------- | + * [ data[] ]<---------o + * [ ... ] + * [ ... ] + * -------------------------------------------------- + */ + + if (header->header_marker != INTEL_GSC_CPD_HEADER_MARKER) { + huc_err(huc, "invalid marker for CPD header: 0x%08x!\n", + header->header_marker); + return -EINVAL; + } + + /* we only have binaries with header v2 and entry v1 for now */ + if (header->header_version != 2 || header->entry_version != 1) { + huc_err(huc, "invalid CPD header/entry version %u:%u!\n", + header->header_version, header->entry_version); + return -EINVAL; + } + + if (header->header_length < sizeof(struct intel_gsc_cpd_header_v2)) { + huc_err(huc, "invalid CPD header length %u!\n", + header->header_length); + return -EINVAL; + } + + min_size = header->header_length + sizeof(*entry) * header->num_of_entries; + if (size < min_size) { + huc_err(huc, "FW too small! %zu < %zu\n", size, min_size); + return -ENODATA; + } + + entry = data + header->header_length; + + for (i = 0; i < header->num_of_entries; i++, entry++) { + if (strcmp(entry->name, "HUCP.man") == 0) + get_version_from_gsc_manifest(&huc_fw->file_selected.ver, + data + entry_offset(entry)); + + if (strcmp(entry->name, "huc_fw") == 0) { + u32 offset = entry_offset(entry); + + if (offset < size && css_valid(data + offset, size - offset)) + huc_fw->dma_start_offset = offset; + } + } + + return 0; +} int intel_huc_fw_load_and_auth_via_gsc(struct intel_huc *huc) { @@ -25,7 +256,7 @@ int intel_huc_fw_load_and_auth_via_gsc(struct intel_huc *huc) * component gets re-bound and this function called again. If so, just * mark the HuC as loaded. */ - if (intel_huc_is_authenticated(huc)) { + if (intel_huc_is_authenticated(huc, INTEL_HUC_AUTH_BY_GSC)) { intel_uc_fw_change_status(&huc->fw, INTEL_UC_FIRMWARE_RUNNING); return 0; } @@ -38,7 +269,7 @@ int intel_huc_fw_load_and_auth_via_gsc(struct intel_huc *huc) intel_uc_fw_change_status(&huc->fw, INTEL_UC_FIRMWARE_TRANSFERRED); - return intel_huc_wait_for_auth_complete(huc); + return intel_huc_wait_for_auth_complete(huc, INTEL_HUC_AUTH_BY_GSC); } /** diff --git a/drivers/gpu/drm/i915/gt/uc/intel_huc_fw.h b/drivers/gpu/drm/i915/gt/uc/intel_huc_fw.h index db42e238b45f..307ab45e6b09 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_huc_fw.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_huc_fw.h @@ -7,8 +7,12 @@ #define _INTEL_HUC_FW_H_ struct intel_huc; +struct intel_uc_fw; + +#include <linux/types.h> int intel_huc_fw_load_and_auth_via_gsc(struct intel_huc *huc); +int intel_huc_fw_auth_via_gsccs(struct intel_huc *huc); int intel_huc_fw_upload(struct intel_huc *huc); - +int intel_huc_fw_get_binary_info(struct intel_uc_fw *huc_fw, const void *data, size_t size); #endif diff --git a/drivers/gpu/drm/i915/gt/uc/intel_huc_print.h b/drivers/gpu/drm/i915/gt/uc/intel_huc_print.h new file mode 100644 index 000000000000..915d310ee1df --- /dev/null +++ b/drivers/gpu/drm/i915/gt/uc/intel_huc_print.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef __INTEL_HUC_PRINT__ +#define __INTEL_HUC_PRINT__ + +#include "gt/intel_gt.h" +#include "gt/intel_gt_print.h" + +#define huc_printk(_huc, _level, _fmt, ...) \ + gt_##_level(huc_to_gt(_huc), "HuC: " _fmt, ##__VA_ARGS__) +#define huc_err(_huc, _fmt, ...) huc_printk((_huc), err, _fmt, ##__VA_ARGS__) +#define huc_warn(_huc, _fmt, ...) huc_printk((_huc), warn, _fmt, ##__VA_ARGS__) +#define huc_notice(_huc, _fmt, ...) huc_printk((_huc), notice, _fmt, ##__VA_ARGS__) +#define huc_info(_huc, _fmt, ...) huc_printk((_huc), info, _fmt, ##__VA_ARGS__) +#define huc_dbg(_huc, _fmt, ...) huc_printk((_huc), dbg, _fmt, ##__VA_ARGS__) +#define huc_probe_error(_huc, _fmt, ...) huc_printk((_huc), probe_error, _fmt, ##__VA_ARGS__) + +#endif /* __INTEL_HUC_PRINT__ */ diff --git a/drivers/gpu/drm/i915/gt/uc/intel_uc.c b/drivers/gpu/drm/i915/gt/uc/intel_uc.c index 4ccb4be4c9cb..18250fb64bd8 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_uc.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_uc.c @@ -18,6 +18,7 @@ #include "intel_uc.h" #include "i915_drv.h" +#include "i915_hwmon.h" static const struct intel_uc_ops uc_ops_off; static const struct intel_uc_ops uc_ops_on; @@ -431,6 +432,9 @@ static bool uc_is_wopcm_locked(struct intel_uc *uc) static int __uc_check_hw(struct intel_uc *uc) { + if (uc->fw_table_invalid) + return -EIO; + if (!intel_uc_supports_guc(uc)) return 0; @@ -461,6 +465,7 @@ static int __uc_init_hw(struct intel_uc *uc) struct intel_guc *guc = &uc->guc; struct intel_huc *huc = &uc->huc; int ret, attempts; + bool pl1en = false; GEM_BUG_ON(!intel_uc_supports_guc(uc)); GEM_BUG_ON(!intel_uc_wants_guc(uc)); @@ -491,6 +496,9 @@ static int __uc_init_hw(struct intel_uc *uc) else attempts = 1; + /* Disable a potentially low PL1 power limit to allow freq to be raised */ + i915_hwmon_power_max_disable(gt->i915, &pl1en); + intel_rps_raise_unslice(&uc_to_gt(uc)->rps); while (attempts--) { @@ -500,7 +508,7 @@ static int __uc_init_hw(struct intel_uc *uc) */ ret = __uc_sanitize(uc); if (ret) - goto err_out; + goto err_rps; intel_huc_fw_upload(huc); intel_guc_ads_reset(guc); @@ -530,7 +538,7 @@ static int __uc_init_hw(struct intel_uc *uc) if (intel_huc_is_loaded_by_gsc(huc)) intel_huc_update_auth_status(huc); else - intel_huc_auth(huc); + intel_huc_auth(huc, INTEL_HUC_AUTH_BY_GUC); if (intel_uc_uses_guc_submission(uc)) { ret = intel_guc_submission_enable(guc); @@ -547,6 +555,8 @@ static int __uc_init_hw(struct intel_uc *uc) intel_rps_lower_unslice(&uc_to_gt(uc)->rps); } + i915_hwmon_power_max_restore(gt->i915, pl1en); + guc_info(guc, "submission %s\n", str_enabled_disabled(intel_uc_uses_guc_submission(uc))); guc_info(guc, "SLPC %s\n", str_enabled_disabled(intel_uc_uses_guc_slpc(uc))); @@ -559,10 +569,12 @@ err_submission: intel_guc_submission_disable(guc); err_log_capture: __uc_capture_load_err_log(uc); -err_out: +err_rps: /* Return GT back to RPn */ intel_rps_lower_unslice(&uc_to_gt(uc)->rps); + i915_hwmon_power_max_restore(gt->i915, pl1en); +err_out: __uc_sanitize(uc); if (!ret) { @@ -688,6 +700,12 @@ void intel_uc_suspend(struct intel_uc *uc) } } +static void __uc_resume_mappings(struct intel_uc *uc) +{ + intel_uc_fw_resume_mapping(&uc->guc.fw); + intel_uc_fw_resume_mapping(&uc->huc.fw); +} + static int __uc_resume(struct intel_uc *uc, bool enable_communication) { struct intel_guc *guc = &uc->guc; @@ -755,4 +773,6 @@ static const struct intel_uc_ops uc_ops_on = { .init_hw = __uc_init_hw, .fini_hw = __uc_fini_hw, + + .resume_mappings = __uc_resume_mappings, }; diff --git a/drivers/gpu/drm/i915/gt/uc/intel_uc.h b/drivers/gpu/drm/i915/gt/uc/intel_uc.h index 5d0f1bcc381e..014bb7d83689 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_uc.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_uc.h @@ -24,6 +24,7 @@ struct intel_uc_ops { void (*fini)(struct intel_uc *uc); int (*init_hw)(struct intel_uc *uc); void (*fini_hw)(struct intel_uc *uc); + void (*resume_mappings)(struct intel_uc *uc); }; struct intel_uc { @@ -36,6 +37,7 @@ struct intel_uc { struct drm_i915_gem_object *load_err_log; bool reset_in_progress; + bool fw_table_invalid; }; void intel_uc_init_early(struct intel_uc *uc); @@ -113,6 +115,7 @@ intel_uc_ops_function(init, init, int, 0); intel_uc_ops_function(fini, fini, void, ); intel_uc_ops_function(init_hw, init_hw, int, 0); intel_uc_ops_function(fini_hw, fini_hw, void, ); +intel_uc_ops_function(resume_mappings, resume_mappings, void, ); #undef intel_uc_ops_function #endif diff --git a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c index c36e68e23a14..944725e62414 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c @@ -17,6 +17,12 @@ #include "i915_drv.h" #include "i915_reg.h" +#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM) +#define UNEXPECTED gt_probe_error +#else +#define UNEXPECTED gt_notice +#endif + static inline struct intel_gt * ____uc_fw_to_gt(struct intel_uc_fw *uc_fw, enum intel_uc_fw_type type) { @@ -79,14 +85,15 @@ void intel_uc_fw_change_status(struct intel_uc_fw *uc_fw, * security fixes, etc. to be enabled. */ #define INTEL_GUC_FIRMWARE_DEFS(fw_def, guc_maj, guc_mmp) \ - fw_def(DG2, 0, guc_maj(dg2, 70, 5)) \ - fw_def(ALDERLAKE_P, 0, guc_maj(adlp, 70, 5)) \ + fw_def(METEORLAKE, 0, guc_maj(mtl, 70, 6, 6)) \ + fw_def(DG2, 0, guc_maj(dg2, 70, 5, 1)) \ + fw_def(ALDERLAKE_P, 0, guc_maj(adlp, 70, 5, 1)) \ fw_def(ALDERLAKE_P, 0, guc_mmp(adlp, 70, 1, 1)) \ fw_def(ALDERLAKE_P, 0, guc_mmp(adlp, 69, 0, 3)) \ - fw_def(ALDERLAKE_S, 0, guc_maj(tgl, 70, 5)) \ + fw_def(ALDERLAKE_S, 0, guc_maj(tgl, 70, 5, 1)) \ fw_def(ALDERLAKE_S, 0, guc_mmp(tgl, 70, 1, 1)) \ fw_def(ALDERLAKE_S, 0, guc_mmp(tgl, 69, 0, 3)) \ - fw_def(DG1, 0, guc_maj(dg1, 70, 5)) \ + fw_def(DG1, 0, guc_maj(dg1, 70, 5, 1)) \ fw_def(ROCKETLAKE, 0, guc_mmp(tgl, 70, 1, 1)) \ fw_def(TIGERLAKE, 0, guc_mmp(tgl, 70, 1, 1)) \ fw_def(JASPERLAKE, 0, guc_mmp(ehl, 70, 1, 1)) \ @@ -101,6 +108,7 @@ void intel_uc_fw_change_status(struct intel_uc_fw *uc_fw, fw_def(SKYLAKE, 0, guc_mmp(skl, 70, 1, 1)) #define INTEL_HUC_FIRMWARE_DEFS(fw_def, huc_raw, huc_mmp, huc_gsc) \ + fw_def(METEORLAKE, 0, huc_gsc(mtl)) \ fw_def(DG2, 0, huc_gsc(dg2)) \ fw_def(ALDERLAKE_P, 0, huc_raw(tgl)) \ fw_def(ALDERLAKE_P, 0, huc_mmp(tgl, 7, 9, 3)) \ @@ -140,7 +148,7 @@ void intel_uc_fw_change_status(struct intel_uc_fw *uc_fw, __stringify(patch_) ".bin" /* Minor for internal driver use, not part of file name */ -#define MAKE_GUC_FW_PATH_MAJOR(prefix_, major_, minor_) \ +#define MAKE_GUC_FW_PATH_MAJOR(prefix_, major_, minor_, patch_) \ __MAKE_UC_FW_PATH_MAJOR(prefix_, "guc", major_) #define MAKE_GUC_FW_PATH_MMP(prefix_, major_, minor_, patch_) \ @@ -179,7 +187,7 @@ struct __packed uc_fw_blob { u8 major; u8 minor; u8 patch; - bool loaded_via_gsc; + bool has_gsc_headers; }; #define UC_FW_BLOB_BASE(major_, minor_, patch_, path_) \ @@ -190,15 +198,15 @@ struct __packed uc_fw_blob { #define UC_FW_BLOB_NEW(major_, minor_, patch_, gsc_, path_) \ { UC_FW_BLOB_BASE(major_, minor_, patch_, path_) \ - .legacy = false, .loaded_via_gsc = gsc_ } + .legacy = false, .has_gsc_headers = gsc_ } #define UC_FW_BLOB_OLD(major_, minor_, patch_, path_) \ { UC_FW_BLOB_BASE(major_, minor_, patch_, path_) \ .legacy = true } -#define GUC_FW_BLOB(prefix_, major_, minor_) \ - UC_FW_BLOB_NEW(major_, minor_, 0, false, \ - MAKE_GUC_FW_PATH_MAJOR(prefix_, major_, minor_)) +#define GUC_FW_BLOB(prefix_, major_, minor_, patch_) \ + UC_FW_BLOB_NEW(major_, minor_, patch_, false, \ + MAKE_GUC_FW_PATH_MAJOR(prefix_, major_, minor_, patch_)) #define GUC_FW_BLOB_MMP(prefix_, major_, minor_, patch_) \ UC_FW_BLOB_OLD(major_, minor_, patch_, \ @@ -232,20 +240,22 @@ struct fw_blobs_by_type { u32 count; }; +static const struct uc_fw_platform_requirement blobs_guc[] = { + INTEL_GUC_FIRMWARE_DEFS(MAKE_FW_LIST, GUC_FW_BLOB, GUC_FW_BLOB_MMP) +}; + +static const struct uc_fw_platform_requirement blobs_huc[] = { + INTEL_HUC_FIRMWARE_DEFS(MAKE_FW_LIST, HUC_FW_BLOB, HUC_FW_BLOB_MMP, HUC_FW_BLOB_GSC) +}; + +static const struct fw_blobs_by_type blobs_all[INTEL_UC_FW_NUM_TYPES] = { + [INTEL_UC_FW_TYPE_GUC] = { blobs_guc, ARRAY_SIZE(blobs_guc) }, + [INTEL_UC_FW_TYPE_HUC] = { blobs_huc, ARRAY_SIZE(blobs_huc) }, +}; + static void __uc_fw_auto_select(struct drm_i915_private *i915, struct intel_uc_fw *uc_fw) { - static const struct uc_fw_platform_requirement blobs_guc[] = { - INTEL_GUC_FIRMWARE_DEFS(MAKE_FW_LIST, GUC_FW_BLOB, GUC_FW_BLOB_MMP) - }; - static const struct uc_fw_platform_requirement blobs_huc[] = { - INTEL_HUC_FIRMWARE_DEFS(MAKE_FW_LIST, HUC_FW_BLOB, HUC_FW_BLOB_MMP, HUC_FW_BLOB_GSC) - }; - static const struct fw_blobs_by_type blobs_all[INTEL_UC_FW_NUM_TYPES] = { - [INTEL_UC_FW_TYPE_GUC] = { blobs_guc, ARRAY_SIZE(blobs_guc) }, - [INTEL_UC_FW_TYPE_HUC] = { blobs_huc, ARRAY_SIZE(blobs_huc) }, - }; - static bool verified[INTEL_UC_FW_NUM_TYPES]; const struct uc_fw_platform_requirement *fw_blobs; enum intel_platform p = INTEL_INFO(i915)->platform; u32 fw_count; @@ -285,6 +295,11 @@ __uc_fw_auto_select(struct drm_i915_private *i915, struct intel_uc_fw *uc_fw) continue; if (uc_fw->file_selected.path) { + /* + * Continuing an earlier search after a found blob failed to load. + * Once the previously chosen path has been found, clear it out + * and let the search continue from there. + */ if (uc_fw->file_selected.path == blob->path) uc_fw->file_selected.path = NULL; @@ -295,7 +310,8 @@ __uc_fw_auto_select(struct drm_i915_private *i915, struct intel_uc_fw *uc_fw) uc_fw->file_wanted.path = blob->path; uc_fw->file_wanted.ver.major = blob->major; uc_fw->file_wanted.ver.minor = blob->minor; - uc_fw->loaded_via_gsc = blob->loaded_via_gsc; + uc_fw->file_wanted.ver.patch = blob->patch; + uc_fw->has_gsc_headers = blob->has_gsc_headers; found = true; break; } @@ -304,76 +320,111 @@ __uc_fw_auto_select(struct drm_i915_private *i915, struct intel_uc_fw *uc_fw) /* Failed to find a match for the last attempt?! */ uc_fw->file_selected.path = NULL; } +} - /* make sure the list is ordered as expected */ - if (IS_ENABLED(CONFIG_DRM_I915_SELFTEST) && !verified[uc_fw->type]) { - verified[uc_fw->type] = true; +static bool validate_fw_table_type(struct drm_i915_private *i915, enum intel_uc_fw_type type) +{ + const struct uc_fw_platform_requirement *fw_blobs; + u32 fw_count; + int i, j; + + if (type >= ARRAY_SIZE(blobs_all)) { + drm_err(&i915->drm, "No blob array for %s\n", intel_uc_fw_type_repr(type)); + return false; + } + + fw_blobs = blobs_all[type].blobs; + fw_count = blobs_all[type].count; - for (i = 1; i < fw_count; i++) { - /* Next platform is good: */ - if (fw_blobs[i].p < fw_blobs[i - 1].p) + if (!fw_count) + return true; + + /* make sure the list is ordered as expected */ + for (i = 1; i < fw_count; i++) { + /* Versionless file names must be unique per platform: */ + for (j = i + 1; j < fw_count; j++) { + /* Same platform? */ + if (fw_blobs[i].p != fw_blobs[j].p) continue; - /* Next platform revision is good: */ - if (fw_blobs[i].p == fw_blobs[i - 1].p && - fw_blobs[i].rev < fw_blobs[i - 1].rev) + if (fw_blobs[i].blob.path != fw_blobs[j].blob.path) continue; - /* Platform/revision must be in order: */ - if (fw_blobs[i].p != fw_blobs[i - 1].p || - fw_blobs[i].rev != fw_blobs[i - 1].rev) - goto bad; + drm_err(&i915->drm, "Duplicate %s blobs: %s r%u %s%d.%d.%d [%s] matches %s%d.%d.%d [%s]\n", + intel_uc_fw_type_repr(type), + intel_platform_name(fw_blobs[j].p), fw_blobs[j].rev, + fw_blobs[j].blob.legacy ? "L" : "v", + fw_blobs[j].blob.major, fw_blobs[j].blob.minor, + fw_blobs[j].blob.patch, fw_blobs[j].blob.path, + fw_blobs[i].blob.legacy ? "L" : "v", + fw_blobs[i].blob.major, fw_blobs[i].blob.minor, + fw_blobs[i].blob.patch, fw_blobs[i].blob.path); + } - /* Next major version is good: */ - if (fw_blobs[i].blob.major < fw_blobs[i - 1].blob.major) - continue; + /* Next platform is good: */ + if (fw_blobs[i].p < fw_blobs[i - 1].p) + continue; - /* New must be before legacy: */ - if (!fw_blobs[i].blob.legacy && fw_blobs[i - 1].blob.legacy) - goto bad; + /* Next platform revision is good: */ + if (fw_blobs[i].p == fw_blobs[i - 1].p && + fw_blobs[i].rev < fw_blobs[i - 1].rev) + continue; - /* New to legacy also means 0.0 to X.Y (HuC), or X.0 to X.Y (GuC) */ - if (fw_blobs[i].blob.legacy && !fw_blobs[i - 1].blob.legacy) { - if (!fw_blobs[i - 1].blob.major) - continue; + /* Platform/revision must be in order: */ + if (fw_blobs[i].p != fw_blobs[i - 1].p || + fw_blobs[i].rev != fw_blobs[i - 1].rev) + goto bad; - if (fw_blobs[i].blob.major == fw_blobs[i - 1].blob.major) - continue; - } + /* Next major version is good: */ + if (fw_blobs[i].blob.major < fw_blobs[i - 1].blob.major) + continue; - /* Major versions must be in order: */ - if (fw_blobs[i].blob.major != fw_blobs[i - 1].blob.major) - goto bad; + /* New must be before legacy: */ + if (!fw_blobs[i].blob.legacy && fw_blobs[i - 1].blob.legacy) + goto bad; - /* Next minor version is good: */ - if (fw_blobs[i].blob.minor < fw_blobs[i - 1].blob.minor) + /* New to legacy also means 0.0 to X.Y (HuC), or X.0 to X.Y (GuC) */ + if (fw_blobs[i].blob.legacy && !fw_blobs[i - 1].blob.legacy) { + if (!fw_blobs[i - 1].blob.major) continue; - /* Minor versions must be in order: */ - if (fw_blobs[i].blob.minor != fw_blobs[i - 1].blob.minor) - goto bad; - - /* Patch versions must be in order: */ - if (fw_blobs[i].blob.patch <= fw_blobs[i - 1].blob.patch) + if (fw_blobs[i].blob.major == fw_blobs[i - 1].blob.major) continue; + } -bad: - drm_err(&i915->drm, "Invalid %s blob order: %s r%u %s%d.%d.%d comes before %s r%u %s%d.%d.%d\n", - intel_uc_fw_type_repr(uc_fw->type), - intel_platform_name(fw_blobs[i - 1].p), fw_blobs[i - 1].rev, - fw_blobs[i - 1].blob.legacy ? "L" : "v", - fw_blobs[i - 1].blob.major, - fw_blobs[i - 1].blob.minor, - fw_blobs[i - 1].blob.patch, - intel_platform_name(fw_blobs[i].p), fw_blobs[i].rev, - fw_blobs[i].blob.legacy ? "L" : "v", - fw_blobs[i].blob.major, - fw_blobs[i].blob.minor, - fw_blobs[i].blob.patch); + /* Major versions must be in order: */ + if (fw_blobs[i].blob.major != fw_blobs[i - 1].blob.major) + goto bad; - uc_fw->file_selected.path = NULL; - } + /* Next minor version is good: */ + if (fw_blobs[i].blob.minor < fw_blobs[i - 1].blob.minor) + continue; + + /* Minor versions must be in order: */ + if (fw_blobs[i].blob.minor != fw_blobs[i - 1].blob.minor) + goto bad; + + /* Patch versions must be in order and unique: */ + if (fw_blobs[i].blob.patch < fw_blobs[i - 1].blob.patch) + continue; + +bad: + drm_err(&i915->drm, "Invalid %s blob order: %s r%u %s%d.%d.%d comes before %s r%u %s%d.%d.%d\n", + intel_uc_fw_type_repr(type), + intel_platform_name(fw_blobs[i - 1].p), fw_blobs[i - 1].rev, + fw_blobs[i - 1].blob.legacy ? "L" : "v", + fw_blobs[i - 1].blob.major, + fw_blobs[i - 1].blob.minor, + fw_blobs[i - 1].blob.patch, + intel_platform_name(fw_blobs[i].p), fw_blobs[i].rev, + fw_blobs[i].blob.legacy ? "L" : "v", + fw_blobs[i].blob.major, + fw_blobs[i].blob.minor, + fw_blobs[i].blob.patch); + return false; } + + return true; } static const char *__override_guc_firmware_path(struct drm_i915_private *i915) @@ -421,14 +472,17 @@ static void __uc_fw_user_override(struct drm_i915_private *i915, struct intel_uc * intel_uc_fw_init_early - initialize the uC object and select the firmware * @uc_fw: uC firmware * @type: type of uC + * @needs_ggtt_mapping: whether the FW needs to be GGTT mapped for loading * * Initialize the state of our uC object and relevant tracking and select the * firmware to fetch and load. */ void intel_uc_fw_init_early(struct intel_uc_fw *uc_fw, - enum intel_uc_fw_type type) + enum intel_uc_fw_type type, + bool needs_ggtt_mapping) { - struct drm_i915_private *i915 = ____uc_fw_to_gt(uc_fw, type)->i915; + struct intel_gt *gt = ____uc_fw_to_gt(uc_fw, type); + struct drm_i915_private *i915 = gt->i915; /* * we use FIRMWARE_UNINITIALIZED to detect checks against uc_fw->status @@ -439,8 +493,15 @@ void intel_uc_fw_init_early(struct intel_uc_fw *uc_fw, GEM_BUG_ON(uc_fw->file_selected.path); uc_fw->type = type; + uc_fw->needs_ggtt_mapping = needs_ggtt_mapping; if (HAS_GT_UC(i915)) { + if (!validate_fw_table_type(i915, type)) { + gt->uc.fw_table_invalid = true; + intel_uc_fw_change_status(uc_fw, INTEL_UC_FIRMWARE_NOT_SUPPORTED); + return; + } + __uc_fw_auto_select(i915, uc_fw); __uc_fw_user_override(i915, uc_fw); } @@ -488,33 +549,6 @@ static void __force_fw_fetch_failures(struct intel_uc_fw *uc_fw, int e) } } -static int check_gsc_manifest(struct intel_gt *gt, - const struct firmware *fw, - struct intel_uc_fw *uc_fw) -{ - u32 *dw = (u32 *)fw->data; - u32 version_hi, version_lo; - size_t min_size; - - /* Check the size of the blob before examining buffer contents */ - min_size = sizeof(u32) * (HUC_GSC_VERSION_LO_DW + 1); - if (unlikely(fw->size < min_size)) { - gt_warn(gt, "%s firmware %s: invalid size: %zu < %zu\n", - intel_uc_fw_type_repr(uc_fw->type), uc_fw->file_selected.path, - fw->size, min_size); - return -ENODATA; - } - - version_hi = dw[HUC_GSC_VERSION_HI_DW]; - version_lo = dw[HUC_GSC_VERSION_LO_DW]; - - uc_fw->file_selected.ver.major = FIELD_GET(HUC_GSC_MAJOR_VER_HI_MASK, version_hi); - uc_fw->file_selected.ver.minor = FIELD_GET(HUC_GSC_MINOR_VER_HI_MASK, version_hi); - uc_fw->file_selected.ver.patch = FIELD_GET(HUC_GSC_PATCH_VER_LO_MASK, version_lo); - - return 0; -} - static void uc_unpack_css_version(struct intel_uc_fw_ver *ver, u32 css_value) { /* Get version numbers from the CSS header */ @@ -571,22 +605,22 @@ static void guc_read_css_info(struct intel_uc_fw *uc_fw, struct uc_css_header *c uc_fw->private_data_size = css->private_data_size; } -static int check_ccs_header(struct intel_gt *gt, - const struct firmware *fw, - struct intel_uc_fw *uc_fw) +static int __check_ccs_header(struct intel_gt *gt, + const void *fw_data, size_t fw_size, + struct intel_uc_fw *uc_fw) { struct uc_css_header *css; size_t size; /* Check the size of the blob before examining buffer contents */ - if (unlikely(fw->size < sizeof(struct uc_css_header))) { + if (unlikely(fw_size < sizeof(struct uc_css_header))) { gt_warn(gt, "%s firmware %s: invalid size: %zu < %zu\n", intel_uc_fw_type_repr(uc_fw->type), uc_fw->file_selected.path, - fw->size, sizeof(struct uc_css_header)); + fw_size, sizeof(struct uc_css_header)); return -ENODATA; } - css = (struct uc_css_header *)fw->data; + css = (struct uc_css_header *)fw_data; /* Check integrity of size values inside CSS header */ size = (css->header_size_dw - css->key_size_dw - css->modulus_size_dw - @@ -594,7 +628,7 @@ static int check_ccs_header(struct intel_gt *gt, if (unlikely(size != sizeof(struct uc_css_header))) { gt_warn(gt, "%s firmware %s: unexpected header size: %zu != %zu\n", intel_uc_fw_type_repr(uc_fw->type), uc_fw->file_selected.path, - fw->size, sizeof(struct uc_css_header)); + fw_size, sizeof(struct uc_css_header)); return -EPROTO; } @@ -606,10 +640,10 @@ static int check_ccs_header(struct intel_gt *gt, /* At least, it should have header, uCode and RSA. Size of all three. */ size = sizeof(struct uc_css_header) + uc_fw->ucode_size + uc_fw->rsa_size; - if (unlikely(fw->size < size)) { + if (unlikely(fw_size < size)) { gt_warn(gt, "%s firmware %s: invalid size: %zu < %zu\n", intel_uc_fw_type_repr(uc_fw->type), uc_fw->file_selected.path, - fw->size, size); + fw_size, size); return -ENOEXEC; } @@ -630,6 +664,33 @@ static int check_ccs_header(struct intel_gt *gt, return 0; } +static int check_gsc_manifest(struct intel_gt *gt, + const struct firmware *fw, + struct intel_uc_fw *uc_fw) +{ + if (uc_fw->type != INTEL_UC_FW_TYPE_HUC) { + gt_err(gt, "trying to GSC-parse a non-HuC binary"); + return -EINVAL; + } + + intel_huc_fw_get_binary_info(uc_fw, fw->data, fw->size); + + if (uc_fw->dma_start_offset) { + u32 delta = uc_fw->dma_start_offset; + + __check_ccs_header(gt, fw->data + delta, fw->size - delta, uc_fw); + } + + return 0; +} + +static int check_ccs_header(struct intel_gt *gt, + const struct firmware *fw, + struct intel_uc_fw *uc_fw) +{ + return __check_ccs_header(gt, fw->data, fw->size, uc_fw); +} + static bool is_ver_8bit(struct intel_uc_fw_ver *ver) { return ver->major < 0xFF && ver->minor < 0xFF && ver->patch < 0xFF; @@ -677,7 +738,7 @@ static int check_fw_header(struct intel_gt *gt, if (uc_fw->type == INTEL_UC_FW_TYPE_GSC) return 0; - if (uc_fw->loaded_via_gsc) + if (uc_fw->has_gsc_headers) err = check_gsc_manifest(gt, fw, uc_fw); else err = check_ccs_header(gt, fw, uc_fw); @@ -698,7 +759,7 @@ static int try_firmware_load(struct intel_uc_fw *uc_fw, const struct firmware ** if (err) return err; - if ((*fw)->size > INTEL_UC_RSVD_GGTT_PER_FW) { + if (uc_fw->needs_ggtt_mapping && (*fw)->size > INTEL_UC_RSVD_GGTT_PER_FW) { gt_err(gt, "%s firmware %s: size (%zuKB) exceeds max supported size (%uKB)\n", intel_uc_fw_type_repr(uc_fw->type), uc_fw->file_selected.path, (*fw)->size / SZ_1K, INTEL_UC_RSVD_GGTT_PER_FW / SZ_1K); @@ -782,10 +843,10 @@ int intel_uc_fw_fetch(struct intel_uc_fw *uc_fw) if (uc_fw->file_wanted.ver.major && uc_fw->file_selected.ver.major) { /* Check the file's major version was as it claimed */ if (uc_fw->file_selected.ver.major != uc_fw->file_wanted.ver.major) { - gt_notice(gt, "%s firmware %s: unexpected version: %u.%u != %u.%u\n", - intel_uc_fw_type_repr(uc_fw->type), uc_fw->file_selected.path, - uc_fw->file_selected.ver.major, uc_fw->file_selected.ver.minor, - uc_fw->file_wanted.ver.major, uc_fw->file_wanted.ver.minor); + UNEXPECTED(gt, "%s firmware %s: unexpected version: %u.%u != %u.%u\n", + intel_uc_fw_type_repr(uc_fw->type), uc_fw->file_selected.path, + uc_fw->file_selected.ver.major, uc_fw->file_selected.ver.minor, + uc_fw->file_wanted.ver.major, uc_fw->file_wanted.ver.minor); if (!intel_uc_fw_is_overridden(uc_fw)) { err = -ENOEXEC; goto fail; @@ -793,6 +854,9 @@ int intel_uc_fw_fetch(struct intel_uc_fw *uc_fw) } else { if (uc_fw->file_selected.ver.minor < uc_fw->file_wanted.ver.minor) old_ver = true; + else if ((uc_fw->file_selected.ver.minor == uc_fw->file_wanted.ver.minor) && + (uc_fw->file_selected.ver.patch < uc_fw->file_wanted.ver.patch)) + old_ver = true; } } @@ -800,12 +864,16 @@ int intel_uc_fw_fetch(struct intel_uc_fw *uc_fw) /* Preserve the version that was really wanted */ memcpy(&uc_fw->file_wanted, &file_ideal, sizeof(uc_fw->file_wanted)); - gt_notice(gt, "%s firmware %s (%d.%d) is recommended, but only %s (%d.%d) was found\n", - intel_uc_fw_type_repr(uc_fw->type), - uc_fw->file_wanted.path, - uc_fw->file_wanted.ver.major, uc_fw->file_wanted.ver.minor, - uc_fw->file_selected.path, - uc_fw->file_selected.ver.major, uc_fw->file_selected.ver.minor); + UNEXPECTED(gt, "%s firmware %s (%d.%d.%d) is recommended, but only %s (%d.%d.%d) was found\n", + intel_uc_fw_type_repr(uc_fw->type), + uc_fw->file_wanted.path, + uc_fw->file_wanted.ver.major, + uc_fw->file_wanted.ver.minor, + uc_fw->file_wanted.ver.patch, + uc_fw->file_selected.path, + uc_fw->file_selected.ver.major, + uc_fw->file_selected.ver.minor, + uc_fw->file_selected.ver.patch); gt_info(gt, "Consider updating your linux-firmware pkg or downloading from %s\n", INTEL_UC_FIRMWARE_URL); } @@ -876,35 +944,46 @@ static void uc_fw_bind_ggtt(struct intel_uc_fw *uc_fw) { struct drm_i915_gem_object *obj = uc_fw->obj; struct i915_ggtt *ggtt = __uc_fw_to_gt(uc_fw)->ggtt; - struct i915_vma_resource *dummy = &uc_fw->dummy; + struct i915_vma_resource *vma_res = &uc_fw->vma_res; u32 pte_flags = 0; - dummy->start = uc_fw_ggtt_offset(uc_fw); - dummy->node_size = obj->base.size; - dummy->bi.pages = obj->mm.pages; + if (!uc_fw->needs_ggtt_mapping) + return; + + vma_res->start = uc_fw_ggtt_offset(uc_fw); + vma_res->node_size = obj->base.size; + vma_res->bi.pages = obj->mm.pages; GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); /* uc_fw->obj cache domains were not controlled across suspend */ if (i915_gem_object_has_struct_page(obj)) - drm_clflush_sg(dummy->bi.pages); + drm_clflush_sg(vma_res->bi.pages); if (i915_gem_object_is_lmem(obj)) pte_flags |= PTE_LM; if (ggtt->vm.raw_insert_entries) - ggtt->vm.raw_insert_entries(&ggtt->vm, dummy, I915_CACHE_NONE, pte_flags); + ggtt->vm.raw_insert_entries(&ggtt->vm, vma_res, + i915_gem_get_pat_index(ggtt->vm.i915, + I915_CACHE_NONE), + pte_flags); else - ggtt->vm.insert_entries(&ggtt->vm, dummy, I915_CACHE_NONE, pte_flags); + ggtt->vm.insert_entries(&ggtt->vm, vma_res, + i915_gem_get_pat_index(ggtt->vm.i915, + I915_CACHE_NONE), + pte_flags); } static void uc_fw_unbind_ggtt(struct intel_uc_fw *uc_fw) { - struct drm_i915_gem_object *obj = uc_fw->obj; struct i915_ggtt *ggtt = __uc_fw_to_gt(uc_fw)->ggtt; - u64 start = uc_fw_ggtt_offset(uc_fw); + struct i915_vma_resource *vma_res = &uc_fw->vma_res; - ggtt->vm.clear_range(&ggtt->vm, start, obj->base.size); + if (!vma_res->node_size) + return; + + ggtt->vm.clear_range(&ggtt->vm, vma_res->start, vma_res->node_size); } static int uc_fw_xfer(struct intel_uc_fw *uc_fw, u32 dst_offset, u32 dma_flags) @@ -921,7 +1000,7 @@ static int uc_fw_xfer(struct intel_uc_fw *uc_fw, u32 dst_offset, u32 dma_flags) intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL); /* Set the source address for the uCode */ - offset = uc_fw_ggtt_offset(uc_fw); + offset = uc_fw->vma_res.start + uc_fw->dma_start_offset; GEM_BUG_ON(upper_32_bits(offset) & 0xFFFF0000); intel_uncore_write_fw(uncore, DMA_ADDR_0_LOW, lower_32_bits(offset)); intel_uncore_write_fw(uncore, DMA_ADDR_0_HIGH, upper_32_bits(offset)); @@ -995,9 +1074,7 @@ int intel_uc_fw_upload(struct intel_uc_fw *uc_fw, u32 dst_offset, u32 dma_flags) return -ENOEXEC; /* Call custom loader */ - uc_fw_bind_ggtt(uc_fw); err = uc_fw_xfer(uc_fw, dst_offset, dma_flags); - uc_fw_unbind_ggtt(uc_fw); if (err) goto fail; @@ -1101,6 +1178,8 @@ int intel_uc_fw_init(struct intel_uc_fw *uc_fw) goto out_unpin; } + uc_fw_bind_ggtt(uc_fw); + return 0; out_unpin: @@ -1111,6 +1190,7 @@ out: void intel_uc_fw_fini(struct intel_uc_fw *uc_fw) { + uc_fw_unbind_ggtt(uc_fw); uc_fw_rsa_data_destroy(uc_fw); if (i915_gem_object_has_pinned_pages(uc_fw->obj)) @@ -1119,6 +1199,17 @@ void intel_uc_fw_fini(struct intel_uc_fw *uc_fw) intel_uc_fw_change_status(uc_fw, INTEL_UC_FIRMWARE_AVAILABLE); } +void intel_uc_fw_resume_mapping(struct intel_uc_fw *uc_fw) +{ + if (!intel_uc_fw_is_available(uc_fw)) + return; + + if (!i915_gem_object_has_pinned_pages(uc_fw->obj)) + return; + + uc_fw_bind_ggtt(uc_fw); +} + /** * intel_uc_fw_cleanup_fetch - cleanup uC firmware * @uc_fw: uC firmware @@ -1148,7 +1239,7 @@ size_t intel_uc_fw_copy_rsa(struct intel_uc_fw *uc_fw, void *dst, u32 max_len) { struct intel_memory_region *mr = uc_fw->obj->mm.region; u32 size = min_t(u32, uc_fw->rsa_size, max_len); - u32 offset = sizeof(struct uc_css_header) + uc_fw->ucode_size; + u32 offset = uc_fw->dma_start_offset + sizeof(struct uc_css_header) + uc_fw->ucode_size; struct sgt_iter iter; size_t count = 0; int idx; diff --git a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.h b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.h index 6ba00e6b3975..054f02811971 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.h @@ -99,20 +99,28 @@ struct intel_uc_fw { struct drm_i915_gem_object *obj; /** - * @dummy: A vma used in binding the uc fw to ggtt. We can't define this - * vma on the stack as it can lead to a stack overflow, so we define it - * here. Safe to have 1 copy per uc fw because the binding is single - * threaded as it done during driver load (inherently single threaded) - * or during a GT reset (mutex guarantees single threaded). + * @needs_ggtt_mapping: indicates whether the fw object needs to be + * pinned to ggtt. If true, the fw is pinned at init time and unpinned + * during driver unload. */ - struct i915_vma_resource dummy; + bool needs_ggtt_mapping; + + /** + * @vma_res: A vma resource used in binding the uc fw to ggtt. The fw is + * pinned in a reserved area of the ggtt (above the maximum address + * usable by GuC); therefore, we can't use the normal vma functions to + * do the pinning and we instead use this resource to do so. + */ + struct i915_vma_resource vma_res; struct i915_vma *rsa_data; u32 rsa_size; u32 ucode_size; u32 private_data_size; - bool loaded_via_gsc; + u32 dma_start_offset; + + bool has_gsc_headers; }; /* @@ -282,12 +290,14 @@ static inline u32 intel_uc_fw_get_upload_size(struct intel_uc_fw *uc_fw) } void intel_uc_fw_init_early(struct intel_uc_fw *uc_fw, - enum intel_uc_fw_type type); + enum intel_uc_fw_type type, + bool needs_ggtt_mapping); int intel_uc_fw_fetch(struct intel_uc_fw *uc_fw); void intel_uc_fw_cleanup_fetch(struct intel_uc_fw *uc_fw); int intel_uc_fw_upload(struct intel_uc_fw *uc_fw, u32 offset, u32 dma_flags); int intel_uc_fw_init(struct intel_uc_fw *uc_fw); void intel_uc_fw_fini(struct intel_uc_fw *uc_fw); +void intel_uc_fw_resume_mapping(struct intel_uc_fw *uc_fw); size_t intel_uc_fw_copy_rsa(struct intel_uc_fw *uc_fw, void *dst, u32 max_len); int intel_uc_fw_mark_load_failed(struct intel_uc_fw *uc_fw, int err); void intel_uc_fw_dump(const struct intel_uc_fw *uc_fw, struct drm_printer *p); diff --git a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw_abi.h b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw_abi.h index 646fa8aa6cf1..7fe405126249 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw_abi.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw_abi.h @@ -84,10 +84,4 @@ struct uc_css_header { } __packed; static_assert(sizeof(struct uc_css_header) == 128); -#define HUC_GSC_VERSION_HI_DW 44 -#define HUC_GSC_MAJOR_VER_HI_MASK (0xFF << 0) -#define HUC_GSC_MINOR_VER_HI_MASK (0xFF << 16) -#define HUC_GSC_VERSION_LO_DW 45 -#define HUC_GSC_PATCH_VER_LO_MASK (0xFF << 0) - #endif /* _INTEL_UC_FW_ABI_H */ |