diff options
Diffstat (limited to 'drivers/gpu/drm/i915/gt/intel_engine_cs.c')
| -rw-r--r-- | drivers/gpu/drm/i915/gt/intel_engine_cs.c | 510 |
1 files changed, 379 insertions, 131 deletions
diff --git a/drivers/gpu/drm/i915/gt/intel_engine_cs.c b/drivers/gpu/drm/i915/gt/intel_engine_cs.c index 1f7188129cd1..b721bbd23356 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_cs.c +++ b/drivers/gpu/drm/i915/gt/intel_engine_cs.c @@ -9,10 +9,13 @@ #include "gem/i915_gem_context.h" #include "gem/i915_gem_internal.h" +#include "gt/intel_gt_print.h" #include "gt/intel_gt_regs.h" #include "i915_cmd_parser.h" #include "i915_drv.h" +#include "i915_irq.h" +#include "i915_reg.h" #include "intel_breadcrumbs.h" #include "intel_context.h" #include "intel_engine.h" @@ -44,7 +47,7 @@ #define GEN9_LR_CONTEXT_RENDER_SIZE (22 * PAGE_SIZE) #define GEN11_LR_CONTEXT_RENDER_SIZE (14 * PAGE_SIZE) -#define GEN8_LR_CONTEXT_OTHER_SIZE ( 2 * PAGE_SIZE) +#define GEN8_LR_CONTEXT_OTHER_SIZE (2 * PAGE_SIZE) #define MAX_MMIO_BASES 3 struct engine_info { @@ -244,6 +247,13 @@ static const struct engine_info intel_engines[] = { { .graphics_ver = 12, .base = GEN12_COMPUTE3_RING_BASE } } }, + [GSC0] = { + .class = OTHER_CLASS, + .instance = OTHER_GSC_INSTANCE, + .mmio_bases = { + { .graphics_ver = 12, .base = MTL_GSC_RING_BASE } + } + }, }; /** @@ -298,7 +308,7 @@ u32 intel_engine_context_size(struct intel_gt *gt, u8 class) /* * There is a discrepancy here between the size reported * by the register and the size of the context layout - * in the docs. Both are described as authorative! + * in the docs. Both are described as authoritative! * * The discrepancy is on the order of a few cachelines, * but the total is under one page (4k), which is our @@ -306,10 +316,9 @@ u32 intel_engine_context_size(struct intel_gt *gt, u8 class) * out in the wash. */ cxt_size = intel_uncore_read(uncore, CXT_SIZE) + 1; - drm_dbg(>->i915->drm, - "graphics_ver = %d CXT_SIZE = %d bytes [0x%08x]\n", - GRAPHICS_VER(gt->i915), cxt_size * 64, - cxt_size - 1); + gt_dbg(gt, "graphics_ver = %d CXT_SIZE = %d bytes [0x%08x]\n", + GRAPHICS_VER(gt->i915), cxt_size * 64, + cxt_size - 1); return round_up(cxt_size * 64, PAGE_SIZE); case 3: case 2: @@ -324,6 +333,7 @@ u32 intel_engine_context_size(struct intel_gt *gt, u8 class) case VIDEO_DECODE_CLASS: case VIDEO_ENHANCEMENT_CLASS: case COPY_ENGINE_CLASS: + case OTHER_CLASS: if (GRAPHICS_VER(gt->i915) < 8) return 0; return GEN8_LR_CONTEXT_OTHER_SIZE; @@ -415,6 +425,7 @@ static u32 get_reset_domain(u8 ver, enum intel_engine_id id) [CCS1] = GEN11_GRDOM_RENDER, [CCS2] = GEN11_GRDOM_RENDER, [CCS3] = GEN11_GRDOM_RENDER, + [GSC0] = GEN12_GRDOM_GSC, }; GEM_BUG_ON(id >= ARRAY_SIZE(engine_reset_domains) || !engine_reset_domains[id]); @@ -486,6 +497,16 @@ static int intel_engine_setup(struct intel_gt *gt, enum intel_engine_id id, engine->logical_mask = BIT(logical_instance); __sprint_engine_name(engine); + if ((engine->class == COMPUTE_CLASS || engine->class == RENDER_CLASS) && + __ffs(CCS_MASK(engine->gt) | RCS_MASK(engine->gt)) == engine->instance) + engine->flags |= I915_ENGINE_FIRST_RENDER_COMPUTE; + + /* features common between engines sharing EUs */ + if (engine->class == RENDER_CLASS || engine->class == COMPUTE_CLASS) { + engine->flags |= I915_ENGINE_HAS_RCS_REG_STATE; + engine->flags |= I915_ENGINE_HAS_EU_PRIORITY; + } + engine->props.heartbeat_interval_ms = CONFIG_DRM_I915_HEARTBEAT_INTERVAL; engine->props.max_busywait_duration_ns = @@ -497,20 +518,34 @@ static int intel_engine_setup(struct intel_gt *gt, enum intel_engine_id id, engine->props.timeslice_duration_ms = CONFIG_DRM_I915_TIMESLICE_DURATION; - /* Override to uninterruptible for OpenCL workloads. */ - if (GRAPHICS_VER(i915) == 12 && engine->class == RENDER_CLASS) - engine->props.preempt_timeout_ms = 0; - - if ((engine->class == COMPUTE_CLASS && !RCS_MASK(engine->gt) && - __ffs(CCS_MASK(engine->gt)) == engine->instance) || - engine->class == RENDER_CLASS) - engine->flags |= I915_ENGINE_FIRST_RENDER_COMPUTE; - - /* features common between engines sharing EUs */ - if (engine->class == RENDER_CLASS || engine->class == COMPUTE_CLASS) { - engine->flags |= I915_ENGINE_HAS_RCS_REG_STATE; - engine->flags |= I915_ENGINE_HAS_EU_PRIORITY; - } + /* + * Mid-thread pre-emption is not available in Gen12. Unfortunately, + * some compute workloads run quite long threads. That means they get + * reset due to not pre-empting in a timely manner. So, bump the + * pre-emption timeout value to be much higher for compute engines. + */ + if (GRAPHICS_VER(i915) == 12 && (engine->flags & I915_ENGINE_HAS_RCS_REG_STATE)) + engine->props.preempt_timeout_ms = CONFIG_DRM_I915_PREEMPT_TIMEOUT_COMPUTE; + + /* Cap properties according to any system limits */ +#define CLAMP_PROP(field) \ + do { \ + u64 clamp = intel_clamp_##field(engine, engine->props.field); \ + if (clamp != engine->props.field) { \ + drm_notice(&engine->i915->drm, \ + "Warning, clamping %s to %lld to prevent overflow\n", \ + #field, clamp); \ + engine->props.field = clamp; \ + } \ + } while (0) + + CLAMP_PROP(heartbeat_interval_ms); + CLAMP_PROP(max_busywait_duration_ns); + CLAMP_PROP(preempt_timeout_ms); + CLAMP_PROP(stop_timeout_ms); + CLAMP_PROP(timeslice_duration_ms); + +#undef CLAMP_PROP engine->defaults = engine->props; /* never to change again */ @@ -521,7 +556,6 @@ static int intel_engine_setup(struct intel_gt *gt, enum intel_engine_id id, DRIVER_CAPS(i915)->has_logical_contexts = true; ewma__engine_latency_init(&engine->latency); - seqcount_init(&engine->stats.execlists.lock); ATOMIC_INIT_NOTIFIER_HEAD(&engine->context_status_notifier); @@ -534,6 +568,55 @@ static int intel_engine_setup(struct intel_gt *gt, enum intel_engine_id id, return 0; } +u64 intel_clamp_heartbeat_interval_ms(struct intel_engine_cs *engine, u64 value) +{ + value = min_t(u64, value, jiffies_to_msecs(MAX_SCHEDULE_TIMEOUT)); + + return value; +} + +u64 intel_clamp_max_busywait_duration_ns(struct intel_engine_cs *engine, u64 value) +{ + value = min(value, jiffies_to_nsecs(2)); + + return value; +} + +u64 intel_clamp_preempt_timeout_ms(struct intel_engine_cs *engine, u64 value) +{ + /* + * NB: The GuC API only supports 32bit values. However, the limit is further + * reduced due to internal calculations which would otherwise overflow. + */ + if (intel_guc_submission_is_wanted(gt_to_guc(engine->gt))) + value = min_t(u64, value, guc_policy_max_preempt_timeout_ms()); + + value = min_t(u64, value, jiffies_to_msecs(MAX_SCHEDULE_TIMEOUT)); + + return value; +} + +u64 intel_clamp_stop_timeout_ms(struct intel_engine_cs *engine, u64 value) +{ + value = min_t(u64, value, jiffies_to_msecs(MAX_SCHEDULE_TIMEOUT)); + + return value; +} + +u64 intel_clamp_timeslice_duration_ms(struct intel_engine_cs *engine, u64 value) +{ + /* + * NB: The GuC API only supports 32bit values. However, the limit is further + * reduced due to internal calculations which would otherwise overflow. + */ + if (intel_guc_submission_is_wanted(gt_to_guc(engine->gt))) + value = min_t(u64, value, guc_policy_max_exec_quantum_ms()); + + value = min_t(u64, value, jiffies_to_msecs(MAX_SCHEDULE_TIMEOUT)); + + return value; +} + static void __setup_engine_capabilities(struct intel_engine_cs *engine) { struct drm_i915_private *i915 = engine->i915; @@ -594,8 +677,8 @@ void intel_engines_release(struct intel_gt *gt) * in case we aborted before completely initialising the engines. */ GEM_BUG_ON(intel_gt_pm_is_awake(gt)); - if (!INTEL_INFO(gt->i915)->gpu_reset_clobbers_display) - __intel_gt_reset(gt, ALL_ENGINES); + if (!intel_gt_gpu_reset_clobbers_display(gt)) + intel_gt_reset_all_engines(gt); /* Decouple the backend; but keep the layout for late GPU resets */ for_each_engine(engine, gt, id) { @@ -610,6 +693,8 @@ void intel_engines_release(struct intel_gt *gt) memset(&engine->reset, 0, sizeof(engine->reset)); } + + llist_del_all(>->i915->uabi_engines_llist); } void intel_engine_free_request_pool(struct intel_engine_cs *engine) @@ -681,14 +766,13 @@ static void engine_mask_apply_media_fuses(struct intel_gt *gt) * and bits have disable semantices. */ media_fuse = intel_uncore_read(gt->uncore, GEN11_GT_VEBOX_VDBOX_DISABLE); - if (MEDIA_VER_FULL(i915) < IP_VER(12, 50)) + if (MEDIA_VER_FULL(i915) < IP_VER(12, 55)) media_fuse = ~media_fuse; - vdbox_mask = media_fuse & GEN11_GT_VDBOX_DISABLE_MASK; - vebox_mask = (media_fuse & GEN11_GT_VEBOX_DISABLE_MASK) >> - GEN11_GT_VEBOX_DISABLE_SHIFT; + vdbox_mask = REG_FIELD_GET(GEN11_GT_VDBOX_DISABLE_MASK, media_fuse); + vebox_mask = REG_FIELD_GET(GEN11_GT_VEBOX_DISABLE_MASK, media_fuse); - if (MEDIA_VER_FULL(i915) >= IP_VER(12, 50)) { + if (MEDIA_VER_FULL(i915) >= IP_VER(12, 55)) { fuse1 = intel_uncore_read(gt->uncore, HSW_PAVP_FUSE1); gt->info.sfc_mask = REG_FIELD_GET(XEHP_SFC_ENABLE_MASK, fuse1); } else { @@ -703,7 +787,7 @@ static void engine_mask_apply_media_fuses(struct intel_gt *gt) if (!(BIT(i) & vdbox_mask)) { gt->info.engine_mask &= ~BIT(_VCS(i)); - drm_dbg(&i915->drm, "vcs%u fused off\n", i); + gt_dbg(gt, "vcs%u fused off\n", i); continue; } @@ -711,8 +795,7 @@ static void engine_mask_apply_media_fuses(struct intel_gt *gt) gt->info.vdbox_sfc_access |= BIT(i); logical_vdbox++; } - drm_dbg(&i915->drm, "vdbox enable: %04x, instances: %04lx\n", - vdbox_mask, VDBOX_MASK(gt)); + gt_dbg(gt, "vdbox enable: %04x, instances: %04lx\n", vdbox_mask, VDBOX_MASK(gt)); GEM_BUG_ON(vdbox_mask != VDBOX_MASK(gt)); for (i = 0; i < I915_MAX_VECS; i++) { @@ -723,11 +806,10 @@ static void engine_mask_apply_media_fuses(struct intel_gt *gt) if (!(BIT(i) & vebox_mask)) { gt->info.engine_mask &= ~BIT(_VECS(i)); - drm_dbg(&i915->drm, "vecs%u fused off\n", i); + gt_dbg(gt, "vecs%u fused off\n", i); } } - drm_dbg(&i915->drm, "vebox enable: %04x, instances: %04lx\n", - vebox_mask, VEBOX_MASK(gt)); + gt_dbg(gt, "vebox enable: %04x, instances: %04lx\n", vebox_mask, VEBOX_MASK(gt)); GEM_BUG_ON(vebox_mask != VEBOX_MASK(gt)); } @@ -753,39 +835,7 @@ static void engine_mask_apply_compute_fuses(struct intel_gt *gt) */ for_each_clear_bit(i, &ccs_mask, I915_MAX_CCS) { info->engine_mask &= ~BIT(_CCS(i)); - drm_dbg(&i915->drm, "ccs%u fused off\n", i); - } -} - -static void engine_mask_apply_copy_fuses(struct intel_gt *gt) -{ - struct drm_i915_private *i915 = gt->i915; - struct intel_gt_info *info = >->info; - unsigned long meml3_mask; - unsigned long quad; - - if (!(GRAPHICS_VER_FULL(i915) >= IP_VER(12, 60) && - GRAPHICS_VER_FULL(i915) < IP_VER(12, 70))) - return; - - meml3_mask = intel_uncore_read(gt->uncore, GEN10_MIRROR_FUSE3); - meml3_mask = REG_FIELD_GET(GEN12_MEML3_EN_MASK, meml3_mask); - - /* - * Link Copy engines may be fused off according to meml3_mask. Each - * bit is a quad that houses 2 Link Copy and two Sub Copy engines. - */ - for_each_clear_bit(quad, &meml3_mask, GEN12_MAX_MSLICES) { - unsigned int instance = quad * 2 + 1; - intel_engine_mask_t mask = GENMASK(_BCS(instance + 1), - _BCS(instance)); - - if (mask & info->engine_mask) { - drm_dbg(&i915->drm, "bcs%u fused off\n", instance); - drm_dbg(&i915->drm, "bcs%u fused off\n", instance + 1); - - info->engine_mask &= ~mask; - } + gt_dbg(gt, "ccs%u fused off\n", i); } } @@ -794,7 +844,7 @@ static void engine_mask_apply_copy_fuses(struct intel_gt *gt) * Note that we have a catch-22 situation where we need to be able to access * the blitter forcewake domain to read the engine fuses, but at the same time * we need to know which engines are available on the system to know which - * forcewake domains are present. We solve this by intializing the forcewake + * forcewake domains are present. We solve this by initializing the forcewake * domains based on the full engine mask in the platform capabilities before * calling this function and pruning the domains for fused-off engines * afterwards. @@ -807,7 +857,46 @@ static intel_engine_mask_t init_engine_mask(struct intel_gt *gt) engine_mask_apply_media_fuses(gt); engine_mask_apply_compute_fuses(gt); - engine_mask_apply_copy_fuses(gt); + + /* + * The only use of the GSC CS is to load and communicate with the GSC + * FW, so we have no use for it if we don't have the FW. + * + * IMPORTANT: in cases where we don't have the GSC FW, we have a + * catch-22 situation that breaks media C6 due to 2 requirements: + * 1) once turned on, the GSC power well will not go to sleep unless the + * GSC FW is loaded. + * 2) to enable idling (which is required for media C6) we need to + * initialize the IDLE_MSG register for the GSC CS and do at least 1 + * submission, which will wake up the GSC power well. + */ + if (__HAS_ENGINE(info->engine_mask, GSC0) && !intel_uc_wants_gsc_uc(>->uc)) { + gt_notice(gt, "No GSC FW selected, disabling GSC CS and media C6\n"); + info->engine_mask &= ~BIT(GSC0); + } + + /* + * Do not create the command streamer for CCS slices beyond the first. + * All the workload submitted to the first engine will be shared among + * all the slices. + * + * Once the user will be allowed to customize the CCS mode, then this + * check needs to be removed. + */ + if (IS_DG2(gt->i915)) { + u8 first_ccs = __ffs(CCS_MASK(gt)); + + /* + * Store the number of active cslices before + * changing the CCS engine configuration + */ + gt->ccs.cslices = CCS_MASK(gt); + + /* Mask off all the CCS engine */ + info->engine_mask &= ~GENMASK(CCS3, CCS0); + /* Put back in the first CCS engine */ + info->engine_mask |= BIT(_CCS(first_ccs)); + } return info->engine_mask; } @@ -994,8 +1083,7 @@ static int init_status_page(struct intel_engine_cs *engine) */ obj = i915_gem_object_create_internal(engine->i915, PAGE_SIZE); if (IS_ERR(obj)) { - drm_err(&engine->i915->drm, - "Failed to allocate status page\n"); + gt_err(engine->gt, "Failed to allocate status page\n"); return PTR_ERR(obj); } @@ -1040,12 +1128,130 @@ err_put: return ret; } +static int intel_engine_init_tlb_invalidation(struct intel_engine_cs *engine) +{ + static const union intel_engine_tlb_inv_reg gen8_regs[] = { + [RENDER_CLASS].reg = GEN8_RTCR, + [VIDEO_DECODE_CLASS].reg = GEN8_M1TCR, /* , GEN8_M2TCR */ + [VIDEO_ENHANCEMENT_CLASS].reg = GEN8_VTCR, + [COPY_ENGINE_CLASS].reg = GEN8_BTCR, + }; + static const union intel_engine_tlb_inv_reg gen12_regs[] = { + [RENDER_CLASS].reg = GEN12_GFX_TLB_INV_CR, + [VIDEO_DECODE_CLASS].reg = GEN12_VD_TLB_INV_CR, + [VIDEO_ENHANCEMENT_CLASS].reg = GEN12_VE_TLB_INV_CR, + [COPY_ENGINE_CLASS].reg = GEN12_BLT_TLB_INV_CR, + [COMPUTE_CLASS].reg = GEN12_COMPCTX_TLB_INV_CR, + }; + static const union intel_engine_tlb_inv_reg xehp_regs[] = { + [RENDER_CLASS].mcr_reg = XEHP_GFX_TLB_INV_CR, + [VIDEO_DECODE_CLASS].mcr_reg = XEHP_VD_TLB_INV_CR, + [VIDEO_ENHANCEMENT_CLASS].mcr_reg = XEHP_VE_TLB_INV_CR, + [COPY_ENGINE_CLASS].mcr_reg = XEHP_BLT_TLB_INV_CR, + [COMPUTE_CLASS].mcr_reg = XEHP_COMPCTX_TLB_INV_CR, + }; + static const union intel_engine_tlb_inv_reg xelpmp_regs[] = { + [VIDEO_DECODE_CLASS].reg = GEN12_VD_TLB_INV_CR, + [VIDEO_ENHANCEMENT_CLASS].reg = GEN12_VE_TLB_INV_CR, + [OTHER_CLASS].reg = XELPMP_GSC_TLB_INV_CR, + }; + struct drm_i915_private *i915 = engine->i915; + const unsigned int instance = engine->instance; + const unsigned int class = engine->class; + const union intel_engine_tlb_inv_reg *regs; + union intel_engine_tlb_inv_reg reg; + unsigned int num = 0; + u32 val; + + /* + * New platforms should not be added with catch-all-newer (>=) + * condition so that any later platform added triggers the below warning + * and in turn mandates a human cross-check of whether the invalidation + * flows have compatible semantics. + * + * For instance with the 11.00 -> 12.00 transition three out of five + * respective engine registers were moved to masked type. Then after the + * 12.00 -> 12.50 transition multi cast handling is required too. + */ + + if (engine->gt->type == GT_MEDIA) { + if (MEDIA_VER_FULL(i915) == IP_VER(13, 0)) { + regs = xelpmp_regs; + num = ARRAY_SIZE(xelpmp_regs); + } + } else { + if (GRAPHICS_VER_FULL(i915) == IP_VER(12, 74) || + GRAPHICS_VER_FULL(i915) == IP_VER(12, 71) || + GRAPHICS_VER_FULL(i915) == IP_VER(12, 70) || + GRAPHICS_VER_FULL(i915) == IP_VER(12, 55)) { + regs = xehp_regs; + num = ARRAY_SIZE(xehp_regs); + } else if (GRAPHICS_VER_FULL(i915) == IP_VER(12, 0) || + GRAPHICS_VER_FULL(i915) == IP_VER(12, 10)) { + regs = gen12_regs; + num = ARRAY_SIZE(gen12_regs); + } else if (GRAPHICS_VER(i915) >= 8 && GRAPHICS_VER(i915) <= 11) { + regs = gen8_regs; + num = ARRAY_SIZE(gen8_regs); + } else if (GRAPHICS_VER(i915) < 8) { + return 0; + } + } + + if (gt_WARN_ONCE(engine->gt, !num, + "Platform does not implement TLB invalidation!")) + return -ENODEV; + + if (gt_WARN_ON_ONCE(engine->gt, + class >= num || + (!regs[class].reg.reg && + !regs[class].mcr_reg.reg))) + return -ERANGE; + + reg = regs[class]; + + if (regs == xelpmp_regs && class == OTHER_CLASS) { + /* + * There's only a single GSC instance, but it uses register bit + * 1 instead of either 0 or OTHER_GSC_INSTANCE. + */ + GEM_WARN_ON(instance != OTHER_GSC_INSTANCE); + val = 1; + } else if (regs == gen8_regs && class == VIDEO_DECODE_CLASS && instance == 1) { + reg.reg = GEN8_M2TCR; + val = 0; + } else { + val = instance; + } + + val = BIT(val); + + engine->tlb_inv.mcr = regs == xehp_regs; + engine->tlb_inv.reg = reg; + engine->tlb_inv.done = val; + + if (GRAPHICS_VER(i915) >= 12 && + (engine->class == VIDEO_DECODE_CLASS || + engine->class == VIDEO_ENHANCEMENT_CLASS || + engine->class == COMPUTE_CLASS || + engine->class == OTHER_CLASS)) + engine->tlb_inv.request = _MASKED_BIT_ENABLE(val); + else + engine->tlb_inv.request = val; + + return 0; +} + static int engine_setup_common(struct intel_engine_cs *engine) { int err; init_llist_head(&engine->barrier_tasks); + err = intel_engine_init_tlb_invalidation(engine); + if (err) + return err; + err = init_status_page(engine); if (err) return err; @@ -1111,6 +1317,7 @@ static int measure_breadcrumb_dw(struct intel_context *ce) if (!frame) return -ENOMEM; + frame->rq.i915 = engine->i915; frame->rq.engine = engine; frame->rq.context = ce; rcu_assign_pointer(frame->rq.timeline, ce->timeline); @@ -1197,6 +1404,20 @@ void intel_engine_destroy_pinned_context(struct intel_context *ce) } static struct intel_context * +create_ggtt_bind_context(struct intel_engine_cs *engine) +{ + static struct lock_class_key kernel; + + /* + * MI_UPDATE_GTT can insert up to 511 PTE entries and there could be multiple + * bind requests at a time so get a bigger ring. + */ + return intel_engine_create_pinned_context(engine, engine->gt->vm, SZ_512K, + I915_GEM_HWS_GGTT_BIND_ADDR, + &kernel, "ggtt_bind_context"); +} + +static struct intel_context * create_kernel_context(struct intel_engine_cs *engine) { static struct lock_class_key kernel; @@ -1206,8 +1427,8 @@ create_kernel_context(struct intel_engine_cs *engine) &kernel, "kernel_context"); } -/** - * intel_engines_init_common - initialize cengine state which might require hw access +/* + * engine_init_common - initialize engine state which might require hw access * @engine: Engine to initialize. * * Initializes @engine@ structure members shared between legacy and execlists @@ -1219,7 +1440,7 @@ create_kernel_context(struct intel_engine_cs *engine) */ static int engine_init_common(struct intel_engine_cs *engine) { - struct intel_context *ce; + struct intel_context *ce, *bce = NULL; int ret; engine->set_default_submission(engine); @@ -1235,17 +1456,34 @@ static int engine_init_common(struct intel_engine_cs *engine) ce = create_kernel_context(engine); if (IS_ERR(ce)) return PTR_ERR(ce); + /* + * Create a separate pinned context for GGTT update with blitter engine + * if a platform require such service. MI_UPDATE_GTT works on other + * engines as well but BCS should be less busy engine so pick that for + * GGTT updates. + */ + if (i915_ggtt_require_binder(engine->i915) && engine->id == BCS0) { + bce = create_ggtt_bind_context(engine); + if (IS_ERR(bce)) { + ret = PTR_ERR(bce); + goto err_ce_context; + } + } ret = measure_breadcrumb_dw(ce); if (ret < 0) - goto err_context; + goto err_bce_context; engine->emit_fini_breadcrumb_dw = ret; engine->kernel_context = ce; + engine->bind_context = bce; return 0; -err_context: +err_bce_context: + if (bce) + intel_engine_destroy_pinned_context(bce); +err_ce_context: intel_engine_destroy_pinned_context(ce); return ret; } @@ -1274,8 +1512,13 @@ int intel_engines_init(struct intel_gt *gt) return err; err = setup(engine); - if (err) + if (err) { + intel_engine_cleanup_common(engine); return err; + } + + /* The backend should now be responsible for cleanup */ + GEM_BUG_ON(engine->release == NULL); err = engine_init_common(engine); if (err) @@ -1288,8 +1531,8 @@ int intel_engines_init(struct intel_gt *gt) } /** - * intel_engines_cleanup_common - cleans up the engine state created by - * the common initiailizers. + * intel_engine_cleanup_common - cleans up the engine state created by + * the common initializers. * @engine: Engine to cleanup. * * This cleans up everything created by the common helpers. @@ -1310,6 +1553,10 @@ void intel_engine_cleanup_common(struct intel_engine_cs *engine) if (engine->kernel_context) intel_engine_destroy_pinned_context(engine->kernel_context); + if (engine->bind_context) + intel_engine_destroy_pinned_context(engine->bind_context); + + GEM_BUG_ON(!llist_empty(&engine->barrier_tasks)); cleanup_status_page(engine); @@ -1386,10 +1633,10 @@ static int __intel_engine_stop_cs(struct intel_engine_cs *engine, intel_uncore_write_fw(uncore, mode, _MASKED_BIT_ENABLE(STOP_RING)); /* - * Wa_22011802037 : gen11, gen12, Prior to doing a reset, ensure CS is + * Wa_22011802037: Prior to doing a reset, ensure CS is * stopped, set ring stop bit and prefetch disable bit to halt CS */ - if (IS_GRAPHICS_VER(engine->i915, 11, 12)) + if (intel_engine_reset_needs_wa_22011802037(engine->gt)) intel_uncore_write_fw(uncore, RING_MODE_GEN7(engine->mmio_base), _MASKED_BIT_ENABLE(GEN12_GFX_PREFETCH_DISABLE)); @@ -1474,11 +1721,8 @@ static u32 __cs_pending_mi_force_wakes(struct intel_engine_cs *engine) }; u32 val; - if (!_reg[engine->id].reg) { - drm_err(&engine->i915->drm, - "MSG IDLE undefined for engine id %u\n", engine->id); + if (!_reg[engine->id].reg) return 0; - } val = intel_uncore_read(engine->uncore, _reg[engine->id]); @@ -1554,11 +1798,11 @@ void intel_engine_get_instdone(const struct intel_engine_cs *engine, for_each_ss_steering(iter, engine->gt, slice, subslice) { instdone->sampler[slice][subslice] = intel_gt_mcr_read(engine->gt, - GEN7_SAMPLER_INSTDONE, + GEN8_SAMPLER_INSTDONE, slice, subslice); instdone->row[slice][subslice] = intel_gt_mcr_read(engine->gt, - GEN7_ROW_INSTDONE, + GEN8_ROW_INSTDONE, slice, subslice); } @@ -1832,13 +2076,13 @@ static const char *repr_timer(const struct timer_list *t) static void intel_engine_print_registers(struct intel_engine_cs *engine, struct drm_printer *m) { - struct drm_i915_private *dev_priv = engine->i915; + struct drm_i915_private *i915 = engine->i915; struct intel_engine_execlists * const execlists = &engine->execlists; u64 addr; - if (engine->id == RENDER_CLASS && IS_GRAPHICS_VER(dev_priv, 4, 7)) + if (engine->id == RENDER_CLASS && IS_GRAPHICS_VER(i915, 4, 7)) drm_printf(m, "\tCCID: 0x%08x\n", ENGINE_READ(engine, CCID)); - if (HAS_EXECLISTS(dev_priv)) { + if (HAS_EXECLISTS(i915)) { drm_printf(m, "\tEL_STAT_HI: 0x%08x\n", ENGINE_READ(engine, RING_EXECLIST_STATUS_HI)); drm_printf(m, "\tEL_STAT_LO: 0x%08x\n", @@ -1859,7 +2103,7 @@ static void intel_engine_print_registers(struct intel_engine_cs *engine, ENGINE_READ(engine, RING_MI_MODE) & (MODE_IDLE) ? " [idle]" : ""); } - if (GRAPHICS_VER(dev_priv) >= 6) { + if (GRAPHICS_VER(i915) >= 6) { drm_printf(m, "\tRING_IMR: 0x%08x\n", ENGINE_READ(engine, RING_IMR)); drm_printf(m, "\tRING_ESR: 0x%08x\n", @@ -1876,15 +2120,15 @@ static void intel_engine_print_registers(struct intel_engine_cs *engine, addr = intel_engine_get_last_batch_head(engine); drm_printf(m, "\tBBADDR: 0x%08x_%08x\n", upper_32_bits(addr), lower_32_bits(addr)); - if (GRAPHICS_VER(dev_priv) >= 8) + if (GRAPHICS_VER(i915) >= 8) addr = ENGINE_READ64(engine, RING_DMA_FADD, RING_DMA_FADD_UDW); - else if (GRAPHICS_VER(dev_priv) >= 4) + else if (GRAPHICS_VER(i915) >= 4) addr = ENGINE_READ(engine, RING_DMA_FADD); else addr = ENGINE_READ(engine, DMA_FADD_I8XX); drm_printf(m, "\tDMA_FADDR: 0x%08x_%08x\n", upper_32_bits(addr), lower_32_bits(addr)); - if (GRAPHICS_VER(dev_priv) >= 4) { + if (GRAPHICS_VER(i915) >= 4) { drm_printf(m, "\tIPEIR: 0x%08x\n", ENGINE_READ(engine, RING_IPEIR)); drm_printf(m, "\tIPEHR: 0x%08x\n", @@ -1894,7 +2138,7 @@ static void intel_engine_print_registers(struct intel_engine_cs *engine, drm_printf(m, "\tIPEHR: 0x%08x\n", ENGINE_READ(engine, IPEHR)); } - if (HAS_EXECLISTS(dev_priv) && !intel_engine_uses_guc(engine)) { + if (HAS_EXECLISTS(i915) && !intel_engine_uses_guc(engine)) { struct i915_request * const *port, *rq; const u32 *hws = &engine->status_page.addr[I915_HWS_CSB_BUF0_INDEX]; @@ -1960,7 +2204,7 @@ static void intel_engine_print_registers(struct intel_engine_cs *engine, } rcu_read_unlock(); i915_sched_engine_active_unlock_bh(engine->sched_engine); - } else if (GRAPHICS_VER(dev_priv) > 6) { + } else if (GRAPHICS_VER(i915) > 6) { drm_printf(m, "\tPP_DIR_BASE: 0x%08x\n", ENGINE_READ(engine, RING_PP_DIR_BASE)); drm_printf(m, "\tPP_DIR_BASE_READ: 0x%08x\n", @@ -2004,17 +2248,6 @@ static void print_request_ring(struct drm_printer *m, struct i915_request *rq) } } -static unsigned long list_count(struct list_head *list) -{ - struct list_head *pos; - unsigned long count = 0; - - list_for_each(pos, list) - count++; - - return count; -} - static unsigned long read_ul(void *p, size_t x) { return *(unsigned long *)(p + x); @@ -2106,11 +2339,11 @@ void intel_engine_dump_active_requests(struct list_head *requests, } } -static void engine_dump_active_requests(struct intel_engine_cs *engine, struct drm_printer *m) +static void engine_dump_active_requests(struct intel_engine_cs *engine, + struct drm_printer *m) { + struct intel_context *hung_ce = NULL; struct i915_request *hung_rq = NULL; - struct intel_context *ce; - bool guc; /* * No need for an engine->irq_seqno_barrier() before the seqno reads. @@ -2119,27 +2352,22 @@ static void engine_dump_active_requests(struct intel_engine_cs *engine, struct d * But the intention here is just to report an instantaneous snapshot * so that's fine. */ - lockdep_assert_held(&engine->sched_engine->lock); + intel_engine_get_hung_entity(engine, &hung_ce, &hung_rq); drm_printf(m, "\tRequests:\n"); - guc = intel_uc_uses_guc_submission(&engine->gt->uc); - if (guc) { - ce = intel_engine_get_hung_context(engine); - if (ce) - hung_rq = intel_context_find_active_request(ce); - } else { - hung_rq = intel_engine_execlist_find_hung_request(engine); - } - if (hung_rq) engine_dump_request(hung_rq, m, "\t\thung"); + else if (hung_ce) + drm_printf(m, "\t\tGot hung ce but no hung rq!\n"); - if (guc) + if (intel_uc_uses_guc_submission(&engine->gt->uc)) intel_guc_dump_active_requests(engine, hung_rq, m); else - intel_engine_dump_active_requests(&engine->sched_engine->requests, - hung_rq, m); + intel_execlists_dump_active_requests(engine, hung_rq, m); + + if (hung_rq) + i915_request_put(hung_rq); } void intel_engine_dump(struct intel_engine_cs *engine, @@ -2149,7 +2377,6 @@ void intel_engine_dump(struct intel_engine_cs *engine, struct i915_gpu_error * const error = &engine->i915->gpu_error; struct i915_request *rq; intel_wakeref_t wakeref; - unsigned long flags; ktime_t dummy; if (header) { @@ -2186,13 +2413,8 @@ void intel_engine_dump(struct intel_engine_cs *engine, i915_reset_count(error)); print_properties(engine, m); - spin_lock_irqsave(&engine->sched_engine->lock, flags); engine_dump_active_requests(engine, m); - drm_printf(m, "\tOn hold?: %lu\n", - list_count(&engine->sched_engine->hold)); - spin_unlock_irqrestore(&engine->sched_engine->lock, flags); - drm_printf(m, "\tMMIO base: 0x%08x\n", engine->mmio_base); wakeref = intel_runtime_pm_get_if_in_use(engine->uncore->rpm); if (wakeref) { @@ -2238,8 +2460,7 @@ intel_engine_create_virtual(struct intel_engine_cs **siblings, return siblings[0]->cops->create_virtual(siblings, count, flags); } -struct i915_request * -intel_engine_execlist_find_hung_request(struct intel_engine_cs *engine) +static struct i915_request *engine_execlist_find_hung_request(struct intel_engine_cs *engine) { struct i915_request *request, *active = NULL; @@ -2291,6 +2512,33 @@ intel_engine_execlist_find_hung_request(struct intel_engine_cs *engine) return active; } +void intel_engine_get_hung_entity(struct intel_engine_cs *engine, + struct intel_context **ce, struct i915_request **rq) +{ + unsigned long flags; + + *ce = intel_engine_get_hung_context(engine); + if (*ce) { + intel_engine_clear_hung_context(engine); + + *rq = intel_context_get_active_request(*ce); + return; + } + + /* + * Getting here with GuC enabled means it is a forced error capture + * with no actual hang. So, no need to attempt the execlist search. + */ + if (intel_uc_uses_guc_submission(&engine->gt->uc)) + return; + + spin_lock_irqsave(&engine->sched_engine->lock, flags); + *rq = engine_execlist_find_hung_request(engine); + if (*rq) + *rq = i915_request_get_rcu(*rq); + spin_unlock_irqrestore(&engine->sched_engine->lock, flags); +} + void xehp_enable_ccs_engines(struct intel_engine_cs *engine) { /* |