diff options
Diffstat (limited to 'drivers/gpu/drm/xe/xe_lrc.c')
| -rw-r--r-- | drivers/gpu/drm/xe/xe_lrc.c | 858 |
1 files changed, 778 insertions, 80 deletions
diff --git a/drivers/gpu/drm/xe/xe_lrc.c b/drivers/gpu/drm/xe/xe_lrc.c index 7ad853b0788a..6e7b70532d11 100644 --- a/drivers/gpu/drm/xe/xe_lrc.c +++ b/drivers/gpu/drm/xe/xe_lrc.c @@ -5,10 +5,14 @@ #include "xe_lrc.h" +#include <generated/xe_wa_oob.h> + +#include <linux/ascii85.h> + #include "instructions/xe_mi_commands.h" #include "instructions/xe_gfxpipe_commands.h" +#include "instructions/xe_gfx_state_commands.h" #include "regs/xe_engine_regs.h" -#include "regs/xe_gpu_commands.h" #include "regs/xe_lrc_layout.h" #include "xe_bb.h" #include "xe_bo.h" @@ -20,16 +24,23 @@ #include "xe_hw_fence.h" #include "xe_map.h" #include "xe_memirq.h" +#include "xe_mmio.h" #include "xe_sriov.h" +#include "xe_trace_lrc.h" #include "xe_vm.h" +#include "xe_wa.h" -#define LRC_VALID (1 << 0) -#define LRC_PRIVILEGE (1 << 8) -#define LRC_ADDRESSING_MODE_SHIFT 3 +#define LRC_VALID BIT_ULL(0) +#define LRC_PRIVILEGE BIT_ULL(8) +#define LRC_ADDRESSING_MODE GENMASK_ULL(4, 3) #define LRC_LEGACY_64B_CONTEXT 3 -#define ENGINE_CLASS_SHIFT 61 -#define ENGINE_INSTANCE_SHIFT 48 +#define LRC_ENGINE_CLASS GENMASK_ULL(63, 61) +#define LRC_ENGINE_INSTANCE GENMASK_ULL(53, 48) + +#define LRC_PPHWSP_SIZE SZ_4K +#define LRC_INDIRECT_RING_STATE_SIZE SZ_4K +#define LRC_WA_BB_SIZE SZ_4K static struct xe_device * lrc_to_xe(struct xe_lrc *lrc) @@ -37,20 +48,28 @@ lrc_to_xe(struct xe_lrc *lrc) return gt_to_xe(lrc->fence_ctx.gt); } -size_t xe_lrc_size(struct xe_device *xe, enum xe_engine_class class) +size_t xe_gt_lrc_size(struct xe_gt *gt, enum xe_engine_class class) { + struct xe_device *xe = gt_to_xe(gt); + size_t size; + + /* Per-process HW status page (PPHWSP) */ + size = LRC_PPHWSP_SIZE; + + /* Engine context image */ switch (class) { case XE_ENGINE_CLASS_RENDER: if (GRAPHICS_VER(xe) >= 20) - return 4 * SZ_4K; + size += 3 * SZ_4K; else - return 14 * SZ_4K; + size += 13 * SZ_4K; + break; case XE_ENGINE_CLASS_COMPUTE: - /* 14 pages since graphics_ver == 11 */ if (GRAPHICS_VER(xe) >= 20) - return 3 * SZ_4K; + size += 2 * SZ_4K; else - return 14 * SZ_4K; + size += 13 * SZ_4K; + break; default: WARN(1, "Unknown engine class: %d", class); fallthrough; @@ -58,8 +77,14 @@ size_t xe_lrc_size(struct xe_device *xe, enum xe_engine_class class) case XE_ENGINE_CLASS_VIDEO_DECODE: case XE_ENGINE_CLASS_VIDEO_ENHANCE: case XE_ENGINE_CLASS_OTHER: - return 2 * SZ_4K; + size += 1 * SZ_4K; } + + /* Add indirect ring state page */ + if (xe_gt_has_indirect_ring_state(gt)) + size += LRC_INDIRECT_RING_STATE_SIZE; + + return size; } /* @@ -97,7 +122,6 @@ static void set_offsets(u32 *regs, #define REG16(x) \ (((x) >> 9) | BIT(7) | BUILD_BUG_ON_ZERO(x >= 0x10000)), \ (((x) >> 2) & 0x7f) -#define END 0 { const u32 base = hwe->mmio_base; @@ -168,7 +192,7 @@ static const u8 gen12_xcs_offsets[] = { REG16(0x274), REG16(0x270), - END + 0 }; static const u8 dg2_xcs_offsets[] = { @@ -202,7 +226,7 @@ static const u8 dg2_xcs_offsets[] = { REG16(0x274), REG16(0x270), - END + 0 }; static const u8 gen12_rcs_offsets[] = { @@ -298,7 +322,7 @@ static const u8 gen12_rcs_offsets[] = { REG(0x084), NOP(1), - END + 0 }; static const u8 xehp_rcs_offsets[] = { @@ -339,7 +363,7 @@ static const u8 xehp_rcs_offsets[] = { LRI(1, 0), REG(0x0c8), - END + 0 }; static const u8 dg2_rcs_offsets[] = { @@ -382,7 +406,7 @@ static const u8 dg2_rcs_offsets[] = { LRI(1, 0), REG(0x0c8), - END + 0 }; static const u8 mtl_rcs_offsets[] = { @@ -425,7 +449,7 @@ static const u8 mtl_rcs_offsets[] = { LRI(1, 0), REG(0x0c8), - END + 0 }; #define XE2_CTX_COMMON \ @@ -471,7 +495,7 @@ static const u8 xe2_rcs_offsets[] = { LRI(1, 0), /* [0x47] */ REG(0x0c8), /* [0x48] R_PWR_CLK_STATE */ - END + 0 }; static const u8 xe2_bcs_offsets[] = { @@ -482,16 +506,41 @@ static const u8 xe2_bcs_offsets[] = { REG16(0x200), /* [0x42] BCS_SWCTRL */ REG16(0x204), /* [0x44] BLIT_CCTL */ - END + 0 }; static const u8 xe2_xcs_offsets[] = { XE2_CTX_COMMON, - END + 0 +}; + +static const u8 xe2_indirect_ring_state_offsets[] = { + NOP(1), /* [0x00] */ + LRI(5, POSTED), /* [0x01] */ + REG(0x034), /* [0x02] RING_BUFFER_HEAD */ + REG(0x030), /* [0x04] RING_BUFFER_TAIL */ + REG(0x038), /* [0x06] RING_BUFFER_START */ + REG(0x048), /* [0x08] RING_BUFFER_START_UDW */ + REG(0x03c), /* [0x0a] RING_BUFFER_CONTROL */ + + NOP(5), /* [0x0c] */ + LRI(9, POSTED), /* [0x11] */ + REG(0x168), /* [0x12] BB_ADDR_UDW */ + REG(0x140), /* [0x14] BB_ADDR */ + REG(0x110), /* [0x16] BB_STATE */ + REG16(0x588), /* [0x18] BB_STACK_WRITE_PORT */ + REG16(0x588), /* [0x20] BB_STACK_WRITE_PORT */ + REG16(0x588), /* [0x22] BB_STACK_WRITE_PORT */ + REG16(0x588), /* [0x24] BB_STACK_WRITE_PORT */ + REG16(0x588), /* [0x26] BB_STACK_WRITE_PORT */ + REG16(0x588), /* [0x28] BB_STACK_WRITE_PORT */ + + NOP(12), /* [0x00] */ + + 0 }; -#undef END #undef REG16 #undef REG #undef LRI @@ -527,19 +576,23 @@ static const u8 *reg_offsets(struct xe_device *xe, enum xe_engine_class class) static void set_context_control(u32 *regs, struct xe_hw_engine *hwe) { - regs[CTX_CONTEXT_CONTROL] = _MASKED_BIT_ENABLE(CTX_CTRL_INHIBIT_SYN_CTX_SWITCH) | - _MASKED_BIT_DISABLE(CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT) | - CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT; + regs[CTX_CONTEXT_CONTROL] = _MASKED_BIT_ENABLE(CTX_CTRL_INHIBIT_SYN_CTX_SWITCH | + CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT); + + if (xe_gt_has_indirect_ring_state(hwe->gt)) + regs[CTX_CONTEXT_CONTROL] |= + _MASKED_BIT_ENABLE(CTX_CTRL_INDIRECT_RING_STATE_ENABLE); /* TODO: Timestamp */ } static void set_memory_based_intr(u32 *regs, struct xe_hw_engine *hwe) { - struct xe_memirq *memirq = >_to_tile(hwe->gt)->sriov.vf.memirq; + struct xe_memirq *memirq = >_to_tile(hwe->gt)->memirq; struct xe_device *xe = gt_to_xe(hwe->gt); + u8 num_regs; - if (!IS_SRIOV_VF(xe) || !xe_device_has_memirq(xe)) + if (!xe_device_uses_memirq(xe)) return; regs[CTX_LRM_INT_MASK_ENABLE] = MI_LOAD_REGISTER_MEM | @@ -547,12 +600,18 @@ static void set_memory_based_intr(u32 *regs, struct xe_hw_engine *hwe) regs[CTX_INT_MASK_ENABLE_REG] = RING_IMR(0).addr; regs[CTX_INT_MASK_ENABLE_PTR] = xe_memirq_enable_ptr(memirq); - regs[CTX_LRI_INT_REPORT_PTR] = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(2) | + num_regs = xe_device_has_msix(xe) ? 3 : 2; + regs[CTX_LRI_INT_REPORT_PTR] = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(num_regs) | MI_LRI_LRM_CS_MMIO | MI_LRI_FORCE_POSTED; regs[CTX_INT_STATUS_REPORT_REG] = RING_INT_STATUS_RPT_PTR(0).addr; - regs[CTX_INT_STATUS_REPORT_PTR] = xe_memirq_status_ptr(memirq); + regs[CTX_INT_STATUS_REPORT_PTR] = xe_memirq_status_ptr(memirq, hwe); regs[CTX_INT_SRC_REPORT_REG] = RING_INT_SRC_RPT_PTR(0).addr; - regs[CTX_INT_SRC_REPORT_PTR] = xe_memirq_source_ptr(memirq); + regs[CTX_INT_SRC_REPORT_PTR] = xe_memirq_source_ptr(memirq, hwe); + + if (xe_device_has_msix(xe)) { + regs[CTX_CS_INT_VEC_REG] = CS_INT_VEC(0).addr; + /* CTX_CS_INT_VEC_DATA will be set in xe_lrc_init */ + } } static int lrc_ring_mi_mode(struct xe_hw_engine *hwe) @@ -574,6 +633,11 @@ static void reset_stop_ring(u32 *regs, struct xe_hw_engine *hwe) regs[x + 1] |= STOP_RING << 16; } +static inline bool xe_lrc_has_indirect_ring_state(struct xe_lrc *lrc) +{ + return lrc->flags & XE_LRC_FLAG_INDIRECT_RING_STATE; +} + static inline u32 __xe_lrc_ring_offset(struct xe_lrc *lrc) { return 0; @@ -586,11 +650,18 @@ u32 xe_lrc_pphwsp_offset(struct xe_lrc *lrc) /* Make the magic macros work */ #define __xe_lrc_pphwsp_offset xe_lrc_pphwsp_offset +#define __xe_lrc_regs_offset xe_lrc_regs_offset #define LRC_SEQNO_PPHWSP_OFFSET 512 #define LRC_START_SEQNO_PPHWSP_OFFSET (LRC_SEQNO_PPHWSP_OFFSET + 8) +#define LRC_CTX_JOB_TIMESTAMP_OFFSET (LRC_START_SEQNO_PPHWSP_OFFSET + 8) #define LRC_PARALLEL_PPHWSP_OFFSET 2048 -#define LRC_PPHWSP_SIZE SZ_4K +#define LRC_ENGINE_ID_PPHWSP_OFFSET 2096 + +u32 xe_lrc_regs_offset(struct xe_lrc *lrc) +{ + return xe_lrc_pphwsp_offset(lrc) + LRC_PPHWSP_SIZE; +} static size_t lrc_reg_size(struct xe_device *xe) { @@ -617,15 +688,37 @@ static inline u32 __xe_lrc_start_seqno_offset(struct xe_lrc *lrc) return xe_lrc_pphwsp_offset(lrc) + LRC_START_SEQNO_PPHWSP_OFFSET; } +static u32 __xe_lrc_ctx_job_timestamp_offset(struct xe_lrc *lrc) +{ + /* This is stored in the driver-defined portion of PPHWSP */ + return xe_lrc_pphwsp_offset(lrc) + LRC_CTX_JOB_TIMESTAMP_OFFSET; +} + static inline u32 __xe_lrc_parallel_offset(struct xe_lrc *lrc) { /* The parallel is stored in the driver-defined portion of PPHWSP */ return xe_lrc_pphwsp_offset(lrc) + LRC_PARALLEL_PPHWSP_OFFSET; } -static inline u32 __xe_lrc_regs_offset(struct xe_lrc *lrc) +static inline u32 __xe_lrc_engine_id_offset(struct xe_lrc *lrc) { - return xe_lrc_pphwsp_offset(lrc) + LRC_PPHWSP_SIZE; + return xe_lrc_pphwsp_offset(lrc) + LRC_ENGINE_ID_PPHWSP_OFFSET; +} + +static u32 __xe_lrc_ctx_timestamp_offset(struct xe_lrc *lrc) +{ + return __xe_lrc_regs_offset(lrc) + CTX_TIMESTAMP * sizeof(u32); +} + +static u32 __xe_lrc_ctx_timestamp_udw_offset(struct xe_lrc *lrc) +{ + return __xe_lrc_regs_offset(lrc) + CTX_TIMESTAMP_UDW * sizeof(u32); +} + +static inline u32 __xe_lrc_indirect_ring_offset(struct xe_lrc *lrc) +{ + /* Indirect ring state page is at the very end of LRC */ + return lrc->size - LRC_INDIRECT_RING_STATE_SIZE; } #define DECL_MAP_ADDR_HELPERS(elem) \ @@ -637,7 +730,7 @@ static inline struct iosys_map __xe_lrc_##elem##_map(struct xe_lrc *lrc) \ iosys_map_incr(&map, __xe_lrc_##elem##_offset(lrc)); \ return map; \ } \ -static inline u32 __xe_lrc_##elem##_ggtt_addr(struct xe_lrc *lrc) \ +static inline u32 __maybe_unused __xe_lrc_##elem##_ggtt_addr(struct xe_lrc *lrc) \ { \ return xe_bo_ggtt_addr(lrc->bo) + __xe_lrc_##elem##_offset(lrc); \ } \ @@ -647,15 +740,120 @@ DECL_MAP_ADDR_HELPERS(pphwsp) DECL_MAP_ADDR_HELPERS(seqno) DECL_MAP_ADDR_HELPERS(regs) DECL_MAP_ADDR_HELPERS(start_seqno) +DECL_MAP_ADDR_HELPERS(ctx_job_timestamp) +DECL_MAP_ADDR_HELPERS(ctx_timestamp) +DECL_MAP_ADDR_HELPERS(ctx_timestamp_udw) DECL_MAP_ADDR_HELPERS(parallel) +DECL_MAP_ADDR_HELPERS(indirect_ring) +DECL_MAP_ADDR_HELPERS(engine_id) #undef DECL_MAP_ADDR_HELPERS +/** + * xe_lrc_ctx_timestamp_ggtt_addr() - Get ctx timestamp GGTT address + * @lrc: Pointer to the lrc. + * + * Returns: ctx timestamp GGTT address + */ +u32 xe_lrc_ctx_timestamp_ggtt_addr(struct xe_lrc *lrc) +{ + return __xe_lrc_ctx_timestamp_ggtt_addr(lrc); +} + +/** + * xe_lrc_ctx_timestamp_udw_ggtt_addr() - Get ctx timestamp udw GGTT address + * @lrc: Pointer to the lrc. + * + * Returns: ctx timestamp udw GGTT address + */ +u32 xe_lrc_ctx_timestamp_udw_ggtt_addr(struct xe_lrc *lrc) +{ + return __xe_lrc_ctx_timestamp_udw_ggtt_addr(lrc); +} + +/** + * xe_lrc_ctx_timestamp() - Read ctx timestamp value + * @lrc: Pointer to the lrc. + * + * Returns: ctx timestamp value + */ +u64 xe_lrc_ctx_timestamp(struct xe_lrc *lrc) +{ + struct xe_device *xe = lrc_to_xe(lrc); + struct iosys_map map; + u32 ldw, udw = 0; + + map = __xe_lrc_ctx_timestamp_map(lrc); + ldw = xe_map_read32(xe, &map); + + if (xe->info.has_64bit_timestamp) { + map = __xe_lrc_ctx_timestamp_udw_map(lrc); + udw = xe_map_read32(xe, &map); + } + + return (u64)udw << 32 | ldw; +} + +/** + * xe_lrc_ctx_job_timestamp_ggtt_addr() - Get ctx job timestamp GGTT address + * @lrc: Pointer to the lrc. + * + * Returns: ctx timestamp job GGTT address + */ +u32 xe_lrc_ctx_job_timestamp_ggtt_addr(struct xe_lrc *lrc) +{ + return __xe_lrc_ctx_job_timestamp_ggtt_addr(lrc); +} + +/** + * xe_lrc_ctx_job_timestamp() - Read ctx job timestamp value + * @lrc: Pointer to the lrc. + * + * Returns: ctx timestamp job value + */ +u32 xe_lrc_ctx_job_timestamp(struct xe_lrc *lrc) +{ + struct xe_device *xe = lrc_to_xe(lrc); + struct iosys_map map; + + map = __xe_lrc_ctx_job_timestamp_map(lrc); + return xe_map_read32(xe, &map); +} + u32 xe_lrc_ggtt_addr(struct xe_lrc *lrc) { return __xe_lrc_pphwsp_ggtt_addr(lrc); } +u32 xe_lrc_indirect_ring_ggtt_addr(struct xe_lrc *lrc) +{ + if (!xe_lrc_has_indirect_ring_state(lrc)) + return 0; + + return __xe_lrc_indirect_ring_ggtt_addr(lrc); +} + +static u32 xe_lrc_read_indirect_ctx_reg(struct xe_lrc *lrc, int reg_nr) +{ + struct xe_device *xe = lrc_to_xe(lrc); + struct iosys_map map; + + map = __xe_lrc_indirect_ring_map(lrc); + iosys_map_incr(&map, reg_nr * sizeof(u32)); + return xe_map_read32(xe, &map); +} + +static void xe_lrc_write_indirect_ctx_reg(struct xe_lrc *lrc, + int reg_nr, u32 val) +{ + struct xe_device *xe = lrc_to_xe(lrc); + struct iosys_map map; + + map = __xe_lrc_indirect_ring_map(lrc); + iosys_map_incr(&map, reg_nr * sizeof(u32)); + xe_map_write32(xe, &map, val); +} + u32 xe_lrc_read_ctx_reg(struct xe_lrc *lrc, int reg_nr) { struct xe_device *xe = lrc_to_xe(lrc); @@ -678,37 +876,128 @@ void xe_lrc_write_ctx_reg(struct xe_lrc *lrc, int reg_nr, u32 val) static void *empty_lrc_data(struct xe_hw_engine *hwe) { - struct xe_device *xe = gt_to_xe(hwe->gt); + struct xe_gt *gt = hwe->gt; void *data; u32 *regs; - data = kzalloc(xe_lrc_size(xe, hwe->class), GFP_KERNEL); + data = kzalloc(xe_gt_lrc_size(gt, hwe->class), GFP_KERNEL); if (!data) return NULL; /* 1st page: Per-Process of HW status Page */ regs = data + LRC_PPHWSP_SIZE; - set_offsets(regs, reg_offsets(xe, hwe->class), hwe); + set_offsets(regs, reg_offsets(gt_to_xe(gt), hwe->class), hwe); set_context_control(regs, hwe); set_memory_based_intr(regs, hwe); reset_stop_ring(regs, hwe); + if (xe_gt_has_indirect_ring_state(gt)) { + regs = data + xe_gt_lrc_size(gt, hwe->class) - + LRC_INDIRECT_RING_STATE_SIZE; + set_offsets(regs, xe2_indirect_ring_state_offsets, hwe); + } return data; } static void xe_lrc_set_ppgtt(struct xe_lrc *lrc, struct xe_vm *vm) { - u64 desc = xe_vm_pdp4_descriptor(vm, lrc->tile); + u64 desc = xe_vm_pdp4_descriptor(vm, gt_to_tile(lrc->gt)); xe_lrc_write_ctx_reg(lrc, CTX_PDP0_UDW, upper_32_bits(desc)); xe_lrc_write_ctx_reg(lrc, CTX_PDP0_LDW, lower_32_bits(desc)); } +static void xe_lrc_finish(struct xe_lrc *lrc) +{ + xe_hw_fence_ctx_finish(&lrc->fence_ctx); + xe_bo_unpin_map_no_vm(lrc->bo); +} + +static size_t wa_bb_offset(struct xe_lrc *lrc) +{ + return lrc->bo->size - LRC_WA_BB_SIZE; +} + +/* + * xe_lrc_setup_utilization() - Setup wa bb to assist in calculating active + * context run ticks. + * @lrc: Pointer to the lrc. + * + * Context Timestamp (CTX_TIMESTAMP) in the LRC accumulates the run ticks of the + * context, but only gets updated when the context switches out. In order to + * check how long a context has been active before it switches out, two things + * are required: + * + * (1) Determine if the context is running: + * To do so, we program the WA BB to set an initial value for CTX_TIMESTAMP in + * the LRC. The value chosen is 1 since 0 is the initial value when the LRC is + * initialized. During a query, we just check for this value to determine if the + * context is active. If the context switched out, it would overwrite this + * location with the actual CTX_TIMESTAMP MMIO value. Note that WA BB runs as + * the last part of context restore, so reusing this LRC location will not + * clobber anything. + * + * (2) Calculate the time that the context has been active for: + * The CTX_TIMESTAMP ticks only when the context is active. If a context is + * active, we just use the CTX_TIMESTAMP MMIO as the new value of utilization. + * While doing so, we need to read the CTX_TIMESTAMP MMIO for the specific + * engine instance. Since we do not know which instance the context is running + * on until it is scheduled, we also read the ENGINE_ID MMIO in the WA BB and + * store it in the PPHSWP. + */ +#define CONTEXT_ACTIVE 1ULL +static int xe_lrc_setup_utilization(struct xe_lrc *lrc) +{ + const size_t max_size = LRC_WA_BB_SIZE; + u32 *cmd, *buf = NULL; + + if (lrc->bo->vmap.is_iomem) { + buf = kmalloc(max_size, GFP_KERNEL); + if (!buf) + return -ENOMEM; + cmd = buf; + } else { + cmd = lrc->bo->vmap.vaddr + wa_bb_offset(lrc); + } + + *cmd++ = MI_STORE_REGISTER_MEM | MI_SRM_USE_GGTT | MI_SRM_ADD_CS_OFFSET; + *cmd++ = ENGINE_ID(0).addr; + *cmd++ = __xe_lrc_engine_id_ggtt_addr(lrc); + *cmd++ = 0; + + *cmd++ = MI_STORE_DATA_IMM | MI_SDI_GGTT | MI_SDI_NUM_DW(1); + *cmd++ = __xe_lrc_ctx_timestamp_ggtt_addr(lrc); + *cmd++ = 0; + *cmd++ = lower_32_bits(CONTEXT_ACTIVE); + + if (lrc_to_xe(lrc)->info.has_64bit_timestamp) { + *cmd++ = MI_STORE_DATA_IMM | MI_SDI_GGTT | MI_SDI_NUM_DW(1); + *cmd++ = __xe_lrc_ctx_timestamp_udw_ggtt_addr(lrc); + *cmd++ = 0; + *cmd++ = upper_32_bits(CONTEXT_ACTIVE); + } + + *cmd++ = MI_BATCH_BUFFER_END; + + if (buf) { + xe_map_memcpy_to(gt_to_xe(lrc->gt), &lrc->bo->vmap, + wa_bb_offset(lrc), buf, + (cmd - buf) * sizeof(*cmd)); + kfree(buf); + } + + xe_lrc_write_ctx_reg(lrc, CTX_BB_PER_CTX_PTR, xe_bo_ggtt_addr(lrc->bo) + + wa_bb_offset(lrc) + 1); + + return 0; +} + #define PVC_CTX_ASID (0x2e + 1) #define PVC_CTX_ACC_CTR_THOLD (0x2a + 1) -int xe_lrc_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe, - struct xe_exec_queue *q, struct xe_vm *vm, u32 ring_size) +static int xe_lrc_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe, + struct xe_vm *vm, u32 ring_size, u16 msix_vec, + u32 init_flags) { struct xe_gt *gt = hwe->gt; struct xe_tile *tile = gt_to_tile(gt); @@ -716,23 +1005,34 @@ int xe_lrc_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe, struct iosys_map map; void *init_data = NULL; u32 arb_enable; + u32 lrc_size; + u32 bo_flags; int err; + kref_init(&lrc->refcount); + lrc->gt = gt; lrc->flags = 0; + lrc_size = ring_size + xe_gt_lrc_size(gt, hwe->class); + if (xe_gt_has_indirect_ring_state(gt)) + lrc->flags |= XE_LRC_FLAG_INDIRECT_RING_STATE; + + bo_flags = XE_BO_FLAG_VRAM_IF_DGFX(tile) | XE_BO_FLAG_GGTT | + XE_BO_FLAG_GGTT_INVALIDATE; + if (vm && vm->xef) /* userspace */ + bo_flags |= XE_BO_FLAG_PINNED_LATE_RESTORE; /* * FIXME: Perma-pinning LRC as we don't yet support moving GGTT address * via VM bind calls. */ - lrc->bo = xe_bo_create_pin_map(xe, tile, vm, - ring_size + xe_lrc_size(xe, hwe->class), - ttm_bo_type_kernel, - XE_BO_CREATE_VRAM_IF_DGFX(tile) | - XE_BO_CREATE_GGTT_BIT); + lrc->bo = xe_bo_create_pin_map(xe, tile, NULL, + lrc_size + LRC_WA_BB_SIZE, + ttm_bo_type_kernel, + bo_flags); if (IS_ERR(lrc->bo)) return PTR_ERR(lrc->bo); - lrc->tile = gt_to_tile(hwe->gt); + lrc->size = lrc_size; lrc->ring.size = ring_size; lrc->ring.tail = 0; @@ -756,10 +1056,10 @@ int xe_lrc_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe, xe_map_memset(xe, &map, 0, 0, LRC_PPHWSP_SIZE); /* PPHWSP */ xe_map_memcpy_to(xe, &map, LRC_PPHWSP_SIZE, gt->default_lrc[hwe->class] + LRC_PPHWSP_SIZE, - xe_lrc_size(xe, hwe->class) - LRC_PPHWSP_SIZE); + xe_gt_lrc_size(gt, hwe->class) - LRC_PPHWSP_SIZE); } else { xe_map_memcpy_to(xe, &map, 0, init_data, - xe_lrc_size(xe, hwe->class)); + xe_gt_lrc_size(gt, hwe->class)); kfree(init_data); } @@ -770,16 +1070,53 @@ int xe_lrc_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe, xe_drm_client_add_bo(vm->xef->client, lrc->bo); } - xe_lrc_write_ctx_reg(lrc, CTX_RING_START, __xe_lrc_ring_ggtt_addr(lrc)); - xe_lrc_write_ctx_reg(lrc, CTX_RING_HEAD, 0); - xe_lrc_write_ctx_reg(lrc, CTX_RING_TAIL, lrc->ring.tail); - xe_lrc_write_ctx_reg(lrc, CTX_RING_CTL, - RING_CTL_SIZE(lrc->ring.size) | RING_VALID); + if (xe_device_has_msix(xe)) { + xe_lrc_write_ctx_reg(lrc, CTX_INT_STATUS_REPORT_PTR, + xe_memirq_status_ptr(&tile->memirq, hwe)); + xe_lrc_write_ctx_reg(lrc, CTX_INT_SRC_REPORT_PTR, + xe_memirq_source_ptr(&tile->memirq, hwe)); + xe_lrc_write_ctx_reg(lrc, CTX_CS_INT_VEC_DATA, msix_vec << 16 | msix_vec); + } + + if (xe_gt_has_indirect_ring_state(gt)) { + xe_lrc_write_ctx_reg(lrc, CTX_INDIRECT_RING_STATE, + __xe_lrc_indirect_ring_ggtt_addr(lrc)); + + xe_lrc_write_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_START, + __xe_lrc_ring_ggtt_addr(lrc)); + xe_lrc_write_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_START_UDW, 0); + xe_lrc_write_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_HEAD, 0); + xe_lrc_write_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_TAIL, lrc->ring.tail); + xe_lrc_write_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_CTL, + RING_CTL_SIZE(lrc->ring.size) | RING_VALID); + } else { + xe_lrc_write_ctx_reg(lrc, CTX_RING_START, __xe_lrc_ring_ggtt_addr(lrc)); + xe_lrc_write_ctx_reg(lrc, CTX_RING_HEAD, 0); + xe_lrc_write_ctx_reg(lrc, CTX_RING_TAIL, lrc->ring.tail); + xe_lrc_write_ctx_reg(lrc, CTX_RING_CTL, + RING_CTL_SIZE(lrc->ring.size) | RING_VALID); + } + + if (init_flags & XE_LRC_CREATE_RUNALONE) + xe_lrc_write_ctx_reg(lrc, CTX_CONTEXT_CONTROL, + xe_lrc_read_ctx_reg(lrc, CTX_CONTEXT_CONTROL) | + _MASKED_BIT_ENABLE(CTX_CTRL_RUN_ALONE)); + + if (init_flags & XE_LRC_CREATE_PXP) + xe_lrc_write_ctx_reg(lrc, CTX_CONTEXT_CONTROL, + xe_lrc_read_ctx_reg(lrc, CTX_CONTEXT_CONTROL) | + _MASKED_BIT_ENABLE(CTX_CTRL_PXP_ENABLE)); + + lrc->ctx_timestamp = 0; + xe_lrc_write_ctx_reg(lrc, CTX_TIMESTAMP, 0); + if (lrc_to_xe(lrc)->info.has_64bit_timestamp) + xe_lrc_write_ctx_reg(lrc, CTX_TIMESTAMP_UDW, 0); + if (xe->info.has_asid && vm) xe_lrc_write_ctx_reg(lrc, PVC_CTX_ASID, vm->usm.asid); lrc->desc = LRC_VALID; - lrc->desc |= LRC_LEGACY_64B_CONTEXT << LRC_ADDRESSING_MODE_SHIFT; + lrc->desc |= FIELD_PREP(LRC_ADDRESSING_MODE, LRC_LEGACY_64B_CONTEXT); /* TODO: Priority */ /* While this appears to have something about privileged batches or @@ -789,8 +1126,8 @@ int xe_lrc_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe, lrc->desc |= LRC_PRIVILEGE; if (GRAPHICS_VERx100(xe) < 1250) { - lrc->desc |= (u64)hwe->instance << ENGINE_INSTANCE_SHIFT; - lrc->desc |= (u64)hwe->class << ENGINE_CLASS_SHIFT; + lrc->desc |= FIELD_PREP(LRC_ENGINE_INSTANCE, hwe->instance); + lrc->desc |= FIELD_PREP(LRC_ENGINE_CLASS, hwe->class); } arb_enable = MI_ARB_ON_OFF | MI_ARB_ENABLE; @@ -802,6 +1139,10 @@ int xe_lrc_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe, map = __xe_lrc_start_seqno_map(lrc); xe_map_write32(lrc_to_xe(lrc), &map, lrc->fence_ctx.next_seqno - 1); + err = xe_lrc_setup_utilization(lrc); + if (err) + goto err_lrc_finish; + return 0; err_lrc_finish: @@ -809,23 +1150,91 @@ err_lrc_finish: return err; } -void xe_lrc_finish(struct xe_lrc *lrc) +/** + * xe_lrc_create - Create a LRC + * @hwe: Hardware Engine + * @vm: The VM (address space) + * @ring_size: LRC ring size + * @msix_vec: MSI-X interrupt vector (for platforms that support it) + * @flags: LRC initialization flags + * + * Allocate and initialize the Logical Ring Context (LRC). + * + * Return pointer to created LRC upon success and an error pointer + * upon failure. + */ +struct xe_lrc *xe_lrc_create(struct xe_hw_engine *hwe, struct xe_vm *vm, + u32 ring_size, u16 msix_vec, u32 flags) { - xe_hw_fence_ctx_finish(&lrc->fence_ctx); - xe_bo_lock(lrc->bo, false); - xe_bo_unpin(lrc->bo); - xe_bo_unlock(lrc->bo); - xe_bo_put(lrc->bo); + struct xe_lrc *lrc; + int err; + + lrc = kzalloc(sizeof(*lrc), GFP_KERNEL); + if (!lrc) + return ERR_PTR(-ENOMEM); + + err = xe_lrc_init(lrc, hwe, vm, ring_size, msix_vec, flags); + if (err) { + kfree(lrc); + return ERR_PTR(err); + } + + return lrc; +} + +/** + * xe_lrc_destroy - Destroy the LRC + * @ref: reference to LRC + * + * Called when ref == 0, release resources held by the Logical Ring Context + * (LRC) and free the LRC memory. + */ +void xe_lrc_destroy(struct kref *ref) +{ + struct xe_lrc *lrc = container_of(ref, struct xe_lrc, refcount); + + xe_lrc_finish(lrc); + kfree(lrc); +} + +void xe_lrc_set_ring_tail(struct xe_lrc *lrc, u32 tail) +{ + if (xe_lrc_has_indirect_ring_state(lrc)) + xe_lrc_write_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_TAIL, tail); + else + xe_lrc_write_ctx_reg(lrc, CTX_RING_TAIL, tail); +} + +u32 xe_lrc_ring_tail(struct xe_lrc *lrc) +{ + if (xe_lrc_has_indirect_ring_state(lrc)) + return xe_lrc_read_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_TAIL) & TAIL_ADDR; + else + return xe_lrc_read_ctx_reg(lrc, CTX_RING_TAIL) & TAIL_ADDR; +} + +static u32 xe_lrc_ring_start(struct xe_lrc *lrc) +{ + if (xe_lrc_has_indirect_ring_state(lrc)) + return xe_lrc_read_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_START); + else + return xe_lrc_read_ctx_reg(lrc, CTX_RING_START); } void xe_lrc_set_ring_head(struct xe_lrc *lrc, u32 head) { - xe_lrc_write_ctx_reg(lrc, CTX_RING_HEAD, head); + if (xe_lrc_has_indirect_ring_state(lrc)) + xe_lrc_write_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_HEAD, head); + else + xe_lrc_write_ctx_reg(lrc, CTX_RING_HEAD, head); } u32 xe_lrc_ring_head(struct xe_lrc *lrc) { - return xe_lrc_read_ctx_reg(lrc, CTX_RING_HEAD) & HEAD_ADDR; + if (xe_lrc_has_indirect_ring_state(lrc)) + return xe_lrc_read_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_HEAD) & HEAD_ADDR; + else + return xe_lrc_read_ctx_reg(lrc, CTX_RING_HEAD) & HEAD_ADDR; } u32 xe_lrc_ring_space(struct xe_lrc *lrc) @@ -885,10 +1294,43 @@ u32 xe_lrc_seqno_ggtt_addr(struct xe_lrc *lrc) return __xe_lrc_seqno_ggtt_addr(lrc); } -struct dma_fence *xe_lrc_create_seqno_fence(struct xe_lrc *lrc) +/** + * xe_lrc_alloc_seqno_fence() - Allocate an lrc seqno fence. + * + * Allocate but don't initialize an lrc seqno fence. + * + * Return: Pointer to the allocated fence or + * negative error pointer on error. + */ +struct dma_fence *xe_lrc_alloc_seqno_fence(void) +{ + return xe_hw_fence_alloc(); +} + +/** + * xe_lrc_free_seqno_fence() - Free an lrc seqno fence. + * @fence: Pointer to the fence to free. + * + * Frees an lrc seqno fence that hasn't yet been + * initialized. + */ +void xe_lrc_free_seqno_fence(struct dma_fence *fence) { - return &xe_hw_fence_create(&lrc->fence_ctx, - __xe_lrc_seqno_map(lrc))->dma; + xe_hw_fence_free(fence); +} + +/** + * xe_lrc_init_seqno_fence() - Initialize an lrc seqno fence. + * @lrc: Pointer to the lrc. + * @fence: Pointer to the fence to initialize. + * + * Initializes a pre-allocated lrc seqno fence. + * After initialization, the fence is subject to normal + * dma-fence refcounting. + */ +void xe_lrc_init_seqno_fence(struct xe_lrc *lrc, struct dma_fence *fence) +{ + xe_hw_fence_init(fence, &lrc->fence_ctx, __xe_lrc_seqno_map(lrc)); } s32 xe_lrc_seqno(struct xe_lrc *lrc) @@ -920,6 +1362,21 @@ struct iosys_map xe_lrc_parallel_map(struct xe_lrc *lrc) return __xe_lrc_parallel_map(lrc); } +/** + * xe_lrc_engine_id() - Read engine id value + * @lrc: Pointer to the lrc. + * + * Returns: context id value + */ +static u32 xe_lrc_engine_id(struct xe_lrc *lrc) +{ + struct xe_device *xe = lrc_to_xe(lrc); + struct iosys_map map; + + map = __xe_lrc_engine_id_map(lrc); + return xe_map_read32(xe, &map); +} + static int instr_dw(u32 cmd_header) { /* GFXPIPE "SINGLE_DW" opcodes are a single dword */ @@ -1037,6 +1494,8 @@ static int dump_gfxpipe_command(struct drm_printer *p, MATCH(GPGPU_CSR_BASE_ADDRESS); MATCH(STATE_COMPUTE_MODE); MATCH3D(3DSTATE_BTD); + MATCH(STATE_SYSTEM_MEM_FENCE_ADDRESS); + MATCH(STATE_CONTEXT_DATA_BASE_ADDRESS); MATCH3D(3DSTATE_VF_STATISTICS); @@ -1061,6 +1520,7 @@ static int dump_gfxpipe_command(struct drm_printer *p, MATCH3D(3DSTATE_WM); MATCH3D(3DSTATE_CONSTANT_VS); MATCH3D(3DSTATE_CONSTANT_GS); + MATCH3D(3DSTATE_CONSTANT_PS); MATCH3D(3DSTATE_SAMPLE_MASK); MATCH3D(3DSTATE_CONSTANT_HS); MATCH3D(3DSTATE_CONSTANT_DS); @@ -1124,6 +1584,7 @@ static int dump_gfxpipe_command(struct drm_printer *p, MATCH3D(3DSTATE_CLIP_MESH); MATCH3D(3DSTATE_SBE_MESH); MATCH3D(3DSTATE_CPSIZE_CONTROL_BUFFER); + MATCH3D(3DSTATE_COARSE_PIXEL); MATCH3D(3DSTATE_DRAWING_RECTANGLE); MATCH3D(3DSTATE_CHROMA_KEY); @@ -1153,6 +1614,31 @@ static int dump_gfxpipe_command(struct drm_printer *p, } } +static int dump_gfx_state_command(struct drm_printer *p, + struct xe_gt *gt, + u32 *dw, + int remaining_dw) +{ + u32 numdw = instr_dw(*dw); + u32 opcode = REG_FIELD_GET(GFX_STATE_OPCODE, *dw); + + /* + * Make sure we haven't mis-parsed a number of dwords that exceeds the + * remaining size of the LRC. + */ + if (xe_gt_WARN_ON(gt, numdw > remaining_dw)) + numdw = remaining_dw; + + switch (*dw & (XE_INSTR_GFX_STATE | GFX_STATE_OPCODE)) { + MATCH(STATE_WRITE_INLINE); + + default: + drm_printf(p, "[%#010x] unknown GFX_STATE command (opcode=%#x), likely %d dwords\n", + *dw, opcode, numdw); + return numdw; + } +} + void xe_lrc_dump_default(struct drm_printer *p, struct xe_gt *gt, enum xe_engine_class hwe_class) @@ -1170,13 +1656,15 @@ void xe_lrc_dump_default(struct drm_printer *p, * hardware status page. */ dw = gt->default_lrc[hwe_class] + LRC_PPHWSP_SIZE; - remaining_dw = (xe_lrc_size(gt_to_xe(gt), hwe_class) - LRC_PPHWSP_SIZE) / 4; + remaining_dw = (xe_gt_lrc_size(gt, hwe_class) - LRC_PPHWSP_SIZE) / 4; while (remaining_dw > 0) { if ((*dw & XE_INSTR_CMD_TYPE) == XE_INSTR_MI) { num_dw = dump_mi_command(p, gt, dw, remaining_dw); } else if ((*dw & XE_INSTR_CMD_TYPE) == XE_INSTR_GFXPIPE) { num_dw = dump_gfxpipe_command(p, gt, dw, remaining_dw); + } else if ((*dw & XE_INSTR_CMD_TYPE) == XE_INSTR_GFX_STATE) { + num_dw = dump_gfx_state_command(p, gt, dw, remaining_dw); } else { num_dw = min(instr_dw(*dw), remaining_dw); drm_printf(p, "[%#10x] Unknown instruction of type %#x, likely %d dwords\n", @@ -1255,19 +1743,31 @@ void xe_lrc_emit_hwe_state_instructions(struct xe_exec_queue *q, struct xe_bb *b int state_table_size = 0; /* - * At the moment we only need to emit non-register state for the RCS - * engine. + * Wa_14019789679 + * + * If the driver doesn't explicitly emit the SVG instructions while + * setting up the default LRC, the context switch will write 0's + * (noops) into the LRC memory rather than the expected instruction + * headers. Application contexts start out as a copy of the default + * LRC, and if they also do not emit specific settings for some SVG + * state, then on context restore they'll unintentionally inherit + * whatever state setting the previous context had programmed into the + * hardware (i.e., the lack of a 3DSTATE_* instruction in the LRC will + * prevent the hardware from resetting that state back to any specific + * value). + * + * The official workaround only requires emitting 3DSTATE_MESH_CONTROL + * since that's a specific state setting that can easily cause GPU + * hangs if unintentionally inherited. However to be safe we'll + * continue to emit all of the SVG state since it's best not to leak + * any of the state between contexts, even if that leakage is harmless. */ - if (q->hwe->class != XE_ENGINE_CLASS_RENDER) - return; - - switch (GRAPHICS_VERx100(xe)) { - case 1255: - case 1270 ... 2004: + if (XE_WA(gt, 14019789679) && q->hwe->class == XE_ENGINE_CLASS_RENDER) { state_table = xe_hpg_svg_state; state_table_size = ARRAY_SIZE(xe_hpg_svg_state); - break; - default: + } + + if (!state_table) { xe_gt_dbg(gt, "No non-register state to emit on graphics ver %d.%02d\n", GRAPHICS_VER(xe), GRAPHICS_VERx100(xe) % 100); return; @@ -1300,3 +1800,201 @@ void xe_lrc_emit_hwe_state_instructions(struct xe_exec_queue *q, struct xe_bb *b bb->len += num_dw; } } + +struct xe_lrc_snapshot *xe_lrc_snapshot_capture(struct xe_lrc *lrc) +{ + struct xe_lrc_snapshot *snapshot = kmalloc(sizeof(*snapshot), GFP_NOWAIT); + + if (!snapshot) + return NULL; + + snapshot->context_desc = xe_lrc_ggtt_addr(lrc); + snapshot->ring_addr = __xe_lrc_ring_ggtt_addr(lrc); + snapshot->indirect_context_desc = xe_lrc_indirect_ring_ggtt_addr(lrc); + snapshot->head = xe_lrc_ring_head(lrc); + snapshot->tail.internal = lrc->ring.tail; + snapshot->tail.memory = xe_lrc_ring_tail(lrc); + snapshot->start = xe_lrc_ring_start(lrc); + snapshot->start_seqno = xe_lrc_start_seqno(lrc); + snapshot->seqno = xe_lrc_seqno(lrc); + snapshot->lrc_bo = xe_bo_get(lrc->bo); + snapshot->lrc_offset = xe_lrc_pphwsp_offset(lrc); + snapshot->lrc_size = lrc->bo->size - snapshot->lrc_offset - + LRC_WA_BB_SIZE; + snapshot->lrc_snapshot = NULL; + snapshot->ctx_timestamp = lower_32_bits(xe_lrc_ctx_timestamp(lrc)); + snapshot->ctx_job_timestamp = xe_lrc_ctx_job_timestamp(lrc); + return snapshot; +} + +void xe_lrc_snapshot_capture_delayed(struct xe_lrc_snapshot *snapshot) +{ + struct xe_bo *bo; + struct iosys_map src; + + if (!snapshot) + return; + + bo = snapshot->lrc_bo; + snapshot->lrc_bo = NULL; + + snapshot->lrc_snapshot = kvmalloc(snapshot->lrc_size, GFP_KERNEL); + if (!snapshot->lrc_snapshot) + goto put_bo; + + xe_bo_lock(bo, false); + if (!ttm_bo_vmap(&bo->ttm, &src)) { + xe_map_memcpy_from(xe_bo_device(bo), + snapshot->lrc_snapshot, &src, snapshot->lrc_offset, + snapshot->lrc_size); + ttm_bo_vunmap(&bo->ttm, &src); + } else { + kvfree(snapshot->lrc_snapshot); + snapshot->lrc_snapshot = NULL; + } + xe_bo_unlock(bo); +put_bo: + xe_bo_put(bo); +} + +void xe_lrc_snapshot_print(struct xe_lrc_snapshot *snapshot, struct drm_printer *p) +{ + unsigned long i; + + if (!snapshot) + return; + + drm_printf(p, "\tHW Context Desc: 0x%08x\n", snapshot->context_desc); + drm_printf(p, "\tHW Ring address: 0x%08x\n", + snapshot->ring_addr); + drm_printf(p, "\tHW Indirect Ring State: 0x%08x\n", + snapshot->indirect_context_desc); + drm_printf(p, "\tLRC Head: (memory) %u\n", snapshot->head); + drm_printf(p, "\tLRC Tail: (internal) %u, (memory) %u\n", + snapshot->tail.internal, snapshot->tail.memory); + drm_printf(p, "\tRing start: (memory) 0x%08x\n", snapshot->start); + drm_printf(p, "\tStart seqno: (memory) %d\n", snapshot->start_seqno); + drm_printf(p, "\tSeqno: (memory) %d\n", snapshot->seqno); + drm_printf(p, "\tTimestamp: 0x%08x\n", snapshot->ctx_timestamp); + drm_printf(p, "\tJob Timestamp: 0x%08x\n", snapshot->ctx_job_timestamp); + + if (!snapshot->lrc_snapshot) + return; + + drm_printf(p, "\t[HWSP].length: 0x%x\n", LRC_PPHWSP_SIZE); + drm_puts(p, "\t[HWSP].data: "); + for (i = 0; i < LRC_PPHWSP_SIZE; i += sizeof(u32)) { + u32 *val = snapshot->lrc_snapshot + i; + char dumped[ASCII85_BUFSZ]; + + drm_puts(p, ascii85_encode(*val, dumped)); + } + + drm_printf(p, "\n\t[HWCTX].length: 0x%lx\n", snapshot->lrc_size - LRC_PPHWSP_SIZE); + drm_puts(p, "\t[HWCTX].data: "); + for (; i < snapshot->lrc_size; i += sizeof(u32)) { + u32 *val = snapshot->lrc_snapshot + i; + char dumped[ASCII85_BUFSZ]; + + drm_puts(p, ascii85_encode(*val, dumped)); + } + drm_puts(p, "\n"); +} + +void xe_lrc_snapshot_free(struct xe_lrc_snapshot *snapshot) +{ + if (!snapshot) + return; + + kvfree(snapshot->lrc_snapshot); + if (snapshot->lrc_bo) + xe_bo_put(snapshot->lrc_bo); + + kfree(snapshot); +} + +static int get_ctx_timestamp(struct xe_lrc *lrc, u32 engine_id, u64 *reg_ctx_ts) +{ + u16 class = REG_FIELD_GET(ENGINE_CLASS_ID, engine_id); + u16 instance = REG_FIELD_GET(ENGINE_INSTANCE_ID, engine_id); + struct xe_hw_engine *hwe; + u64 val; + + hwe = xe_gt_hw_engine(lrc->gt, class, instance, false); + if (xe_gt_WARN_ONCE(lrc->gt, !hwe || xe_hw_engine_is_reserved(hwe), + "Unexpected engine class:instance %d:%d for context utilization\n", + class, instance)) + return -1; + + if (lrc_to_xe(lrc)->info.has_64bit_timestamp) + val = xe_mmio_read64_2x32(&hwe->gt->mmio, + RING_CTX_TIMESTAMP(hwe->mmio_base)); + else + val = xe_mmio_read32(&hwe->gt->mmio, + RING_CTX_TIMESTAMP(hwe->mmio_base)); + + *reg_ctx_ts = val; + + return 0; +} + +/** + * xe_lrc_update_timestamp() - Update ctx timestamp + * @lrc: Pointer to the lrc. + * @old_ts: Old timestamp value + * + * Populate @old_ts current saved ctx timestamp, read new ctx timestamp and + * update saved value. With support for active contexts, the calculation may be + * slightly racy, so follow a read-again logic to ensure that the context is + * still active before returning the right timestamp. + * + * Returns: New ctx timestamp value + */ +u64 xe_lrc_update_timestamp(struct xe_lrc *lrc, u64 *old_ts) +{ + u64 lrc_ts, reg_ts; + u32 engine_id; + + *old_ts = lrc->ctx_timestamp; + + lrc_ts = xe_lrc_ctx_timestamp(lrc); + /* CTX_TIMESTAMP mmio read is invalid on VF, so return the LRC value */ + if (IS_SRIOV_VF(lrc_to_xe(lrc))) { + lrc->ctx_timestamp = lrc_ts; + goto done; + } + + if (lrc_ts == CONTEXT_ACTIVE) { + engine_id = xe_lrc_engine_id(lrc); + if (!get_ctx_timestamp(lrc, engine_id, ®_ts)) + lrc->ctx_timestamp = reg_ts; + + /* read lrc again to ensure context is still active */ + lrc_ts = xe_lrc_ctx_timestamp(lrc); + } + + /* + * If context switched out, just use the lrc_ts. Note that this needs to + * be a separate if condition. + */ + if (lrc_ts != CONTEXT_ACTIVE) + lrc->ctx_timestamp = lrc_ts; + +done: + trace_xe_lrc_update_timestamp(lrc, *old_ts); + + return lrc->ctx_timestamp; +} + +/** + * xe_lrc_ring_is_idle() - LRC is idle + * @lrc: Pointer to the lrc. + * + * Compare LRC ring head and tail to determine if idle. + * + * Return: True is ring is idle, False otherwise + */ +bool xe_lrc_ring_is_idle(struct xe_lrc *lrc) +{ + return xe_lrc_ring_head(lrc) == xe_lrc_ring_tail(lrc); +} |