diff options
Diffstat (limited to 'drivers/gpu/drm/xe')
61 files changed, 1925 insertions, 268 deletions
diff --git a/drivers/gpu/drm/xe/Makefile b/drivers/gpu/drm/xe/Makefile index b9670ae09a9e..edfd812e0f41 100644 --- a/drivers/gpu/drm/xe/Makefile +++ b/drivers/gpu/drm/xe/Makefile @@ -40,6 +40,7 @@ xe-y += xe_bb.o \ xe_ggtt.o \ xe_gpu_scheduler.o \ xe_gsc.o \ + xe_gsc_debugfs.o \ xe_gsc_proxy.o \ xe_gsc_submit.o \ xe_gt.o \ diff --git a/drivers/gpu/drm/xe/compat-i915-headers/i915_drv.h b/drivers/gpu/drm/xe/compat-i915-headers/i915_drv.h index b7b12b20e390..f27a2c75b56d 100644 --- a/drivers/gpu/drm/xe/compat-i915-headers/i915_drv.h +++ b/drivers/gpu/drm/xe/compat-i915-headers/i915_drv.h @@ -21,11 +21,6 @@ static inline struct drm_i915_private *to_i915(const struct drm_device *dev) return container_of(dev, struct drm_i915_private, drm); } -static inline struct drm_i915_private *kdev_to_i915(struct device *kdev) -{ - return dev_get_drvdata(kdev); -} - #define IS_PLATFORM(xe, x) ((xe)->info.platform == x) #define INTEL_INFO(dev_priv) (&((dev_priv)->info)) #define IS_I830(dev_priv) (dev_priv && 0) @@ -80,9 +75,9 @@ static inline struct drm_i915_private *kdev_to_i915(struct device *kdev) #define IS_MOBILE(xe) (xe && 0) -#define IS_LP(xe) (0) -#define IS_GEN9_LP(xe) (0) -#define IS_GEN9_BC(xe) (0) +#define IS_LP(xe) ((xe) && 0) +#define IS_GEN9_LP(xe) ((xe) && 0) +#define IS_GEN9_BC(xe) ((xe) && 0) #define IS_TIGERLAKE_UY(xe) (xe && 0) #define IS_COMETLAKE_ULX(xe) (xe && 0) @@ -110,8 +105,6 @@ struct i915_sched_attr { }; #define i915_gem_fence_wait_priority(fence, attr) do { (void) attr; } while (0) -#define pdev_to_i915 pdev_to_xe_device - #define FORCEWAKE_ALL XE_FORCEWAKE_ALL #ifdef CONFIG_ARM64 diff --git a/drivers/gpu/drm/xe/compat-i915-headers/intel_pcode.h b/drivers/gpu/drm/xe/compat-i915-headers/intel_pcode.h index 0c47661bdc6a..a473aa6697d0 100644 --- a/drivers/gpu/drm/xe/compat-i915-headers/intel_pcode.h +++ b/drivers/gpu/drm/xe/compat-i915-headers/intel_pcode.h @@ -13,7 +13,7 @@ static inline int snb_pcode_write_timeout(struct intel_uncore *uncore, u32 mbox, u32 val, int fast_timeout_us, int slow_timeout_ms) { - return xe_pcode_write_timeout(__compat_uncore_to_gt(uncore), mbox, val, + return xe_pcode_write_timeout(__compat_uncore_to_tile(uncore), mbox, val, slow_timeout_ms ?: 1); } @@ -21,13 +21,13 @@ static inline int snb_pcode_write(struct intel_uncore *uncore, u32 mbox, u32 val) { - return xe_pcode_write(__compat_uncore_to_gt(uncore), mbox, val); + return xe_pcode_write(__compat_uncore_to_tile(uncore), mbox, val); } static inline int snb_pcode_read(struct intel_uncore *uncore, u32 mbox, u32 *val, u32 *val1) { - return xe_pcode_read(__compat_uncore_to_gt(uncore), mbox, val, val1); + return xe_pcode_read(__compat_uncore_to_tile(uncore), mbox, val, val1); } static inline int @@ -35,7 +35,7 @@ skl_pcode_request(struct intel_uncore *uncore, u32 mbox, u32 request, u32 reply_mask, u32 reply, int timeout_base_ms) { - return xe_pcode_request(__compat_uncore_to_gt(uncore), mbox, request, reply_mask, reply, + return xe_pcode_request(__compat_uncore_to_tile(uncore), mbox, request, reply_mask, reply, timeout_base_ms); } diff --git a/drivers/gpu/drm/xe/compat-i915-headers/intel_uncore.h b/drivers/gpu/drm/xe/compat-i915-headers/intel_uncore.h index 083c4da2ea41..eb5b5f0e4bd9 100644 --- a/drivers/gpu/drm/xe/compat-i915-headers/intel_uncore.h +++ b/drivers/gpu/drm/xe/compat-i915-headers/intel_uncore.h @@ -17,6 +17,13 @@ static inline struct xe_gt *__compat_uncore_to_gt(struct intel_uncore *uncore) return xe_root_mmio_gt(xe); } +static inline struct xe_tile *__compat_uncore_to_tile(struct intel_uncore *uncore) +{ + struct xe_device *xe = container_of(uncore, struct xe_device, uncore); + + return xe_device_get_root_tile(xe); +} + static inline u32 intel_uncore_read(struct intel_uncore *uncore, i915_reg_t i915_reg) { diff --git a/drivers/gpu/drm/xe/display/intel_fbdev_fb.c b/drivers/gpu/drm/xe/display/intel_fbdev_fb.c index cd8948c08661..99499d6c0256 100644 --- a/drivers/gpu/drm/xe/display/intel_fbdev_fb.c +++ b/drivers/gpu/drm/xe/display/intel_fbdev_fb.c @@ -8,7 +8,6 @@ #include "intel_display_types.h" #include "intel_fbdev_fb.h" #include "xe_bo.h" -#include "xe_gt.h" #include "xe_ttm_stolen_mgr.h" #include "xe_wa.h" diff --git a/drivers/gpu/drm/xe/display/xe_display.c b/drivers/gpu/drm/xe/display/xe_display.c index 78a884ddd499..75736faf2a80 100644 --- a/drivers/gpu/drm/xe/display/xe_display.c +++ b/drivers/gpu/drm/xe/display/xe_display.c @@ -10,7 +10,7 @@ #include <drm/drm_drv.h> #include <drm/drm_managed.h> -#include <drm/xe_drm.h> +#include <uapi/drm/xe_drm.h> #include "soc/intel_dram.h" #include "i915_drv.h" /* FIXME: HAS_DISPLAY() depends on this */ @@ -341,16 +341,14 @@ void xe_display_pm_suspend(struct xe_device *xe, bool runtime) xe_display_flush_cleanup_work(xe); - xe_display_flush_cleanup_work(xe); - intel_dp_mst_suspend(xe); intel_hpd_cancel_work(xe); - if (!runtime && has_display(xe)) + if (!runtime && has_display(xe)) { intel_display_driver_suspend_access(xe); - - intel_encoder_suspend_all(&xe->display); + intel_encoder_suspend_all(&xe->display); + } intel_opregion_suspend(display, s2idle ? PCI_D1 : PCI_D3cold); diff --git a/drivers/gpu/drm/xe/display/xe_dsb_buffer.c b/drivers/gpu/drm/xe/display/xe_dsb_buffer.c index ccd0d87d438a..f99d901a3214 100644 --- a/drivers/gpu/drm/xe/display/xe_dsb_buffer.c +++ b/drivers/gpu/drm/xe/display/xe_dsb_buffer.c @@ -9,7 +9,6 @@ #include "xe_bo.h" #include "xe_device.h" #include "xe_device_types.h" -#include "xe_gt.h" u32 intel_dsb_buffer_ggtt_offset(struct intel_dsb_buffer *dsb_buf) { diff --git a/drivers/gpu/drm/xe/display/xe_fb_pin.c b/drivers/gpu/drm/xe/display/xe_fb_pin.c index d650c5ac41a4..b58fc4ba2aac 100644 --- a/drivers/gpu/drm/xe/display/xe_fb_pin.c +++ b/drivers/gpu/drm/xe/display/xe_fb_pin.c @@ -12,7 +12,6 @@ #include "xe_bo.h" #include "xe_device.h" #include "xe_ggtt.h" -#include "xe_gt.h" #include "xe_pm.h" static void diff --git a/drivers/gpu/drm/xe/display/xe_hdcp_gsc.c b/drivers/gpu/drm/xe/display/xe_hdcp_gsc.c index 0af667ebebf9..6619a40aed15 100644 --- a/drivers/gpu/drm/xe/display/xe_hdcp_gsc.c +++ b/drivers/gpu/drm/xe/display/xe_hdcp_gsc.c @@ -16,7 +16,6 @@ #include "xe_force_wake.h" #include "xe_gsc_proxy.h" #include "xe_gsc_submit.h" -#include "xe_gt.h" #include "xe_map.h" #include "xe_pm.h" #include "xe_uc_fw.h" diff --git a/drivers/gpu/drm/xe/regs/xe_gsc_regs.h b/drivers/gpu/drm/xe/regs/xe_gsc_regs.h index e2a925be137c..7702364b65f1 100644 --- a/drivers/gpu/drm/xe/regs/xe_gsc_regs.h +++ b/drivers/gpu/drm/xe/regs/xe_gsc_regs.h @@ -32,8 +32,12 @@ #define HECI1_FWSTS1_CURRENT_STATE_RESET 0 #define HECI1_FWSTS1_PROXY_STATE_NORMAL 5 #define HECI1_FWSTS1_INIT_COMPLETE REG_BIT(9) +#define HECI_FWSTS2(base) XE_REG((base) + 0xc48) +#define HECI_FWSTS3(base) XE_REG((base) + 0xc60) +#define HECI_FWSTS4(base) XE_REG((base) + 0xc64) #define HECI_FWSTS5(base) XE_REG((base) + 0xc68) #define HECI1_FWSTS5_HUC_AUTH_DONE REG_BIT(19) +#define HECI_FWSTS6(base) XE_REG((base) + 0xc6c) #define HECI_H_GS1(base) XE_REG((base) + 0xc4c) #define HECI_H_GS1_ER_PREP REG_BIT(0) diff --git a/drivers/gpu/drm/xe/tests/xe_dma_buf.c b/drivers/gpu/drm/xe/tests/xe_dma_buf.c index 13db6c0530b3..cedd3e88a6fb 100644 --- a/drivers/gpu/drm/xe/tests/xe_dma_buf.c +++ b/drivers/gpu/drm/xe/tests/xe_dma_buf.c @@ -3,7 +3,7 @@ * Copyright © 2022 Intel Corporation */ -#include <drm/xe_drm.h> +#include <uapi/drm/xe_drm.h> #include <kunit/test.h> #include <kunit/visibility.h> diff --git a/drivers/gpu/drm/xe/xe_assert.h b/drivers/gpu/drm/xe/xe_assert.h index 8b0cc1bc9327..e22bbf57fca7 100644 --- a/drivers/gpu/drm/xe/xe_assert.h +++ b/drivers/gpu/drm/xe/xe_assert.h @@ -81,7 +81,7 @@ #if IS_ENABLED(CONFIG_DRM_XE_DEBUG) #define __xe_assert_msg(xe, condition, msg, arg...) ({ \ - (void)drm_WARN(&(xe)->drm, !(condition), "[" DRM_NAME "] Assertion `%s` failed!\n" msg, \ + (void)drm_WARN(&(xe)->drm, !(condition), "Assertion `%s` failed!\n" msg, \ __stringify(condition), ## arg); \ }) #else diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c index 25d0c939ba31..06911e9a3bf5 100644 --- a/drivers/gpu/drm/xe/xe_bo.c +++ b/drivers/gpu/drm/xe/xe_bo.c @@ -13,7 +13,7 @@ #include <drm/ttm/ttm_device.h> #include <drm/ttm/ttm_placement.h> #include <drm/ttm/ttm_tt.h> -#include <drm/xe_drm.h> +#include <uapi/drm/xe_drm.h> #include "xe_device.h" #include "xe_dma_buf.h" @@ -758,7 +758,16 @@ static int xe_bo_move(struct ttm_buffer_object *ttm_bo, bool evict, xe_assert(xe, migrate); trace_xe_bo_move(bo, new_mem->mem_type, old_mem_type, move_lacks_source); - xe_pm_runtime_get_noresume(xe); + if (xe_rpm_reclaim_safe(xe)) { + /* + * We might be called through swapout in the validation path of + * another TTM device, so unconditionally acquire rpm here. + */ + xe_pm_runtime_get(xe); + } else { + drm_WARN_ON(&xe->drm, handle_system_ccs); + xe_pm_runtime_get_noresume(xe); + } if (xe_bo_is_pinned(bo) && !xe_bo_is_user(bo)) { /* diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c index b6db7e082d88..1a0d7fdd094b 100644 --- a/drivers/gpu/drm/xe/xe_device.c +++ b/drivers/gpu/drm/xe/xe_device.c @@ -15,7 +15,7 @@ #include <drm/drm_ioctl.h> #include <drm/drm_managed.h> #include <drm/drm_print.h> -#include <drm/xe_drm.h> +#include <uapi/drm/xe_drm.h> #include "display/xe_display.h" #include "instructions/xe_gpu_commands.h" diff --git a/drivers/gpu/drm/xe/xe_device.h b/drivers/gpu/drm/xe/xe_device.h index f052c06a2d2f..894f04770454 100644 --- a/drivers/gpu/drm/xe/xe_device.h +++ b/drivers/gpu/drm/xe/xe_device.h @@ -17,12 +17,16 @@ static inline struct xe_device *to_xe_device(const struct drm_device *dev) static inline struct xe_device *kdev_to_xe_device(struct device *kdev) { - return dev_get_drvdata(kdev); + struct drm_device *drm = dev_get_drvdata(kdev); + + return drm ? to_xe_device(drm) : NULL; } static inline struct xe_device *pdev_to_xe_device(struct pci_dev *pdev) { - return pci_get_drvdata(pdev); + struct drm_device *drm = pci_get_drvdata(pdev); + + return drm ? to_xe_device(drm) : NULL; } static inline struct xe_device *xe_device_const_cast(const struct xe_device *xe) diff --git a/drivers/gpu/drm/xe/xe_device_types.h b/drivers/gpu/drm/xe/xe_device_types.h index e73fb0c23932..ec7eb7811126 100644 --- a/drivers/gpu/drm/xe/xe_device_types.h +++ b/drivers/gpu/drm/xe/xe_device_types.h @@ -208,6 +208,12 @@ struct xe_tile { } vf; } sriov; + /** @pcode: tile's PCODE */ + struct { + /** @pcode.lock: protecting tile's PCODE mailbox data */ + struct mutex lock; + } pcode; + /** @migrate: Migration helper for vram blits and clearing */ struct xe_migrate *migrate; diff --git a/drivers/gpu/drm/xe/xe_drm_client.c b/drivers/gpu/drm/xe/xe_drm_client.c index 7ddd59908334..e64f4b645e2e 100644 --- a/drivers/gpu/drm/xe/xe_drm_client.c +++ b/drivers/gpu/drm/xe/xe_drm_client.c @@ -5,7 +5,7 @@ #include "xe_drm_client.h" #include <drm/drm_print.h> -#include <drm/xe_drm.h> +#include <uapi/drm/xe_drm.h> #include <linux/kernel.h> #include <linux/slab.h> #include <linux/types.h> diff --git a/drivers/gpu/drm/xe/xe_exec.c b/drivers/gpu/drm/xe/xe_exec.c index 484acfbe0e61..7b38485817dc 100644 --- a/drivers/gpu/drm/xe/xe_exec.c +++ b/drivers/gpu/drm/xe/xe_exec.c @@ -8,7 +8,7 @@ #include <drm/drm_device.h> #include <drm/drm_exec.h> #include <drm/drm_file.h> -#include <drm/xe_drm.h> +#include <uapi/drm/xe_drm.h> #include <linux/delay.h> #include "xe_bo.h" diff --git a/drivers/gpu/drm/xe/xe_exec_queue.c b/drivers/gpu/drm/xe/xe_exec_queue.c index e53937fafd14..5a9cbc97f0be 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue.c +++ b/drivers/gpu/drm/xe/xe_exec_queue.c @@ -9,7 +9,7 @@ #include <drm/drm_device.h> #include <drm/drm_file.h> -#include <drm/xe_drm.h> +#include <uapi/drm/xe_drm.h> #include "xe_device.h" #include "xe_gt.h" diff --git a/drivers/gpu/drm/xe/xe_execlist.c b/drivers/gpu/drm/xe/xe_execlist.c index 7502e3486eaf..6a59165b9569 100644 --- a/drivers/gpu/drm/xe/xe_execlist.c +++ b/drivers/gpu/drm/xe/xe_execlist.c @@ -123,8 +123,8 @@ static void __xe_execlist_port_idle(struct xe_execlist_port *port) if (!port->running_exl) return; - xe_lrc_write_ring(port->hwe->kernel_lrc, noop, sizeof(noop)); - __start_lrc(port->hwe, port->hwe->kernel_lrc, 0); + xe_lrc_write_ring(port->lrc, noop, sizeof(noop)); + __start_lrc(port->hwe, port->lrc, 0); port->running_exl = NULL; } @@ -254,14 +254,22 @@ struct xe_execlist_port *xe_execlist_port_create(struct xe_device *xe, { struct drm_device *drm = &xe->drm; struct xe_execlist_port *port; - int i; + int i, err; port = drmm_kzalloc(drm, sizeof(*port), GFP_KERNEL); - if (!port) - return ERR_PTR(-ENOMEM); + if (!port) { + err = -ENOMEM; + goto err; + } port->hwe = hwe; + port->lrc = xe_lrc_create(hwe, NULL, SZ_16K); + if (IS_ERR(port->lrc)) { + err = PTR_ERR(port->lrc); + goto err; + } + spin_lock_init(&port->lock); for (i = 0; i < ARRAY_SIZE(port->active); i++) INIT_LIST_HEAD(&port->active[i]); @@ -277,6 +285,9 @@ struct xe_execlist_port *xe_execlist_port_create(struct xe_device *xe, add_timer(&port->irq_fail); return port; + +err: + return ERR_PTR(err); } void xe_execlist_port_destroy(struct xe_execlist_port *port) @@ -287,6 +298,8 @@ void xe_execlist_port_destroy(struct xe_execlist_port *port) spin_lock_irq(>_to_xe(port->hwe->gt)->irq.lock); port->hwe->irq_handler = NULL; spin_unlock_irq(>_to_xe(port->hwe->gt)->irq.lock); + + xe_lrc_put(port->lrc); } static struct dma_fence * diff --git a/drivers/gpu/drm/xe/xe_execlist_types.h b/drivers/gpu/drm/xe/xe_execlist_types.h index f94bbf4c53e4..415140936f11 100644 --- a/drivers/gpu/drm/xe/xe_execlist_types.h +++ b/drivers/gpu/drm/xe/xe_execlist_types.h @@ -27,6 +27,8 @@ struct xe_execlist_port { struct xe_execlist_exec_queue *running_exl; struct timer_list irq_fail; + + struct xe_lrc *lrc; }; struct xe_execlist_exec_queue { diff --git a/drivers/gpu/drm/xe/xe_gsc.c b/drivers/gpu/drm/xe/xe_gsc.c index 8a137cb83318..6fbea70d3d36 100644 --- a/drivers/gpu/drm/xe/xe_gsc.c +++ b/drivers/gpu/drm/xe/xe_gsc.c @@ -8,6 +8,7 @@ #include <linux/delay.h> #include <drm/drm_managed.h> +#include <drm/drm_print.h> #include <generated/xe_wa_oob.h> @@ -165,10 +166,11 @@ static int query_compatibility_version(struct xe_gsc *gsc) return err; } - compat->major = version_query_rd(xe, &bo->vmap, rd_offset, compat_major); - compat->minor = version_query_rd(xe, &bo->vmap, rd_offset, compat_minor); + compat->major = version_query_rd(xe, &bo->vmap, rd_offset, proj_major); + compat->minor = version_query_rd(xe, &bo->vmap, rd_offset, compat_major); + compat->patch = version_query_rd(xe, &bo->vmap, rd_offset, compat_minor); - xe_gt_info(gt, "found GSC cv%u.%u\n", compat->major, compat->minor); + xe_gt_info(gt, "found GSC cv%u.%u.%u\n", compat->major, compat->minor, compat->patch); out_bo: xe_bo_unpin_map_no_vm(bo); @@ -333,9 +335,11 @@ static int gsc_er_complete(struct xe_gt *gt) if (er_status == GSCI_TIMER_STATUS_TIMER_EXPIRED) { /* * XXX: we should trigger an FLR here, but we don't have support - * for that yet. + * for that yet. Since we can't recover from the error, we + * declare the device as wedged. */ xe_gt_err(gt, "GSC ER timed out!\n"); + xe_device_declare_wedged(gt_to_xe(gt)); return -EIO; } @@ -513,13 +517,28 @@ out_bo: void xe_gsc_load_start(struct xe_gsc *gsc) { struct xe_gt *gt = gsc_to_gt(gsc); + struct xe_device *xe = gt_to_xe(gt); if (!xe_uc_fw_is_loadable(&gsc->fw) || !gsc->q) return; + /* + * The GSC HW is only reset by driver FLR or D3cold entry. We don't + * support the former at runtime, while the latter is only supported on + * DGFX, for which we don't support GSC. Therefore, if GSC failed to + * load previously there is no need to try again because the HW is + * stuck in the error state. + */ + xe_assert(xe, !IS_DGFX(xe)); + if (xe_uc_fw_is_in_error_state(&gsc->fw)) + return; + /* GSC FW survives GT reset and D3Hot */ if (gsc_fw_is_loaded(gt)) { - xe_uc_fw_change_status(&gsc->fw, XE_UC_FIRMWARE_TRANSFERRED); + if (xe_gsc_proxy_init_done(gsc)) + xe_uc_fw_change_status(&gsc->fw, XE_UC_FIRMWARE_RUNNING); + else + xe_uc_fw_change_status(&gsc->fw, XE_UC_FIRMWARE_TRANSFERRED); return; } @@ -571,3 +590,35 @@ void xe_gsc_wa_14015076503(struct xe_gt *gt, bool prep) msleep(200); } } + +/** + * xe_gsc_print_info - print info about GSC FW status + * @gsc: the GSC structure + * @p: the printer to be used to print the info + */ +void xe_gsc_print_info(struct xe_gsc *gsc, struct drm_printer *p) +{ + struct xe_gt *gt = gsc_to_gt(gsc); + int err; + + xe_uc_fw_print(&gsc->fw, p); + + drm_printf(p, "\tfound security version %u\n", gsc->security_version); + + if (!xe_uc_fw_is_enabled(&gsc->fw)) + return; + + err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GSC); + if (err) + return; + + drm_printf(p, "\nHECI1 FWSTS: 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x\n", + xe_mmio_read32(gt, HECI_FWSTS1(MTL_GSC_HECI1_BASE)), + xe_mmio_read32(gt, HECI_FWSTS2(MTL_GSC_HECI1_BASE)), + xe_mmio_read32(gt, HECI_FWSTS3(MTL_GSC_HECI1_BASE)), + xe_mmio_read32(gt, HECI_FWSTS4(MTL_GSC_HECI1_BASE)), + xe_mmio_read32(gt, HECI_FWSTS5(MTL_GSC_HECI1_BASE)), + xe_mmio_read32(gt, HECI_FWSTS6(MTL_GSC_HECI1_BASE))); + + xe_force_wake_put(gt_to_fw(gt), XE_FW_GSC); +} diff --git a/drivers/gpu/drm/xe/xe_gsc.h b/drivers/gpu/drm/xe/xe_gsc.h index 1c7a623faf11..e282b9ef6ec4 100644 --- a/drivers/gpu/drm/xe/xe_gsc.h +++ b/drivers/gpu/drm/xe/xe_gsc.h @@ -8,6 +8,7 @@ #include <linux/types.h> +struct drm_printer; struct xe_gsc; struct xe_gt; struct xe_hw_engine; @@ -21,4 +22,6 @@ void xe_gsc_hwe_irq_handler(struct xe_hw_engine *hwe, u16 intr_vec); void xe_gsc_wa_14015076503(struct xe_gt *gt, bool prep); +void xe_gsc_print_info(struct xe_gsc *gsc, struct drm_printer *p); + #endif diff --git a/drivers/gpu/drm/xe/xe_gsc_debugfs.c b/drivers/gpu/drm/xe/xe_gsc_debugfs.c new file mode 100644 index 000000000000..461d7e99c2b3 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gsc_debugfs.c @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "xe_gsc_debugfs.h" + +#include <drm/drm_debugfs.h> +#include <drm/drm_managed.h> + +#include "xe_device.h" +#include "xe_gt.h" +#include "xe_gsc.h" +#include "xe_macros.h" +#include "xe_pm.h" + +static struct xe_gt * +gsc_to_gt(struct xe_gsc *gsc) +{ + return container_of(gsc, struct xe_gt, uc.gsc); +} + +static struct xe_device * +gsc_to_xe(struct xe_gsc *gsc) +{ + return gt_to_xe(gsc_to_gt(gsc)); +} + +static struct xe_gsc *node_to_gsc(struct drm_info_node *node) +{ + return node->info_ent->data; +} + +static int gsc_info(struct seq_file *m, void *data) +{ + struct xe_gsc *gsc = node_to_gsc(m->private); + struct xe_device *xe = gsc_to_xe(gsc); + struct drm_printer p = drm_seq_file_printer(m); + + xe_pm_runtime_get(xe); + xe_gsc_print_info(gsc, &p); + xe_pm_runtime_put(xe); + + return 0; +} + +static const struct drm_info_list debugfs_list[] = { + {"gsc_info", gsc_info, 0}, +}; + +void xe_gsc_debugfs_register(struct xe_gsc *gsc, struct dentry *parent) +{ + struct drm_minor *minor = gsc_to_xe(gsc)->drm.primary; + struct drm_info_list *local; + int i; + +#define DEBUGFS_SIZE (ARRAY_SIZE(debugfs_list) * sizeof(struct drm_info_list)) + local = drmm_kmalloc(&gsc_to_xe(gsc)->drm, DEBUGFS_SIZE, GFP_KERNEL); + if (!local) + return; + + memcpy(local, debugfs_list, DEBUGFS_SIZE); +#undef DEBUGFS_SIZE + + for (i = 0; i < ARRAY_SIZE(debugfs_list); ++i) + local[i].data = gsc; + + drm_debugfs_create_files(local, + ARRAY_SIZE(debugfs_list), + parent, minor); +} diff --git a/drivers/gpu/drm/xe/xe_gsc_debugfs.h b/drivers/gpu/drm/xe/xe_gsc_debugfs.h new file mode 100644 index 000000000000..c2e2645dc705 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gsc_debugfs.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2024 Intel Corporation + */ + +#ifndef _XE_GSC_DEBUGFS_H_ +#define _XE_GSC_DEBUGFS_H_ + +struct dentry; +struct xe_gsc; + +void xe_gsc_debugfs_register(struct xe_gsc *gsc, struct dentry *parent); + +#endif diff --git a/drivers/gpu/drm/xe/xe_gt.c b/drivers/gpu/drm/xe/xe_gt.c index 08a004d698d4..dd96dec95b19 100644 --- a/drivers/gpu/drm/xe/xe_gt.c +++ b/drivers/gpu/drm/xe/xe_gt.c @@ -8,7 +8,7 @@ #include <linux/minmax.h> #include <drm/drm_managed.h> -#include <drm/xe_drm.h> +#include <uapi/drm/xe_drm.h> #include <generated/xe_wa_oob.h> @@ -48,7 +48,6 @@ #include "xe_migrate.h" #include "xe_mmio.h" #include "xe_pat.h" -#include "xe_pcode.h" #include "xe_pm.h" #include "xe_mocs.h" #include "xe_reg_sr.h" @@ -388,7 +387,6 @@ int xe_gt_init_early(struct xe_gt *gt) xe_tuning_process_gt(gt); xe_force_wake_init_gt(gt, gt_to_fw(gt)); - xe_pcode_init(gt); spin_lock_init(>->global_invl_lock); return 0; @@ -756,12 +754,13 @@ static int gt_reset(struct xe_gt *gt) xe_gt_info(gt, "reset started\n"); + xe_pm_runtime_get(gt_to_xe(gt)); + if (xe_fault_inject_gt_reset()) { err = -ECANCELED; goto err_fail; } - xe_pm_runtime_get(gt_to_xe(gt)); xe_gt_sanitize(gt); err = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); @@ -796,11 +795,11 @@ err_out: XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL)); err_msg: XE_WARN_ON(xe_uc_start(>->uc)); - xe_pm_runtime_put(gt_to_xe(gt)); err_fail: xe_gt_err(gt, "reset failed (%pe)\n", ERR_PTR(err)); xe_device_declare_wedged(gt_to_xe(gt)); + xe_pm_runtime_put(gt_to_xe(gt)); return err; } diff --git a/drivers/gpu/drm/xe/xe_gt_pagefault.c b/drivers/gpu/drm/xe/xe_gt_pagefault.c index 0be4687bfc20..730eec07795e 100644 --- a/drivers/gpu/drm/xe/xe_gt_pagefault.c +++ b/drivers/gpu/drm/xe/xe_gt_pagefault.c @@ -388,20 +388,17 @@ static void pagefault_fini(void *arg) { struct xe_gt *gt = arg; struct xe_device *xe = gt_to_xe(gt); - int i; if (!xe->info.has_usm) return; destroy_workqueue(gt->usm.acc_wq); destroy_workqueue(gt->usm.pf_wq); - - for (i = 0; i < NUM_PF_QUEUE; ++i) - kfree(gt->usm.pf_queue[i].data); } static int xe_alloc_pf_queue(struct xe_gt *gt, struct pf_queue *pf_queue) { + struct xe_device *xe = gt_to_xe(gt); xe_dss_mask_t all_dss; int num_dss, num_eus; @@ -417,7 +414,8 @@ static int xe_alloc_pf_queue(struct xe_gt *gt, struct pf_queue *pf_queue) (num_eus + XE_NUM_HW_ENGINES) * PF_MSG_LEN_DW; pf_queue->gt = gt; - pf_queue->data = kcalloc(pf_queue->num_dw, sizeof(u32), GFP_KERNEL); + pf_queue->data = devm_kcalloc(xe->drm.dev, pf_queue->num_dw, + sizeof(u32), GFP_KERNEL); if (!pf_queue->data) return -ENOMEM; diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf.c index ef239440963c..905f409db74b 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_pf.c +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf.c @@ -9,6 +9,7 @@ #include "xe_gt_sriov_pf.h" #include "xe_gt_sriov_pf_config.h" +#include "xe_gt_sriov_pf_control.h" #include "xe_gt_sriov_pf_helpers.h" #include "xe_gt_sriov_pf_service.h" #include "xe_mmio.h" @@ -57,6 +58,10 @@ int xe_gt_sriov_pf_init_early(struct xe_gt *gt) if (err) return err; + err = xe_gt_sriov_pf_control_init(gt); + if (err) + return err; + return 0; } @@ -93,4 +98,5 @@ void xe_gt_sriov_pf_init_hw(struct xe_gt *gt) void xe_gt_sriov_pf_restart(struct xe_gt *gt) { xe_gt_sriov_pf_config_restart(gt); + xe_gt_sriov_pf_control_restart(gt); } diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c index 41ed07b153b5..a95e546b7744 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c @@ -29,6 +29,7 @@ #include "xe_guc_submit.h" #include "xe_lmtt.h" #include "xe_map.h" +#include "xe_migrate.h" #include "xe_sriov.h" #include "xe_ttm_vram_mgr.h" #include "xe_wopcm.h" @@ -276,6 +277,14 @@ static u32 encode_config(u32 *cfg, const struct xe_gt_sriov_config *config) cfg[n++] = PREP_GUC_KLV_TAG(VF_CFG_PREEMPT_TIMEOUT); cfg[n++] = config->preempt_timeout; +#define encode_threshold_config(TAG, ...) ({ \ + cfg[n++] = PREP_GUC_KLV_TAG(VF_CFG_THRESHOLD_##TAG); \ + cfg[n++] = config->thresholds[MAKE_XE_GUC_KLV_THRESHOLD_INDEX(TAG)]; \ +}); + + MAKE_XE_GUC_KLV_THRESHOLDS_SET(encode_threshold_config); +#undef encode_threshold_config + return n; } @@ -1833,6 +1842,18 @@ u32 xe_gt_sriov_pf_config_get_threshold(struct xe_gt *gt, unsigned int vfid, return value; } +static void pf_reset_config_thresholds(struct xe_gt *gt, struct xe_gt_sriov_config *config) +{ + lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); + +#define reset_threshold_config(TAG, ...) ({ \ + config->thresholds[MAKE_XE_GUC_KLV_THRESHOLD_INDEX(TAG)] = 0; \ +}); + + MAKE_XE_GUC_KLV_THRESHOLDS_SET(reset_threshold_config); +#undef reset_threshold_config +} + static void pf_release_vf_config(struct xe_gt *gt, unsigned int vfid) { struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid); @@ -1848,6 +1869,7 @@ static void pf_release_vf_config(struct xe_gt *gt, unsigned int vfid) pf_release_config_ctxs(gt, config); pf_release_config_dbs(gt, config); pf_reset_config_sched(gt, config); + pf_reset_config_thresholds(gt, config); } /** @@ -1881,6 +1903,87 @@ int xe_gt_sriov_pf_config_release(struct xe_gt *gt, unsigned int vfid, bool forc return force ? 0 : err; } +static void pf_sanitize_ggtt(struct xe_ggtt_node *ggtt_region, unsigned int vfid) +{ + if (xe_ggtt_node_allocated(ggtt_region)) + xe_ggtt_assign(ggtt_region, vfid); +} + +static int pf_sanitize_lmem(struct xe_tile *tile, struct xe_bo *bo, long timeout) +{ + struct xe_migrate *m = tile->migrate; + struct dma_fence *fence; + int err; + + if (!bo) + return 0; + + xe_bo_lock(bo, false); + fence = xe_migrate_clear(m, bo, bo->ttm.resource, XE_MIGRATE_CLEAR_FLAG_FULL); + if (IS_ERR(fence)) { + err = PTR_ERR(fence); + } else if (!fence) { + err = -ENOMEM; + } else { + long ret = dma_fence_wait_timeout(fence, false, timeout); + + err = ret > 0 ? 0 : ret < 0 ? ret : -ETIMEDOUT; + dma_fence_put(fence); + if (!err) + xe_gt_sriov_dbg_verbose(tile->primary_gt, "LMEM cleared in %dms\n", + jiffies_to_msecs(timeout - ret)); + } + xe_bo_unlock(bo); + + return err; +} + +static int pf_sanitize_vf_resources(struct xe_gt *gt, u32 vfid, long timeout) +{ + struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid); + struct xe_tile *tile = gt_to_tile(gt); + struct xe_device *xe = gt_to_xe(gt); + int err = 0; + + /* + * Only GGTT and LMEM requires to be cleared by the PF. + * GuC doorbell IDs and context IDs do not need any clearing. + */ + if (!xe_gt_is_media_type(gt)) { + pf_sanitize_ggtt(config->ggtt_region, vfid); + if (IS_DGFX(xe)) + err = pf_sanitize_lmem(tile, config->lmem_obj, timeout); + } + + return err; +} + +/** + * xe_gt_sriov_pf_config_sanitize() - Sanitize VF's resources. + * @gt: the &xe_gt + * @vfid: the VF identifier (can't be PF) + * @timeout: maximum timeout to wait for completion in jiffies + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_config_sanitize(struct xe_gt *gt, unsigned int vfid, long timeout) +{ + int err; + + xe_gt_assert(gt, vfid != PFID); + + mutex_lock(xe_gt_sriov_pf_master_mutex(gt)); + err = pf_sanitize_vf_resources(gt, vfid, timeout); + mutex_unlock(xe_gt_sriov_pf_master_mutex(gt)); + + if (unlikely(err)) + xe_gt_sriov_notice(gt, "VF%u resource sanitizing failed (%pe)\n", + vfid, ERR_PTR(err)); + return err; +} + /** * xe_gt_sriov_pf_config_push - Reprovision VF's configuration. * @gt: the &xe_gt diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.h b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.h index c0e6e4743dc2..42e64769f666 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.h +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.h @@ -50,6 +50,7 @@ int xe_gt_sriov_pf_config_set_threshold(struct xe_gt *gt, unsigned int vfid, enum xe_guc_klv_threshold_index index, u32 value); int xe_gt_sriov_pf_config_set_fair(struct xe_gt *gt, unsigned int vfid, unsigned int num_vfs); +int xe_gt_sriov_pf_config_sanitize(struct xe_gt *gt, unsigned int vfid, long timeout); int xe_gt_sriov_pf_config_release(struct xe_gt *gt, unsigned int vfid, bool force); int xe_gt_sriov_pf_config_push(struct xe_gt *gt, unsigned int vfid, bool refresh); diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_control.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf_control.c index ebf06e037750..02f7328bd6ce 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_control.c +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_control.c @@ -3,11 +3,17 @@ * Copyright © 2023-2024 Intel Corporation */ +#include <drm/drm_managed.h> + #include "abi/guc_actions_sriov_abi.h" #include "xe_device.h" #include "xe_gt.h" +#include "xe_gt_sriov_pf_config.h" #include "xe_gt_sriov_pf_control.h" +#include "xe_gt_sriov_pf_helpers.h" +#include "xe_gt_sriov_pf_monitor.h" +#include "xe_gt_sriov_pf_service.h" #include "xe_gt_sriov_printk.h" #include "xe_guc_ct.h" #include "xe_sriov.h" @@ -41,10 +47,6 @@ static int guc_action_vf_control_cmd(struct xe_guc *guc, u32 vfid, u32 cmd) }; int ret; - /* XXX those two commands are now sent from the G2H handler */ - if (cmd == GUC_PF_TRIGGER_VF_FLR_START || cmd == GUC_PF_TRIGGER_VF_FLR_FINISH) - return xe_guc_ct_send_g2h_handler(&guc->ct, request, ARRAY_SIZE(request)); - ret = xe_guc_ct_send_block(&guc->ct, request, ARRAY_SIZE(request)); return ret > 0 ? -EPROTO : ret; } @@ -54,6 +56,8 @@ static int pf_send_vf_control_cmd(struct xe_gt *gt, unsigned int vfid, u32 cmd) int err; xe_gt_assert(gt, vfid != PFID); + xe_gt_sriov_dbg_verbose(gt, "sending VF%u control command %s\n", + vfid, control_cmd_to_string(cmd)); err = guc_action_vf_control_cmd(>->uc.guc, vfid, cmd); if (unlikely(err)) @@ -88,6 +92,456 @@ static int pf_send_vf_flr_finish(struct xe_gt *gt, unsigned int vfid) } /** + * DOC: The VF state machine + * + * The simplified VF state machine could be presented as:: + * + * pause--------------------------o + * / | + * / v + * (READY)<------------------resume-----(PAUSED) + * ^ \ / / + * | \ / / + * | stop---->(STOPPED)<----stop / + * | / / + * | / / + * o--------<-----flr / + * \ / + * o------<--------------------flr + * + * Where: + * + * * READY - represents a state in which VF is fully operable + * * PAUSED - represents a state in which VF activity is temporarily suspended + * * STOPPED - represents a state in which VF activity is definitely halted + * * pause - represents a request to temporarily suspend VF activity + * * resume - represents a request to resume VF activity + * * stop - represents a request to definitely halt VF activity + * * flr - represents a request to perform VF FLR to restore VF activity + * + * However, each state transition requires additional steps that involves + * communication with GuC that might fail or be interrupted by other requests:: + * + * .................................WIP.... + * : : + * pause--------------------->PAUSE_WIP----------------------------o + * / : / \ : | + * / : o----<---stop flr--o : | + * / : | \ / | : V + * (READY,RESUMED)<--------+------------RESUME_WIP<----+--<-----resume--(PAUSED) + * ^ \ \ : | | : / / + * | \ \ : | | : / / + * | \ \ : | | : / / + * | \ \ : o----<----------------------+--<-------stop / + * | \ \ : | | : / + * | \ \ : V | : / + * | \ stop----->STOP_WIP---------flr--->-----o : / + * | \ : | | : / + * | \ : | V : / + * | flr--------+----->----------------->FLR_WIP<-----flr + * | : | / ^ : + * | : | / | : + * o--------<-------:----+-----<----------------o | : + * : | | : + * :....|...........................|.....: + * | | + * V | + * (STOPPED)--------------------flr + * + * For details about each internal WIP state machine see: + * + * * `The VF PAUSE state machine`_ + * * `The VF RESUME state machine`_ + * * `The VF STOP state machine`_ + * * `The VF FLR state machine`_ + */ + +#ifdef CONFIG_DRM_XE_DEBUG_SRIOV +static const char *control_bit_to_string(enum xe_gt_sriov_control_bits bit) +{ + switch (bit) { +#define CASE2STR(_X) \ + case XE_GT_SRIOV_STATE_##_X: return #_X + CASE2STR(WIP); + CASE2STR(FLR_WIP); + CASE2STR(FLR_SEND_START); + CASE2STR(FLR_WAIT_GUC); + CASE2STR(FLR_GUC_DONE); + CASE2STR(FLR_RESET_CONFIG); + CASE2STR(FLR_RESET_DATA); + CASE2STR(FLR_RESET_MMIO); + CASE2STR(FLR_SEND_FINISH); + CASE2STR(FLR_FAILED); + CASE2STR(PAUSE_WIP); + CASE2STR(PAUSE_SEND_PAUSE); + CASE2STR(PAUSE_WAIT_GUC); + CASE2STR(PAUSE_GUC_DONE); + CASE2STR(PAUSE_FAILED); + CASE2STR(PAUSED); + CASE2STR(RESUME_WIP); + CASE2STR(RESUME_SEND_RESUME); + CASE2STR(RESUME_FAILED); + CASE2STR(RESUMED); + CASE2STR(STOP_WIP); + CASE2STR(STOP_SEND_STOP); + CASE2STR(STOP_FAILED); + CASE2STR(STOPPED); + CASE2STR(MISMATCH); +#undef CASE2STR + default: return "?"; + } +} +#endif + +static unsigned long pf_get_default_timeout(enum xe_gt_sriov_control_bits bit) +{ + switch (bit) { + case XE_GT_SRIOV_STATE_FLR_WAIT_GUC: + case XE_GT_SRIOV_STATE_PAUSE_WAIT_GUC: + return HZ / 2; + case XE_GT_SRIOV_STATE_FLR_WIP: + case XE_GT_SRIOV_STATE_FLR_RESET_CONFIG: + return 5 * HZ; + default: + return HZ; + } +} + +static struct xe_gt_sriov_control_state *pf_pick_vf_control(struct xe_gt *gt, unsigned int vfid) +{ + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + xe_gt_assert(gt, vfid <= xe_gt_sriov_pf_get_totalvfs(gt)); + + return >->sriov.pf.vfs[vfid].control; +} + +static unsigned long *pf_peek_vf_state(struct xe_gt *gt, unsigned int vfid) +{ + struct xe_gt_sriov_control_state *cs = pf_pick_vf_control(gt, vfid); + + return &cs->state; +} + +static bool pf_check_vf_state(struct xe_gt *gt, unsigned int vfid, + enum xe_gt_sriov_control_bits bit) +{ + return test_bit(bit, pf_peek_vf_state(gt, vfid)); +} + +static void pf_dump_vf_state(struct xe_gt *gt, unsigned int vfid) +{ + unsigned long state = *pf_peek_vf_state(gt, vfid); + enum xe_gt_sriov_control_bits bit; + + if (state) { + xe_gt_sriov_dbg_verbose(gt, "VF%u state %#lx%s%*pbl\n", + vfid, state, state ? " bits " : "", + (int)BITS_PER_LONG, &state); + for_each_set_bit(bit, &state, BITS_PER_LONG) + xe_gt_sriov_dbg_verbose(gt, "VF%u state %s(%d)\n", + vfid, control_bit_to_string(bit), bit); + } else { + xe_gt_sriov_dbg_verbose(gt, "VF%u state READY\n", vfid); + } +} + +static bool pf_expect_vf_state(struct xe_gt *gt, unsigned int vfid, + enum xe_gt_sriov_control_bits bit) +{ + bool result = pf_check_vf_state(gt, vfid, bit); + + if (unlikely(!result)) + pf_dump_vf_state(gt, vfid); + + return result; +} + +static bool pf_expect_vf_not_state(struct xe_gt *gt, unsigned int vfid, + enum xe_gt_sriov_control_bits bit) +{ + bool result = !pf_check_vf_state(gt, vfid, bit); + + if (unlikely(!result)) + pf_dump_vf_state(gt, vfid); + + return result; +} + +static bool pf_enter_vf_state(struct xe_gt *gt, unsigned int vfid, + enum xe_gt_sriov_control_bits bit) +{ + if (!test_and_set_bit(bit, pf_peek_vf_state(gt, vfid))) { + xe_gt_sriov_dbg_verbose(gt, "VF%u state %s(%d) enter\n", + vfid, control_bit_to_string(bit), bit); + return true; + } + return false; +} + +static bool pf_exit_vf_state(struct xe_gt *gt, unsigned int vfid, + enum xe_gt_sriov_control_bits bit) +{ + if (test_and_clear_bit(bit, pf_peek_vf_state(gt, vfid))) { + xe_gt_sriov_dbg_verbose(gt, "VF%u state %s(%d) exit\n", + vfid, control_bit_to_string(bit), bit); + return true; + } + return false; +} + +static void pf_escape_vf_state(struct xe_gt *gt, unsigned int vfid, + enum xe_gt_sriov_control_bits bit) +{ + if (pf_exit_vf_state(gt, vfid, bit)) + xe_gt_sriov_dbg_verbose(gt, "VF%u state %s(%d) escaped by %ps\n", + vfid, control_bit_to_string(bit), bit, + __builtin_return_address(0)); +} + +static void pf_enter_vf_mismatch(struct xe_gt *gt, unsigned int vfid) +{ + if (pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_MISMATCH)) { + xe_gt_sriov_dbg(gt, "VF%u state mismatch detected by %ps\n", + vfid, __builtin_return_address(0)); + pf_dump_vf_state(gt, vfid); + } +} + +static void pf_exit_vf_mismatch(struct xe_gt *gt, unsigned int vfid) +{ + if (pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_MISMATCH)) + xe_gt_sriov_dbg(gt, "VF%u state mismatch cleared by %ps\n", + vfid, __builtin_return_address(0)); + + pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_STOP_FAILED); + pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_PAUSE_FAILED); + pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_RESUME_FAILED); + pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_FAILED); +} + +#define pf_enter_vf_state_machine_bug(gt, vfid) ({ \ + pf_enter_vf_mismatch((gt), (vfid)); \ +}) + +static void pf_queue_control_worker(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + + xe_gt_assert(gt, IS_SRIOV_PF(xe)); + + queue_work(xe->sriov.wq, >->sriov.pf.control.worker); +} + +static void pf_queue_vf(struct xe_gt *gt, unsigned int vfid) +{ + struct xe_gt_sriov_pf_control *pfc = >->sriov.pf.control; + + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + + spin_lock(&pfc->lock); + list_move_tail(>->sriov.pf.vfs[vfid].control.link, &pfc->list); + spin_unlock(&pfc->lock); + + pf_queue_control_worker(gt); +} + +static void pf_exit_vf_flr_wip(struct xe_gt *gt, unsigned int vfid); +static void pf_exit_vf_stop_wip(struct xe_gt *gt, unsigned int vfid); +static void pf_exit_vf_pause_wip(struct xe_gt *gt, unsigned int vfid); +static void pf_exit_vf_resume_wip(struct xe_gt *gt, unsigned int vfid); + +static bool pf_enter_vf_wip(struct xe_gt *gt, unsigned int vfid) +{ + if (pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_WIP)) { + struct xe_gt_sriov_control_state *cs = pf_pick_vf_control(gt, vfid); + + reinit_completion(&cs->done); + return true; + } + return false; +} + +static void pf_exit_vf_wip(struct xe_gt *gt, unsigned int vfid) +{ + if (pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_WIP)) { + struct xe_gt_sriov_control_state *cs = pf_pick_vf_control(gt, vfid); + + pf_exit_vf_flr_wip(gt, vfid); + pf_exit_vf_stop_wip(gt, vfid); + pf_exit_vf_pause_wip(gt, vfid); + pf_exit_vf_resume_wip(gt, vfid); + + complete_all(&cs->done); + } +} + +static int pf_wait_vf_wip_done(struct xe_gt *gt, unsigned int vfid, unsigned long timeout) +{ + struct xe_gt_sriov_control_state *cs = pf_pick_vf_control(gt, vfid); + + return wait_for_completion_timeout(&cs->done, timeout) ? 0 : -ETIMEDOUT; +} + +static void pf_enter_vf_ready(struct xe_gt *gt, unsigned int vfid) +{ + pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_PAUSED); + pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_STOPPED); + pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_RESUMED); + pf_exit_vf_mismatch(gt, vfid); + pf_exit_vf_wip(gt, vfid); +} + +/** + * DOC: The VF PAUSE state machine + * + * The VF PAUSE state machine looks like:: + * + * (READY,RESUMED)<-------------<---------------------o---------o + * | \ \ + * pause \ \ + * | \ \ + * ....V...........................PAUSE_WIP........ \ \ + * : \ : o \ + * : \ o------<-----busy : | \ + * : \ / / : | | + * : PAUSE_SEND_PAUSE ---failed--->----------o--->(PAUSE_FAILED) | + * : | \ : | | + * : acked rejected---->----------o--->(MISMATCH) / + * : | : / + * : v : / + * : PAUSE_WAIT_GUC : / + * : | : / + * : done : / + * : | : / + * : v : / + * : PAUSE_GUC_DONE o-----restart + * : / : + * : / : + * :....o..............o...............o...........: + * | | | + * completed flr stop + * | | | + * V .....V..... ......V..... + * (PAUSED) : FLR_WIP : : STOP_WIP : + * :.........: :..........: + * + * For the full state machine view, see `The VF state machine`_. + */ + +static void pf_exit_vf_pause_wip(struct xe_gt *gt, unsigned int vfid) +{ + if (pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_PAUSE_WIP)) { + pf_escape_vf_state(gt, vfid, XE_GT_SRIOV_STATE_PAUSE_SEND_PAUSE); + pf_escape_vf_state(gt, vfid, XE_GT_SRIOV_STATE_PAUSE_WAIT_GUC); + pf_escape_vf_state(gt, vfid, XE_GT_SRIOV_STATE_PAUSE_GUC_DONE); + } +} + +static void pf_enter_vf_paused(struct xe_gt *gt, unsigned int vfid) +{ + if (!pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_PAUSED)) + pf_enter_vf_state_machine_bug(gt, vfid); + + pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_RESUMED); + pf_exit_vf_mismatch(gt, vfid); + pf_exit_vf_wip(gt, vfid); +} + +static void pf_enter_vf_pause_completed(struct xe_gt *gt, unsigned int vfid) +{ + pf_enter_vf_paused(gt, vfid); +} + +static void pf_enter_vf_pause_failed(struct xe_gt *gt, unsigned int vfid) +{ + pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_PAUSE_FAILED); + pf_exit_vf_wip(gt, vfid); +} + +static void pf_enter_vf_pause_rejected(struct xe_gt *gt, unsigned int vfid) +{ + pf_enter_vf_mismatch(gt, vfid); + pf_enter_vf_pause_failed(gt, vfid); +} + +static bool pf_exit_vf_pause_guc_done(struct xe_gt *gt, unsigned int vfid) +{ + if (!pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_PAUSE_GUC_DONE)) + return false; + + pf_enter_vf_pause_completed(gt, vfid); + return true; +} + +static void pf_enter_vf_pause_guc_done(struct xe_gt *gt, unsigned int vfid) +{ + if (pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_PAUSE_GUC_DONE)) + pf_queue_vf(gt, vfid); +} + +static void pf_enter_pause_wait_guc(struct xe_gt *gt, unsigned int vfid) +{ + if (!pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_PAUSE_WAIT_GUC)) + pf_enter_vf_state_machine_bug(gt, vfid); +} + +static bool pf_exit_pause_wait_guc(struct xe_gt *gt, unsigned int vfid) +{ + return pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_PAUSE_WAIT_GUC); +} + +static void pf_enter_vf_pause_send_pause(struct xe_gt *gt, unsigned int vfid) +{ + if (!pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_PAUSE_SEND_PAUSE)) + pf_enter_vf_state_machine_bug(gt, vfid); + + pf_queue_vf(gt, vfid); +} + +static bool pf_exit_vf_pause_send_pause(struct xe_gt *gt, unsigned int vfid) +{ + int err; + + if (!pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_PAUSE_SEND_PAUSE)) + return false; + + /* GuC may actually send a PAUSE_DONE before we get a RESPONSE */ + pf_enter_pause_wait_guc(gt, vfid); + + err = pf_send_vf_pause(gt, vfid); + if (err) { + /* send failed, so we shouldn't expect PAUSE_DONE from GuC */ + pf_exit_pause_wait_guc(gt, vfid); + + if (err == -EBUSY) + pf_enter_vf_pause_send_pause(gt, vfid); + else if (err == -EIO) + pf_enter_vf_pause_rejected(gt, vfid); + else + pf_enter_vf_pause_failed(gt, vfid); + } else { + /* + * we have already moved to WAIT_GUC, maybe even to GUC_DONE + * but since GuC didn't complain, we may clear MISMATCH + */ + pf_exit_vf_mismatch(gt, vfid); + } + + return true; +} + +static bool pf_enter_vf_pause_wip(struct xe_gt *gt, unsigned int vfid) +{ + if (pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_PAUSE_WIP)) { + pf_enter_vf_wip(gt, vfid); + pf_enter_vf_pause_send_pause(gt, vfid); + return true; + } + + return false; +} + +/** * xe_gt_sriov_pf_control_pause_vf - Pause a VF. * @gt: the &xe_gt * @vfid: the VF identifier @@ -98,7 +552,140 @@ static int pf_send_vf_flr_finish(struct xe_gt *gt, unsigned int vfid) */ int xe_gt_sriov_pf_control_pause_vf(struct xe_gt *gt, unsigned int vfid) { - return pf_send_vf_pause(gt, vfid); + unsigned long timeout = pf_get_default_timeout(XE_GT_SRIOV_STATE_PAUSE_WIP); + int err; + + if (pf_check_vf_state(gt, vfid, XE_GT_SRIOV_STATE_STOPPED)) { + xe_gt_sriov_dbg(gt, "VF%u is stopped!\n", vfid); + return -EPERM; + } + + if (pf_check_vf_state(gt, vfid, XE_GT_SRIOV_STATE_PAUSED)) { + xe_gt_sriov_dbg(gt, "VF%u was already paused!\n", vfid); + return -ESTALE; + } + + if (!pf_enter_vf_pause_wip(gt, vfid)) { + xe_gt_sriov_dbg(gt, "VF%u pause already in progress!\n", vfid); + return -EALREADY; + } + + err = pf_wait_vf_wip_done(gt, vfid, timeout); + if (err) { + xe_gt_sriov_dbg(gt, "VF%u pause didn't finish in %u ms (%pe)\n", + vfid, jiffies_to_msecs(timeout), ERR_PTR(err)); + return err; + } + + if (pf_expect_vf_state(gt, vfid, XE_GT_SRIOV_STATE_PAUSED)) { + xe_gt_sriov_info(gt, "VF%u paused!\n", vfid); + return 0; + } + + if (pf_check_vf_state(gt, vfid, XE_GT_SRIOV_STATE_PAUSE_FAILED)) { + xe_gt_sriov_dbg(gt, "VF%u pause failed!\n", vfid); + return -EIO; + } + + xe_gt_sriov_dbg(gt, "VF%u pause was canceled!\n", vfid); + return -ECANCELED; +} + +/** + * DOC: The VF RESUME state machine + * + * The VF RESUME state machine looks like:: + * + * (PAUSED)<-----------------<------------------------o + * | \ + * resume \ + * | \ + * ....V............................RESUME_WIP...... \ + * : \ : o + * : \ o-------<-----busy : | + * : \ / / : | + * : RESUME_SEND_RESUME ---failed--->--------o--->(RESUME_FAILED) + * : / \ : | + * : acked rejected---->---------o--->(MISMATCH) + * : / : + * :....o..............o...............o.....o.....: + * | | | \ + * completed flr stop restart-->(READY) + * | | | + * V .....V..... ......V..... + * (RESUMED) : FLR_WIP : : STOP_WIP : + * :.........: :..........: + * + * For the full state machine view, see `The VF state machine`_. + */ + +static void pf_exit_vf_resume_wip(struct xe_gt *gt, unsigned int vfid) +{ + if (pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_RESUME_WIP)) + pf_escape_vf_state(gt, vfid, XE_GT_SRIOV_STATE_RESUME_SEND_RESUME); +} + +static void pf_enter_vf_resumed(struct xe_gt *gt, unsigned int vfid) +{ + pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_RESUMED); + pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_PAUSED); + pf_exit_vf_mismatch(gt, vfid); + pf_exit_vf_wip(gt, vfid); +} + +static void pf_enter_vf_resume_completed(struct xe_gt *gt, unsigned int vfid) +{ + pf_enter_vf_resumed(gt, vfid); +} + +static void pf_enter_vf_resume_failed(struct xe_gt *gt, unsigned int vfid) +{ + pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_RESUME_FAILED); + pf_exit_vf_wip(gt, vfid); +} + +static void pf_enter_vf_resume_rejected(struct xe_gt *gt, unsigned int vfid) +{ + pf_enter_vf_mismatch(gt, vfid); + pf_enter_vf_resume_failed(gt, vfid); +} + +static void pf_enter_vf_resume_send_resume(struct xe_gt *gt, unsigned int vfid) +{ + if (!pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_RESUME_SEND_RESUME)) + pf_enter_vf_state_machine_bug(gt, vfid); + + pf_queue_vf(gt, vfid); +} + +static bool pf_exit_vf_resume_send_resume(struct xe_gt *gt, unsigned int vfid) +{ + int err; + + if (!pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_RESUME_SEND_RESUME)) + return false; + + err = pf_send_vf_resume(gt, vfid); + if (err == -EBUSY) + pf_enter_vf_resume_send_resume(gt, vfid); + else if (err == -EIO) + pf_enter_vf_resume_rejected(gt, vfid); + else if (err) + pf_enter_vf_resume_failed(gt, vfid); + else + pf_enter_vf_resume_completed(gt, vfid); + return true; +} + +static bool pf_enter_vf_resume_wip(struct xe_gt *gt, unsigned int vfid) +{ + if (pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_RESUME_WIP)) { + pf_enter_vf_wip(gt, vfid); + pf_enter_vf_resume_send_resume(gt, vfid); + return true; + } + + return false; } /** @@ -112,7 +699,134 @@ int xe_gt_sriov_pf_control_pause_vf(struct xe_gt *gt, unsigned int vfid) */ int xe_gt_sriov_pf_control_resume_vf(struct xe_gt *gt, unsigned int vfid) { - return pf_send_vf_resume(gt, vfid); + unsigned long timeout = pf_get_default_timeout(XE_GT_SRIOV_STATE_RESUME_WIP); + int err; + + if (!pf_check_vf_state(gt, vfid, XE_GT_SRIOV_STATE_PAUSED)) { + xe_gt_sriov_dbg(gt, "VF%u is not paused!\n", vfid); + return -EPERM; + } + + if (!pf_enter_vf_resume_wip(gt, vfid)) { + xe_gt_sriov_dbg(gt, "VF%u resume already in progress!\n", vfid); + return -EALREADY; + } + + err = pf_wait_vf_wip_done(gt, vfid, timeout); + if (err) + return err; + + if (pf_expect_vf_state(gt, vfid, XE_GT_SRIOV_STATE_RESUMED)) { + xe_gt_sriov_info(gt, "VF%u resumed!\n", vfid); + return 0; + } + + if (pf_check_vf_state(gt, vfid, XE_GT_SRIOV_STATE_RESUME_FAILED)) { + xe_gt_sriov_dbg(gt, "VF%u resume failed!\n", vfid); + return -EIO; + } + + xe_gt_sriov_dbg(gt, "VF%u resume was canceled!\n", vfid); + return -ECANCELED; +} + +/** + * DOC: The VF STOP state machine + * + * The VF STOP state machine looks like:: + * + * (READY,PAUSED,RESUMED)<-------<--------------------o + * | \ + * stop \ + * | \ + * ....V..............................STOP_WIP...... \ + * : \ : o + * : \ o----<----busy : | + * : \ / / : | + * : STOP_SEND_STOP--------failed--->--------o--->(STOP_FAILED) + * : / \ : | + * : acked rejected-------->--------o--->(MISMATCH) + * : / : + * :....o..............o...............o...........: + * | | | + * completed flr restart + * | | | + * V .....V..... V + * (STOPPED) : FLR_WIP : (READY) + * :.........: + * + * For the full state machine view, see `The VF state machine`_. + */ + +static void pf_exit_vf_stop_wip(struct xe_gt *gt, unsigned int vfid) +{ + if (pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_STOP_WIP)) + pf_escape_vf_state(gt, vfid, XE_GT_SRIOV_STATE_STOP_SEND_STOP); +} + +static void pf_enter_vf_stopped(struct xe_gt *gt, unsigned int vfid) +{ + if (!pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_STOPPED)) + pf_enter_vf_state_machine_bug(gt, vfid); + + pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_RESUMED); + pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_PAUSED); + pf_exit_vf_mismatch(gt, vfid); + pf_exit_vf_wip(gt, vfid); +} + +static void pf_enter_vf_stop_completed(struct xe_gt *gt, unsigned int vfid) +{ + pf_enter_vf_stopped(gt, vfid); +} + +static void pf_enter_vf_stop_failed(struct xe_gt *gt, unsigned int vfid) +{ + pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_STOP_FAILED); + pf_exit_vf_wip(gt, vfid); +} + +static void pf_enter_vf_stop_rejected(struct xe_gt *gt, unsigned int vfid) +{ + pf_enter_vf_mismatch(gt, vfid); + pf_enter_vf_stop_failed(gt, vfid); +} + +static void pf_enter_vf_stop_send_stop(struct xe_gt *gt, unsigned int vfid) +{ + if (!pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_STOP_SEND_STOP)) + pf_enter_vf_state_machine_bug(gt, vfid); + + pf_queue_vf(gt, vfid); +} + +static bool pf_exit_vf_stop_send_stop(struct xe_gt *gt, unsigned int vfid) +{ + int err; + + if (!pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_STOP_SEND_STOP)) + return false; + + err = pf_send_vf_stop(gt, vfid); + if (err == -EBUSY) + pf_enter_vf_stop_send_stop(gt, vfid); + else if (err == -EIO) + pf_enter_vf_stop_rejected(gt, vfid); + else if (err) + pf_enter_vf_stop_failed(gt, vfid); + else + pf_enter_vf_stop_completed(gt, vfid); + return true; +} + +static bool pf_enter_vf_stop_wip(struct xe_gt *gt, unsigned int vfid) +{ + if (pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_STOP_WIP)) { + pf_enter_vf_wip(gt, vfid); + pf_enter_vf_stop_send_stop(gt, vfid); + return true; + } + return false; } /** @@ -126,7 +840,280 @@ int xe_gt_sriov_pf_control_resume_vf(struct xe_gt *gt, unsigned int vfid) */ int xe_gt_sriov_pf_control_stop_vf(struct xe_gt *gt, unsigned int vfid) { - return pf_send_vf_stop(gt, vfid); + unsigned long timeout = pf_get_default_timeout(XE_GT_SRIOV_STATE_STOP_WIP); + int err; + + if (pf_check_vf_state(gt, vfid, XE_GT_SRIOV_STATE_STOPPED)) { + xe_gt_sriov_dbg(gt, "VF%u was already stopped!\n", vfid); + return -ESTALE; + } + + if (!pf_enter_vf_stop_wip(gt, vfid)) { + xe_gt_sriov_dbg(gt, "VF%u stop already in progress!\n", vfid); + return -EALREADY; + } + + err = pf_wait_vf_wip_done(gt, vfid, timeout); + if (err) + return err; + + if (pf_expect_vf_state(gt, vfid, XE_GT_SRIOV_STATE_STOPPED)) { + xe_gt_sriov_info(gt, "VF%u stopped!\n", vfid); + return 0; + } + + if (pf_check_vf_state(gt, vfid, XE_GT_SRIOV_STATE_STOP_FAILED)) { + xe_gt_sriov_dbg(gt, "VF%u stop failed!\n", vfid); + return -EIO; + } + + xe_gt_sriov_dbg(gt, "VF%u stop was canceled!\n", vfid); + return -ECANCELED; +} + +/** + * DOC: The VF FLR state machine + * + * The VF FLR state machine looks like:: + * + * (READY,PAUSED,STOPPED)<------------<--------------o + * | \ + * flr \ + * | \ + * ....V..........................FLR_WIP........... \ + * : \ : \ + * : \ o----<----busy : | + * : \ / / : | + * : FLR_SEND_START---failed----->-----------o--->(FLR_FAILED)<---o + * : | \ : | | + * : acked rejected----->-----------o--->(MISMATCH) | + * : | : ^ | + * : v : | | + * : FLR_WAIT_GUC : | | + * : | : | | + * : done : | | + * : | : | | + * : v : | | + * : FLR_GUC_DONE : | | + * : | : | | + * : FLR_RESET_CONFIG---failed--->-----------o--------+-----------o + * : | : | | + * : FLR_RESET_DATA : | | + * : | : | | + * : FLR_RESET_MMIO : | | + * : | : | | + * : | o----<----busy : | | + * : |/ / : | | + * : FLR_SEND_FINISH----failed--->-----------o--------+-----------o + * : / \ : | + * : acked rejected----->-----------o--------o + * : / : + * :....o..............................o...........: + * | | + * completed restart + * | / + * V / + * (READY)<----------<------------o + * + * For the full state machine view, see `The VF state machine`_. + */ + +static void pf_enter_vf_flr_send_start(struct xe_gt *gt, unsigned int vfid) +{ + if (!pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_SEND_START)) + pf_enter_vf_state_machine_bug(gt, vfid); + + pf_queue_vf(gt, vfid); +} + +static void pf_enter_vf_flr_wip(struct xe_gt *gt, unsigned int vfid) +{ + if (!pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_WIP)) { + xe_gt_sriov_dbg(gt, "VF%u FLR is already in progress\n", vfid); + return; + } + + pf_enter_vf_wip(gt, vfid); + pf_enter_vf_flr_send_start(gt, vfid); +} + +static void pf_exit_vf_flr_wip(struct xe_gt *gt, unsigned int vfid) +{ + if (pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_WIP)) { + pf_escape_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_SEND_FINISH); + pf_escape_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_RESET_MMIO); + pf_escape_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_RESET_DATA); + pf_escape_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_RESET_CONFIG); + pf_escape_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_GUC_DONE); + pf_escape_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_WAIT_GUC); + pf_escape_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_SEND_START); + } +} + +static void pf_enter_vf_flr_completed(struct xe_gt *gt, unsigned int vfid) +{ + pf_enter_vf_ready(gt, vfid); +} + +static void pf_enter_vf_flr_failed(struct xe_gt *gt, unsigned int vfid) +{ + if (pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_FAILED)) + xe_gt_sriov_notice(gt, "VF%u FLR failed!\n", vfid); + pf_exit_vf_wip(gt, vfid); +} + +static void pf_enter_vf_flr_rejected(struct xe_gt *gt, unsigned int vfid) +{ + pf_enter_vf_mismatch(gt, vfid); + pf_enter_vf_flr_failed(gt, vfid); +} + +static void pf_enter_vf_flr_send_finish(struct xe_gt *gt, unsigned int vfid) +{ + if (!pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_SEND_FINISH)) + pf_enter_vf_state_machine_bug(gt, vfid); + + pf_queue_vf(gt, vfid); +} + +static bool pf_exit_vf_flr_send_finish(struct xe_gt *gt, unsigned int vfid) +{ + int err; + + if (!pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_SEND_FINISH)) + return false; + + err = pf_send_vf_flr_finish(gt, vfid); + if (err == -EBUSY) + pf_enter_vf_flr_send_finish(gt, vfid); + else if (err == -EIO) + pf_enter_vf_flr_rejected(gt, vfid); + else if (err) + pf_enter_vf_flr_failed(gt, vfid); + else + pf_enter_vf_flr_completed(gt, vfid); + return true; +} + +static void pf_enter_vf_flr_reset_mmio(struct xe_gt *gt, unsigned int vfid) +{ + if (!pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_RESET_MMIO)) + pf_enter_vf_state_machine_bug(gt, vfid); + + pf_queue_vf(gt, vfid); +} + +static bool pf_exit_vf_flr_reset_mmio(struct xe_gt *gt, unsigned int vfid) +{ + if (!pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_RESET_MMIO)) + return false; + + /* XXX: placeholder */ + + pf_enter_vf_flr_send_finish(gt, vfid); + return true; +} + +static void pf_enter_vf_flr_reset_data(struct xe_gt *gt, unsigned int vfid) +{ + if (!pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_RESET_DATA)) + pf_enter_vf_state_machine_bug(gt, vfid); + + pf_queue_vf(gt, vfid); +} + +static bool pf_exit_vf_flr_reset_data(struct xe_gt *gt, unsigned int vfid) +{ + if (!pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_RESET_DATA)) + return false; + + xe_gt_sriov_pf_service_reset(gt, vfid); + xe_gt_sriov_pf_monitor_flr(gt, vfid); + + pf_enter_vf_flr_reset_mmio(gt, vfid); + return true; +} + +static void pf_enter_vf_flr_reset_config(struct xe_gt *gt, unsigned int vfid) +{ + if (!pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_RESET_CONFIG)) + pf_enter_vf_state_machine_bug(gt, vfid); + + pf_queue_vf(gt, vfid); +} + +static bool pf_exit_vf_flr_reset_config(struct xe_gt *gt, unsigned int vfid) +{ + unsigned long timeout = pf_get_default_timeout(XE_GT_SRIOV_STATE_FLR_RESET_CONFIG); + int err; + + if (!pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_RESET_CONFIG)) + return false; + + err = xe_gt_sriov_pf_config_sanitize(gt, vfid, timeout); + if (err) + pf_enter_vf_flr_failed(gt, vfid); + else + pf_enter_vf_flr_reset_data(gt, vfid); + return true; +} + +static void pf_enter_vf_flr_wait_guc(struct xe_gt *gt, unsigned int vfid) +{ + if (!pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_WAIT_GUC)) + pf_enter_vf_state_machine_bug(gt, vfid); +} + +static bool pf_exit_vf_flr_wait_guc(struct xe_gt *gt, unsigned int vfid) +{ + return pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_WAIT_GUC); +} + +static bool pf_exit_vf_flr_send_start(struct xe_gt *gt, unsigned int vfid) +{ + int err; + + if (!pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_SEND_START)) + return false; + + /* GuC may actually send a FLR_DONE before we get a RESPONSE */ + pf_enter_vf_flr_wait_guc(gt, vfid); + + err = pf_send_vf_flr_start(gt, vfid); + if (err) { + /* send failed, so we shouldn't expect FLR_DONE from GuC */ + pf_exit_vf_flr_wait_guc(gt, vfid); + + if (err == -EBUSY) + pf_enter_vf_flr_send_start(gt, vfid); + else if (err == -EIO) + pf_enter_vf_flr_rejected(gt, vfid); + else + pf_enter_vf_flr_failed(gt, vfid); + } else { + /* + * we have already moved to WAIT_GUC, maybe even to GUC_DONE + * but since GuC didn't complain, we may clear MISMATCH + */ + pf_exit_vf_mismatch(gt, vfid); + } + + return true; +} + +static bool pf_exit_vf_flr_guc_done(struct xe_gt *gt, unsigned int vfid) +{ + if (!pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_GUC_DONE)) + return false; + + pf_enter_vf_flr_reset_config(gt, vfid); + return true; +} + +static void pf_enter_vf_flr_guc_done(struct xe_gt *gt, unsigned int vfid) +{ + if (pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_GUC_DONE)) + pf_queue_vf(gt, vfid); } /** @@ -140,46 +1127,56 @@ int xe_gt_sriov_pf_control_stop_vf(struct xe_gt *gt, unsigned int vfid) */ int xe_gt_sriov_pf_control_trigger_flr(struct xe_gt *gt, unsigned int vfid) { + unsigned long timeout = pf_get_default_timeout(XE_GT_SRIOV_STATE_FLR_WIP); int err; - /* XXX pf_send_vf_flr_start() expects ct->lock */ - mutex_lock(>->uc.guc.ct.lock); - err = pf_send_vf_flr_start(gt, vfid); - mutex_unlock(>->uc.guc.ct.lock); + pf_enter_vf_flr_wip(gt, vfid); - return err; + err = pf_wait_vf_wip_done(gt, vfid, timeout); + if (err) { + xe_gt_sriov_notice(gt, "VF%u FLR didn't finish in %u ms (%pe)\n", + vfid, jiffies_to_msecs(timeout), ERR_PTR(err)); + return err; + } + + if (!pf_expect_vf_not_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_FAILED)) + return -EIO; + + return 0; } /** * DOC: The VF FLR Flow with GuC * - * PF GUC PCI - * ======================================================== - * | | | - * (1) | [ ] <----- FLR --| - * | [ ] : - * (2) [ ] <-------- NOTIFY FLR --[ ] - * [ ] | - * (3) [ ] | - * [ ] | - * [ ]-- START FLR ---------> [ ] - * | [ ] - * (4) | [ ] - * | [ ] - * [ ] <--------- FLR DONE -- [ ] - * [ ] | - * (5) [ ] | - * [ ] | - * [ ]-- FINISH FLR --------> [ ] - * | | - * - * Step 1: PCI HW generates interrupt to the GuC about VF FLR - * Step 2: GuC FW sends G2H notification to the PF about VF FLR - * Step 2a: on some platforms G2H is only received from root GuC - * Step 3: PF sends H2G request to the GuC to start VF FLR sequence - * Step 3a: on some platforms PF must send H2G to all other GuCs - * Step 4: GuC FW performs VF FLR cleanups and notifies the PF when done - * Step 5: PF performs VF FLR cleanups and notifies the GuC FW when finished + * The VF FLR flow includes several steps:: + * + * PF GUC PCI + * ======================================================== + * | | | + * (1) | [ ] <----- FLR --| + * | [ ] : + * (2) [ ] <-------- NOTIFY FLR --[ ] + * [ ] | + * (3) [ ] | + * [ ] | + * [ ]-- START FLR ---------> [ ] + * | [ ] + * (4) | [ ] + * | [ ] + * [ ] <--------- FLR DONE -- [ ] + * [ ] | + * (5) [ ] | + * [ ] | + * [ ]-- FINISH FLR --------> [ ] + * | | + * + * * Step 1: PCI HW generates interrupt to the GuC about VF FLR + * * Step 2: GuC FW sends G2H notification to the PF about VF FLR + * * Step 2a: on some platforms G2H is only received from root GuC + * * Step 3: PF sends H2G request to the GuC to start VF FLR sequence + * * Step 3a: on some platforms PF must send H2G to all other GuCs + * * Step 4: GuC FW performs VF FLR cleanups and notifies the PF when done + * * Step 5: PF performs VF FLR cleanups and notifies the GuC FW when finished */ static bool needs_dispatch_flr(struct xe_device *xe) @@ -197,19 +1194,41 @@ static void pf_handle_vf_flr(struct xe_gt *gt, u32 vfid) if (needs_dispatch_flr(xe)) { for_each_gt(gtit, xe, gtid) - pf_send_vf_flr_start(gtit, vfid); + pf_enter_vf_flr_wip(gtit, vfid); } else { - pf_send_vf_flr_start(gt, vfid); + pf_enter_vf_flr_wip(gt, vfid); } } static void pf_handle_vf_flr_done(struct xe_gt *gt, u32 vfid) { - pf_send_vf_flr_finish(gt, vfid); + if (!pf_exit_vf_flr_wait_guc(gt, vfid)) { + xe_gt_sriov_dbg(gt, "Received out of order 'VF%u FLR done'\n", vfid); + pf_enter_vf_mismatch(gt, vfid); + return; + } + + pf_enter_vf_flr_guc_done(gt, vfid); +} + +static void pf_handle_vf_pause_done(struct xe_gt *gt, u32 vfid) +{ + if (!pf_exit_pause_wait_guc(gt, vfid)) { + xe_gt_sriov_dbg(gt, "Received out of order 'VF%u PAUSE done'\n", vfid); + pf_enter_vf_mismatch(gt, vfid); + return; + } + + pf_enter_vf_pause_guc_done(gt, vfid); } static int pf_handle_vf_event(struct xe_gt *gt, u32 vfid, u32 eventid) { + xe_gt_sriov_dbg_verbose(gt, "received VF%u event %#x\n", vfid, eventid); + + if (vfid > xe_gt_sriov_pf_get_totalvfs(gt)) + return -EPROTO; + switch (eventid) { case GUC_PF_NOTIFY_VF_FLR: pf_handle_vf_flr(gt, vfid); @@ -218,6 +1237,7 @@ static int pf_handle_vf_event(struct xe_gt *gt, u32 vfid, u32 eventid) pf_handle_vf_flr_done(gt, vfid); break; case GUC_PF_NOTIFY_VF_PAUSE_DONE: + pf_handle_vf_pause_done(gt, vfid); break; case GUC_PF_NOTIFY_VF_FIXUP_DONE: break; @@ -276,3 +1296,159 @@ int xe_gt_sriov_pf_control_process_guc2pf(struct xe_gt *gt, const u32 *msg, u32 return vfid ? pf_handle_vf_event(gt, vfid, eventid) : pf_handle_pf_event(gt, eventid); } + +static bool pf_process_vf_state_machine(struct xe_gt *gt, unsigned int vfid) +{ + if (pf_exit_vf_flr_send_start(gt, vfid)) + return true; + + if (pf_check_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_WAIT_GUC)) { + xe_gt_sriov_dbg_verbose(gt, "VF%u in %s\n", vfid, + control_bit_to_string(XE_GT_SRIOV_STATE_FLR_WAIT_GUC)); + return false; + } + + if (pf_exit_vf_flr_guc_done(gt, vfid)) + return true; + + if (pf_exit_vf_flr_reset_config(gt, vfid)) + return true; + + if (pf_exit_vf_flr_reset_data(gt, vfid)) + return true; + + if (pf_exit_vf_flr_reset_mmio(gt, vfid)) + return true; + + if (pf_exit_vf_flr_send_finish(gt, vfid)) + return true; + + if (pf_exit_vf_stop_send_stop(gt, vfid)) + return true; + + if (pf_exit_vf_pause_send_pause(gt, vfid)) + return true; + + if (pf_check_vf_state(gt, vfid, XE_GT_SRIOV_STATE_PAUSE_WAIT_GUC)) { + xe_gt_sriov_dbg_verbose(gt, "VF%u in %s\n", vfid, + control_bit_to_string(XE_GT_SRIOV_STATE_PAUSE_WAIT_GUC)); + return true; + } + + if (pf_exit_vf_pause_guc_done(gt, vfid)) + return true; + + if (pf_exit_vf_resume_send_resume(gt, vfid)) + return true; + + return false; +} + +static unsigned int pf_control_state_index(struct xe_gt *gt, + struct xe_gt_sriov_control_state *cs) +{ + return container_of(cs, struct xe_gt_sriov_metadata, control) - gt->sriov.pf.vfs; +} + +static void pf_worker_find_work(struct xe_gt *gt) +{ + struct xe_gt_sriov_pf_control *pfc = >->sriov.pf.control; + struct xe_gt_sriov_control_state *cs; + unsigned int vfid; + bool empty; + bool more; + + spin_lock(&pfc->lock); + cs = list_first_entry_or_null(&pfc->list, struct xe_gt_sriov_control_state, link); + if (cs) + list_del_init(&cs->link); + empty = list_empty(&pfc->list); + spin_unlock(&pfc->lock); + + if (!cs) + return; + + /* VF metadata structures are indexed by the VFID */ + vfid = pf_control_state_index(gt, cs); + xe_gt_assert(gt, vfid <= xe_gt_sriov_pf_get_totalvfs(gt)); + + more = pf_process_vf_state_machine(gt, vfid); + if (more) + pf_queue_vf(gt, vfid); + else if (!empty) + pf_queue_control_worker(gt); +} + +static void control_worker_func(struct work_struct *w) +{ + struct xe_gt *gt = container_of(w, struct xe_gt, sriov.pf.control.worker); + + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + pf_worker_find_work(gt); +} + +static void pf_stop_worker(struct xe_gt *gt) +{ + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + cancel_work_sync(>->sriov.pf.control.worker); +} + +static void control_fini_action(struct drm_device *dev, void *data) +{ + struct xe_gt *gt = data; + + pf_stop_worker(gt); +} + +/** + * xe_gt_sriov_pf_control_init() - Initialize PF's control data. + * @gt: the &xe_gt + * + * This function is for PF only. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_control_init(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + unsigned int n, totalvfs; + + xe_gt_assert(gt, IS_SRIOV_PF(xe)); + + totalvfs = xe_sriov_pf_get_totalvfs(xe); + for (n = 0; n <= totalvfs; n++) { + struct xe_gt_sriov_control_state *cs = pf_pick_vf_control(gt, n); + + init_completion(&cs->done); + INIT_LIST_HEAD(&cs->link); + } + + spin_lock_init(>->sriov.pf.control.lock); + INIT_LIST_HEAD(>->sriov.pf.control.list); + INIT_WORK(>->sriov.pf.control.worker, control_worker_func); + + return drmm_add_action_or_reset(&xe->drm, control_fini_action, gt); +} + +/** + * xe_gt_sriov_pf_control_restart() - Restart SR-IOV control data after a GT reset. + * @gt: the &xe_gt + * + * Any per-VF status maintained by the PF or any ongoing VF control activity + * performed by the PF must be reset or cancelled when the GT is reset. + * + * This function is for PF only. + */ +void xe_gt_sriov_pf_control_restart(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + unsigned int n, totalvfs; + + xe_gt_assert(gt, IS_SRIOV_PF(xe)); + + pf_stop_worker(gt); + + totalvfs = xe_sriov_pf_get_totalvfs(xe); + for (n = 1; n <= totalvfs; n++) + pf_enter_vf_ready(gt, n); +} diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_control.h b/drivers/gpu/drm/xe/xe_gt_sriov_pf_control.h index 405d1586f991..c85e64f099cc 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_control.h +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_control.h @@ -11,6 +11,9 @@ struct xe_gt; +int xe_gt_sriov_pf_control_init(struct xe_gt *gt); +void xe_gt_sriov_pf_control_restart(struct xe_gt *gt); + int xe_gt_sriov_pf_control_pause_vf(struct xe_gt *gt, unsigned int vfid); int xe_gt_sriov_pf_control_resume_vf(struct xe_gt *gt, unsigned int vfid); int xe_gt_sriov_pf_control_stop_vf(struct xe_gt *gt, unsigned int vfid); diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_control_types.h b/drivers/gpu/drm/xe/xe_gt_sriov_pf_control_types.h new file mode 100644 index 000000000000..11830aafea45 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_control_types.h @@ -0,0 +1,107 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2024 Intel Corporation + */ + +#ifndef _XE_GT_SRIOV_PF_CONTROL_TYPES_H_ +#define _XE_GT_SRIOV_PF_CONTROL_TYPES_H_ + +#include <linux/completion.h> +#include <linux/spinlock.h> +#include <linux/workqueue_types.h> + +/** + * enum xe_gt_sriov_control_bits - Various bits used by the PF to represent a VF state + * + * @XE_GT_SRIOV_STATE_WIP: indicates that some operations are in progress. + * @XE_GT_SRIOV_STATE_FLR_WIP: indicates that a VF FLR is in progress. + * @XE_GT_SRIOV_STATE_FLR_SEND_START: indicates that the PF wants to send a FLR START command. + * @XE_GT_SRIOV_STATE_FLR_WAIT_GUC: indicates that the PF awaits for a response from the GuC. + * @XE_GT_SRIOV_STATE_FLR_GUC_DONE: indicates that the PF has received a response from the GuC. + * @XE_GT_SRIOV_STATE_FLR_RESET_CONFIG: indicates that the PF needs to clear VF's resources. + * @XE_GT_SRIOV_STATE_FLR_RESET_DATA: indicates that the PF needs to clear VF's data. + * @XE_GT_SRIOV_STATE_FLR_RESET_MMIO: indicates that the PF needs to reset VF's registers. + * @XE_GT_SRIOV_STATE_FLR_SEND_FINISH: indicates that the PF wants to send a FLR FINISH message. + * @XE_GT_SRIOV_STATE_FLR_FAILED: indicates that VF FLR sequence failed. + * @XE_GT_SRIOV_STATE_PAUSE_WIP: indicates that a VF pause operation is in progress. + * @XE_GT_SRIOV_STATE_PAUSE_SEND_PAUSE: indicates that the PF is about to send a PAUSE command. + * @XE_GT_SRIOV_STATE_PAUSE_WAIT_GUC: indicates that the PF awaits for a response from the GuC. + * @XE_GT_SRIOV_STATE_PAUSE_GUC_DONE: indicates that the PF has received a response from the GuC. + * @XE_GT_SRIOV_STATE_PAUSE_FAILED: indicates that a VF pause operation has failed. + * @XE_GT_SRIOV_STATE_PAUSED: indicates that the VF is paused. + * @XE_GT_SRIOV_STATE_RESUME_WIP: indicates the a VF resume operation is in progress. + * @XE_GT_SRIOV_STATE_RESUME_SEND_RESUME: indicates that the PF is about to send RESUME command. + * @XE_GT_SRIOV_STATE_RESUME_FAILED: indicates that a VF resume operation has failed. + * @XE_GT_SRIOV_STATE_RESUMED: indicates that the VF was resumed. + * @XE_GT_SRIOV_STATE_STOP_WIP: indicates that a VF stop operation is in progress. + * @XE_GT_SRIOV_STATE_STOP_SEND_STOP: indicates that the PF wants to send a STOP command. + * @XE_GT_SRIOV_STATE_STOP_FAILED: indicates that the VF stop operation has failed + * @XE_GT_SRIOV_STATE_STOPPED: indicates that the VF was stopped. + * @XE_GT_SRIOV_STATE_MISMATCH: indicates that the PF has detected a VF state mismatch. + */ +enum xe_gt_sriov_control_bits { + XE_GT_SRIOV_STATE_WIP = 1, + + XE_GT_SRIOV_STATE_FLR_WIP, + XE_GT_SRIOV_STATE_FLR_SEND_START, + XE_GT_SRIOV_STATE_FLR_WAIT_GUC, + XE_GT_SRIOV_STATE_FLR_GUC_DONE, + XE_GT_SRIOV_STATE_FLR_RESET_CONFIG, + XE_GT_SRIOV_STATE_FLR_RESET_DATA, + XE_GT_SRIOV_STATE_FLR_RESET_MMIO, + XE_GT_SRIOV_STATE_FLR_SEND_FINISH, + XE_GT_SRIOV_STATE_FLR_FAILED, + + XE_GT_SRIOV_STATE_PAUSE_WIP, + XE_GT_SRIOV_STATE_PAUSE_SEND_PAUSE, + XE_GT_SRIOV_STATE_PAUSE_WAIT_GUC, + XE_GT_SRIOV_STATE_PAUSE_GUC_DONE, + XE_GT_SRIOV_STATE_PAUSE_FAILED, + XE_GT_SRIOV_STATE_PAUSED, + + XE_GT_SRIOV_STATE_RESUME_WIP, + XE_GT_SRIOV_STATE_RESUME_SEND_RESUME, + XE_GT_SRIOV_STATE_RESUME_FAILED, + XE_GT_SRIOV_STATE_RESUMED, + + XE_GT_SRIOV_STATE_STOP_WIP, + XE_GT_SRIOV_STATE_STOP_SEND_STOP, + XE_GT_SRIOV_STATE_STOP_FAILED, + XE_GT_SRIOV_STATE_STOPPED, + + XE_GT_SRIOV_STATE_MISMATCH = BITS_PER_LONG - 1, +}; + +/** + * struct xe_gt_sriov_control_state - GT-level per-VF control state. + * + * Used by the PF driver to maintain per-VF control data. + */ +struct xe_gt_sriov_control_state { + /** @state: VF state bits */ + unsigned long state; + + /** @done: completion of async operations */ + struct completion done; + + /** @link: link into worker list */ + struct list_head link; +}; + +/** + * struct xe_gt_sriov_pf_control - GT-level control data. + * + * Used by the PF driver to maintain its data. + */ +struct xe_gt_sriov_pf_control { + /** @worker: worker that executes a VF operations */ + struct work_struct worker; + + /** @list: list of VF entries that have a pending work */ + struct list_head list; + + /** @lock: protects VF pending list */ + spinlock_t lock; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_types.h b/drivers/gpu/drm/xe/xe_gt_sriov_pf_types.h index 40cbaea3ef44..28e1b130bf87 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_types.h +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_types.h @@ -9,6 +9,7 @@ #include <linux/types.h> #include "xe_gt_sriov_pf_config_types.h" +#include "xe_gt_sriov_pf_control_types.h" #include "xe_gt_sriov_pf_monitor_types.h" #include "xe_gt_sriov_pf_policy_types.h" #include "xe_gt_sriov_pf_service_types.h" @@ -23,6 +24,9 @@ struct xe_gt_sriov_metadata { /** @monitor: per-VF monitoring data. */ struct xe_gt_sriov_monitor monitor; + /** @control: per-VF control data. */ + struct xe_gt_sriov_control_state control; + /** @version: negotiated VF/PF ABI version */ struct xe_gt_sriov_pf_service_version version; }; @@ -30,12 +34,14 @@ struct xe_gt_sriov_metadata { /** * struct xe_gt_sriov_pf - GT level PF virtualization data. * @service: service data. + * @control: control data. * @policy: policy data. * @spare: PF-only provisioning configuration. * @vfs: metadata for all VFs. */ struct xe_gt_sriov_pf { struct xe_gt_sriov_pf_service service; + struct xe_gt_sriov_pf_control control; struct xe_gt_sriov_pf_policy policy; struct xe_gt_sriov_spare_config spare; struct xe_gt_sriov_metadata *vfs; diff --git a/drivers/gpu/drm/xe/xe_gt_types.h b/drivers/gpu/drm/xe/xe_gt_types.h index 31946d7fe701..3d1c51de0268 100644 --- a/drivers/gpu/drm/xe/xe_gt_types.h +++ b/drivers/gpu/drm/xe/xe_gt_types.h @@ -329,12 +329,6 @@ struct xe_gt { /** @eclass: per hardware engine class interface on the GT */ struct xe_hw_engine_class_intf eclass[XE_ENGINE_CLASS_MAX]; - /** @pcode: GT's PCODE */ - struct { - /** @pcode.lock: protecting GT's PCODE mailbox data */ - struct mutex lock; - } pcode; - /** @sysfs: sysfs' kobj used by xe_gt_sysfs */ struct kobject *sysfs; diff --git a/drivers/gpu/drm/xe/xe_guc_pc.c b/drivers/gpu/drm/xe/xe_guc_pc.c index def503abeed5..034b29984d5e 100644 --- a/drivers/gpu/drm/xe/xe_guc_pc.c +++ b/drivers/gpu/drm/xe/xe_guc_pc.c @@ -915,7 +915,7 @@ static void pc_init_pcode_freq(struct xe_guc_pc *pc) u32 min = DIV_ROUND_CLOSEST(pc->rpn_freq, GT_FREQUENCY_MULTIPLIER); u32 max = DIV_ROUND_CLOSEST(pc->rp0_freq, GT_FREQUENCY_MULTIPLIER); - XE_WARN_ON(xe_pcode_init_min_freq_table(pc_to_gt(pc), min, max)); + XE_WARN_ON(xe_pcode_init_min_freq_table(gt_to_tile(pc_to_gt(pc)), min, max)); } static int pc_init_freqs(struct xe_guc_pc *pc) diff --git a/drivers/gpu/drm/xe/xe_hw_engine.c b/drivers/gpu/drm/xe/xe_hw_engine.c index 18980238a2ea..c9c3beb3ce8d 100644 --- a/drivers/gpu/drm/xe/xe_hw_engine.c +++ b/drivers/gpu/drm/xe/xe_hw_engine.c @@ -8,7 +8,7 @@ #include <linux/nospec.h> #include <drm/drm_managed.h> -#include <drm/xe_drm.h> +#include <uapi/drm/xe_drm.h> #include "regs/xe_engine_regs.h" #include "regs/xe_gt_regs.h" @@ -273,7 +273,6 @@ static void hw_engine_fini(void *arg) if (hwe->exl_port) xe_execlist_port_destroy(hwe->exl_port); - xe_lrc_put(hwe->kernel_lrc); hwe->gt = NULL; } @@ -558,21 +557,13 @@ static int hw_engine_init(struct xe_gt *gt, struct xe_hw_engine *hwe, goto err_name; } - hwe->kernel_lrc = xe_lrc_create(hwe, NULL, SZ_16K); - if (IS_ERR(hwe->kernel_lrc)) { - err = PTR_ERR(hwe->kernel_lrc); - goto err_hwsp; - } - if (!xe_device_uc_enabled(xe)) { hwe->exl_port = xe_execlist_port_create(xe, hwe); if (IS_ERR(hwe->exl_port)) { err = PTR_ERR(hwe->exl_port); - goto err_kernel_lrc; + goto err_hwsp; } - } - - if (xe_device_uc_enabled(xe)) { + } else { /* GSCCS has a special interrupt for reset */ if (hwe->class == XE_ENGINE_CLASS_OTHER) hwe->irq_handler = xe_gsc_hwe_irq_handler; @@ -587,8 +578,6 @@ static int hw_engine_init(struct xe_gt *gt, struct xe_hw_engine *hwe, return devm_add_action_or_reset(xe->drm.dev, hw_engine_fini, hwe); -err_kernel_lrc: - xe_lrc_put(hwe->kernel_lrc); err_hwsp: xe_bo_unpin_map_no_vm(hwe->hwsp); err_name: diff --git a/drivers/gpu/drm/xe/xe_hw_engine_types.h b/drivers/gpu/drm/xe/xe_hw_engine_types.h index 39f24012d0f4..8be6d420ece4 100644 --- a/drivers/gpu/drm/xe/xe_hw_engine_types.h +++ b/drivers/gpu/drm/xe/xe_hw_engine_types.h @@ -136,8 +136,6 @@ struct xe_hw_engine { enum xe_force_wake_domains domain; /** @hwsp: hardware status page buffer object */ struct xe_bo *hwsp; - /** @kernel_lrc: Kernel LRC (should be replaced /w an xe_engine) */ - struct xe_lrc *kernel_lrc; /** @exl_port: execlists port */ struct xe_execlist_port *exl_port; /** @fence_irq: fence IRQ to run when a hw engine IRQ is received */ diff --git a/drivers/gpu/drm/xe/xe_hwmon.c b/drivers/gpu/drm/xe/xe_hwmon.c index 832ea81faeee..aa11728e7e79 100644 --- a/drivers/gpu/drm/xe/xe_hwmon.c +++ b/drivers/gpu/drm/xe/xe_hwmon.c @@ -12,7 +12,6 @@ #include "regs/xe_mchbar_regs.h" #include "regs/xe_pcode_regs.h" #include "xe_device.h" -#include "xe_gt.h" #include "xe_hwmon.h" #include "xe_mmio.h" #include "xe_pcode.h" @@ -65,8 +64,8 @@ struct xe_hwmon_energy_info { struct xe_hwmon { /** @hwmon_dev: hwmon device for xe */ struct device *hwmon_dev; - /** @gt: primary gt */ - struct xe_gt *gt; + /** @xe: Xe device */ + struct xe_device *xe; /** @hwmon_lock: lock for rw attributes*/ struct mutex hwmon_lock; /** @scl_shift_power: pkg power unit */ @@ -82,7 +81,7 @@ struct xe_hwmon { static struct xe_reg xe_hwmon_get_reg(struct xe_hwmon *hwmon, enum xe_hwmon_reg hwmon_reg, int channel) { - struct xe_device *xe = gt_to_xe(hwmon->gt); + struct xe_device *xe = hwmon->xe; switch (hwmon_reg) { case REG_PKG_RAPL_LIMIT: @@ -148,8 +147,9 @@ static struct xe_reg xe_hwmon_get_reg(struct xe_hwmon *hwmon, enum xe_hwmon_reg static void xe_hwmon_power_max_read(struct xe_hwmon *hwmon, int channel, long *value) { u64 reg_val, min, max; - struct xe_device *xe = gt_to_xe(hwmon->gt); + struct xe_device *xe = hwmon->xe; struct xe_reg rapl_limit, pkg_power_sku; + struct xe_gt *mmio = xe_root_mmio_gt(xe); rapl_limit = xe_hwmon_get_reg(hwmon, REG_PKG_RAPL_LIMIT, channel); pkg_power_sku = xe_hwmon_get_reg(hwmon, REG_PKG_POWER_SKU, channel); @@ -166,7 +166,7 @@ static void xe_hwmon_power_max_read(struct xe_hwmon *hwmon, int channel, long *v mutex_lock(&hwmon->hwmon_lock); - reg_val = xe_mmio_read32(hwmon->gt, rapl_limit); + reg_val = xe_mmio_read32(mmio, rapl_limit); /* Check if PL1 limit is disabled */ if (!(reg_val & PKG_PWR_LIM_1_EN)) { *value = PL1_DISABLE; @@ -176,7 +176,7 @@ static void xe_hwmon_power_max_read(struct xe_hwmon *hwmon, int channel, long *v reg_val = REG_FIELD_GET(PKG_PWR_LIM_1, reg_val); *value = mul_u64_u32_shr(reg_val, SF_POWER, hwmon->scl_shift_power); - reg_val = xe_mmio_read64_2x32(hwmon->gt, pkg_power_sku); + reg_val = xe_mmio_read64_2x32(mmio, pkg_power_sku); min = REG_FIELD_GET(PKG_MIN_PWR, reg_val); min = mul_u64_u32_shr(min, SF_POWER, hwmon->scl_shift_power); max = REG_FIELD_GET(PKG_MAX_PWR, reg_val); @@ -190,6 +190,7 @@ unlock: static int xe_hwmon_power_max_write(struct xe_hwmon *hwmon, int channel, long value) { + struct xe_gt *mmio = xe_root_mmio_gt(hwmon->xe); int ret = 0; u64 reg_val; struct xe_reg rapl_limit; @@ -200,10 +201,10 @@ static int xe_hwmon_power_max_write(struct xe_hwmon *hwmon, int channel, long va /* Disable PL1 limit and verify, as limit cannot be disabled on all platforms */ if (value == PL1_DISABLE) { - reg_val = xe_mmio_rmw32(hwmon->gt, rapl_limit, PKG_PWR_LIM_1_EN, 0); - reg_val = xe_mmio_read32(hwmon->gt, rapl_limit); + reg_val = xe_mmio_rmw32(mmio, rapl_limit, PKG_PWR_LIM_1_EN, 0); + reg_val = xe_mmio_read32(mmio, rapl_limit); if (reg_val & PKG_PWR_LIM_1_EN) { - drm_warn(>_to_xe(hwmon->gt)->drm, "PL1 disable is not supported!\n"); + drm_warn(&hwmon->xe->drm, "PL1 disable is not supported!\n"); ret = -EOPNOTSUPP; } goto unlock; @@ -212,7 +213,7 @@ static int xe_hwmon_power_max_write(struct xe_hwmon *hwmon, int channel, long va /* Computation in 64-bits to avoid overflow. Round to nearest. */ reg_val = DIV_ROUND_CLOSEST_ULL((u64)value << hwmon->scl_shift_power, SF_POWER); reg_val = PKG_PWR_LIM_1_EN | REG_FIELD_PREP(PKG_PWR_LIM_1, reg_val); - reg_val = xe_mmio_rmw32(hwmon->gt, rapl_limit, PKG_PWR_LIM_1_EN | PKG_PWR_LIM_1, reg_val); + reg_val = xe_mmio_rmw32(mmio, rapl_limit, PKG_PWR_LIM_1_EN | PKG_PWR_LIM_1, reg_val); unlock: mutex_unlock(&hwmon->hwmon_lock); @@ -221,6 +222,7 @@ unlock: static void xe_hwmon_power_rated_max_read(struct xe_hwmon *hwmon, int channel, long *value) { + struct xe_gt *mmio = xe_root_mmio_gt(hwmon->xe); struct xe_reg reg = xe_hwmon_get_reg(hwmon, REG_PKG_POWER_SKU, channel); u64 reg_val; @@ -229,7 +231,7 @@ static void xe_hwmon_power_rated_max_read(struct xe_hwmon *hwmon, int channel, l * for this register can be skipped. * See xe_hwmon_power_is_visible. */ - reg_val = xe_mmio_read32(hwmon->gt, reg); + reg_val = xe_mmio_read32(mmio, reg); reg_val = REG_FIELD_GET(PKG_TDP, reg_val); *value = mul_u64_u32_shr(reg_val, SF_POWER, hwmon->scl_shift_power); } @@ -257,11 +259,12 @@ static void xe_hwmon_power_rated_max_read(struct xe_hwmon *hwmon, int channel, l static void xe_hwmon_energy_get(struct xe_hwmon *hwmon, int channel, long *energy) { + struct xe_gt *mmio = xe_root_mmio_gt(hwmon->xe); struct xe_hwmon_energy_info *ei = &hwmon->ei[channel]; u64 reg_val; - reg_val = xe_mmio_read32(hwmon->gt, xe_hwmon_get_reg(hwmon, REG_PKG_ENERGY_STATUS, - channel)); + reg_val = xe_mmio_read32(mmio, xe_hwmon_get_reg(hwmon, REG_PKG_ENERGY_STATUS, + channel)); if (reg_val >= ei->reg_val_prev) ei->accum_energy += reg_val - ei->reg_val_prev; @@ -279,19 +282,20 @@ xe_hwmon_power_max_interval_show(struct device *dev, struct device_attribute *at char *buf) { struct xe_hwmon *hwmon = dev_get_drvdata(dev); + struct xe_gt *mmio = xe_root_mmio_gt(hwmon->xe); u32 x, y, x_w = 2; /* 2 bits */ u64 r, tau4, out; int sensor_index = to_sensor_dev_attr(attr)->index; - xe_pm_runtime_get(gt_to_xe(hwmon->gt)); + xe_pm_runtime_get(hwmon->xe); mutex_lock(&hwmon->hwmon_lock); - r = xe_mmio_read32(hwmon->gt, xe_hwmon_get_reg(hwmon, REG_PKG_RAPL_LIMIT, sensor_index)); + r = xe_mmio_read32(mmio, xe_hwmon_get_reg(hwmon, REG_PKG_RAPL_LIMIT, sensor_index)); mutex_unlock(&hwmon->hwmon_lock); - xe_pm_runtime_put(gt_to_xe(hwmon->gt)); + xe_pm_runtime_put(hwmon->xe); x = REG_FIELD_GET(PKG_PWR_LIM_1_TIME_X, r); y = REG_FIELD_GET(PKG_PWR_LIM_1_TIME_Y, r); @@ -319,6 +323,7 @@ xe_hwmon_power_max_interval_store(struct device *dev, struct device_attribute *a const char *buf, size_t count) { struct xe_hwmon *hwmon = dev_get_drvdata(dev); + struct xe_gt *mmio = xe_root_mmio_gt(hwmon->xe); u32 x, y, rxy, x_w = 2; /* 2 bits */ u64 tau4, r, max_win; unsigned long val; @@ -371,16 +376,16 @@ xe_hwmon_power_max_interval_store(struct device *dev, struct device_attribute *a rxy = REG_FIELD_PREP(PKG_PWR_LIM_1_TIME_X, x) | REG_FIELD_PREP(PKG_PWR_LIM_1_TIME_Y, y); - xe_pm_runtime_get(gt_to_xe(hwmon->gt)); + xe_pm_runtime_get(hwmon->xe); mutex_lock(&hwmon->hwmon_lock); - r = xe_mmio_rmw32(hwmon->gt, xe_hwmon_get_reg(hwmon, REG_PKG_RAPL_LIMIT, sensor_index), + r = xe_mmio_rmw32(mmio, xe_hwmon_get_reg(hwmon, REG_PKG_RAPL_LIMIT, sensor_index), PKG_PWR_LIM_1_TIME, rxy); mutex_unlock(&hwmon->hwmon_lock); - xe_pm_runtime_put(gt_to_xe(hwmon->gt)); + xe_pm_runtime_put(hwmon->xe); return count; } @@ -406,11 +411,11 @@ static umode_t xe_hwmon_attributes_visible(struct kobject *kobj, struct xe_hwmon *hwmon = dev_get_drvdata(dev); int ret = 0; - xe_pm_runtime_get(gt_to_xe(hwmon->gt)); + xe_pm_runtime_get(hwmon->xe); ret = xe_reg_is_valid(xe_hwmon_get_reg(hwmon, REG_PKG_RAPL_LIMIT, index)) ? attr->mode : 0; - xe_pm_runtime_put(gt_to_xe(hwmon->gt)); + xe_pm_runtime_put(hwmon->xe); return ret; } @@ -435,22 +440,26 @@ static const struct hwmon_channel_info * const hwmon_info[] = { }; /* I1 is exposed as power_crit or as curr_crit depending on bit 31 */ -static int xe_hwmon_pcode_read_i1(struct xe_gt *gt, u32 *uval) +static int xe_hwmon_pcode_read_i1(const struct xe_hwmon *hwmon, u32 *uval) { + struct xe_tile *root_tile = xe_device_get_root_tile(hwmon->xe); + /* Avoid Illegal Subcommand error */ - if (gt_to_xe(gt)->info.platform == XE_DG2) + if (hwmon->xe->info.platform == XE_DG2) return -ENXIO; - return xe_pcode_read(gt, PCODE_MBOX(PCODE_POWER_SETUP, + return xe_pcode_read(root_tile, PCODE_MBOX(PCODE_POWER_SETUP, POWER_SETUP_SUBCOMMAND_READ_I1, 0), uval, NULL); } -static int xe_hwmon_pcode_write_i1(struct xe_gt *gt, u32 uval) +static int xe_hwmon_pcode_write_i1(const struct xe_hwmon *hwmon, u32 uval) { - return xe_pcode_write(gt, PCODE_MBOX(PCODE_POWER_SETUP, + struct xe_tile *root_tile = xe_device_get_root_tile(hwmon->xe); + + return xe_pcode_write(root_tile, PCODE_MBOX(PCODE_POWER_SETUP, POWER_SETUP_SUBCOMMAND_WRITE_I1, 0), - uval); + (uval & POWER_SETUP_I1_DATA_MASK)); } static int xe_hwmon_power_curr_crit_read(struct xe_hwmon *hwmon, int channel, @@ -461,7 +470,7 @@ static int xe_hwmon_power_curr_crit_read(struct xe_hwmon *hwmon, int channel, mutex_lock(&hwmon->hwmon_lock); - ret = xe_hwmon_pcode_read_i1(hwmon->gt, &uval); + ret = xe_hwmon_pcode_read_i1(hwmon, &uval); if (ret) goto unlock; @@ -481,7 +490,7 @@ static int xe_hwmon_power_curr_crit_write(struct xe_hwmon *hwmon, int channel, mutex_lock(&hwmon->hwmon_lock); uval = DIV_ROUND_CLOSEST_ULL(value << POWER_SETUP_I1_SHIFT, scale_factor); - ret = xe_hwmon_pcode_write_i1(hwmon->gt, uval); + ret = xe_hwmon_pcode_write_i1(hwmon, uval); mutex_unlock(&hwmon->hwmon_lock); return ret; @@ -489,9 +498,10 @@ static int xe_hwmon_power_curr_crit_write(struct xe_hwmon *hwmon, int channel, static void xe_hwmon_get_voltage(struct xe_hwmon *hwmon, int channel, long *value) { + struct xe_gt *mmio = xe_root_mmio_gt(hwmon->xe); u64 reg_val; - reg_val = xe_mmio_read32(hwmon->gt, xe_hwmon_get_reg(hwmon, REG_GT_PERF_STATUS, channel)); + reg_val = xe_mmio_read32(mmio, xe_hwmon_get_reg(hwmon, REG_GT_PERF_STATUS, channel)); /* HW register value in units of 2.5 millivolt */ *value = DIV_ROUND_CLOSEST(REG_FIELD_GET(VOLTAGE_MASK, reg_val) * 2500, SF_VOLTAGE); } @@ -510,7 +520,7 @@ xe_hwmon_power_is_visible(struct xe_hwmon *hwmon, u32 attr, int channel) channel)) ? 0444 : 0; case hwmon_power_crit: if (channel == CHANNEL_PKG) - return (xe_hwmon_pcode_read_i1(hwmon->gt, &uval) || + return (xe_hwmon_pcode_read_i1(hwmon, &uval) || !(uval & POWER_SETUP_I1_WATTS)) ? 0 : 0644; break; case hwmon_power_label: @@ -563,10 +573,10 @@ xe_hwmon_curr_is_visible(const struct xe_hwmon *hwmon, u32 attr, int channel) switch (attr) { case hwmon_curr_crit: - return (xe_hwmon_pcode_read_i1(hwmon->gt, &uval) || + return (xe_hwmon_pcode_read_i1(hwmon, &uval) || (uval & POWER_SETUP_I1_WATTS)) ? 0 : 0644; case hwmon_curr_label: - return (xe_hwmon_pcode_read_i1(hwmon->gt, &uval) || + return (xe_hwmon_pcode_read_i1(hwmon, &uval) || (uval & POWER_SETUP_I1_WATTS)) ? 0 : 0444; break; default: @@ -654,7 +664,7 @@ xe_hwmon_is_visible(const void *drvdata, enum hwmon_sensor_types type, struct xe_hwmon *hwmon = (struct xe_hwmon *)drvdata; int ret; - xe_pm_runtime_get(gt_to_xe(hwmon->gt)); + xe_pm_runtime_get(hwmon->xe); switch (type) { case hwmon_power: @@ -674,7 +684,7 @@ xe_hwmon_is_visible(const void *drvdata, enum hwmon_sensor_types type, break; } - xe_pm_runtime_put(gt_to_xe(hwmon->gt)); + xe_pm_runtime_put(hwmon->xe); return ret; } @@ -686,7 +696,7 @@ xe_hwmon_read(struct device *dev, enum hwmon_sensor_types type, u32 attr, struct xe_hwmon *hwmon = dev_get_drvdata(dev); int ret; - xe_pm_runtime_get(gt_to_xe(hwmon->gt)); + xe_pm_runtime_get(hwmon->xe); switch (type) { case hwmon_power: @@ -706,7 +716,7 @@ xe_hwmon_read(struct device *dev, enum hwmon_sensor_types type, u32 attr, break; } - xe_pm_runtime_put(gt_to_xe(hwmon->gt)); + xe_pm_runtime_put(hwmon->xe); return ret; } @@ -718,7 +728,7 @@ xe_hwmon_write(struct device *dev, enum hwmon_sensor_types type, u32 attr, struct xe_hwmon *hwmon = dev_get_drvdata(dev); int ret; - xe_pm_runtime_get(gt_to_xe(hwmon->gt)); + xe_pm_runtime_get(hwmon->xe); switch (type) { case hwmon_power: @@ -732,7 +742,7 @@ xe_hwmon_write(struct device *dev, enum hwmon_sensor_types type, u32 attr, break; } - xe_pm_runtime_put(gt_to_xe(hwmon->gt)); + xe_pm_runtime_put(hwmon->xe); return ret; } @@ -771,6 +781,7 @@ static const struct hwmon_chip_info hwmon_chip_info = { static void xe_hwmon_get_preregistration_info(struct xe_device *xe) { + struct xe_gt *mmio = xe_root_mmio_gt(xe); struct xe_hwmon *hwmon = xe->hwmon; long energy; u64 val_sku_unit = 0; @@ -783,7 +794,7 @@ xe_hwmon_get_preregistration_info(struct xe_device *xe) */ pkg_power_sku_unit = xe_hwmon_get_reg(hwmon, REG_PKG_POWER_SKU_UNIT, 0); if (xe_reg_is_valid(pkg_power_sku_unit)) { - val_sku_unit = xe_mmio_read32(hwmon->gt, pkg_power_sku_unit); + val_sku_unit = xe_mmio_read32(mmio, pkg_power_sku_unit); hwmon->scl_shift_power = REG_FIELD_GET(PKG_PWR_UNIT, val_sku_unit); hwmon->scl_shift_energy = REG_FIELD_GET(PKG_ENERGY_UNIT, val_sku_unit); hwmon->scl_shift_time = REG_FIELD_GET(PKG_TIME_UNIT, val_sku_unit); @@ -828,8 +839,8 @@ void xe_hwmon_register(struct xe_device *xe) if (devm_add_action_or_reset(dev, xe_hwmon_mutex_destroy, hwmon)) return; - /* primary GT to access device level properties */ - hwmon->gt = xe->tiles[0].primary_gt; + /* There's only one instance of hwmon per device */ + hwmon->xe = xe; xe_hwmon_get_preregistration_info(xe); diff --git a/drivers/gpu/drm/xe/xe_migrate.c b/drivers/gpu/drm/xe/xe_migrate.c index cbf54be224c9..cfd31ae49cc1 100644 --- a/drivers/gpu/drm/xe/xe_migrate.c +++ b/drivers/gpu/drm/xe/xe_migrate.c @@ -10,7 +10,7 @@ #include <drm/drm_managed.h> #include <drm/ttm/ttm_tt.h> -#include <drm/xe_drm.h> +#include <uapi/drm/xe_drm.h> #include <generated/xe_wa_oob.h> diff --git a/drivers/gpu/drm/xe/xe_oa.c b/drivers/gpu/drm/xe/xe_oa.c index 4d4541e0b24c..63286ed8457f 100644 --- a/drivers/gpu/drm/xe/xe_oa.c +++ b/drivers/gpu/drm/xe/xe_oa.c @@ -10,7 +10,7 @@ #include <drm/drm_drv.h> #include <drm/drm_managed.h> -#include <drm/xe_drm.h> +#include <uapi/drm/xe_drm.h> #include "abi/guc_actions_slpc_abi.h" #include "instructions/xe_mi_commands.h" diff --git a/drivers/gpu/drm/xe/xe_oa_types.h b/drivers/gpu/drm/xe/xe_oa_types.h index 540c3ec53a6d..8862eca73fbe 100644 --- a/drivers/gpu/drm/xe/xe_oa_types.h +++ b/drivers/gpu/drm/xe/xe_oa_types.h @@ -11,7 +11,7 @@ #include <linux/mutex.h> #include <linux/types.h> -#include <drm/xe_drm.h> +#include <uapi/drm/xe_drm.h> #include "regs/xe_reg_defs.h" #include "xe_hw_engine_types.h" diff --git a/drivers/gpu/drm/xe/xe_observation.c b/drivers/gpu/drm/xe/xe_observation.c index a78c92a44ec2..8ec1b84cbb9e 100644 --- a/drivers/gpu/drm/xe/xe_observation.c +++ b/drivers/gpu/drm/xe/xe_observation.c @@ -6,7 +6,7 @@ #include <linux/errno.h> #include <linux/sysctl.h> -#include <drm/xe_drm.h> +#include <uapi/drm/xe_drm.h> #include "xe_oa.h" #include "xe_observation.h" diff --git a/drivers/gpu/drm/xe/xe_pat.c b/drivers/gpu/drm/xe/xe_pat.c index 722278cc23fc..f291a1730024 100644 --- a/drivers/gpu/drm/xe/xe_pat.c +++ b/drivers/gpu/drm/xe/xe_pat.c @@ -5,7 +5,7 @@ #include "xe_pat.h" -#include <drm/xe_drm.h> +#include <uapi/drm/xe_drm.h> #include <generated/xe_wa_oob.h> diff --git a/drivers/gpu/drm/xe/xe_pci.c b/drivers/gpu/drm/xe/xe_pci.c index f276194d9c4e..937c3e064f0d 100644 --- a/drivers/gpu/drm/xe/xe_pci.c +++ b/drivers/gpu/drm/xe/xe_pci.c @@ -793,7 +793,7 @@ static int xe_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) if (IS_ERR(xe)) return PTR_ERR(xe); - pci_set_drvdata(pdev, xe); + pci_set_drvdata(pdev, &xe->drm); xe_pm_assert_unbounded_bridge(xe); subplatform_desc = find_subplatform(xe, desc); diff --git a/drivers/gpu/drm/xe/xe_pcode.c b/drivers/gpu/drm/xe/xe_pcode.c index 9c4eefdf6642..7397d556996a 100644 --- a/drivers/gpu/drm/xe/xe_pcode.c +++ b/drivers/gpu/drm/xe/xe_pcode.c @@ -12,7 +12,6 @@ #include "xe_assert.h" #include "xe_device.h" -#include "xe_gt.h" #include "xe_mmio.h" #include "xe_pcode_api.h" @@ -30,7 +29,7 @@ * - PCODE for display operations */ -static int pcode_mailbox_status(struct xe_gt *gt) +static int pcode_mailbox_status(struct xe_tile *tile) { u32 err; static const struct pcode_err_decode err_decode[] = { @@ -45,9 +44,9 @@ static int pcode_mailbox_status(struct xe_gt *gt) [PCODE_ERROR_MASK] = {-EPROTO, "Unknown"}, }; - err = xe_mmio_read32(gt, PCODE_MAILBOX) & PCODE_ERROR_MASK; + err = xe_mmio_read32(tile->primary_gt, PCODE_MAILBOX) & PCODE_ERROR_MASK; if (err) { - drm_err(>_to_xe(gt)->drm, "PCODE Mailbox failed: %d %s", err, + drm_err(&tile_to_xe(tile)->drm, "PCODE Mailbox failed: %d %s", err, err_decode[err].str ?: "Unknown"); return err_decode[err].errno ?: -EPROTO; } @@ -55,84 +54,85 @@ static int pcode_mailbox_status(struct xe_gt *gt) return 0; } -static int __pcode_mailbox_rw(struct xe_gt *gt, u32 mbox, u32 *data0, u32 *data1, +static int __pcode_mailbox_rw(struct xe_tile *tile, u32 mbox, u32 *data0, u32 *data1, unsigned int timeout_ms, bool return_data, bool atomic) { + struct xe_gt *mmio = tile->primary_gt; int err; - if (gt_to_xe(gt)->info.skip_pcode) + if (tile_to_xe(tile)->info.skip_pcode) return 0; - if ((xe_mmio_read32(gt, PCODE_MAILBOX) & PCODE_READY) != 0) + if ((xe_mmio_read32(mmio, PCODE_MAILBOX) & PCODE_READY) != 0) return -EAGAIN; - xe_mmio_write32(gt, PCODE_DATA0, *data0); - xe_mmio_write32(gt, PCODE_DATA1, data1 ? *data1 : 0); - xe_mmio_write32(gt, PCODE_MAILBOX, PCODE_READY | mbox); + xe_mmio_write32(mmio, PCODE_DATA0, *data0); + xe_mmio_write32(mmio, PCODE_DATA1, data1 ? *data1 : 0); + xe_mmio_write32(mmio, PCODE_MAILBOX, PCODE_READY | mbox); - err = xe_mmio_wait32(gt, PCODE_MAILBOX, PCODE_READY, 0, + err = xe_mmio_wait32(mmio, PCODE_MAILBOX, PCODE_READY, 0, timeout_ms * USEC_PER_MSEC, NULL, atomic); if (err) return err; if (return_data) { - *data0 = xe_mmio_read32(gt, PCODE_DATA0); + *data0 = xe_mmio_read32(mmio, PCODE_DATA0); if (data1) - *data1 = xe_mmio_read32(gt, PCODE_DATA1); + *data1 = xe_mmio_read32(mmio, PCODE_DATA1); } - return pcode_mailbox_status(gt); + return pcode_mailbox_status(tile); } -static int pcode_mailbox_rw(struct xe_gt *gt, u32 mbox, u32 *data0, u32 *data1, +static int pcode_mailbox_rw(struct xe_tile *tile, u32 mbox, u32 *data0, u32 *data1, unsigned int timeout_ms, bool return_data, bool atomic) { - if (gt_to_xe(gt)->info.skip_pcode) + if (tile_to_xe(tile)->info.skip_pcode) return 0; - lockdep_assert_held(>->pcode.lock); + lockdep_assert_held(&tile->pcode.lock); - return __pcode_mailbox_rw(gt, mbox, data0, data1, timeout_ms, return_data, atomic); + return __pcode_mailbox_rw(tile, mbox, data0, data1, timeout_ms, return_data, atomic); } -int xe_pcode_write_timeout(struct xe_gt *gt, u32 mbox, u32 data, int timeout) +int xe_pcode_write_timeout(struct xe_tile *tile, u32 mbox, u32 data, int timeout) { int err; - mutex_lock(>->pcode.lock); - err = pcode_mailbox_rw(gt, mbox, &data, NULL, timeout, false, false); - mutex_unlock(>->pcode.lock); + mutex_lock(&tile->pcode.lock); + err = pcode_mailbox_rw(tile, mbox, &data, NULL, timeout, false, false); + mutex_unlock(&tile->pcode.lock); return err; } -int xe_pcode_read(struct xe_gt *gt, u32 mbox, u32 *val, u32 *val1) +int xe_pcode_read(struct xe_tile *tile, u32 mbox, u32 *val, u32 *val1) { int err; - mutex_lock(>->pcode.lock); - err = pcode_mailbox_rw(gt, mbox, val, val1, 1, true, false); - mutex_unlock(>->pcode.lock); + mutex_lock(&tile->pcode.lock); + err = pcode_mailbox_rw(tile, mbox, val, val1, 1, true, false); + mutex_unlock(&tile->pcode.lock); return err; } -static int pcode_try_request(struct xe_gt *gt, u32 mbox, +static int pcode_try_request(struct xe_tile *tile, u32 mbox, u32 request, u32 reply_mask, u32 reply, u32 *status, bool atomic, int timeout_us, bool locked) { int slept, wait = 10; - xe_gt_assert(gt, timeout_us > 0); + xe_tile_assert(tile, timeout_us > 0); for (slept = 0; slept < timeout_us; slept += wait) { if (locked) - *status = pcode_mailbox_rw(gt, mbox, &request, NULL, 1, true, + *status = pcode_mailbox_rw(tile, mbox, &request, NULL, 1, true, atomic); else - *status = __pcode_mailbox_rw(gt, mbox, &request, NULL, 1, true, + *status = __pcode_mailbox_rw(tile, mbox, &request, NULL, 1, true, atomic); if ((*status == 0) && ((request & reply_mask) == reply)) return 0; @@ -149,7 +149,7 @@ static int pcode_try_request(struct xe_gt *gt, u32 mbox, /** * xe_pcode_request - send PCODE request until acknowledgment - * @gt: gt + * @tile: tile * @mbox: PCODE mailbox ID the request is targeted for * @request: request ID * @reply_mask: mask used to check for request acknowledgment @@ -166,17 +166,17 @@ static int pcode_try_request(struct xe_gt *gt, u32 mbox, * Returns 0 on success, %-ETIMEDOUT in case of a timeout, <0 in case of some * other error as reported by PCODE. */ -int xe_pcode_request(struct xe_gt *gt, u32 mbox, u32 request, - u32 reply_mask, u32 reply, int timeout_base_ms) +int xe_pcode_request(struct xe_tile *tile, u32 mbox, u32 request, + u32 reply_mask, u32 reply, int timeout_base_ms) { u32 status; int ret; - xe_gt_assert(gt, timeout_base_ms <= 3); + xe_tile_assert(tile, timeout_base_ms <= 3); - mutex_lock(>->pcode.lock); + mutex_lock(&tile->pcode.lock); - ret = pcode_try_request(gt, mbox, request, reply_mask, reply, &status, + ret = pcode_try_request(tile, mbox, request, reply_mask, reply, &status, false, timeout_base_ms * 1000, true); if (!ret) goto out; @@ -191,20 +191,20 @@ int xe_pcode_request(struct xe_gt *gt, u32 mbox, u32 request, * requests, and for any quirks of the PCODE firmware that delays * the request completion. */ - drm_err(>_to_xe(gt)->drm, + drm_err(&tile_to_xe(tile)->drm, "PCODE timeout, retrying with preemption disabled\n"); preempt_disable(); - ret = pcode_try_request(gt, mbox, request, reply_mask, reply, &status, + ret = pcode_try_request(tile, mbox, request, reply_mask, reply, &status, true, 50 * 1000, true); preempt_enable(); out: - mutex_unlock(>->pcode.lock); + mutex_unlock(&tile->pcode.lock); return status ? status : ret; } /** * xe_pcode_init_min_freq_table - Initialize PCODE's QOS frequency table - * @gt: gt instance + * @tile: tile instance * @min_gt_freq: Minimal (RPn) GT frequency in units of 50MHz. * @max_gt_freq: Maximal (RP0) GT frequency in units of 50MHz. * @@ -227,30 +227,30 @@ out: * - -EACCES, "PCODE Rejected" * - -EPROTO, "Unknown" */ -int xe_pcode_init_min_freq_table(struct xe_gt *gt, u32 min_gt_freq, +int xe_pcode_init_min_freq_table(struct xe_tile *tile, u32 min_gt_freq, u32 max_gt_freq) { int ret; u32 freq; - if (!gt_to_xe(gt)->info.has_llc) + if (!tile_to_xe(tile)->info.has_llc) return 0; if (max_gt_freq <= min_gt_freq) return -EINVAL; - mutex_lock(>->pcode.lock); + mutex_lock(&tile->pcode.lock); for (freq = min_gt_freq; freq <= max_gt_freq; freq++) { u32 data = freq << PCODE_FREQ_RING_RATIO_SHIFT | freq; - ret = pcode_mailbox_rw(gt, PCODE_WRITE_MIN_FREQ_TABLE, + ret = pcode_mailbox_rw(tile, PCODE_WRITE_MIN_FREQ_TABLE, &data, NULL, 1, false, false); if (ret) goto unlock; } unlock: - mutex_unlock(>->pcode.lock); + mutex_unlock(&tile->pcode.lock); return ret; } @@ -270,7 +270,7 @@ unlock: int xe_pcode_ready(struct xe_device *xe, bool locked) { u32 status, request = DGFX_GET_INIT_STATUS; - struct xe_gt *gt = xe_root_mmio_gt(xe); + struct xe_tile *tile = xe_device_get_root_tile(xe); int timeout_us = 180000000; /* 3 min */ int ret; @@ -281,15 +281,15 @@ int xe_pcode_ready(struct xe_device *xe, bool locked) return 0; if (locked) - mutex_lock(>->pcode.lock); + mutex_lock(&tile->pcode.lock); - ret = pcode_try_request(gt, DGFX_PCODE_STATUS, request, + ret = pcode_try_request(tile, DGFX_PCODE_STATUS, request, DGFX_INIT_STATUS_COMPLETE, DGFX_INIT_STATUS_COMPLETE, &status, false, timeout_us, locked); if (locked) - mutex_unlock(>->pcode.lock); + mutex_unlock(&tile->pcode.lock); if (ret) drm_err(&xe->drm, @@ -300,14 +300,14 @@ int xe_pcode_ready(struct xe_device *xe, bool locked) /** * xe_pcode_init: initialize components of PCODE - * @gt: gt instance + * @tile: tile instance * * This function initializes the xe_pcode component. * To be called once only during probe. */ -void xe_pcode_init(struct xe_gt *gt) +void xe_pcode_init(struct xe_tile *tile) { - drmm_mutex_init(>_to_xe(gt)->drm, >->pcode.lock); + drmm_mutex_init(&tile_to_xe(tile)->drm, &tile->pcode.lock); } /** diff --git a/drivers/gpu/drm/xe/xe_pcode.h b/drivers/gpu/drm/xe/xe_pcode.h index 3f54c6d2a57d..ba33991d72a7 100644 --- a/drivers/gpu/drm/xe/xe_pcode.h +++ b/drivers/gpu/drm/xe/xe_pcode.h @@ -7,21 +7,21 @@ #define _XE_PCODE_H_ #include <linux/types.h> -struct xe_gt; +struct xe_tile; struct xe_device; -void xe_pcode_init(struct xe_gt *gt); +void xe_pcode_init(struct xe_tile *tile); int xe_pcode_probe_early(struct xe_device *xe); int xe_pcode_ready(struct xe_device *xe, bool locked); -int xe_pcode_init_min_freq_table(struct xe_gt *gt, u32 min_gt_freq, +int xe_pcode_init_min_freq_table(struct xe_tile *tile, u32 min_gt_freq, u32 max_gt_freq); -int xe_pcode_read(struct xe_gt *gt, u32 mbox, u32 *val, u32 *val1); -int xe_pcode_write_timeout(struct xe_gt *gt, u32 mbox, u32 val, +int xe_pcode_read(struct xe_tile *tile, u32 mbox, u32 *val, u32 *val1); +int xe_pcode_write_timeout(struct xe_tile *tile, u32 mbox, u32 val, int timeout_ms); -#define xe_pcode_write(gt, mbox, val) \ - xe_pcode_write_timeout(gt, mbox, val, 1) +#define xe_pcode_write(tile, mbox, val) \ + xe_pcode_write_timeout(tile, mbox, val, 1) -int xe_pcode_request(struct xe_gt *gt, u32 mbox, u32 request, +int xe_pcode_request(struct xe_tile *tile, u32 mbox, u32 request, u32 reply_mask, u32 reply, int timeout_ms); #define PCODE_MBOX(mbcmd, param1, param2)\ diff --git a/drivers/gpu/drm/xe/xe_pm.c b/drivers/gpu/drm/xe/xe_pm.c index 2e2accd76fb2..e518557e0eec 100644 --- a/drivers/gpu/drm/xe/xe_pm.c +++ b/drivers/gpu/drm/xe/xe_pm.c @@ -79,7 +79,14 @@ static struct lockdep_map xe_pm_runtime_nod3cold_map = { }; #endif -static bool __maybe_unused xe_rpm_reclaim_safe(const struct xe_device *xe) +/** + * xe_rpm_reclaim_safe() - Whether runtime resume can be done from reclaim context + * @xe: The xe device. + * + * Return: true if it is safe to runtime resume from reclaim context. + * false otherwise. + */ +bool xe_rpm_reclaim_safe(const struct xe_device *xe) { return !xe->d3cold.capable && !xe->info.has_sriov; } @@ -392,8 +399,6 @@ int xe_pm_runtime_suspend(struct xe_device *xe) xe_display_pm_runtime_suspend(xe); if (xe->d3cold.allowed) { - xe_display_pm_suspend(xe, true); - err = xe_bo_evict_all(xe); if (err) goto out; diff --git a/drivers/gpu/drm/xe/xe_pm.h b/drivers/gpu/drm/xe/xe_pm.h index 9aef673b1c8a..998d1ed64556 100644 --- a/drivers/gpu/drm/xe/xe_pm.h +++ b/drivers/gpu/drm/xe/xe_pm.h @@ -31,6 +31,7 @@ bool xe_pm_runtime_resume_and_get(struct xe_device *xe); void xe_pm_assert_unbounded_bridge(struct xe_device *xe); int xe_pm_set_vram_threshold(struct xe_device *xe, u32 threshold); void xe_pm_d3cold_allowed_toggle(struct xe_device *xe); +bool xe_rpm_reclaim_safe(const struct xe_device *xe); struct task_struct *xe_pm_read_callback_task(struct xe_device *xe); int xe_pm_module_init(void); diff --git a/drivers/gpu/drm/xe/xe_pt.c b/drivers/gpu/drm/xe/xe_pt.c index 579ed31b46db..d6353e8969f0 100644 --- a/drivers/gpu/drm/xe/xe_pt.c +++ b/drivers/gpu/drm/xe/xe_pt.c @@ -3,6 +3,8 @@ * Copyright © 2022 Intel Corporation */ +#include <linux/dma-fence-array.h> + #include "xe_pt.h" #include "regs/xe_gtt_defs.h" @@ -1627,9 +1629,11 @@ xe_pt_update_ops_rfence_interval(struct xe_vm_pgtable_update_ops *pt_update_ops, static int vma_reserve_fences(struct xe_device *xe, struct xe_vma *vma) { + int shift = xe_device_get_root_tile(xe)->media_gt ? 1 : 0; + if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm) return dma_resv_reserve_fences(xe_vma_bo(vma)->ttm.base.resv, - xe->info.tile_count); + xe->info.tile_count << shift); return 0; } @@ -1816,6 +1820,7 @@ int xe_pt_update_ops_prepare(struct xe_tile *tile, struct xe_vma_ops *vops) struct xe_vm_pgtable_update_ops *pt_update_ops = &vops->pt_update_ops[tile->id]; struct xe_vma_op *op; + int shift = tile->media_gt ? 1 : 0; int err; lockdep_assert_held(&vops->vm->lock); @@ -1824,7 +1829,7 @@ int xe_pt_update_ops_prepare(struct xe_tile *tile, struct xe_vma_ops *vops) xe_pt_update_ops_init(pt_update_ops); err = dma_resv_reserve_fences(xe_vm_resv(vops->vm), - tile_to_xe(tile)->info.tile_count); + tile_to_xe(tile)->info.tile_count << shift); if (err) return err; @@ -1849,13 +1854,20 @@ int xe_pt_update_ops_prepare(struct xe_tile *tile, struct xe_vma_ops *vops) static void bind_op_commit(struct xe_vm *vm, struct xe_tile *tile, struct xe_vm_pgtable_update_ops *pt_update_ops, - struct xe_vma *vma, struct dma_fence *fence) + struct xe_vma *vma, struct dma_fence *fence, + struct dma_fence *fence2) { - if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm) + if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm) { dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence, pt_update_ops->wait_vm_bookkeep ? DMA_RESV_USAGE_KERNEL : DMA_RESV_USAGE_BOOKKEEP); + if (fence2) + dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence2, + pt_update_ops->wait_vm_bookkeep ? + DMA_RESV_USAGE_KERNEL : + DMA_RESV_USAGE_BOOKKEEP); + } vma->tile_present |= BIT(tile->id); vma->tile_staged &= ~BIT(tile->id); if (xe_vma_is_userptr(vma)) { @@ -1875,13 +1887,20 @@ static void bind_op_commit(struct xe_vm *vm, struct xe_tile *tile, static void unbind_op_commit(struct xe_vm *vm, struct xe_tile *tile, struct xe_vm_pgtable_update_ops *pt_update_ops, - struct xe_vma *vma, struct dma_fence *fence) + struct xe_vma *vma, struct dma_fence *fence, + struct dma_fence *fence2) { - if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm) + if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm) { dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence, pt_update_ops->wait_vm_bookkeep ? DMA_RESV_USAGE_KERNEL : DMA_RESV_USAGE_BOOKKEEP); + if (fence2) + dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence2, + pt_update_ops->wait_vm_bookkeep ? + DMA_RESV_USAGE_KERNEL : + DMA_RESV_USAGE_BOOKKEEP); + } vma->tile_present &= ~BIT(tile->id); if (!vma->tile_present) { list_del_init(&vma->combined_links.rebind); @@ -1898,7 +1917,8 @@ static void unbind_op_commit(struct xe_vm *vm, struct xe_tile *tile, static void op_commit(struct xe_vm *vm, struct xe_tile *tile, struct xe_vm_pgtable_update_ops *pt_update_ops, - struct xe_vma_op *op, struct dma_fence *fence) + struct xe_vma_op *op, struct dma_fence *fence, + struct dma_fence *fence2) { xe_vm_assert_held(vm); @@ -1907,26 +1927,28 @@ static void op_commit(struct xe_vm *vm, if (!op->map.immediate && xe_vm_in_fault_mode(vm)) break; - bind_op_commit(vm, tile, pt_update_ops, op->map.vma, fence); + bind_op_commit(vm, tile, pt_update_ops, op->map.vma, fence, + fence2); break; case DRM_GPUVA_OP_REMAP: unbind_op_commit(vm, tile, pt_update_ops, - gpuva_to_vma(op->base.remap.unmap->va), fence); + gpuva_to_vma(op->base.remap.unmap->va), fence, + fence2); if (op->remap.prev) bind_op_commit(vm, tile, pt_update_ops, op->remap.prev, - fence); + fence, fence2); if (op->remap.next) bind_op_commit(vm, tile, pt_update_ops, op->remap.next, - fence); + fence, fence2); break; case DRM_GPUVA_OP_UNMAP: unbind_op_commit(vm, tile, pt_update_ops, - gpuva_to_vma(op->base.unmap.va), fence); + gpuva_to_vma(op->base.unmap.va), fence, fence2); break; case DRM_GPUVA_OP_PREFETCH: bind_op_commit(vm, tile, pt_update_ops, - gpuva_to_vma(op->base.prefetch.va), fence); + gpuva_to_vma(op->base.prefetch.va), fence, fence2); break; default: drm_warn(&vm->xe->drm, "NOT POSSIBLE"); @@ -1963,7 +1985,9 @@ xe_pt_update_ops_run(struct xe_tile *tile, struct xe_vma_ops *vops) struct xe_vm_pgtable_update_ops *pt_update_ops = &vops->pt_update_ops[tile->id]; struct dma_fence *fence; - struct invalidation_fence *ifence = NULL; + struct invalidation_fence *ifence = NULL, *mfence = NULL; + struct dma_fence **fences = NULL; + struct dma_fence_array *cf = NULL; struct xe_range_fence *rfence; struct xe_vma_op *op; int err = 0, i; @@ -1996,6 +2020,23 @@ xe_pt_update_ops_run(struct xe_tile *tile, struct xe_vma_ops *vops) err = -ENOMEM; goto kill_vm_tile1; } + if (tile->media_gt) { + mfence = kzalloc(sizeof(*ifence), GFP_KERNEL); + if (!mfence) { + err = -ENOMEM; + goto free_ifence; + } + fences = kmalloc_array(2, sizeof(*fences), GFP_KERNEL); + if (!fences) { + err = -ENOMEM; + goto free_ifence; + } + cf = dma_fence_array_alloc(2); + if (!cf) { + err = -ENOMEM; + goto free_ifence; + } + } } rfence = kzalloc(sizeof(*rfence), GFP_KERNEL); @@ -2027,19 +2068,50 @@ xe_pt_update_ops_run(struct xe_tile *tile, struct xe_vma_ops *vops) /* tlb invalidation must be done before signaling rebind */ if (ifence) { + if (mfence) + dma_fence_get(fence); invalidation_fence_init(tile->primary_gt, ifence, fence, pt_update_ops->start, pt_update_ops->last, vm->usm.asid); - fence = &ifence->base.base; + if (mfence) { + invalidation_fence_init(tile->media_gt, mfence, fence, + pt_update_ops->start, + pt_update_ops->last, vm->usm.asid); + fences[0] = &ifence->base.base; + fences[1] = &mfence->base.base; + dma_fence_array_init(cf, 2, fences, + vm->composite_fence_ctx, + vm->composite_fence_seqno++, + false); + fence = &cf->base; + } else { + fence = &ifence->base.base; + } } - dma_resv_add_fence(xe_vm_resv(vm), fence, - pt_update_ops->wait_vm_bookkeep ? - DMA_RESV_USAGE_KERNEL : - DMA_RESV_USAGE_BOOKKEEP); + if (!mfence) { + dma_resv_add_fence(xe_vm_resv(vm), fence, + pt_update_ops->wait_vm_bookkeep ? + DMA_RESV_USAGE_KERNEL : + DMA_RESV_USAGE_BOOKKEEP); - list_for_each_entry(op, &vops->list, link) - op_commit(vops->vm, tile, pt_update_ops, op, fence); + list_for_each_entry(op, &vops->list, link) + op_commit(vops->vm, tile, pt_update_ops, op, fence, NULL); + } else { + dma_resv_add_fence(xe_vm_resv(vm), &ifence->base.base, + pt_update_ops->wait_vm_bookkeep ? + DMA_RESV_USAGE_KERNEL : + DMA_RESV_USAGE_BOOKKEEP); + + dma_resv_add_fence(xe_vm_resv(vm), &mfence->base.base, + pt_update_ops->wait_vm_bookkeep ? + DMA_RESV_USAGE_KERNEL : + DMA_RESV_USAGE_BOOKKEEP); + + list_for_each_entry(op, &vops->list, link) + op_commit(vops->vm, tile, pt_update_ops, op, + &ifence->base.base, &mfence->base.base); + } if (pt_update_ops->needs_userptr_lock) up_read(&vm->userptr.notifier_lock); @@ -2049,6 +2121,9 @@ xe_pt_update_ops_run(struct xe_tile *tile, struct xe_vma_ops *vops) free_rfence: kfree(rfence); free_ifence: + kfree(cf); + kfree(fences); + kfree(mfence); kfree(ifence); kill_vm_tile1: if (err != -EAGAIN && tile->id) diff --git a/drivers/gpu/drm/xe/xe_query.c b/drivers/gpu/drm/xe/xe_query.c index 73ef6e4c2dc9..28d9bb3b825d 100644 --- a/drivers/gpu/drm/xe/xe_query.c +++ b/drivers/gpu/drm/xe/xe_query.c @@ -9,7 +9,7 @@ #include <linux/sched/clock.h> #include <drm/ttm/ttm_placement.h> -#include <drm/xe_drm.h> +#include <uapi/drm/xe_drm.h> #include "regs/xe_engine_regs.h" #include "regs/xe_gt_regs.h" diff --git a/drivers/gpu/drm/xe/xe_rtp.c b/drivers/gpu/drm/xe/xe_rtp.c index e78ba324dd18..86c705d18c0d 100644 --- a/drivers/gpu/drm/xe/xe_rtp.c +++ b/drivers/gpu/drm/xe/xe_rtp.c @@ -7,7 +7,7 @@ #include <kunit/visibility.h> -#include <drm/xe_drm.h> +#include <uapi/drm/xe_drm.h> #include "xe_gt.h" #include "xe_gt_topology.h" diff --git a/drivers/gpu/drm/xe/xe_sched_job.c b/drivers/gpu/drm/xe/xe_sched_job.c index 55d47450b2c6..eeccc1c318ae 100644 --- a/drivers/gpu/drm/xe/xe_sched_job.c +++ b/drivers/gpu/drm/xe/xe_sched_job.c @@ -5,7 +5,7 @@ #include "xe_sched_job.h" -#include <drm/xe_drm.h> +#include <uapi/drm/xe_drm.h> #include <linux/dma-fence-chain.h> #include <linux/slab.h> diff --git a/drivers/gpu/drm/xe/xe_sync.c b/drivers/gpu/drm/xe/xe_sync.c index 436faff09bac..bb3c2a830362 100644 --- a/drivers/gpu/drm/xe/xe_sync.c +++ b/drivers/gpu/drm/xe/xe_sync.c @@ -12,7 +12,7 @@ #include <drm/drm_print.h> #include <drm/drm_syncobj.h> -#include <drm/xe_drm.h> +#include <uapi/drm/xe_drm.h> #include "xe_device_types.h" #include "xe_exec_queue.h" diff --git a/drivers/gpu/drm/xe/xe_tile.c b/drivers/gpu/drm/xe/xe_tile.c index 15ea0a942f67..dda5268507d8 100644 --- a/drivers/gpu/drm/xe/xe_tile.c +++ b/drivers/gpu/drm/xe/xe_tile.c @@ -9,6 +9,7 @@ #include "xe_ggtt.h" #include "xe_gt.h" #include "xe_migrate.h" +#include "xe_pcode.h" #include "xe_sa.h" #include "xe_tile.h" #include "xe_tile_sysfs.h" @@ -124,6 +125,8 @@ int xe_tile_init_early(struct xe_tile *tile, struct xe_device *xe, u8 id) if (IS_ERR(tile->primary_gt)) return PTR_ERR(tile->primary_gt); + xe_pcode_init(tile); + return 0; } diff --git a/drivers/gpu/drm/xe/xe_uc_debugfs.c b/drivers/gpu/drm/xe/xe_uc_debugfs.c index 78eb8db73791..24a4209051ee 100644 --- a/drivers/gpu/drm/xe/xe_uc_debugfs.c +++ b/drivers/gpu/drm/xe/xe_uc_debugfs.c @@ -8,6 +8,7 @@ #include <drm/drm_debugfs.h> #include "xe_gt.h" +#include "xe_gsc_debugfs.h" #include "xe_guc_debugfs.h" #include "xe_huc_debugfs.h" #include "xe_macros.h" @@ -23,6 +24,7 @@ void xe_uc_debugfs_register(struct xe_uc *uc, struct dentry *parent) return; } + xe_gsc_debugfs_register(&uc->gsc, root); xe_guc_debugfs_register(&uc->guc, root); xe_huc_debugfs_register(&uc->huc, root); } diff --git a/drivers/gpu/drm/xe/xe_uc_fw.c b/drivers/gpu/drm/xe/xe_uc_fw.c index 4bb2a4a80ddc..d431d0031185 100644 --- a/drivers/gpu/drm/xe/xe_uc_fw.c +++ b/drivers/gpu/drm/xe/xe_uc_fw.c @@ -129,8 +129,8 @@ struct fw_blobs_by_type { /* for the GSC FW we match the compatibility version and not the release one */ #define XE_GSC_FIRMWARE_DEFS(fw_def, major_ver) \ - fw_def(LUNARLAKE, major_ver(xe, gsc, lnl, 1, 0, 0)) \ - fw_def(METEORLAKE, major_ver(i915, gsc, mtl, 1, 0, 0)) + fw_def(LUNARLAKE, major_ver(xe, gsc, lnl, 104, 1, 0)) \ + fw_def(METEORLAKE, major_ver(i915, gsc, mtl, 102, 1, 0)) #define MAKE_FW_PATH(dir__, uc__, shortname__, version__) \ __stringify(dir__) "/" __stringify(shortname__) "_" __stringify(uc__) version__ ".bin" @@ -141,6 +141,8 @@ struct fw_blobs_by_type { MAKE_FW_PATH(dir_, uc_, shortname_, "_" __stringify(a)) #define fw_filename_no_ver(dir_, uc_, shortname_) \ MAKE_FW_PATH(dir_, uc_, shortname_, "") +#define fw_filename_gsc(dir_, uc_, shortname_, a, b, c) \ + MAKE_FW_PATH(dir_, uc_, shortname_, "_" __stringify(b)) #define uc_fw_entry_mmp_ver(dir_, uc_, shortname_, a, b, c) \ { fw_filename_mmp_ver(dir_, uc_, shortname_, a, b, c), \ @@ -151,6 +153,9 @@ struct fw_blobs_by_type { #define uc_fw_entry_no_ver(dir_, uc_, shortname_) \ { fw_filename_no_ver(dir_, uc_, shortname_), \ 0, 0 } +#define uc_fw_entry_gsc(dir_, uc_, shortname_, a, b, c) \ + { fw_filename_gsc(dir_, uc_, shortname_, a, b, c), \ + a, b, c } /* All blobs need to be declared via MODULE_FIRMWARE() */ #define XE_UC_MODULE_FIRMWARE(platform__, fw_filename) \ @@ -166,7 +171,7 @@ XE_GUC_FIRMWARE_DEFS(XE_UC_MODULE_FIRMWARE, fw_filename_mmp_ver, fw_filename_major_ver) XE_HUC_FIRMWARE_DEFS(XE_UC_MODULE_FIRMWARE, fw_filename_mmp_ver, fw_filename_no_ver) -XE_GSC_FIRMWARE_DEFS(XE_UC_MODULE_FIRMWARE, fw_filename_major_ver) +XE_GSC_FIRMWARE_DEFS(XE_UC_MODULE_FIRMWARE, fw_filename_gsc) static struct xe_gt * __uc_fw_to_gt(struct xe_uc_fw *uc_fw, enum xe_uc_fw_type type) @@ -209,7 +214,7 @@ uc_fw_auto_select(struct xe_device *xe, struct xe_uc_fw *uc_fw) uc_fw_entry_no_ver) }; static const struct uc_fw_entry entries_gsc[] = { - XE_GSC_FIRMWARE_DEFS(XE_UC_FW_ENTRY, uc_fw_entry_major_ver) + XE_GSC_FIRMWARE_DEFS(XE_UC_FW_ENTRY, uc_fw_entry_gsc) }; static const struct fw_blobs_by_type blobs_all[XE_UC_FW_NUM_TYPES] = { [XE_UC_FW_TYPE_GUC] = { entries_guc, ARRAY_SIZE(entries_guc) }, diff --git a/drivers/gpu/drm/xe/xe_uc_fw.h b/drivers/gpu/drm/xe/xe_uc_fw.h index c108e9d08e70..6195e353f269 100644 --- a/drivers/gpu/drm/xe/xe_uc_fw.h +++ b/drivers/gpu/drm/xe/xe_uc_fw.h @@ -65,7 +65,7 @@ const char *xe_uc_fw_status_repr(enum xe_uc_fw_status status) return "<invalid>"; } -static inline int xe_uc_fw_status_to_error(enum xe_uc_fw_status status) +static inline int xe_uc_fw_status_to_error(const enum xe_uc_fw_status status) { switch (status) { case XE_UC_FIRMWARE_NOT_SUPPORTED: @@ -108,7 +108,7 @@ static inline const char *xe_uc_fw_type_repr(enum xe_uc_fw_type type) } static inline enum xe_uc_fw_status -__xe_uc_fw_status(struct xe_uc_fw *uc_fw) +__xe_uc_fw_status(const struct xe_uc_fw *uc_fw) { /* shouldn't call this before checking hw/blob availability */ XE_WARN_ON(uc_fw->status == XE_UC_FIRMWARE_UNINITIALIZED); @@ -156,6 +156,11 @@ static inline bool xe_uc_fw_is_overridden(const struct xe_uc_fw *uc_fw) return uc_fw->user_overridden; } +static inline bool xe_uc_fw_is_in_error_state(const struct xe_uc_fw *uc_fw) +{ + return xe_uc_fw_status_to_error(__xe_uc_fw_status(uc_fw)) < 0; +} + static inline void xe_uc_fw_sanitize(struct xe_uc_fw *uc_fw) { if (xe_uc_fw_is_loadable(uc_fw)) diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index 3eb76d874eb2..7acd5fc9d032 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -12,7 +12,7 @@ #include <drm/drm_print.h> #include <drm/ttm/ttm_execbuf_util.h> #include <drm/ttm/ttm_tt.h> -#include <drm/xe_drm.h> +#include <uapi/drm/xe_drm.h> #include <linux/ascii85.h> #include <linux/delay.h> #include <linux/kthread.h> diff --git a/drivers/gpu/drm/xe/xe_vram_freq.c b/drivers/gpu/drm/xe/xe_vram_freq.c index 99ff95e408e0..b26e26d73dae 100644 --- a/drivers/gpu/drm/xe/xe_vram_freq.c +++ b/drivers/gpu/drm/xe/xe_vram_freq.c @@ -34,7 +34,6 @@ static ssize_t max_freq_show(struct device *dev, struct device_attribute *attr, char *buf) { struct xe_tile *tile = dev_to_tile(dev); - struct xe_gt *gt = tile->primary_gt; u32 val, mbox; int err; @@ -42,7 +41,7 @@ static ssize_t max_freq_show(struct device *dev, struct device_attribute *attr, | REG_FIELD_PREP(PCODE_MB_PARAM1, PCODE_MBOX_FC_SC_READ_FUSED_P0) | REG_FIELD_PREP(PCODE_MB_PARAM2, PCODE_MBOX_DOMAIN_HBM); - err = xe_pcode_read(gt, mbox, &val, NULL); + err = xe_pcode_read(tile, mbox, &val, NULL); if (err) return err; @@ -57,7 +56,6 @@ static ssize_t min_freq_show(struct device *dev, struct device_attribute *attr, char *buf) { struct xe_tile *tile = dev_to_tile(dev); - struct xe_gt *gt = tile->primary_gt; u32 val, mbox; int err; @@ -65,7 +63,7 @@ static ssize_t min_freq_show(struct device *dev, struct device_attribute *attr, | REG_FIELD_PREP(PCODE_MB_PARAM1, PCODE_MBOX_FC_SC_READ_FUSED_PN) | REG_FIELD_PREP(PCODE_MB_PARAM2, PCODE_MBOX_DOMAIN_HBM); - err = xe_pcode_read(gt, mbox, &val, NULL); + err = xe_pcode_read(tile, mbox, &val, NULL); if (err) return err; diff --git a/drivers/gpu/drm/xe/xe_wait_user_fence.c b/drivers/gpu/drm/xe/xe_wait_user_fence.c index f69721339201..d46fa8374980 100644 --- a/drivers/gpu/drm/xe/xe_wait_user_fence.c +++ b/drivers/gpu/drm/xe/xe_wait_user_fence.c @@ -8,7 +8,7 @@ #include <drm/drm_device.h> #include <drm/drm_file.h> #include <drm/drm_utils.h> -#include <drm/xe_drm.h> +#include <uapi/drm/xe_drm.h> #include "xe_device.h" #include "xe_gt.h" |