diff options
Diffstat (limited to 'drivers/gpu/drm/xe/xe_pci.c')
| -rw-r--r-- | drivers/gpu/drm/xe/xe_pci.c | 874 |
1 files changed, 579 insertions, 295 deletions
diff --git a/drivers/gpu/drm/xe/xe_pci.c b/drivers/gpu/drm/xe/xe_pci.c index f326dbb1cecd..9c9ea10d994c 100644 --- a/drivers/gpu/drm/xe/xe_pci.c +++ b/drivers/gpu/drm/xe/xe_pci.c @@ -13,20 +13,27 @@ #include <drm/drm_color_mgmt.h> #include <drm/drm_drv.h> -#include <drm/xe_pciids.h> +#include <drm/intel/pciids.h> #include "display/xe_display.h" #include "regs/xe_gt_regs.h" +#include "regs/xe_regs.h" +#include "xe_configfs.h" #include "xe_device.h" #include "xe_drv.h" #include "xe_gt.h" +#include "xe_gt_sriov_vf.h" +#include "xe_guc.h" #include "xe_macros.h" #include "xe_mmio.h" #include "xe_module.h" +#include "xe_pci_sriov.h" #include "xe_pci_types.h" #include "xe_pm.h" +#include "xe_printk.h" #include "xe_sriov.h" #include "xe_step.h" +#include "xe_survivability_mode.h" #include "xe_tile.h" enum toggle_d3cold { @@ -34,101 +41,32 @@ enum toggle_d3cold { D3COLD_ENABLE, }; -struct xe_subplatform_desc { - enum xe_subplatform subplatform; - const char *name; - const u16 *pciidlist; -}; - -struct xe_gt_desc { - enum xe_gt_type type; - u32 mmio_adj_limit; - u32 mmio_adj_offset; -}; - -struct xe_device_desc { - /* Should only ever be set for platforms without GMD_ID */ - const struct xe_graphics_desc *graphics; - /* Should only ever be set for platforms without GMD_ID */ - const struct xe_media_desc *media; - - const char *platform_name; - const struct xe_subplatform_desc *subplatforms; - - enum xe_platform platform; - - u8 require_force_probe:1; - u8 is_dgfx:1; - - u8 has_display:1; - u8 has_heci_gscfi:1; - u8 has_llc:1; - u8 has_mmio_ext:1; - u8 has_sriov:1; - u8 skip_guc_pc:1; - u8 skip_mtcfg:1; - u8 skip_pcode:1; -}; - __diag_push(); __diag_ignore_all("-Woverride-init", "Allow field overrides in table"); #define PLATFORM(x) \ - .platform = (x), \ + .platform = XE_##x, \ .platform_name = #x #define NOP(x) x static const struct xe_graphics_desc graphics_xelp = { - .name = "Xe_LP", - .ver = 12, - .rel = 0, - .hw_engine_mask = BIT(XE_HW_ENGINE_RCS0) | BIT(XE_HW_ENGINE_BCS0), - - .dma_mask_size = 39, - .va_bits = 48, - .vm_max_level = 3, -}; - -static const struct xe_graphics_desc graphics_xelpp = { - .name = "Xe_LP+", - .ver = 12, - .rel = 10, - - .hw_engine_mask = BIT(XE_HW_ENGINE_RCS0) | BIT(XE_HW_ENGINE_BCS0), - - .dma_mask_size = 39, - .va_bits = 48, - .vm_max_level = 3, }; #define XE_HP_FEATURES \ - .has_range_tlb_invalidation = true, \ - .has_flat_ccs = true, \ - .dma_mask_size = 46, \ - .va_bits = 48, \ - .vm_max_level = 3 + .has_range_tlb_inval = true static const struct xe_graphics_desc graphics_xehpg = { - .name = "Xe_HPG", - .ver = 12, - .rel = 55, - .hw_engine_mask = BIT(XE_HW_ENGINE_RCS0) | BIT(XE_HW_ENGINE_BCS0) | BIT(XE_HW_ENGINE_CCS0) | BIT(XE_HW_ENGINE_CCS1) | BIT(XE_HW_ENGINE_CCS2) | BIT(XE_HW_ENGINE_CCS3), XE_HP_FEATURES, - .vram_flags = XE_VRAM_FLAGS_NEED64K, }; static const struct xe_graphics_desc graphics_xehpc = { - .name = "Xe_HPC", - .ver = 12, - .rel = 60, - .hw_engine_mask = BIT(XE_HW_ENGINE_BCS0) | BIT(XE_HW_ENGINE_BCS1) | BIT(XE_HW_ENGINE_BCS2) | BIT(XE_HW_ENGINE_BCS3) | @@ -139,235 +77,345 @@ static const struct xe_graphics_desc graphics_xehpc = { BIT(XE_HW_ENGINE_CCS2) | BIT(XE_HW_ENGINE_CCS3), XE_HP_FEATURES, - .dma_mask_size = 52, - .max_remote_tiles = 1, - .va_bits = 57, - .vm_max_level = 4, - .vram_flags = XE_VRAM_FLAGS_NEED64K, .has_asid = 1, - .has_flat_ccs = 0, + .has_atomic_enable_pte_bit = 1, .has_usm = 1, }; static const struct xe_graphics_desc graphics_xelpg = { - .name = "Xe_LPG", .hw_engine_mask = BIT(XE_HW_ENGINE_RCS0) | BIT(XE_HW_ENGINE_BCS0) | BIT(XE_HW_ENGINE_CCS0), XE_HP_FEATURES, - .has_flat_ccs = 0, }; #define XE2_GFX_FEATURES \ - .dma_mask_size = 46, \ .has_asid = 1, \ - .has_flat_ccs = 1, \ - .has_range_tlb_invalidation = 1, \ + .has_atomic_enable_pte_bit = 1, \ + .has_range_tlb_inval = 1, \ .has_usm = 1, \ - .va_bits = 48, \ - .vm_max_level = 4, \ + .has_64bit_timestamp = 1, \ .hw_engine_mask = \ BIT(XE_HW_ENGINE_RCS0) | \ BIT(XE_HW_ENGINE_BCS8) | BIT(XE_HW_ENGINE_BCS0) | \ GENMASK(XE_HW_ENGINE_CCS3, XE_HW_ENGINE_CCS0) static const struct xe_graphics_desc graphics_xe2 = { - .name = "Xe2_LPG / Xe2_HPG", - XE2_GFX_FEATURES, }; -static const struct xe_media_desc media_xem = { - .name = "Xe_M", - .ver = 12, - .rel = 0, - +static const struct xe_graphics_desc graphics_xe3p_xpc = { + XE2_GFX_FEATURES, .hw_engine_mask = - GENMASK(XE_HW_ENGINE_VCS7, XE_HW_ENGINE_VCS0) | - GENMASK(XE_HW_ENGINE_VECS3, XE_HW_ENGINE_VECS0), + GENMASK(XE_HW_ENGINE_BCS8, XE_HW_ENGINE_BCS1) | + GENMASK(XE_HW_ENGINE_CCS3, XE_HW_ENGINE_CCS0), }; -static const struct xe_media_desc media_xehpm = { - .name = "Xe_HPM", - .ver = 12, - .rel = 55, - +static const struct xe_media_desc media_xem = { .hw_engine_mask = GENMASK(XE_HW_ENGINE_VCS7, XE_HW_ENGINE_VCS0) | GENMASK(XE_HW_ENGINE_VECS3, XE_HW_ENGINE_VECS0), }; static const struct xe_media_desc media_xelpmp = { - .name = "Xe_LPM+", .hw_engine_mask = GENMASK(XE_HW_ENGINE_VCS7, XE_HW_ENGINE_VCS0) | GENMASK(XE_HW_ENGINE_VECS3, XE_HW_ENGINE_VECS0) | BIT(XE_HW_ENGINE_GSCCS0) }; -static const struct xe_media_desc media_xe2 = { - .name = "Xe2_LPM / Xe2_HPM", - .hw_engine_mask = - GENMASK(XE_HW_ENGINE_VCS7, XE_HW_ENGINE_VCS0) | - GENMASK(XE_HW_ENGINE_VECS3, XE_HW_ENGINE_VECS0), /* TODO: GSC0 */ +/* Pre-GMDID Graphics IPs */ +static const struct xe_ip graphics_ip_xelp = { 1200, "Xe_LP", &graphics_xelp }; +static const struct xe_ip graphics_ip_xelpp = { 1210, "Xe_LP+", &graphics_xelp }; +static const struct xe_ip graphics_ip_xehpg = { 1255, "Xe_HPG", &graphics_xehpg }; +static const struct xe_ip graphics_ip_xehpc = { 1260, "Xe_HPC", &graphics_xehpc }; + +/* GMDID-based Graphics IPs */ +static const struct xe_ip graphics_ips[] = { + { 1270, "Xe_LPG", &graphics_xelpg }, + { 1271, "Xe_LPG", &graphics_xelpg }, + { 1274, "Xe_LPG+", &graphics_xelpg }, + { 2001, "Xe2_HPG", &graphics_xe2 }, + { 2002, "Xe2_HPG", &graphics_xe2 }, + { 2004, "Xe2_LPG", &graphics_xe2 }, + { 3000, "Xe3_LPG", &graphics_xe2 }, + { 3001, "Xe3_LPG", &graphics_xe2 }, + { 3003, "Xe3_LPG", &graphics_xe2 }, + { 3004, "Xe3_LPG", &graphics_xe2 }, + { 3005, "Xe3_LPG", &graphics_xe2 }, + { 3511, "Xe3p_XPC", &graphics_xe3p_xpc }, +}; + +/* Pre-GMDID Media IPs */ +static const struct xe_ip media_ip_xem = { 1200, "Xe_M", &media_xem }; +static const struct xe_ip media_ip_xehpm = { 1255, "Xe_HPM", &media_xem }; + +/* GMDID-based Media IPs */ +static const struct xe_ip media_ips[] = { + { 1300, "Xe_LPM+", &media_xelpmp }, + { 1301, "Xe2_HPM", &media_xelpmp }, + { 2000, "Xe2_LPM", &media_xelpmp }, + { 3000, "Xe3_LPM", &media_xelpmp }, + { 3002, "Xe3_LPM", &media_xelpmp }, + { 3500, "Xe3p_LPM", &media_xelpmp }, + { 3503, "Xe3p_HPM", &media_xelpmp }, }; static const struct xe_device_desc tgl_desc = { - .graphics = &graphics_xelp, - .media = &media_xem, - PLATFORM(XE_TIGERLAKE), + .pre_gmdid_graphics_ip = &graphics_ip_xelp, + .pre_gmdid_media_ip = &media_ip_xem, + PLATFORM(TIGERLAKE), + .dma_mask_size = 39, .has_display = true, .has_llc = true, + .has_sriov = true, + .max_gt_per_tile = 1, .require_force_probe = true, + .va_bits = 48, + .vm_max_level = 3, }; static const struct xe_device_desc rkl_desc = { - .graphics = &graphics_xelp, - .media = &media_xem, - PLATFORM(XE_ROCKETLAKE), + .pre_gmdid_graphics_ip = &graphics_ip_xelp, + .pre_gmdid_media_ip = &media_ip_xem, + PLATFORM(ROCKETLAKE), + .dma_mask_size = 39, .has_display = true, .has_llc = true, + .max_gt_per_tile = 1, .require_force_probe = true, + .va_bits = 48, + .vm_max_level = 3, }; -static const u16 adls_rpls_ids[] = { XE_RPLS_IDS(NOP), 0 }; +static const u16 adls_rpls_ids[] = { INTEL_RPLS_IDS(NOP), 0 }; static const struct xe_device_desc adl_s_desc = { - .graphics = &graphics_xelp, - .media = &media_xem, - PLATFORM(XE_ALDERLAKE_S), + .pre_gmdid_graphics_ip = &graphics_ip_xelp, + .pre_gmdid_media_ip = &media_ip_xem, + PLATFORM(ALDERLAKE_S), + .dma_mask_size = 39, .has_display = true, .has_llc = true, + .has_sriov = true, + .max_gt_per_tile = 1, .require_force_probe = true, .subplatforms = (const struct xe_subplatform_desc[]) { { XE_SUBPLATFORM_ALDERLAKE_S_RPLS, "RPLS", adls_rpls_ids }, {}, }, + .va_bits = 48, + .vm_max_level = 3, }; -static const u16 adlp_rplu_ids[] = { XE_RPLU_IDS(NOP), 0 }; +static const u16 adlp_rplu_ids[] = { INTEL_RPLU_IDS(NOP), 0 }; static const struct xe_device_desc adl_p_desc = { - .graphics = &graphics_xelp, - .media = &media_xem, - PLATFORM(XE_ALDERLAKE_P), + .pre_gmdid_graphics_ip = &graphics_ip_xelp, + .pre_gmdid_media_ip = &media_ip_xem, + PLATFORM(ALDERLAKE_P), + .dma_mask_size = 39, .has_display = true, .has_llc = true, + .has_sriov = true, + .max_gt_per_tile = 1, .require_force_probe = true, .subplatforms = (const struct xe_subplatform_desc[]) { { XE_SUBPLATFORM_ALDERLAKE_P_RPLU, "RPLU", adlp_rplu_ids }, {}, }, + .va_bits = 48, + .vm_max_level = 3, }; static const struct xe_device_desc adl_n_desc = { - .graphics = &graphics_xelp, - .media = &media_xem, - PLATFORM(XE_ALDERLAKE_N), + .pre_gmdid_graphics_ip = &graphics_ip_xelp, + .pre_gmdid_media_ip = &media_ip_xem, + PLATFORM(ALDERLAKE_N), + .dma_mask_size = 39, .has_display = true, .has_llc = true, + .has_sriov = true, + .max_gt_per_tile = 1, .require_force_probe = true, + .va_bits = 48, + .vm_max_level = 3, }; #define DGFX_FEATURES \ .is_dgfx = 1 static const struct xe_device_desc dg1_desc = { - .graphics = &graphics_xelpp, - .media = &media_xem, + .pre_gmdid_graphics_ip = &graphics_ip_xelpp, + .pre_gmdid_media_ip = &media_ip_xem, DGFX_FEATURES, - PLATFORM(XE_DG1), + PLATFORM(DG1), + .dma_mask_size = 39, .has_display = true, + .has_gsc_nvm = 1, .has_heci_gscfi = 1, + .max_gt_per_tile = 1, .require_force_probe = true, + .va_bits = 48, + .vm_max_level = 3, }; -static const u16 dg2_g10_ids[] = { XE_DG2_G10_IDS(NOP), XE_ATS_M150_IDS(NOP), 0 }; -static const u16 dg2_g11_ids[] = { XE_DG2_G11_IDS(NOP), XE_ATS_M75_IDS(NOP), 0 }; -static const u16 dg2_g12_ids[] = { XE_DG2_G12_IDS(NOP), 0 }; +static const u16 dg2_g10_ids[] = { INTEL_DG2_G10_IDS(NOP), INTEL_ATS_M150_IDS(NOP), 0 }; +static const u16 dg2_g11_ids[] = { INTEL_DG2_G11_IDS(NOP), INTEL_ATS_M75_IDS(NOP), 0 }; +static const u16 dg2_g12_ids[] = { INTEL_DG2_G12_IDS(NOP), 0 }; #define DG2_FEATURES \ DGFX_FEATURES, \ - PLATFORM(XE_DG2), \ + PLATFORM(DG2), \ + .has_flat_ccs = 1, \ + .has_gsc_nvm = 1, \ .has_heci_gscfi = 1, \ .subplatforms = (const struct xe_subplatform_desc[]) { \ { XE_SUBPLATFORM_DG2_G10, "G10", dg2_g10_ids }, \ { XE_SUBPLATFORM_DG2_G11, "G11", dg2_g11_ids }, \ { XE_SUBPLATFORM_DG2_G12, "G12", dg2_g12_ids }, \ { } \ - } + }, \ + .va_bits = 48, \ + .vm_max_level = 3, \ + .vram_flags = XE_VRAM_FLAGS_NEED64K static const struct xe_device_desc ats_m_desc = { - .graphics = &graphics_xehpg, - .media = &media_xehpm, + .pre_gmdid_graphics_ip = &graphics_ip_xehpg, + .pre_gmdid_media_ip = &media_ip_xehpm, + .dma_mask_size = 46, + .max_gt_per_tile = 1, .require_force_probe = true, DG2_FEATURES, .has_display = false, + .has_sriov = true, }; static const struct xe_device_desc dg2_desc = { - .graphics = &graphics_xehpg, - .media = &media_xehpm, + .pre_gmdid_graphics_ip = &graphics_ip_xehpg, + .pre_gmdid_media_ip = &media_ip_xehpm, + .dma_mask_size = 46, + .max_gt_per_tile = 1, .require_force_probe = true, DG2_FEATURES, .has_display = true, + .has_fan_control = true, + .has_mbx_power_limits = false, }; static const __maybe_unused struct xe_device_desc pvc_desc = { - .graphics = &graphics_xehpc, + .pre_gmdid_graphics_ip = &graphics_ip_xehpc, DGFX_FEATURES, - PLATFORM(XE_PVC), + PLATFORM(PVC), + .dma_mask_size = 52, .has_display = false, + .has_gsc_nvm = 1, .has_heci_gscfi = 1, + .max_gt_per_tile = 1, + .max_remote_tiles = 1, .require_force_probe = true, + .va_bits = 57, + .vm_max_level = 4, + .vram_flags = XE_VRAM_FLAGS_NEED64K, + .has_mbx_power_limits = false, }; static const struct xe_device_desc mtl_desc = { /* .graphics and .media determined via GMD_ID */ .require_force_probe = true, - PLATFORM(XE_METEORLAKE), + PLATFORM(METEORLAKE), + .dma_mask_size = 46, .has_display = true, + .has_pxp = true, + .max_gt_per_tile = 2, + .va_bits = 48, + .vm_max_level = 3, }; static const struct xe_device_desc lnl_desc = { - PLATFORM(XE_LUNARLAKE), + PLATFORM(LUNARLAKE), + .dma_mask_size = 46, .has_display = true, - .require_force_probe = true, + .has_flat_ccs = 1, + .has_pxp = true, + .has_mem_copy_instr = true, + .max_gt_per_tile = 2, + .needs_scratch = true, + .va_bits = 48, + .vm_max_level = 4, }; -static const struct xe_device_desc bmg_desc __maybe_unused = { +static const u16 bmg_g21_ids[] = { INTEL_BMG_G21_IDS(NOP), 0 }; + +static const struct xe_device_desc bmg_desc = { DGFX_FEATURES, - PLATFORM(XE_BATTLEMAGE), - .require_force_probe = true, + PLATFORM(BATTLEMAGE), + .dma_mask_size = 46, + .has_display = true, + .has_fan_control = true, + .has_flat_ccs = 1, + .has_mbx_power_limits = true, + .has_gsc_nvm = 1, + .has_heci_cscfi = 1, + .has_late_bind = true, + .has_sriov = true, + .has_mem_copy_instr = true, + .max_gt_per_tile = 2, + .needs_scratch = true, + .subplatforms = (const struct xe_subplatform_desc[]) { + { XE_SUBPLATFORM_BATTLEMAGE_G21, "G21", bmg_g21_ids }, + { } + }, + .va_bits = 48, + .vm_max_level = 4, }; -#undef PLATFORM -__diag_pop(); +static const struct xe_device_desc ptl_desc = { + PLATFORM(PANTHERLAKE), + .dma_mask_size = 46, + .has_display = true, + .has_flat_ccs = 1, + .has_sriov = true, + .has_mem_copy_instr = true, + .max_gt_per_tile = 2, + .needs_scratch = true, + .needs_shared_vf_gt_wq = true, + .va_bits = 48, + .vm_max_level = 4, +}; -/* Map of GMD_ID values to graphics IP */ -static const struct gmdid_map graphics_ip_map[] = { - { 1270, &graphics_xelpg }, - { 1271, &graphics_xelpg }, - { 1274, &graphics_xelpg }, /* Xe_LPG+ */ - { 2001, &graphics_xe2 }, - { 2004, &graphics_xe2 }, +static const struct xe_device_desc nvls_desc = { + PLATFORM(NOVALAKE_S), + .dma_mask_size = 46, + .has_display = true, + .has_flat_ccs = 1, + .has_mem_copy_instr = true, + .max_gt_per_tile = 2, + .require_force_probe = true, + .va_bits = 48, + .vm_max_level = 4, }; -/* Map of GMD_ID values to media IP */ -static const struct gmdid_map media_ip_map[] = { - { 1300, &media_xelpmp }, - { 1301, &media_xe2 }, - { 2000, &media_xe2 }, +static const struct xe_device_desc cri_desc = { + DGFX_FEATURES, + PLATFORM(CRESCENTISLAND), + .dma_mask_size = 52, + .has_display = false, + .has_flat_ccs = false, + .has_mbx_power_limits = true, + .has_sriov = true, + .max_gt_per_tile = 2, + .require_force_probe = true, + .va_bits = 57, + .vm_max_level = 4, }; -#define INTEL_VGA_DEVICE(id, info) { \ - PCI_DEVICE(PCI_VENDOR_ID_INTEL, id), \ - PCI_BASE_CLASS_DISPLAY << 16, 0xff << 16, \ - (unsigned long) info } +#undef PLATFORM +__diag_pop(); /* * Make sure any device matches here are from most specific to most @@ -376,24 +424,29 @@ static const struct gmdid_map media_ip_map[] = { * PCI ID matches, otherwise we'll use the wrong info struct above. */ static const struct pci_device_id pciidlist[] = { - XE_TGL_IDS(INTEL_VGA_DEVICE, &tgl_desc), - XE_RKL_IDS(INTEL_VGA_DEVICE, &rkl_desc), - XE_ADLS_IDS(INTEL_VGA_DEVICE, &adl_s_desc), - XE_ADLP_IDS(INTEL_VGA_DEVICE, &adl_p_desc), - XE_ADLN_IDS(INTEL_VGA_DEVICE, &adl_n_desc), - XE_RPLP_IDS(INTEL_VGA_DEVICE, &adl_p_desc), - XE_RPLS_IDS(INTEL_VGA_DEVICE, &adl_s_desc), - XE_DG1_IDS(INTEL_VGA_DEVICE, &dg1_desc), - XE_ATS_M_IDS(INTEL_VGA_DEVICE, &ats_m_desc), - XE_DG2_IDS(INTEL_VGA_DEVICE, &dg2_desc), - XE_MTL_IDS(INTEL_VGA_DEVICE, &mtl_desc), - XE_LNL_IDS(INTEL_VGA_DEVICE, &lnl_desc), + INTEL_TGL_IDS(INTEL_VGA_DEVICE, &tgl_desc), + INTEL_RKL_IDS(INTEL_VGA_DEVICE, &rkl_desc), + INTEL_ADLS_IDS(INTEL_VGA_DEVICE, &adl_s_desc), + INTEL_ADLP_IDS(INTEL_VGA_DEVICE, &adl_p_desc), + INTEL_ADLN_IDS(INTEL_VGA_DEVICE, &adl_n_desc), + INTEL_RPLU_IDS(INTEL_VGA_DEVICE, &adl_p_desc), + INTEL_RPLP_IDS(INTEL_VGA_DEVICE, &adl_p_desc), + INTEL_RPLS_IDS(INTEL_VGA_DEVICE, &adl_s_desc), + INTEL_DG1_IDS(INTEL_VGA_DEVICE, &dg1_desc), + INTEL_ATS_M_IDS(INTEL_VGA_DEVICE, &ats_m_desc), + INTEL_ARL_IDS(INTEL_VGA_DEVICE, &mtl_desc), + INTEL_DG2_IDS(INTEL_VGA_DEVICE, &dg2_desc), + INTEL_MTL_IDS(INTEL_VGA_DEVICE, &mtl_desc), + INTEL_LNL_IDS(INTEL_VGA_DEVICE, &lnl_desc), + INTEL_BMG_IDS(INTEL_VGA_DEVICE, &bmg_desc), + INTEL_PTL_IDS(INTEL_VGA_DEVICE, &ptl_desc), + INTEL_WCL_IDS(INTEL_VGA_DEVICE, &ptl_desc), + INTEL_NVLS_IDS(INTEL_VGA_DEVICE, &nvls_desc), + INTEL_CRI_IDS(INTEL_PCI_DEVICE, &cri_desc), { } }; MODULE_DEVICE_TABLE(pci, pciidlist); -#undef INTEL_VGA_DEVICE - /* is device_id present in comma separated list of ids */ static bool device_id_in_list(u16 device_id, const char *devices, bool negative) { @@ -462,86 +515,131 @@ enum xe_gmdid_type { GMDID_MEDIA }; -static void read_gmdid(struct xe_device *xe, enum xe_gmdid_type type, u32 *ver, u32 *revid) +static int read_gmdid(struct xe_device *xe, enum xe_gmdid_type type, u32 *ver, u32 *revid) { - struct xe_gt *gt = xe_root_mmio_gt(xe); + struct xe_mmio *mmio = xe_root_tile_mmio(xe); struct xe_reg gmdid_reg = GMD_ID; u32 val; KUNIT_STATIC_STUB_REDIRECT(read_gmdid, xe, type, ver, revid); - if (type == GMDID_MEDIA) - gmdid_reg.addr += MEDIA_GT_GSI_OFFSET; + if (IS_SRIOV_VF(xe)) { + /* + * To get the value of the GMDID register, VFs must obtain it + * from the GuC using MMIO communication. + * + * Note that at this point the GTs are not initialized and only + * tile-level access to MMIO registers is possible. To use our + * existing GuC communication functions we must create a dummy + * GT structure and perform at least basic xe_gt and xe_guc + * initialization. + */ + struct xe_gt *gt __free(kfree) = NULL; + int err; + + gt = kzalloc(sizeof(*gt), GFP_KERNEL); + if (!gt) + return -ENOMEM; + + gt->tile = &xe->tiles[0]; + if (type == GMDID_MEDIA) { + gt->info.id = 1; + gt->info.type = XE_GT_TYPE_MEDIA; + } else { + gt->info.id = 0; + gt->info.type = XE_GT_TYPE_MAIN; + } + + xe_gt_mmio_init(gt); + xe_guc_comm_init_early(>->uc.guc); + + err = xe_gt_sriov_vf_bootstrap(gt); + if (err) + return err; + + val = xe_gt_sriov_vf_gmdid(gt); + } else { + /* + * GMD_ID is a GT register, but at this point in the driver + * init we haven't fully initialized the GT yet so we need to + * read the register with the tile's MMIO accessor. That means + * we need to apply the GSI offset manually since it won't get + * automatically added as it would if we were using a GT mmio + * accessor. + */ + if (type == GMDID_MEDIA) + gmdid_reg.addr += MEDIA_GT_GSI_OFFSET; + + val = xe_mmio_read32(mmio, gmdid_reg); + } - val = xe_mmio_read32(gt, gmdid_reg); *ver = REG_FIELD_GET(GMD_ID_ARCH_MASK, val) * 100 + REG_FIELD_GET(GMD_ID_RELEASE_MASK, val); *revid = REG_FIELD_GET(GMD_ID_REVID, val); + + return 0; } -/* - * Pre-GMD_ID platform: device descriptor already points to the appropriate - * graphics descriptor. Simply forward the description and calculate the version - * appropriately. "graphics" should be present in all such platforms, while - * media is optional. - */ -static void handle_pre_gmdid(struct xe_device *xe, - const struct xe_graphics_desc *graphics, - const struct xe_media_desc *media) +static const struct xe_ip *find_graphics_ip(unsigned int verx100) { - xe->info.graphics_verx100 = graphics->ver * 100 + graphics->rel; + KUNIT_STATIC_STUB_REDIRECT(find_graphics_ip, verx100); - if (media) - xe->info.media_verx100 = media->ver * 100 + media->rel; + for (int i = 0; i < ARRAY_SIZE(graphics_ips); i++) + if (graphics_ips[i].verx100 == verx100) + return &graphics_ips[i]; + return NULL; +} +static const struct xe_ip *find_media_ip(unsigned int verx100) +{ + KUNIT_STATIC_STUB_REDIRECT(find_media_ip, verx100); + + for (int i = 0; i < ARRAY_SIZE(media_ips); i++) + if (media_ips[i].verx100 == verx100) + return &media_ips[i]; + return NULL; } /* - * GMD_ID platform: read IP version from hardware and select graphics descriptor + * Read IP version from hardware and select graphics/media IP descriptors * based on the result. */ -static void handle_gmdid(struct xe_device *xe, - const struct xe_graphics_desc **graphics, - const struct xe_media_desc **media, - u32 *graphics_revid, - u32 *media_revid) +static int handle_gmdid(struct xe_device *xe, + const struct xe_ip **graphics_ip, + const struct xe_ip **media_ip, + u32 *graphics_revid, + u32 *media_revid) { u32 ver; + int ret; - read_gmdid(xe, GMDID_GRAPHICS, &ver, graphics_revid); + *graphics_ip = NULL; + *media_ip = NULL; - for (int i = 0; i < ARRAY_SIZE(graphics_ip_map); i++) { - if (ver == graphics_ip_map[i].ver) { - xe->info.graphics_verx100 = ver; - *graphics = graphics_ip_map[i].ip; + ret = read_gmdid(xe, GMDID_GRAPHICS, &ver, graphics_revid); + if (ret) + return ret; - break; - } - } - - if (!xe->info.graphics_verx100) { + *graphics_ip = find_graphics_ip(ver); + if (!*graphics_ip) { drm_err(&xe->drm, "Hardware reports unknown graphics version %u.%02u\n", ver / 100, ver % 100); } - read_gmdid(xe, GMDID_MEDIA, &ver, media_revid); + ret = read_gmdid(xe, GMDID_MEDIA, &ver, media_revid); + if (ret) + return ret; /* Media may legitimately be fused off / not present */ if (ver == 0) - return; - - for (int i = 0; i < ARRAY_SIZE(media_ip_map); i++) { - if (ver == media_ip_map[i].ver) { - xe->info.media_verx100 = ver; - *media = media_ip_map[i].ip; - - break; - } - } + return 0; - if (!xe->info.media_verx100) { + *media_ip = find_media_ip(ver); + if (!*media_ip) { drm_err(&xe->drm, "Hardware reports unknown media version %u.%02u\n", ver / 100, ver % 100); } + + return 0; } /* @@ -554,22 +652,44 @@ static int xe_info_init_early(struct xe_device *xe, { int err; + xe->info.platform_name = desc->platform_name; xe->info.platform = desc->platform; xe->info.subplatform = subplatform_desc ? subplatform_desc->subplatform : XE_SUBPLATFORM_NONE; + xe->info.dma_mask_size = desc->dma_mask_size; + xe->info.va_bits = desc->va_bits; + xe->info.vm_max_level = desc->vm_max_level; + xe->info.vram_flags = desc->vram_flags; + xe->info.is_dgfx = desc->is_dgfx; + xe->info.has_fan_control = desc->has_fan_control; + /* runtime fusing may force flat_ccs to disabled later */ + xe->info.has_flat_ccs = desc->has_flat_ccs; + xe->info.has_mbx_power_limits = desc->has_mbx_power_limits; + xe->info.has_gsc_nvm = desc->has_gsc_nvm; xe->info.has_heci_gscfi = desc->has_heci_gscfi; + xe->info.has_heci_cscfi = desc->has_heci_cscfi; + xe->info.has_late_bind = desc->has_late_bind; xe->info.has_llc = desc->has_llc; - xe->info.has_mmio_ext = desc->has_mmio_ext; - xe->info.has_sriov = desc->has_sriov; + xe->info.has_pxp = desc->has_pxp; + xe->info.has_sriov = xe_configfs_primary_gt_allowed(to_pci_dev(xe->drm.dev)) && + desc->has_sriov; + xe->info.has_mem_copy_instr = desc->has_mem_copy_instr; xe->info.skip_guc_pc = desc->skip_guc_pc; xe->info.skip_mtcfg = desc->skip_mtcfg; xe->info.skip_pcode = desc->skip_pcode; + xe->info.needs_scratch = desc->needs_scratch; + xe->info.needs_shared_vf_gt_wq = desc->needs_shared_vf_gt_wq; + + xe->info.probe_display = IS_ENABLED(CONFIG_DRM_XE_DISPLAY) && + xe_modparam.probe_display && + desc->has_display; - xe->info.enable_display = IS_ENABLED(CONFIG_DRM_XE_DISPLAY) && - xe_modparam.enable_display && - desc->has_display; + xe_assert(xe, desc->max_gt_per_tile > 0); + xe_assert(xe, desc->max_gt_per_tile <= XE_MAX_GT_PER_TILE); + xe->info.max_gt_per_tile = desc->max_gt_per_tile; + xe->info.tile_count = 1 + desc->max_remote_tiles; err = xe_tile_init_early(xe_device_get_root_tile(xe), xe, 0); if (err) @@ -579,33 +699,137 @@ static int xe_info_init_early(struct xe_device *xe, } /* + * Possibly override number of tile based on configuration register. + */ +static void xe_info_probe_tile_count(struct xe_device *xe) +{ + struct xe_mmio *mmio; + u8 tile_count; + u32 mtcfg; + + KUNIT_STATIC_STUB_REDIRECT(xe_info_probe_tile_count, xe); + + /* + * Probe for tile count only for platforms that support multiple + * tiles. + */ + if (xe->info.tile_count == 1) + return; + + if (xe->info.skip_mtcfg) + return; + + mmio = xe_root_tile_mmio(xe); + + /* + * Although the per-tile mmio regs are not yet initialized, this + * is fine as it's going to the root tile's mmio, that's + * guaranteed to be initialized earlier in xe_mmio_probe_early() + */ + mtcfg = xe_mmio_read32(mmio, XEHP_MTCFG_ADDR); + tile_count = REG_FIELD_GET(TILE_COUNT, mtcfg) + 1; + + if (tile_count < xe->info.tile_count) { + drm_info(&xe->drm, "tile_count: %d, reduced_tile_count %d\n", + xe->info.tile_count, tile_count); + xe->info.tile_count = tile_count; + } +} + +static struct xe_gt *alloc_primary_gt(struct xe_tile *tile, + const struct xe_graphics_desc *graphics_desc, + const struct xe_media_desc *media_desc) +{ + struct xe_device *xe = tile_to_xe(tile); + struct xe_gt *gt; + + if (!xe_configfs_primary_gt_allowed(to_pci_dev(xe->drm.dev))) { + xe_info(xe, "Primary GT disabled via configfs\n"); + return NULL; + } + + gt = xe_gt_alloc(tile); + if (IS_ERR(gt)) + return gt; + + gt->info.type = XE_GT_TYPE_MAIN; + gt->info.id = tile->id * xe->info.max_gt_per_tile; + gt->info.has_indirect_ring_state = graphics_desc->has_indirect_ring_state; + gt->info.engine_mask = graphics_desc->hw_engine_mask; + + /* + * Before media version 13, the media IP was part of the primary GT + * so we need to add the media engines to the primary GT's engine list. + */ + if (MEDIA_VER(xe) < 13 && media_desc) + gt->info.engine_mask |= media_desc->hw_engine_mask; + + return gt; +} + +static struct xe_gt *alloc_media_gt(struct xe_tile *tile, + const struct xe_media_desc *media_desc) +{ + struct xe_device *xe = tile_to_xe(tile); + struct xe_gt *gt; + + if (!xe_configfs_media_gt_allowed(to_pci_dev(xe->drm.dev))) { + xe_info(xe, "Media GT disabled via configfs\n"); + return NULL; + } + + if (MEDIA_VER(xe) < 13 || !media_desc) + return NULL; + + gt = xe_gt_alloc(tile); + if (IS_ERR(gt)) + return gt; + + gt->info.type = XE_GT_TYPE_MEDIA; + gt->info.id = tile->id * xe->info.max_gt_per_tile + 1; + gt->info.has_indirect_ring_state = media_desc->has_indirect_ring_state; + gt->info.engine_mask = media_desc->hw_engine_mask; + + return gt; +} + +/* * Initialize device info content that does require knowledge about * graphics / media IP version. * Make sure that GT / tile structures allocated by the driver match the data * present in device info. */ static int xe_info_init(struct xe_device *xe, - const struct xe_graphics_desc *graphics_desc, - const struct xe_media_desc *media_desc) + const struct xe_device_desc *desc) { u32 graphics_gmdid_revid = 0, media_gmdid_revid = 0; + const struct xe_ip *graphics_ip; + const struct xe_ip *media_ip; + const struct xe_graphics_desc *graphics_desc; + const struct xe_media_desc *media_desc; struct xe_tile *tile; struct xe_gt *gt; + int ret; u8 id; /* * If this platform supports GMD_ID, we'll detect the proper IP - * descriptor to use from hardware registers. desc->graphics will only - * ever be set at this point for platforms before GMD_ID. In that case - * the IP descriptions and versions are simply derived from that. + * descriptor to use from hardware registers. + * desc->pre_gmdid_graphics_ip will only ever be set at this point for + * platforms before GMD_ID. In that case the IP descriptions and + * versions are simply derived from that. */ - if (graphics_desc) { - handle_pre_gmdid(xe, graphics_desc, media_desc); + if (desc->pre_gmdid_graphics_ip) { + graphics_ip = desc->pre_gmdid_graphics_ip; + media_ip = desc->pre_gmdid_media_ip; xe->info.step = xe_step_pre_gmdid_get(xe); } else { - xe_assert(xe, !media_desc); - handle_gmdid(xe, &graphics_desc, &media_desc, - &graphics_gmdid_revid, &media_gmdid_revid); + xe_assert(xe, !desc->pre_gmdid_media_ip); + ret = handle_gmdid(xe, &graphics_ip, &media_ip, + &graphics_gmdid_revid, &media_gmdid_revid); + if (ret) + return ret; + xe->info.step = xe_step_gmdid_get(xe, graphics_gmdid_revid, media_gmdid_revid); @@ -616,32 +840,32 @@ static int xe_info_init(struct xe_device *xe, * error and we should abort driver load. Failing to detect media * IP is non-fatal; we'll just proceed without enabling media support. */ - if (!graphics_desc) + if (!graphics_ip) return -ENODEV; - xe->info.graphics_name = graphics_desc->name; - xe->info.media_name = media_desc ? media_desc->name : "none"; - xe->info.tile_mmio_ext_size = graphics_desc->tile_mmio_ext_size; + xe->info.graphics_verx100 = graphics_ip->verx100; + xe->info.graphics_name = graphics_ip->name; + graphics_desc = graphics_ip->desc; + + if (media_ip) { + xe->info.media_verx100 = media_ip->verx100; + xe->info.media_name = media_ip->name; + media_desc = media_ip->desc; + } else { + xe->info.media_name = "none"; + media_desc = NULL; + } - xe->info.dma_mask_size = graphics_desc->dma_mask_size; - xe->info.vram_flags = graphics_desc->vram_flags; - xe->info.va_bits = graphics_desc->va_bits; - xe->info.vm_max_level = graphics_desc->vm_max_level; xe->info.has_asid = graphics_desc->has_asid; - xe->info.has_flat_ccs = graphics_desc->has_flat_ccs; - xe->info.has_range_tlb_invalidation = graphics_desc->has_range_tlb_invalidation; + xe->info.has_atomic_enable_pte_bit = graphics_desc->has_atomic_enable_pte_bit; + if (xe->info.platform != XE_PVC) + xe->info.has_device_atomics_on_smem = 1; + + xe->info.has_range_tlb_inval = graphics_desc->has_range_tlb_inval; xe->info.has_usm = graphics_desc->has_usm; + xe->info.has_64bit_timestamp = graphics_desc->has_64bit_timestamp; - /* - * All platforms have at least one primary GT. Any platform with media - * version 13 or higher has an additional dedicated media GT. And - * depending on the graphics IP there may be additional "remote tiles." - * All of these together determine the overall GT count. - * - * FIXME: 'tile_count' here is misnamed since the rest of the driver - * treats it as the number of GTs rather than just the number of tiles. - */ - xe->info.tile_count = 1 + graphics_desc->max_remote_tiles; + xe_info_probe_tile_count(xe); for_each_remote_tile(tile, xe, id) { int err; @@ -651,57 +875,78 @@ static int xe_info_init(struct xe_device *xe, return err; } + /* Allocate any GT and VRAM structures necessary for the platform. */ for_each_tile(tile, xe, id) { - gt = tile->primary_gt; - gt->info.id = xe->info.gt_count++; - gt->info.type = XE_GT_TYPE_MAIN; - gt->info.__engine_mask = graphics_desc->hw_engine_mask; - if (MEDIA_VER(xe) < 13 && media_desc) - gt->info.__engine_mask |= media_desc->hw_engine_mask; - - if (MEDIA_VER(xe) < 13 || !media_desc) - continue; + int err; - /* - * Allocate and setup media GT for platforms with standalone - * media. - */ - tile->media_gt = xe_gt_alloc(tile); - if (IS_ERR(tile->media_gt)) - return PTR_ERR(tile->media_gt); + err = xe_tile_alloc_vram(tile); + if (err) + return err; - gt = tile->media_gt; - gt->info.type = XE_GT_TYPE_MEDIA; - gt->info.__engine_mask = media_desc->hw_engine_mask; - gt->mmio.adj_offset = MEDIA_GT_GSI_OFFSET; - gt->mmio.adj_limit = MEDIA_GT_GSI_LENGTH; + tile->primary_gt = alloc_primary_gt(tile, graphics_desc, media_desc); + if (IS_ERR(tile->primary_gt)) + return PTR_ERR(tile->primary_gt); /* - * FIXME: At the moment multi-tile and standalone media are - * mutually exclusive on current platforms. We'll need to - * come up with a better way to number GTs if we ever wind - * up with platforms that support both together. + * It's not currently possible to probe a device with the + * primary GT disabled. With some work, this may be future in + * the possible for igpu platforms (although probably not for + * dgpu's since access to the primary GT's BCS engines is + * required for VRAM management). */ - drm_WARN_ON(&xe->drm, id != 0); - gt->info.id = xe->info.gt_count++; + if (!tile->primary_gt) { + drm_err(&xe->drm, "Cannot probe device with without a primary GT\n"); + return -ENODEV; + } + + tile->media_gt = alloc_media_gt(tile, media_desc); + if (IS_ERR(tile->media_gt)) + return PTR_ERR(tile->media_gt); } + /* + * Now that we have tiles and GTs defined, let's loop over valid GTs + * in order to define gt_count. + */ + for_each_gt(gt, xe, id) + xe->info.gt_count++; + return 0; } static void xe_pci_remove(struct pci_dev *pdev) { - struct xe_device *xe; + struct xe_device *xe = pdev_to_xe_device(pdev); + + if (IS_SRIOV_PF(xe)) + xe_pci_sriov_configure(pdev, 0); - xe = pci_get_drvdata(pdev); - if (!xe) /* driver load aborted, nothing to cleanup */ + if (xe_survivability_mode_is_boot_enabled(xe)) return; xe_device_remove(xe); - xe_pm_runtime_fini(xe); - pci_set_drvdata(pdev, NULL); + xe_pm_fini(xe); } +/* + * Probe the PCI device, initialize various parts of the driver. + * + * Fault injection is used to test the error paths of some initialization + * functions called either directly from xe_pci_probe() or indirectly for + * example through xe_device_probe(). Those functions use the kernel fault + * injection capabilities infrastructure, see + * Documentation/fault-injection/fault-injection.rst for details. The macro + * ALLOW_ERROR_INJECTION() is used to conditionally skip function execution + * at runtime and use a provided return value. The first requirement for + * error injectable functions is proper handling of the error code by the + * caller for recovery, which is always the case here. The second + * requirement is that no state is changed before the first error return. + * It is not strictly fulfilled for all initialization functions using the + * ALLOW_ERROR_INJECTION() macro but this is acceptable because for those + * error cases at probe time, the error code is simply propagated up by the + * caller. Therefore there is no consequence on those specific callers when + * function error injection skips the whole function. + */ static int xe_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) { const struct xe_device_desc *desc = (const void *)ent->driver_data; @@ -709,6 +954,8 @@ static int xe_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) struct xe_device *xe; int err; + xe_configfs_check_device(pdev); + if (desc->require_force_probe && !id_forced(pdev->device)) { dev_info(&pdev->dev, "Your graphics device %04x is not officially supported\n" @@ -738,7 +985,7 @@ static int xe_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) if (IS_ERR(xe)) return PTR_ERR(xe); - pci_set_drvdata(pdev, xe); + pci_set_drvdata(pdev, &xe->drm); xe_pm_assert_unbounded_bridge(xe); subplatform_desc = find_subplatform(xe, desc); @@ -749,17 +996,30 @@ static int xe_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) if (err) return err; + xe_vram_resize_bar(xe); + err = xe_device_probe_early(xe); + /* + * In Boot Survivability mode, no drm card is exposed and driver + * is loaded with bare minimum to allow for firmware to be + * flashed through mei. Return success, if survivability mode + * is enabled due to pcode failure or configfs being set + */ + if (xe_survivability_mode_is_boot_enabled(xe)) + return 0; + if (err) return err; - err = xe_info_init(xe, desc->graphics, desc->media); + err = xe_info_init(xe, desc); if (err) return err; - xe_display_probe(xe); + err = xe_display_probe(xe); + if (err) + return err; - drm_dbg(&xe->drm, "%s %s %04x:%04x dgfx:%d gfx:%s (%d.%02d) media:%s (%d.%02d) display:%s dma_m_s:%d tc:%d gscfi:%d", + drm_dbg(&xe->drm, "%s %s %04x:%04x dgfx:%d gfx:%s (%d.%02d) media:%s (%d.%02d) display:%s dma_m_s:%d tc:%d gscfi:%d cscfi:%d", desc->platform_name, subplatform_desc ? subplatform_desc->name : "", xe->info.devid, xe->info.revid, @@ -770,14 +1030,13 @@ static int xe_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) xe->info.media_name, xe->info.media_verx100 / 100, xe->info.media_verx100 % 100, - str_yes_no(xe->info.enable_display), + str_yes_no(xe->info.probe_display), xe->info.dma_mask_size, xe->info.tile_count, - xe->info.has_heci_gscfi); + xe->info.has_heci_gscfi, xe->info.has_heci_cscfi); - drm_dbg(&xe->drm, "Stepping = (G:%s, M:%s, D:%s, B:%s)\n", + drm_dbg(&xe->drm, "Stepping = (G:%s, M:%s, B:%s)\n", xe_step_name(xe->info.step.graphics), xe_step_name(xe->info.step.media), - xe_step_name(xe->info.step.display), xe_step_name(xe->info.step.basedie)); drm_dbg(&xe->drm, "SR-IOV support: %s (mode: %s)\n", @@ -837,9 +1096,13 @@ static void d3cold_toggle(struct pci_dev *pdev, enum toggle_d3cold toggle) static int xe_pci_suspend(struct device *dev) { struct pci_dev *pdev = to_pci_dev(dev); + struct xe_device *xe = pdev_to_xe_device(pdev); int err; - err = xe_pm_suspend(pdev_to_xe_device(pdev)); + if (xe_survivability_mode_is_boot_enabled(xe)) + return -EBUSY; + + err = xe_pm_suspend(xe); if (err) return err; @@ -852,6 +1115,7 @@ static int xe_pci_suspend(struct device *dev) pci_save_state(pdev); pci_disable_device(pdev); + pci_set_power_state(pdev, PCI_D3cold); return 0; } @@ -868,6 +1132,8 @@ static int xe_pci_resume(struct device *dev) if (err) return err; + pci_restore_state(pdev); + err = pci_enable_device(pdev); if (err) return err; @@ -951,11 +1217,29 @@ static struct pci_driver xe_pci_driver = { .probe = xe_pci_probe, .remove = xe_pci_remove, .shutdown = xe_pci_shutdown, + .sriov_configure = xe_pci_sriov_configure, #ifdef CONFIG_PM_SLEEP .driver.pm = &xe_pm_ops, #endif }; +/** + * xe_pci_to_pf_device() - Get PF &xe_device. + * @pdev: the VF &pci_dev device + * + * Return: pointer to PF &xe_device, NULL otherwise. + */ +struct xe_device *xe_pci_to_pf_device(struct pci_dev *pdev) +{ + struct drm_device *drm; + + drm = pci_iov_get_pf_drvdata(pdev, &xe_pci_driver); + if (IS_ERR(drm)) + return NULL; + + return to_xe_device(drm); +} + int xe_register_pci_driver(void) { return pci_register_driver(&xe_pci_driver); |