diff options
Diffstat (limited to 'drivers/gpu/drm/i915/i915_drv.c')
| -rw-r--r-- | drivers/gpu/drm/i915/i915_drv.c | 626 |
1 files changed, 575 insertions, 51 deletions
diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c index 82f4d1f47d3b..651e65e051c0 100644 --- a/drivers/gpu/drm/i915/i915_drv.c +++ b/drivers/gpu/drm/i915/i915_drv.c @@ -36,6 +36,7 @@ #include <linux/console.h> #include <linux/module.h> +#include <linux/pm_runtime.h> #include <drm/drm_crtc_helper.h> static struct drm_driver driver; @@ -49,12 +50,30 @@ static struct drm_driver driver; .dpll_md_offsets = { DPLL_A_MD_OFFSET, DPLL_B_MD_OFFSET }, \ .palette_offsets = { PALETTE_A_OFFSET, PALETTE_B_OFFSET } +#define GEN_CHV_PIPEOFFSETS \ + .pipe_offsets = { PIPE_A_OFFSET, PIPE_B_OFFSET, \ + CHV_PIPE_C_OFFSET }, \ + .trans_offsets = { TRANSCODER_A_OFFSET, TRANSCODER_B_OFFSET, \ + CHV_TRANSCODER_C_OFFSET, }, \ + .dpll_offsets = { DPLL_A_OFFSET, DPLL_B_OFFSET, \ + CHV_DPLL_C_OFFSET }, \ + .dpll_md_offsets = { DPLL_A_MD_OFFSET, DPLL_B_MD_OFFSET, \ + CHV_DPLL_C_MD_OFFSET }, \ + .palette_offsets = { PALETTE_A_OFFSET, PALETTE_B_OFFSET, \ + CHV_PALETTE_C_OFFSET } + +#define CURSOR_OFFSETS \ + .cursor_offsets = { CURSOR_A_OFFSET, CURSOR_B_OFFSET, CHV_CURSOR_C_OFFSET } + +#define IVB_CURSOR_OFFSETS \ + .cursor_offsets = { CURSOR_A_OFFSET, IVB_CURSOR_B_OFFSET, IVB_CURSOR_C_OFFSET } static const struct intel_device_info intel_i830_info = { .gen = 2, .is_mobile = 1, .cursor_needs_physical = 1, .num_pipes = 2, .has_overlay = 1, .overlay_needs_physical = 1, .ring_mask = RENDER_RING, GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, }; static const struct intel_device_info intel_845g_info = { @@ -62,6 +81,7 @@ static const struct intel_device_info intel_845g_info = { .has_overlay = 1, .overlay_needs_physical = 1, .ring_mask = RENDER_RING, GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, }; static const struct intel_device_info intel_i85x_info = { @@ -71,6 +91,7 @@ static const struct intel_device_info intel_i85x_info = { .has_fbc = 1, .ring_mask = RENDER_RING, GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, }; static const struct intel_device_info intel_i865g_info = { @@ -78,6 +99,7 @@ static const struct intel_device_info intel_i865g_info = { .has_overlay = 1, .overlay_needs_physical = 1, .ring_mask = RENDER_RING, GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, }; static const struct intel_device_info intel_i915g_info = { @@ -85,6 +107,7 @@ static const struct intel_device_info intel_i915g_info = { .has_overlay = 1, .overlay_needs_physical = 1, .ring_mask = RENDER_RING, GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, }; static const struct intel_device_info intel_i915gm_info = { .gen = 3, .is_mobile = 1, .num_pipes = 2, @@ -94,12 +117,14 @@ static const struct intel_device_info intel_i915gm_info = { .has_fbc = 1, .ring_mask = RENDER_RING, GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, }; static const struct intel_device_info intel_i945g_info = { .gen = 3, .has_hotplug = 1, .cursor_needs_physical = 1, .num_pipes = 2, .has_overlay = 1, .overlay_needs_physical = 1, .ring_mask = RENDER_RING, GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, }; static const struct intel_device_info intel_i945gm_info = { .gen = 3, .is_i945gm = 1, .is_mobile = 1, .num_pipes = 2, @@ -109,6 +134,7 @@ static const struct intel_device_info intel_i945gm_info = { .has_fbc = 1, .ring_mask = RENDER_RING, GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, }; static const struct intel_device_info intel_i965g_info = { @@ -117,6 +143,7 @@ static const struct intel_device_info intel_i965g_info = { .has_overlay = 1, .ring_mask = RENDER_RING, GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, }; static const struct intel_device_info intel_i965gm_info = { @@ -126,6 +153,7 @@ static const struct intel_device_info intel_i965gm_info = { .supports_tv = 1, .ring_mask = RENDER_RING, GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, }; static const struct intel_device_info intel_g33_info = { @@ -134,6 +162,7 @@ static const struct intel_device_info intel_g33_info = { .has_overlay = 1, .ring_mask = RENDER_RING, GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, }; static const struct intel_device_info intel_g45_info = { @@ -141,6 +170,7 @@ static const struct intel_device_info intel_g45_info = { .has_pipe_cxsr = 1, .has_hotplug = 1, .ring_mask = RENDER_RING | BSD_RING, GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, }; static const struct intel_device_info intel_gm45_info = { @@ -150,6 +180,7 @@ static const struct intel_device_info intel_gm45_info = { .supports_tv = 1, .ring_mask = RENDER_RING | BSD_RING, GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, }; static const struct intel_device_info intel_pineview_info = { @@ -157,6 +188,7 @@ static const struct intel_device_info intel_pineview_info = { .need_gfx_hws = 1, .has_hotplug = 1, .has_overlay = 1, GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, }; static const struct intel_device_info intel_ironlake_d_info = { @@ -164,6 +196,7 @@ static const struct intel_device_info intel_ironlake_d_info = { .need_gfx_hws = 1, .has_hotplug = 1, .ring_mask = RENDER_RING | BSD_RING, GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, }; static const struct intel_device_info intel_ironlake_m_info = { @@ -172,6 +205,7 @@ static const struct intel_device_info intel_ironlake_m_info = { .has_fbc = 1, .ring_mask = RENDER_RING | BSD_RING, GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, }; static const struct intel_device_info intel_sandybridge_d_info = { @@ -181,6 +215,7 @@ static const struct intel_device_info intel_sandybridge_d_info = { .ring_mask = RENDER_RING | BSD_RING | BLT_RING, .has_llc = 1, GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, }; static const struct intel_device_info intel_sandybridge_m_info = { @@ -190,6 +225,7 @@ static const struct intel_device_info intel_sandybridge_m_info = { .ring_mask = RENDER_RING | BSD_RING | BLT_RING, .has_llc = 1, GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, }; #define GEN7_FEATURES \ @@ -203,6 +239,7 @@ static const struct intel_device_info intel_ivybridge_d_info = { GEN7_FEATURES, .is_ivybridge = 1, GEN_DEFAULT_PIPEOFFSETS, + IVB_CURSOR_OFFSETS, }; static const struct intel_device_info intel_ivybridge_m_info = { @@ -210,6 +247,7 @@ static const struct intel_device_info intel_ivybridge_m_info = { .is_ivybridge = 1, .is_mobile = 1, GEN_DEFAULT_PIPEOFFSETS, + IVB_CURSOR_OFFSETS, }; static const struct intel_device_info intel_ivybridge_q_info = { @@ -217,6 +255,7 @@ static const struct intel_device_info intel_ivybridge_q_info = { .is_ivybridge = 1, .num_pipes = 0, /* legal, last one wins */ GEN_DEFAULT_PIPEOFFSETS, + IVB_CURSOR_OFFSETS, }; static const struct intel_device_info intel_valleyview_m_info = { @@ -228,6 +267,7 @@ static const struct intel_device_info intel_valleyview_m_info = { .has_fbc = 0, /* legal, last one wins */ .has_llc = 0, /* legal, last one wins */ GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, }; static const struct intel_device_info intel_valleyview_d_info = { @@ -238,6 +278,7 @@ static const struct intel_device_info intel_valleyview_d_info = { .has_fbc = 0, /* legal, last one wins */ .has_llc = 0, /* legal, last one wins */ GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, }; static const struct intel_device_info intel_haswell_d_info = { @@ -247,6 +288,7 @@ static const struct intel_device_info intel_haswell_d_info = { .has_fpga_dbg = 1, .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING, GEN_DEFAULT_PIPEOFFSETS, + IVB_CURSOR_OFFSETS, }; static const struct intel_device_info intel_haswell_m_info = { @@ -257,6 +299,7 @@ static const struct intel_device_info intel_haswell_m_info = { .has_fpga_dbg = 1, .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING, GEN_DEFAULT_PIPEOFFSETS, + IVB_CURSOR_OFFSETS, }; static const struct intel_device_info intel_broadwell_d_info = { @@ -267,6 +310,7 @@ static const struct intel_device_info intel_broadwell_d_info = { .has_ddi = 1, .has_fbc = 1, GEN_DEFAULT_PIPEOFFSETS, + IVB_CURSOR_OFFSETS, }; static const struct intel_device_info intel_broadwell_m_info = { @@ -277,6 +321,40 @@ static const struct intel_device_info intel_broadwell_m_info = { .has_ddi = 1, .has_fbc = 1, GEN_DEFAULT_PIPEOFFSETS, + IVB_CURSOR_OFFSETS, +}; + +static const struct intel_device_info intel_broadwell_gt3d_info = { + .gen = 8, .num_pipes = 3, + .need_gfx_hws = 1, .has_hotplug = 1, + .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING | BSD2_RING, + .has_llc = 1, + .has_ddi = 1, + .has_fbc = 1, + GEN_DEFAULT_PIPEOFFSETS, + IVB_CURSOR_OFFSETS, +}; + +static const struct intel_device_info intel_broadwell_gt3m_info = { + .gen = 8, .is_mobile = 1, .num_pipes = 3, + .need_gfx_hws = 1, .has_hotplug = 1, + .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING | BSD2_RING, + .has_llc = 1, + .has_ddi = 1, + .has_fbc = 1, + GEN_DEFAULT_PIPEOFFSETS, + IVB_CURSOR_OFFSETS, +}; + +static const struct intel_device_info intel_cherryview_info = { + .is_preliminary = 1, + .gen = 8, .num_pipes = 3, + .need_gfx_hws = 1, .has_hotplug = 1, + .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING, + .is_valleyview = 1, + .display_mmio_offset = VLV_DISPLAY_BASE, + GEN_CHV_PIPEOFFSETS, + CURSOR_OFFSETS, }; /* @@ -311,8 +389,11 @@ static const struct intel_device_info intel_broadwell_m_info = { INTEL_HSW_M_IDS(&intel_haswell_m_info), \ INTEL_VLV_M_IDS(&intel_valleyview_m_info), \ INTEL_VLV_D_IDS(&intel_valleyview_d_info), \ - INTEL_BDW_M_IDS(&intel_broadwell_m_info), \ - INTEL_BDW_D_IDS(&intel_broadwell_d_info) + INTEL_BDW_GT12M_IDS(&intel_broadwell_m_info), \ + INTEL_BDW_GT12D_IDS(&intel_broadwell_d_info), \ + INTEL_BDW_GT3M_IDS(&intel_broadwell_gt3m_info), \ + INTEL_BDW_GT3D_IDS(&intel_broadwell_gt3d_info), \ + INTEL_CHV_IDS(&intel_cherryview_info) static const struct pci_device_id pciidlist[] = { /* aka */ INTEL_PCI_IDS, @@ -445,18 +526,20 @@ static int i915_drm_freeze(struct drm_device *dev) return error; } - cancel_delayed_work_sync(&dev_priv->rps.delayed_resume_work); - drm_irq_uninstall(dev); dev_priv->enable_hotplug_processing = false; + + intel_disable_gt_powersave(dev); + /* * Disable CRTCs directly since we want to preserve sw state * for _thaw. */ - mutex_lock(&dev->mode_config.mutex); - list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) + drm_modeset_lock_all(dev); + for_each_crtc(dev, crtc) { dev_priv->display.crtc_disable(crtc); - mutex_unlock(&dev->mode_config.mutex); + } + drm_modeset_unlock_all(dev); intel_modeset_suspend_hw(dev); } @@ -519,24 +602,6 @@ void intel_console_resume(struct work_struct *work) console_unlock(); } -static void intel_resume_hotplug(struct drm_device *dev) -{ - struct drm_mode_config *mode_config = &dev->mode_config; - struct intel_encoder *encoder; - - mutex_lock(&mode_config->mutex); - DRM_DEBUG_KMS("running encoder hotplug functions\n"); - - list_for_each_entry(encoder, &mode_config->encoder_list, base.head) - if (encoder->hot_plug) - encoder->hot_plug(encoder); - - mutex_unlock(&mode_config->mutex); - - /* Just fire off a uevent and let userspace tell us what to do */ - drm_helper_hpd_irq_event(dev); -} - static int i915_drm_thaw_early(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; @@ -551,7 +616,6 @@ static int i915_drm_thaw_early(struct drm_device *dev) static int __i915_drm_thaw(struct drm_device *dev, bool restore_gtt_mappings) { struct drm_i915_private *dev_priv = dev->dev_private; - int error = 0; if (drm_core_check_feature(dev, DRIVER_MODESET) && restore_gtt_mappings) { @@ -569,12 +633,14 @@ static int __i915_drm_thaw(struct drm_device *dev, bool restore_gtt_mappings) drm_mode_config_reset(dev); mutex_lock(&dev->struct_mutex); - - error = i915_gem_init_hw(dev); + if (i915_gem_init_hw(dev)) { + DRM_ERROR("failed to re-initialize GPU, declaring wedged!\n"); + atomic_set_mask(I915_WEDGED, &dev_priv->gpu_error.reset_counter); + } mutex_unlock(&dev->struct_mutex); /* We need working interrupts for modeset enabling ... */ - drm_irq_install(dev); + drm_irq_install(dev, dev->pdev->irq); intel_modeset_init_hw(dev); @@ -591,7 +657,7 @@ static int __i915_drm_thaw(struct drm_device *dev, bool restore_gtt_mappings) intel_hpd_init(dev); dev_priv->enable_hotplug_processing = true; /* Config may have changed between suspend and resume */ - intel_resume_hotplug(dev); + drm_helper_hpd_irq_event(dev); } intel_opregion_init(dev); @@ -613,7 +679,7 @@ static int __i915_drm_thaw(struct drm_device *dev, bool restore_gtt_mappings) mutex_unlock(&dev_priv->modeset_restore_lock); intel_runtime_pm_put(dev_priv); - return error; + return 0; } static int i915_drm_thaw(struct drm_device *dev) @@ -746,18 +812,20 @@ int i915_reset(struct drm_device *dev) return ret; } - drm_irq_uninstall(dev); - drm_irq_install(dev); + /* + * FIXME: This races pretty badly against concurrent holders of + * ring interrupts. This is possible since we've started to drop + * dev->struct_mutex in select places when waiting for the gpu. + */ - /* rps/rc6 re-init is necessary to restore state lost after the - * reset and the re-install of drm irq. Skip for ironlake per + /* + * rps/rc6 re-init is necessary to restore state lost after the + * reset and the re-install of gt irqs. Skip for ironlake per * previous concerns that it doesn't respond well to some forms - * of re-init after reset. */ - if (INTEL_INFO(dev)->gen > 5) { - mutex_lock(&dev->struct_mutex); - intel_enable_gt_powersave(dev); - mutex_unlock(&dev->struct_mutex); - } + * of re-init after reset. + */ + if (INTEL_INFO(dev)->gen > 5) + intel_reset_gt_powersave(dev); intel_hpd_init(dev); } else { @@ -891,21 +959,453 @@ static int i915_pm_poweroff(struct device *dev) return i915_drm_freeze(drm_dev); } -static int i915_runtime_suspend(struct device *device) +static int hsw_runtime_suspend(struct drm_i915_private *dev_priv) +{ + hsw_enable_pc8(dev_priv); + + return 0; +} + +static int snb_runtime_resume(struct drm_i915_private *dev_priv) +{ + struct drm_device *dev = dev_priv->dev; + + intel_init_pch_refclk(dev); + + return 0; +} + +static int hsw_runtime_resume(struct drm_i915_private *dev_priv) +{ + hsw_disable_pc8(dev_priv); + + return 0; +} + +/* + * Save all Gunit registers that may be lost after a D3 and a subsequent + * S0i[R123] transition. The list of registers needing a save/restore is + * defined in the VLV2_S0IXRegs document. This documents marks all Gunit + * registers in the following way: + * - Driver: saved/restored by the driver + * - Punit : saved/restored by the Punit firmware + * - No, w/o marking: no need to save/restore, since the register is R/O or + * used internally by the HW in a way that doesn't depend + * keeping the content across a suspend/resume. + * - Debug : used for debugging + * + * We save/restore all registers marked with 'Driver', with the following + * exceptions: + * - Registers out of use, including also registers marked with 'Debug'. + * These have no effect on the driver's operation, so we don't save/restore + * them to reduce the overhead. + * - Registers that are fully setup by an initialization function called from + * the resume path. For example many clock gating and RPS/RC6 registers. + * - Registers that provide the right functionality with their reset defaults. + * + * TODO: Except for registers that based on the above 3 criteria can be safely + * ignored, we save/restore all others, practically treating the HW context as + * a black-box for the driver. Further investigation is needed to reduce the + * saved/restored registers even further, by following the same 3 criteria. + */ +static void vlv_save_gunit_s0ix_state(struct drm_i915_private *dev_priv) +{ + struct vlv_s0ix_state *s = &dev_priv->vlv_s0ix_state; + int i; + + /* GAM 0x4000-0x4770 */ + s->wr_watermark = I915_READ(GEN7_WR_WATERMARK); + s->gfx_prio_ctrl = I915_READ(GEN7_GFX_PRIO_CTRL); + s->arb_mode = I915_READ(ARB_MODE); + s->gfx_pend_tlb0 = I915_READ(GEN7_GFX_PEND_TLB0); + s->gfx_pend_tlb1 = I915_READ(GEN7_GFX_PEND_TLB1); + + for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++) + s->lra_limits[i] = I915_READ(GEN7_LRA_LIMITS_BASE + i * 4); + + s->media_max_req_count = I915_READ(GEN7_MEDIA_MAX_REQ_COUNT); + s->gfx_max_req_count = I915_READ(GEN7_MEDIA_MAX_REQ_COUNT); + + s->render_hwsp = I915_READ(RENDER_HWS_PGA_GEN7); + s->ecochk = I915_READ(GAM_ECOCHK); + s->bsd_hwsp = I915_READ(BSD_HWS_PGA_GEN7); + s->blt_hwsp = I915_READ(BLT_HWS_PGA_GEN7); + + s->tlb_rd_addr = I915_READ(GEN7_TLB_RD_ADDR); + + /* MBC 0x9024-0x91D0, 0x8500 */ + s->g3dctl = I915_READ(VLV_G3DCTL); + s->gsckgctl = I915_READ(VLV_GSCKGCTL); + s->mbctl = I915_READ(GEN6_MBCTL); + + /* GCP 0x9400-0x9424, 0x8100-0x810C */ + s->ucgctl1 = I915_READ(GEN6_UCGCTL1); + s->ucgctl3 = I915_READ(GEN6_UCGCTL3); + s->rcgctl1 = I915_READ(GEN6_RCGCTL1); + s->rcgctl2 = I915_READ(GEN6_RCGCTL2); + s->rstctl = I915_READ(GEN6_RSTCTL); + s->misccpctl = I915_READ(GEN7_MISCCPCTL); + + /* GPM 0xA000-0xAA84, 0x8000-0x80FC */ + s->gfxpause = I915_READ(GEN6_GFXPAUSE); + s->rpdeuhwtc = I915_READ(GEN6_RPDEUHWTC); + s->rpdeuc = I915_READ(GEN6_RPDEUC); + s->ecobus = I915_READ(ECOBUS); + s->pwrdwnupctl = I915_READ(VLV_PWRDWNUPCTL); + s->rp_down_timeout = I915_READ(GEN6_RP_DOWN_TIMEOUT); + s->rp_deucsw = I915_READ(GEN6_RPDEUCSW); + s->rcubmabdtmr = I915_READ(GEN6_RCUBMABDTMR); + s->rcedata = I915_READ(VLV_RCEDATA); + s->spare2gh = I915_READ(VLV_SPAREG2H); + + /* Display CZ domain, 0x4400C-0x4402C, 0x4F000-0x4F11F */ + s->gt_imr = I915_READ(GTIMR); + s->gt_ier = I915_READ(GTIER); + s->pm_imr = I915_READ(GEN6_PMIMR); + s->pm_ier = I915_READ(GEN6_PMIER); + + for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++) + s->gt_scratch[i] = I915_READ(GEN7_GT_SCRATCH_BASE + i * 4); + + /* GT SA CZ domain, 0x100000-0x138124 */ + s->tilectl = I915_READ(TILECTL); + s->gt_fifoctl = I915_READ(GTFIFOCTL); + s->gtlc_wake_ctrl = I915_READ(VLV_GTLC_WAKE_CTRL); + s->gtlc_survive = I915_READ(VLV_GTLC_SURVIVABILITY_REG); + s->pmwgicz = I915_READ(VLV_PMWGICZ); + + /* Gunit-Display CZ domain, 0x182028-0x1821CF */ + s->gu_ctl0 = I915_READ(VLV_GU_CTL0); + s->gu_ctl1 = I915_READ(VLV_GU_CTL1); + s->clock_gate_dis2 = I915_READ(VLV_GUNIT_CLOCK_GATE2); + + /* + * Not saving any of: + * DFT, 0x9800-0x9EC0 + * SARB, 0xB000-0xB1FC + * GAC, 0x5208-0x524C, 0x14000-0x14C000 + * PCI CFG + */ +} + +static void vlv_restore_gunit_s0ix_state(struct drm_i915_private *dev_priv) +{ + struct vlv_s0ix_state *s = &dev_priv->vlv_s0ix_state; + u32 val; + int i; + + /* GAM 0x4000-0x4770 */ + I915_WRITE(GEN7_WR_WATERMARK, s->wr_watermark); + I915_WRITE(GEN7_GFX_PRIO_CTRL, s->gfx_prio_ctrl); + I915_WRITE(ARB_MODE, s->arb_mode | (0xffff << 16)); + I915_WRITE(GEN7_GFX_PEND_TLB0, s->gfx_pend_tlb0); + I915_WRITE(GEN7_GFX_PEND_TLB1, s->gfx_pend_tlb1); + + for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++) + I915_WRITE(GEN7_LRA_LIMITS_BASE + i * 4, s->lra_limits[i]); + + I915_WRITE(GEN7_MEDIA_MAX_REQ_COUNT, s->media_max_req_count); + I915_WRITE(GEN7_MEDIA_MAX_REQ_COUNT, s->gfx_max_req_count); + + I915_WRITE(RENDER_HWS_PGA_GEN7, s->render_hwsp); + I915_WRITE(GAM_ECOCHK, s->ecochk); + I915_WRITE(BSD_HWS_PGA_GEN7, s->bsd_hwsp); + I915_WRITE(BLT_HWS_PGA_GEN7, s->blt_hwsp); + + I915_WRITE(GEN7_TLB_RD_ADDR, s->tlb_rd_addr); + + /* MBC 0x9024-0x91D0, 0x8500 */ + I915_WRITE(VLV_G3DCTL, s->g3dctl); + I915_WRITE(VLV_GSCKGCTL, s->gsckgctl); + I915_WRITE(GEN6_MBCTL, s->mbctl); + + /* GCP 0x9400-0x9424, 0x8100-0x810C */ + I915_WRITE(GEN6_UCGCTL1, s->ucgctl1); + I915_WRITE(GEN6_UCGCTL3, s->ucgctl3); + I915_WRITE(GEN6_RCGCTL1, s->rcgctl1); + I915_WRITE(GEN6_RCGCTL2, s->rcgctl2); + I915_WRITE(GEN6_RSTCTL, s->rstctl); + I915_WRITE(GEN7_MISCCPCTL, s->misccpctl); + + /* GPM 0xA000-0xAA84, 0x8000-0x80FC */ + I915_WRITE(GEN6_GFXPAUSE, s->gfxpause); + I915_WRITE(GEN6_RPDEUHWTC, s->rpdeuhwtc); + I915_WRITE(GEN6_RPDEUC, s->rpdeuc); + I915_WRITE(ECOBUS, s->ecobus); + I915_WRITE(VLV_PWRDWNUPCTL, s->pwrdwnupctl); + I915_WRITE(GEN6_RP_DOWN_TIMEOUT,s->rp_down_timeout); + I915_WRITE(GEN6_RPDEUCSW, s->rp_deucsw); + I915_WRITE(GEN6_RCUBMABDTMR, s->rcubmabdtmr); + I915_WRITE(VLV_RCEDATA, s->rcedata); + I915_WRITE(VLV_SPAREG2H, s->spare2gh); + + /* Display CZ domain, 0x4400C-0x4402C, 0x4F000-0x4F11F */ + I915_WRITE(GTIMR, s->gt_imr); + I915_WRITE(GTIER, s->gt_ier); + I915_WRITE(GEN6_PMIMR, s->pm_imr); + I915_WRITE(GEN6_PMIER, s->pm_ier); + + for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++) + I915_WRITE(GEN7_GT_SCRATCH_BASE + i * 4, s->gt_scratch[i]); + + /* GT SA CZ domain, 0x100000-0x138124 */ + I915_WRITE(TILECTL, s->tilectl); + I915_WRITE(GTFIFOCTL, s->gt_fifoctl); + /* + * Preserve the GT allow wake and GFX force clock bit, they are not + * be restored, as they are used to control the s0ix suspend/resume + * sequence by the caller. + */ + val = I915_READ(VLV_GTLC_WAKE_CTRL); + val &= VLV_GTLC_ALLOWWAKEREQ; + val |= s->gtlc_wake_ctrl & ~VLV_GTLC_ALLOWWAKEREQ; + I915_WRITE(VLV_GTLC_WAKE_CTRL, val); + + val = I915_READ(VLV_GTLC_SURVIVABILITY_REG); + val &= VLV_GFX_CLK_FORCE_ON_BIT; + val |= s->gtlc_survive & ~VLV_GFX_CLK_FORCE_ON_BIT; + I915_WRITE(VLV_GTLC_SURVIVABILITY_REG, val); + + I915_WRITE(VLV_PMWGICZ, s->pmwgicz); + + /* Gunit-Display CZ domain, 0x182028-0x1821CF */ + I915_WRITE(VLV_GU_CTL0, s->gu_ctl0); + I915_WRITE(VLV_GU_CTL1, s->gu_ctl1); + I915_WRITE(VLV_GUNIT_CLOCK_GATE2, s->clock_gate_dis2); +} + +int vlv_force_gfx_clock(struct drm_i915_private *dev_priv, bool force_on) +{ + u32 val; + int err; + + val = I915_READ(VLV_GTLC_SURVIVABILITY_REG); + WARN_ON(!!(val & VLV_GFX_CLK_FORCE_ON_BIT) == force_on); + +#define COND (I915_READ(VLV_GTLC_SURVIVABILITY_REG) & VLV_GFX_CLK_STATUS_BIT) + /* Wait for a previous force-off to settle */ + if (force_on) { + err = wait_for(!COND, 20); + if (err) { + DRM_ERROR("timeout waiting for GFX clock force-off (%08x)\n", + I915_READ(VLV_GTLC_SURVIVABILITY_REG)); + return err; + } + } + + val = I915_READ(VLV_GTLC_SURVIVABILITY_REG); + val &= ~VLV_GFX_CLK_FORCE_ON_BIT; + if (force_on) + val |= VLV_GFX_CLK_FORCE_ON_BIT; + I915_WRITE(VLV_GTLC_SURVIVABILITY_REG, val); + + if (!force_on) + return 0; + + err = wait_for(COND, 20); + if (err) + DRM_ERROR("timeout waiting for GFX clock force-on (%08x)\n", + I915_READ(VLV_GTLC_SURVIVABILITY_REG)); + + return err; +#undef COND +} + +static int vlv_allow_gt_wake(struct drm_i915_private *dev_priv, bool allow) +{ + u32 val; + int err = 0; + + val = I915_READ(VLV_GTLC_WAKE_CTRL); + val &= ~VLV_GTLC_ALLOWWAKEREQ; + if (allow) + val |= VLV_GTLC_ALLOWWAKEREQ; + I915_WRITE(VLV_GTLC_WAKE_CTRL, val); + POSTING_READ(VLV_GTLC_WAKE_CTRL); + +#define COND (!!(I915_READ(VLV_GTLC_PW_STATUS) & VLV_GTLC_ALLOWWAKEACK) == \ + allow) + err = wait_for(COND, 1); + if (err) + DRM_ERROR("timeout disabling GT waking\n"); + return err; +#undef COND +} + +static int vlv_wait_for_gt_wells(struct drm_i915_private *dev_priv, + bool wait_for_on) +{ + u32 mask; + u32 val; + int err; + + mask = VLV_GTLC_PW_MEDIA_STATUS_MASK | VLV_GTLC_PW_RENDER_STATUS_MASK; + val = wait_for_on ? mask : 0; +#define COND ((I915_READ(VLV_GTLC_PW_STATUS) & mask) == val) + if (COND) + return 0; + + DRM_DEBUG_KMS("waiting for GT wells to go %s (%08x)\n", + wait_for_on ? "on" : "off", + I915_READ(VLV_GTLC_PW_STATUS)); + + /* + * RC6 transitioning can be delayed up to 2 msec (see + * valleyview_enable_rps), use 3 msec for safety. + */ + err = wait_for(COND, 3); + if (err) + DRM_ERROR("timeout waiting for GT wells to go %s\n", + wait_for_on ? "on" : "off"); + + return err; +#undef COND +} + +static void vlv_check_no_gt_access(struct drm_i915_private *dev_priv) +{ + if (!(I915_READ(VLV_GTLC_PW_STATUS) & VLV_GTLC_ALLOWWAKEERR)) + return; + + DRM_ERROR("GT register access while GT waking disabled\n"); + I915_WRITE(VLV_GTLC_PW_STATUS, VLV_GTLC_ALLOWWAKEERR); +} + +static int vlv_runtime_suspend(struct drm_i915_private *dev_priv) +{ + u32 mask; + int err; + + /* + * Bspec defines the following GT well on flags as debug only, so + * don't treat them as hard failures. + */ + (void)vlv_wait_for_gt_wells(dev_priv, false); + + mask = VLV_GTLC_RENDER_CTX_EXISTS | VLV_GTLC_MEDIA_CTX_EXISTS; + WARN_ON((I915_READ(VLV_GTLC_WAKE_CTRL) & mask) != mask); + + vlv_check_no_gt_access(dev_priv); + + err = vlv_force_gfx_clock(dev_priv, true); + if (err) + goto err1; + + err = vlv_allow_gt_wake(dev_priv, false); + if (err) + goto err2; + vlv_save_gunit_s0ix_state(dev_priv); + + err = vlv_force_gfx_clock(dev_priv, false); + if (err) + goto err2; + + return 0; + +err2: + /* For safety always re-enable waking and disable gfx clock forcing */ + vlv_allow_gt_wake(dev_priv, true); +err1: + vlv_force_gfx_clock(dev_priv, false); + + return err; +} + +static int vlv_runtime_resume(struct drm_i915_private *dev_priv) +{ + struct drm_device *dev = dev_priv->dev; + int err; + int ret; + + /* + * If any of the steps fail just try to continue, that's the best we + * can do at this point. Return the first error code (which will also + * leave RPM permanently disabled). + */ + ret = vlv_force_gfx_clock(dev_priv, true); + + vlv_restore_gunit_s0ix_state(dev_priv); + + err = vlv_allow_gt_wake(dev_priv, true); + if (!ret) + ret = err; + + err = vlv_force_gfx_clock(dev_priv, false); + if (!ret) + ret = err; + + vlv_check_no_gt_access(dev_priv); + + intel_init_clock_gating(dev); + i915_gem_restore_fences(dev); + + return ret; +} + +static int intel_runtime_suspend(struct device *device) { struct pci_dev *pdev = to_pci_dev(device); struct drm_device *dev = pci_get_drvdata(pdev); struct drm_i915_private *dev_priv = dev->dev_private; + int ret; + + if (WARN_ON_ONCE(!(dev_priv->rps.enabled && intel_enable_rc6(dev)))) + return -ENODEV; WARN_ON(!HAS_RUNTIME_PM(dev)); assert_force_wake_inactive(dev_priv); DRM_DEBUG_KMS("Suspending device\n"); - if (HAS_PC8(dev)) - hsw_enable_pc8(dev_priv); + /* + * We could deadlock here in case another thread holding struct_mutex + * calls RPM suspend concurrently, since the RPM suspend will wait + * first for this RPM suspend to finish. In this case the concurrent + * RPM resume will be followed by its RPM suspend counterpart. Still + * for consistency return -EAGAIN, which will reschedule this suspend. + */ + if (!mutex_trylock(&dev->struct_mutex)) { + DRM_DEBUG_KMS("device lock contention, deffering suspend\n"); + /* + * Bump the expiration timestamp, otherwise the suspend won't + * be rescheduled. + */ + pm_runtime_mark_last_busy(device); + return -EAGAIN; + } + /* + * We are safe here against re-faults, since the fault handler takes + * an RPM reference. + */ i915_gem_release_all_mmaps(dev_priv); + mutex_unlock(&dev->struct_mutex); + + /* + * rps.work can't be rearmed here, since we get here only after making + * sure the GPU is idle and the RPS freq is set to the minimum. See + * intel_mark_idle(). + */ + cancel_work_sync(&dev_priv->rps.work); + intel_runtime_pm_disable_interrupts(dev); + + if (IS_GEN6(dev)) { + ret = 0; + } else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) { + ret = hsw_runtime_suspend(dev_priv); + } else if (IS_VALLEYVIEW(dev)) { + ret = vlv_runtime_suspend(dev_priv); + } else { + ret = -ENODEV; + WARN_ON(1); + } + + if (ret) { + DRM_ERROR("Runtime suspend failed, disabling it (%d)\n", ret); + intel_runtime_pm_restore_interrupts(dev); + + return ret; + } del_timer_sync(&dev_priv->gpu_error.hangcheck_timer); dev_priv->pm.suspended = true; @@ -923,11 +1423,12 @@ static int i915_runtime_suspend(struct device *device) return 0; } -static int i915_runtime_resume(struct device *device) +static int intel_runtime_resume(struct device *device) { struct pci_dev *pdev = to_pci_dev(device); struct drm_device *dev = pci_get_drvdata(pdev); struct drm_i915_private *dev_priv = dev->dev_private; + int ret; WARN_ON(!HAS_RUNTIME_PM(dev)); @@ -936,11 +1437,33 @@ static int i915_runtime_resume(struct device *device) intel_opregion_notify_adapter(dev, PCI_D0); dev_priv->pm.suspended = false; - if (HAS_PC8(dev)) - hsw_disable_pc8(dev_priv); + if (IS_GEN6(dev)) { + ret = snb_runtime_resume(dev_priv); + } else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) { + ret = hsw_runtime_resume(dev_priv); + } else if (IS_VALLEYVIEW(dev)) { + ret = vlv_runtime_resume(dev_priv); + } else { + WARN_ON(1); + ret = -ENODEV; + } - DRM_DEBUG_KMS("Device resumed\n"); - return 0; + /* + * No point of rolling back things in case of an error, as the best + * we can do is to hope that things will still work (and disable RPM). + */ + i915_gem_init_swizzling(dev); + gen6_update_ring_freq(dev); + + intel_runtime_pm_restore_interrupts(dev); + intel_reset_gt_powersave(dev); + + if (ret) + DRM_ERROR("Runtime resume failed, disabling it (%d)\n", ret); + else + DRM_DEBUG_KMS("Device resumed\n"); + + return ret; } static const struct dev_pm_ops i915_pm_ops = { @@ -954,8 +1477,8 @@ static const struct dev_pm_ops i915_pm_ops = { .poweroff = i915_pm_poweroff, .restore_early = i915_pm_resume_early, .restore = i915_pm_resume, - .runtime_suspend = i915_runtime_suspend, - .runtime_resume = i915_runtime_resume, + .runtime_suspend = intel_runtime_suspend, + .runtime_resume = intel_runtime_resume, }; static const struct vm_operations_struct i915_gem_vm_ops = { @@ -1062,6 +1585,7 @@ static int __init i915_init(void) driver.get_vblank_timestamp = NULL; #ifndef CONFIG_DRM_I915_UMS /* Silently fail loading to not upset userspace. */ + DRM_DEBUG_DRIVER("KMS and UMS disabled.\n"); return 0; #endif } |