diff options
Diffstat (limited to 'drivers/gpu/drm/i915/display/intel_bw.c')
| -rw-r--r-- | drivers/gpu/drm/i915/display/intel_bw.c | 1408 |
1 files changed, 1121 insertions, 287 deletions
diff --git a/drivers/gpu/drm/i915/display/intel_bw.c b/drivers/gpu/drm/i915/display/intel_bw.c index bfb398f0432e..1f6461be50ef 100644 --- a/drivers/gpu/drm/i915/display/intel_bw.c +++ b/drivers/gpu/drm/i915/display/intel_bw.c @@ -4,40 +4,127 @@ */ #include <drm/drm_atomic_state_helper.h> +#include <drm/drm_print.h> -#include "intel_atomic.h" +#include "soc/intel_dram.h" + +#include "i915_drv.h" +#include "i915_reg.h" #include "intel_bw.h" -#include "intel_cdclk.h" +#include "intel_crtc.h" +#include "intel_display_core.h" +#include "intel_display_regs.h" #include "intel_display_types.h" -#include "intel_pm.h" -#include "intel_sideband.h" +#include "intel_display_utils.h" +#include "intel_mchbar_regs.h" +#include "intel_pcode.h" +#include "intel_uncore.h" +#include "skl_watermark.h" + +struct intel_bw_state { + struct intel_global_state base; + + /* + * Contains a bit mask, used to determine, whether correspondent + * pipe allows SAGV or not. + */ + u8 pipe_sagv_reject; + + /* bitmask of active pipes */ + u8 active_pipes; + + /* + * From MTL onwards, to lock a QGV point, punit expects the peak BW of + * the selected QGV point as the parameter in multiples of 100MB/s + */ + u16 qgv_point_peakbw; + + /* + * Current QGV points mask, which restricts + * some particular SAGV states, not to confuse + * with pipe_sagv_mask. + */ + u16 qgv_points_mask; + + unsigned int data_rate[I915_MAX_PIPES]; + u8 num_active_planes[I915_MAX_PIPES]; +}; /* Parameters for Qclk Geyserville (QGV) */ struct intel_qgv_point { u16 dclk, t_rp, t_rdpre, t_rc, t_ras, t_rcd; }; +#define DEPROGBWPCLIMIT 60 + +struct intel_psf_gv_point { + u8 clk; /* clock in multiples of 16.6666 MHz */ +}; + struct intel_qgv_info { struct intel_qgv_point points[I915_NUM_QGV_POINTS]; + struct intel_psf_gv_point psf_points[I915_NUM_PSF_GV_POINTS]; u8 num_points; + u8 num_psf_points; u8 t_bl; + u8 max_numchannels; + u8 channel_width; + u8 deinterleave; }; -static int icl_pcode_read_qgv_point_info(struct drm_i915_private *dev_priv, +static int dg1_mchbar_read_qgv_point_info(struct intel_display *display, + struct intel_qgv_point *sp, + int point) +{ + struct drm_i915_private *i915 = to_i915(display->drm); + u32 dclk_ratio, dclk_reference; + u32 val; + + val = intel_uncore_read(&i915->uncore, SA_PERF_STATUS_0_0_0_MCHBAR_PC); + dclk_ratio = REG_FIELD_GET(DG1_QCLK_RATIO_MASK, val); + if (val & DG1_QCLK_REFERENCE) + dclk_reference = 6; /* 6 * 16.666 MHz = 100 MHz */ + else + dclk_reference = 8; /* 8 * 16.666 MHz = 133 MHz */ + sp->dclk = DIV_ROUND_UP((16667 * dclk_ratio * dclk_reference) + 500, 1000); + + val = intel_uncore_read(&i915->uncore, SKL_MC_BIOS_DATA_0_0_0_MCHBAR_PCU); + if (val & DG1_GEAR_TYPE) + sp->dclk *= 2; + + if (sp->dclk == 0) + return -EINVAL; + + val = intel_uncore_read(&i915->uncore, MCHBAR_CH0_CR_TC_PRE_0_0_0_MCHBAR); + sp->t_rp = REG_FIELD_GET(DG1_DRAM_T_RP_MASK, val); + sp->t_rdpre = REG_FIELD_GET(DG1_DRAM_T_RDPRE_MASK, val); + + val = intel_uncore_read(&i915->uncore, MCHBAR_CH0_CR_TC_PRE_0_0_0_MCHBAR_HIGH); + sp->t_rcd = REG_FIELD_GET(DG1_DRAM_T_RCD_MASK, val); + sp->t_ras = REG_FIELD_GET(DG1_DRAM_T_RAS_MASK, val); + + sp->t_rc = sp->t_rp + sp->t_ras; + + return 0; +} + +static int icl_pcode_read_qgv_point_info(struct intel_display *display, struct intel_qgv_point *sp, int point) { u32 val = 0, val2 = 0; + u16 dclk; int ret; - ret = sandybridge_pcode_read(dev_priv, - ICL_PCODE_MEM_SUBSYSYSTEM_INFO | - ICL_PCODE_MEM_SS_READ_QGV_POINT_INFO(point), - &val, &val2); + ret = intel_pcode_read(display->drm, ICL_PCODE_MEM_SUBSYSYSTEM_INFO | + ICL_PCODE_MEM_SS_READ_QGV_POINT_INFO(point), + &val, &val2); if (ret) return ret; - sp->dclk = val & 0xffff; + dclk = val & 0xffff; + sp->dclk = DIV_ROUND_UP((16667 * dclk) + (DISPLAY_VER(display) >= 12 ? 500 : 0), + 1000); sp->t_rp = (val & 0xff0000) >> 16; sp->t_rcd = (val & 0xff000000) >> 24; @@ -49,73 +136,238 @@ static int icl_pcode_read_qgv_point_info(struct drm_i915_private *dev_priv, return 0; } -int icl_pcode_restrict_qgv_points(struct drm_i915_private *dev_priv, - u32 points_mask) +static int adls_pcode_read_psf_gv_point_info(struct intel_display *display, + struct intel_psf_gv_point *points) +{ + u32 val = 0; + int ret; + int i; + + ret = intel_pcode_read(display->drm, ICL_PCODE_MEM_SUBSYSYSTEM_INFO | + ADL_PCODE_MEM_SS_READ_PSF_GV_INFO, &val, NULL); + if (ret) + return ret; + + for (i = 0; i < I915_NUM_PSF_GV_POINTS; i++) { + points[i].clk = val & 0xff; + val >>= 8; + } + + return 0; +} + +static u16 icl_qgv_points_mask(struct intel_display *display) +{ + unsigned int num_psf_gv_points = display->bw.max[0].num_psf_gv_points; + unsigned int num_qgv_points = display->bw.max[0].num_qgv_points; + u16 qgv_points = 0, psf_points = 0; + + /* + * We can _not_ use the whole ADLS_QGV_PT_MASK here, as PCode rejects + * it with failure if we try masking any unadvertised points. + * So need to operate only with those returned from PCode. + */ + if (num_qgv_points > 0) + qgv_points = GENMASK(num_qgv_points - 1, 0); + + if (num_psf_gv_points > 0) + psf_points = GENMASK(num_psf_gv_points - 1, 0); + + return ICL_PCODE_REQ_QGV_PT(qgv_points) | ADLS_PCODE_REQ_PSF_PT(psf_points); +} + +static bool is_sagv_enabled(struct intel_display *display, u16 points_mask) +{ + return !is_power_of_2(~points_mask & icl_qgv_points_mask(display) & + ICL_PCODE_REQ_QGV_PT_MASK); +} + +static int icl_pcode_restrict_qgv_points(struct intel_display *display, + u32 points_mask) { int ret; + if (DISPLAY_VER(display) >= 14) + return 0; + /* bspec says to keep retrying for at least 1 ms */ - ret = skl_pcode_request(dev_priv, ICL_PCODE_SAGV_DE_MEM_SS_CONFIG, - points_mask, - ICL_PCODE_POINTS_RESTRICTED_MASK, - ICL_PCODE_POINTS_RESTRICTED, - 1); + ret = intel_pcode_request(display->drm, ICL_PCODE_SAGV_DE_MEM_SS_CONFIG, + points_mask, + ICL_PCODE_REP_QGV_MASK | ADLS_PCODE_REP_PSF_MASK, + ICL_PCODE_REP_QGV_SAFE | ADLS_PCODE_REP_PSF_SAFE, + 1); if (ret < 0) { - drm_err(&dev_priv->drm, "Failed to disable qgv points (%d)\n", ret); + drm_err(display->drm, + "Failed to disable qgv points (0x%x) points: 0x%x\n", + ret, points_mask); return ret; } + display->sagv.status = is_sagv_enabled(display, points_mask) ? + I915_SAGV_ENABLED : I915_SAGV_DISABLED; + return 0; } -static int icl_get_qgv_points(struct drm_i915_private *dev_priv, - struct intel_qgv_info *qi) +static int mtl_read_qgv_point_info(struct intel_display *display, + struct intel_qgv_point *sp, int point) +{ + struct drm_i915_private *i915 = to_i915(display->drm); + u32 val, val2; + u16 dclk; + + val = intel_uncore_read(&i915->uncore, + MTL_MEM_SS_INFO_QGV_POINT_LOW(point)); + val2 = intel_uncore_read(&i915->uncore, + MTL_MEM_SS_INFO_QGV_POINT_HIGH(point)); + dclk = REG_FIELD_GET(MTL_DCLK_MASK, val); + sp->dclk = DIV_ROUND_CLOSEST(16667 * dclk, 1000); + sp->t_rp = REG_FIELD_GET(MTL_TRP_MASK, val); + sp->t_rcd = REG_FIELD_GET(MTL_TRCD_MASK, val); + + sp->t_rdpre = REG_FIELD_GET(MTL_TRDPRE_MASK, val2); + sp->t_ras = REG_FIELD_GET(MTL_TRAS_MASK, val2); + + sp->t_rc = sp->t_rp + sp->t_ras; + + return 0; +} + +static int +intel_read_qgv_point_info(struct intel_display *display, + struct intel_qgv_point *sp, + int point) +{ + if (DISPLAY_VER(display) >= 14) + return mtl_read_qgv_point_info(display, sp, point); + else if (display->platform.dg1) + return dg1_mchbar_read_qgv_point_info(display, sp, point); + else + return icl_pcode_read_qgv_point_info(display, sp, point); +} + +static int icl_get_qgv_points(struct intel_display *display, + const struct dram_info *dram_info, + struct intel_qgv_info *qi, + bool is_y_tile) { - const struct dram_info *dram_info = &dev_priv->dram_info; int i, ret; qi->num_points = dram_info->num_qgv_points; + qi->num_psf_points = dram_info->num_psf_gv_points; - if (DISPLAY_VER(dev_priv) == 12) + if (DISPLAY_VER(display) >= 14) { switch (dram_info->type) { case INTEL_DRAM_DDR4: qi->t_bl = 4; + qi->max_numchannels = 2; + qi->channel_width = 64; + qi->deinterleave = 2; break; case INTEL_DRAM_DDR5: qi->t_bl = 8; + qi->max_numchannels = 4; + qi->channel_width = 32; + qi->deinterleave = 2; + break; + case INTEL_DRAM_LPDDR4: + case INTEL_DRAM_LPDDR5: + qi->t_bl = 16; + qi->max_numchannels = 8; + qi->channel_width = 16; + qi->deinterleave = 4; + break; + case INTEL_DRAM_GDDR: + case INTEL_DRAM_GDDR_ECC: + qi->channel_width = 32; + break; + default: + MISSING_CASE(dram_info->type); + return -EINVAL; + } + } else if (DISPLAY_VER(display) >= 12) { + switch (dram_info->type) { + case INTEL_DRAM_DDR4: + qi->t_bl = is_y_tile ? 8 : 4; + qi->max_numchannels = 2; + qi->channel_width = 64; + qi->deinterleave = is_y_tile ? 1 : 2; + break; + case INTEL_DRAM_DDR5: + qi->t_bl = is_y_tile ? 16 : 8; + qi->max_numchannels = 4; + qi->channel_width = 32; + qi->deinterleave = is_y_tile ? 1 : 2; + break; + case INTEL_DRAM_LPDDR4: + if (display->platform.rocketlake) { + qi->t_bl = 8; + qi->max_numchannels = 4; + qi->channel_width = 32; + qi->deinterleave = 2; + break; + } + fallthrough; + case INTEL_DRAM_LPDDR5: + qi->t_bl = 16; + qi->max_numchannels = 8; + qi->channel_width = 16; + qi->deinterleave = is_y_tile ? 2 : 4; break; default: qi->t_bl = 16; + qi->max_numchannels = 1; break; } - else if (DISPLAY_VER(dev_priv) == 11) - qi->t_bl = dev_priv->dram_info.type == INTEL_DRAM_DDR4 ? 4 : 8; + } else if (DISPLAY_VER(display) == 11) { + qi->t_bl = dram_info->type == INTEL_DRAM_DDR4 ? 4 : 8; + qi->max_numchannels = 1; + } - if (drm_WARN_ON(&dev_priv->drm, + if (drm_WARN_ON(display->drm, qi->num_points > ARRAY_SIZE(qi->points))) qi->num_points = ARRAY_SIZE(qi->points); for (i = 0; i < qi->num_points; i++) { struct intel_qgv_point *sp = &qi->points[i]; - ret = icl_pcode_read_qgv_point_info(dev_priv, sp, i); - if (ret) + ret = intel_read_qgv_point_info(display, sp, i); + if (ret) { + drm_dbg_kms(display->drm, "Could not read QGV %d info\n", i); return ret; + } - drm_dbg_kms(&dev_priv->drm, + drm_dbg_kms(display->drm, "QGV %d: DCLK=%d tRP=%d tRDPRE=%d tRAS=%d tRCD=%d tRC=%d\n", i, sp->dclk, sp->t_rp, sp->t_rdpre, sp->t_ras, sp->t_rcd, sp->t_rc); } + if (qi->num_psf_points > 0) { + ret = adls_pcode_read_psf_gv_point_info(display, qi->psf_points); + if (ret) { + drm_err(display->drm, "Failed to read PSF point data; PSF points will not be considered in bandwidth calculations.\n"); + qi->num_psf_points = 0; + } + + for (i = 0; i < qi->num_psf_points; i++) + drm_dbg_kms(display->drm, + "PSF GV %d: CLK=%d\n", + i, qi->psf_points[i].clk); + } + return 0; } -static int icl_calc_bw(int dclk, int num, int den) +static int adl_calc_psf_bw(int clk) { - /* multiples of 16.666MHz (100/6) */ - return DIV_ROUND_CLOSEST(num * dclk * 100, den * 6); + /* + * clk is multiples of 16.666MHz (100/6) + * According to BSpec PSF GV bandwidth is + * calculated as BW = 64 * clk * 16.666Mhz + */ + return DIV_ROUND_CLOSEST(64 * clk * 100, 6); } static int icl_sagv_max_dclk(const struct intel_qgv_info *qi) @@ -131,69 +383,112 @@ static int icl_sagv_max_dclk(const struct intel_qgv_info *qi) struct intel_sa_info { u16 displayrtids; - u8 deburst, deprogbwlimit; + u8 deburst, deprogbwlimit, derating; }; static const struct intel_sa_info icl_sa_info = { .deburst = 8, .deprogbwlimit = 25, /* GB/s */ .displayrtids = 128, + .derating = 10, }; static const struct intel_sa_info tgl_sa_info = { .deburst = 16, .deprogbwlimit = 34, /* GB/s */ .displayrtids = 256, + .derating = 10, }; static const struct intel_sa_info rkl_sa_info = { - .deburst = 16, + .deburst = 8, .deprogbwlimit = 20, /* GB/s */ .displayrtids = 128, + .derating = 10, }; static const struct intel_sa_info adls_sa_info = { .deburst = 16, .deprogbwlimit = 38, /* GB/s */ .displayrtids = 256, + .derating = 10, }; -static int icl_get_bw_info(struct drm_i915_private *dev_priv, const struct intel_sa_info *sa) +static const struct intel_sa_info adlp_sa_info = { + .deburst = 16, + .deprogbwlimit = 38, /* GB/s */ + .displayrtids = 256, + .derating = 20, +}; + +static const struct intel_sa_info mtl_sa_info = { + .deburst = 32, + .deprogbwlimit = 38, /* GB/s */ + .displayrtids = 256, + .derating = 10, +}; + +static const struct intel_sa_info xe2_hpd_sa_info = { + .derating = 30, + .deprogbwlimit = 53, + /* Other values not used by simplified algorithm */ +}; + +static const struct intel_sa_info xe2_hpd_ecc_sa_info = { + .derating = 45, + .deprogbwlimit = 53, + /* Other values not used by simplified algorithm */ +}; + +static const struct intel_sa_info xe3lpd_sa_info = { + .deburst = 32, + .deprogbwlimit = 65, /* GB/s */ + .displayrtids = 256, + .derating = 10, +}; + +static const struct intel_sa_info xe3lpd_3002_sa_info = { + .deburst = 32, + .deprogbwlimit = 22, /* GB/s */ + .displayrtids = 256, + .derating = 10, +}; + +static int icl_get_bw_info(struct intel_display *display, + const struct dram_info *dram_info, + const struct intel_sa_info *sa) { struct intel_qgv_info qi = {}; bool is_y_tile = true; /* assume y tile may be used */ - int num_channels = max_t(u8, 1, dev_priv->dram_info.num_channels); - int deinterleave; - int ipqdepth, ipqdepthpch; + int num_channels = max_t(u8, 1, dram_info->num_channels); + int ipqdepth, ipqdepthpch = 16; int dclk_max; int maxdebw; + int num_groups = ARRAY_SIZE(display->bw.max); int i, ret; - ret = icl_get_qgv_points(dev_priv, &qi); + ret = icl_get_qgv_points(display, dram_info, &qi, is_y_tile); if (ret) { - drm_dbg_kms(&dev_priv->drm, + drm_dbg_kms(display->drm, "Failed to get memory subsystem information, ignoring bandwidth limits"); return ret; } - deinterleave = DIV_ROUND_UP(num_channels, is_y_tile ? 4 : 2); dclk_max = icl_sagv_max_dclk(&qi); - - ipqdepthpch = 16; - - maxdebw = min(sa->deprogbwlimit * 1000, - icl_calc_bw(dclk_max, 16, 1) * 6 / 10); /* 60% */ + maxdebw = min(sa->deprogbwlimit * 1000, dclk_max * 16 * 6 / 10); ipqdepth = min(ipqdepthpch, sa->displayrtids / num_channels); + qi.deinterleave = DIV_ROUND_UP(num_channels, is_y_tile ? 4 : 2); - for (i = 0; i < ARRAY_SIZE(dev_priv->max_bw); i++) { - struct intel_bw_info *bi = &dev_priv->max_bw[i]; + for (i = 0; i < num_groups; i++) { + struct intel_bw_info *bi = &display->bw.max[i]; int clpchgroup; int j; - clpchgroup = (sa->deburst * deinterleave / num_channels) << i; + clpchgroup = (sa->deburst * qi.deinterleave / num_channels) << i; bi->num_planes = (ipqdepth - clpchgroup) / clpchgroup + 1; bi->num_qgv_points = qi.num_points; + bi->num_psf_gv_points = qi.num_psf_points; for (j = 0; j < qi.num_points; j++) { const struct intel_qgv_point *sp = &qi.points[j]; @@ -207,18 +502,132 @@ static int icl_get_bw_info(struct drm_i915_private *dev_priv, const struct intel */ ct = max_t(int, sp->t_rc, sp->t_rp + sp->t_rcd + (clpchgroup - 1) * qi.t_bl + sp->t_rdpre); - bw = icl_calc_bw(sp->dclk, clpchgroup * 32 * num_channels, ct); + bw = DIV_ROUND_UP(sp->dclk * clpchgroup * 32 * num_channels, ct); bi->deratedbw[j] = min(maxdebw, - bw * 9 / 10); /* 90% */ + bw * (100 - sa->derating) / 100); - drm_dbg_kms(&dev_priv->drm, + drm_dbg_kms(display->drm, "BW%d / QGV %d: num_planes=%d deratedbw=%u\n", i, j, bi->num_planes, bi->deratedbw[j]); } + } + /* + * In case if SAGV is disabled in BIOS, we always get 1 + * SAGV point, but we can't send PCode commands to restrict it + * as it will fail and pointless anyway. + */ + if (qi.num_points == 1) + display->sagv.status = I915_SAGV_NOT_CONTROLLED; + else + display->sagv.status = I915_SAGV_ENABLED; - if (bi->num_planes == 1) - break; + return 0; +} + +static int tgl_get_bw_info(struct intel_display *display, + const struct dram_info *dram_info, + const struct intel_sa_info *sa) +{ + struct intel_qgv_info qi = {}; + bool is_y_tile = true; /* assume y tile may be used */ + int num_channels = max_t(u8, 1, dram_info->num_channels); + int ipqdepth, ipqdepthpch = 16; + int dclk_max; + int maxdebw, peakbw; + int clperchgroup; + int num_groups = ARRAY_SIZE(display->bw.max); + int i, ret; + + ret = icl_get_qgv_points(display, dram_info, &qi, is_y_tile); + if (ret) { + drm_dbg_kms(display->drm, + "Failed to get memory subsystem information, ignoring bandwidth limits"); + return ret; + } + + if (DISPLAY_VER(display) < 14 && + (dram_info->type == INTEL_DRAM_LPDDR4 || dram_info->type == INTEL_DRAM_LPDDR5)) + num_channels *= 2; + + qi.deinterleave = qi.deinterleave ? : DIV_ROUND_UP(num_channels, is_y_tile ? 4 : 2); + + if (num_channels < qi.max_numchannels && DISPLAY_VER(display) >= 12) + qi.deinterleave = max(DIV_ROUND_UP(qi.deinterleave, 2), 1); + + if (DISPLAY_VER(display) >= 12 && num_channels > qi.max_numchannels) + drm_warn(display->drm, "Number of channels exceeds max number of channels."); + if (qi.max_numchannels != 0) + num_channels = min_t(u8, num_channels, qi.max_numchannels); + + dclk_max = icl_sagv_max_dclk(&qi); + + peakbw = num_channels * DIV_ROUND_UP(qi.channel_width, 8) * dclk_max; + maxdebw = min(sa->deprogbwlimit * 1000, peakbw * DEPROGBWPCLIMIT / 100); + + ipqdepth = min(ipqdepthpch, sa->displayrtids / num_channels); + /* + * clperchgroup = 4kpagespermempage * clperchperblock, + * clperchperblock = 8 / num_channels * interleave + */ + clperchgroup = 4 * DIV_ROUND_UP(8, num_channels) * qi.deinterleave; + + for (i = 0; i < num_groups; i++) { + struct intel_bw_info *bi = &display->bw.max[i]; + struct intel_bw_info *bi_next; + int clpchgroup; + int j; + + clpchgroup = (sa->deburst * qi.deinterleave / num_channels) << i; + + if (i < num_groups - 1) { + bi_next = &display->bw.max[i + 1]; + + if (clpchgroup < clperchgroup) + bi_next->num_planes = (ipqdepth - clpchgroup) / + clpchgroup + 1; + else + bi_next->num_planes = 0; + } + + bi->num_qgv_points = qi.num_points; + bi->num_psf_gv_points = qi.num_psf_points; + + for (j = 0; j < qi.num_points; j++) { + const struct intel_qgv_point *sp = &qi.points[j]; + int ct, bw; + + /* + * Max row cycle time + * + * FIXME what is the logic behind the + * assumed burst length? + */ + ct = max_t(int, sp->t_rc, sp->t_rp + sp->t_rcd + + (clpchgroup - 1) * qi.t_bl + sp->t_rdpre); + bw = DIV_ROUND_UP(sp->dclk * clpchgroup * 32 * num_channels, ct); + + bi->deratedbw[j] = min(maxdebw, + bw * (100 - sa->derating) / 100); + bi->peakbw[j] = DIV_ROUND_CLOSEST(sp->dclk * + num_channels * + qi.channel_width, 8); + + drm_dbg_kms(display->drm, + "BW%d / QGV %d: num_planes=%d deratedbw=%u peakbw: %u\n", + i, j, bi->num_planes, bi->deratedbw[j], + bi->peakbw[j]); + } + + for (j = 0; j < qi.num_psf_points; j++) { + const struct intel_psf_gv_point *sp = &qi.psf_points[j]; + + bi->psf_bw[j] = adl_calc_psf_bw(sp->clk); + + drm_dbg_kms(display->drm, + "BW%d / PSF GV %d: num_planes=%d bw=%u\n", + i, j, bi->num_planes, bi->psf_bw[j]); + } } /* @@ -227,15 +636,89 @@ static int icl_get_bw_info(struct drm_i915_private *dev_priv, const struct intel * as it will fail and pointless anyway. */ if (qi.num_points == 1) - dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED; + display->sagv.status = I915_SAGV_NOT_CONTROLLED; else - dev_priv->sagv_status = I915_SAGV_ENABLED; + display->sagv.status = I915_SAGV_ENABLED; + + return 0; +} + +static void dg2_get_bw_info(struct intel_display *display) +{ + unsigned int deratedbw = display->platform.dg2_g11 ? 38000 : 50000; + int num_groups = ARRAY_SIZE(display->bw.max); + int i; + + /* + * DG2 doesn't have SAGV or QGV points, just a constant max bandwidth + * that doesn't depend on the number of planes enabled. So fill all the + * plane group with constant bw information for uniformity with other + * platforms. DG2-G10 platforms have a constant 50 GB/s bandwidth, + * whereas DG2-G11 platforms have 38 GB/s. + */ + for (i = 0; i < num_groups; i++) { + struct intel_bw_info *bi = &display->bw.max[i]; + + bi->num_planes = 1; + /* Need only one dummy QGV point per group */ + bi->num_qgv_points = 1; + bi->deratedbw[0] = deratedbw; + } + + display->sagv.status = I915_SAGV_NOT_CONTROLLED; +} + +static int xe2_hpd_get_bw_info(struct intel_display *display, + const struct dram_info *dram_info, + const struct intel_sa_info *sa) +{ + struct intel_qgv_info qi = {}; + int num_channels = dram_info->num_channels; + int peakbw, maxdebw; + int ret, i; + + ret = icl_get_qgv_points(display, dram_info, &qi, true); + if (ret) { + drm_dbg_kms(display->drm, + "Failed to get memory subsystem information, ignoring bandwidth limits"); + return ret; + } + + peakbw = num_channels * qi.channel_width / 8 * icl_sagv_max_dclk(&qi); + maxdebw = min(sa->deprogbwlimit * 1000, peakbw * DEPROGBWPCLIMIT / 10); + + for (i = 0; i < qi.num_points; i++) { + const struct intel_qgv_point *point = &qi.points[i]; + int bw = num_channels * (qi.channel_width / 8) * point->dclk; + + display->bw.max[0].deratedbw[i] = + min(maxdebw, (100 - sa->derating) * bw / 100); + display->bw.max[0].peakbw[i] = bw; + + drm_dbg_kms(display->drm, "QGV %d: deratedbw=%u peakbw: %u\n", + i, display->bw.max[0].deratedbw[i], + display->bw.max[0].peakbw[i]); + } + + /* Bandwidth does not depend on # of planes; set all groups the same */ + display->bw.max[0].num_planes = 1; + display->bw.max[0].num_qgv_points = qi.num_points; + for (i = 1; i < ARRAY_SIZE(display->bw.max); i++) + memcpy(&display->bw.max[i], &display->bw.max[0], + sizeof(display->bw.max[0])); + + /* + * Xe2_HPD should always have exactly two QGV points representing + * battery and plugged-in operation. + */ + drm_WARN_ON(display->drm, qi.num_points != 2); + display->sagv.status = I915_SAGV_ENABLED; return 0; } -static unsigned int icl_max_bw(struct drm_i915_private *dev_priv, - int num_planes, int qgv_point) +static unsigned int icl_max_bw_index(struct intel_display *display, + int num_planes, int qgv_point) { int i; @@ -244,9 +727,9 @@ static unsigned int icl_max_bw(struct drm_i915_private *dev_priv, */ num_planes = max(1, num_planes); - for (i = 0; i < ARRAY_SIZE(dev_priv->max_bw); i++) { + for (i = 0; i < ARRAY_SIZE(display->bw.max); i++) { const struct intel_bw_info *bi = - &dev_priv->max_bw[i]; + &display->bw.max[i]; /* * Pcode will not expose all QGV points when @@ -256,107 +739,147 @@ static unsigned int icl_max_bw(struct drm_i915_private *dev_priv, return UINT_MAX; if (num_planes >= bi->num_planes) - return bi->deratedbw[qgv_point]; + return i; } - return 0; + return UINT_MAX; } -void intel_bw_init_hw(struct drm_i915_private *dev_priv) +static unsigned int tgl_max_bw_index(struct intel_display *display, + int num_planes, int qgv_point) { - if (!HAS_DISPLAY(dev_priv)) - return; - - if (IS_ALDERLAKE_S(dev_priv) || IS_ALDERLAKE_P(dev_priv)) - icl_get_bw_info(dev_priv, &adls_sa_info); - else if (IS_ROCKETLAKE(dev_priv)) - icl_get_bw_info(dev_priv, &rkl_sa_info); - else if (DISPLAY_VER(dev_priv) == 12) - icl_get_bw_info(dev_priv, &tgl_sa_info); - else if (DISPLAY_VER(dev_priv) == 11) - icl_get_bw_info(dev_priv, &icl_sa_info); -} + int i; -static unsigned int intel_bw_crtc_num_active_planes(const struct intel_crtc_state *crtc_state) -{ /* - * We assume cursors are small enough - * to not not cause bandwidth problems. + * Let's return max bw for 0 planes */ - return hweight8(crtc_state->active_planes & ~BIT(PLANE_CURSOR)); -} + num_planes = max(1, num_planes); -static unsigned int intel_bw_crtc_data_rate(const struct intel_crtc_state *crtc_state) -{ - struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); - unsigned int data_rate = 0; - enum plane_id plane_id; + for (i = ARRAY_SIZE(display->bw.max) - 1; i >= 0; i--) { + const struct intel_bw_info *bi = + &display->bw.max[i]; - for_each_plane_id_on_crtc(crtc, plane_id) { /* - * We assume cursors are small enough - * to not not cause bandwidth problems. + * Pcode will not expose all QGV points when + * SAGV is forced to off/min/med/max. */ - if (plane_id == PLANE_CURSOR) - continue; + if (qgv_point >= bi->num_qgv_points) + return UINT_MAX; - data_rate += crtc_state->data_rate[plane_id]; + if (num_planes <= bi->num_planes) + return i; } - return data_rate; + return 0; } -void intel_bw_crtc_update(struct intel_bw_state *bw_state, - const struct intel_crtc_state *crtc_state) +static unsigned int adl_psf_bw(struct intel_display *display, + int psf_gv_point) { - struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); - struct drm_i915_private *i915 = to_i915(crtc->base.dev); + const struct intel_bw_info *bi = + &display->bw.max[0]; - bw_state->data_rate[crtc->pipe] = - intel_bw_crtc_data_rate(crtc_state); - bw_state->num_active_planes[crtc->pipe] = - intel_bw_crtc_num_active_planes(crtc_state); + return bi->psf_bw[psf_gv_point]; +} - drm_dbg_kms(&i915->drm, "pipe %c data rate %u num active planes %u\n", - pipe_name(crtc->pipe), - bw_state->data_rate[crtc->pipe], - bw_state->num_active_planes[crtc->pipe]); +static unsigned int icl_qgv_bw(struct intel_display *display, + int num_active_planes, int qgv_point) +{ + unsigned int idx; + + if (DISPLAY_VER(display) >= 12) + idx = tgl_max_bw_index(display, num_active_planes, qgv_point); + else + idx = icl_max_bw_index(display, num_active_planes, qgv_point); + + if (idx >= ARRAY_SIZE(display->bw.max)) + return 0; + + return display->bw.max[idx].deratedbw[qgv_point]; } -static unsigned int intel_bw_num_active_planes(struct drm_i915_private *dev_priv, +void intel_bw_init_hw(struct intel_display *display) +{ + const struct dram_info *dram_info = intel_dram_info(display->drm); + + if (!HAS_DISPLAY(display)) + return; + + /* + * Starting with Xe3p_LPD, the hardware tells us whether memory has ECC + * enabled that would impact display bandwidth. However, so far there + * are no instructions in Bspec on how to handle that case. Let's + * complain if we ever find such a scenario. + */ + if (DISPLAY_VER(display) >= 35) + drm_WARN_ON(display->drm, dram_info->ecc_impacting_de_bw); + + if (DISPLAY_VER(display) >= 30) { + if (DISPLAY_VERx100(display) == 3002) + tgl_get_bw_info(display, dram_info, &xe3lpd_3002_sa_info); + else + tgl_get_bw_info(display, dram_info, &xe3lpd_sa_info); + } else if (DISPLAY_VERx100(display) >= 1401 && display->platform.dgfx) { + if (dram_info->type == INTEL_DRAM_GDDR_ECC) + xe2_hpd_get_bw_info(display, dram_info, &xe2_hpd_ecc_sa_info); + else + xe2_hpd_get_bw_info(display, dram_info, &xe2_hpd_sa_info); + } else if (DISPLAY_VER(display) >= 14) { + tgl_get_bw_info(display, dram_info, &mtl_sa_info); + } else if (display->platform.dg2) { + dg2_get_bw_info(display); + } else if (display->platform.alderlake_p) { + tgl_get_bw_info(display, dram_info, &adlp_sa_info); + } else if (display->platform.alderlake_s) { + tgl_get_bw_info(display, dram_info, &adls_sa_info); + } else if (display->platform.rocketlake) { + tgl_get_bw_info(display, dram_info, &rkl_sa_info); + } else if (DISPLAY_VER(display) == 12) { + tgl_get_bw_info(display, dram_info, &tgl_sa_info); + } else if (DISPLAY_VER(display) == 11) { + icl_get_bw_info(display, dram_info, &icl_sa_info); + } +} + +static unsigned int intel_bw_num_active_planes(struct intel_display *display, const struct intel_bw_state *bw_state) { unsigned int num_active_planes = 0; enum pipe pipe; - for_each_pipe(dev_priv, pipe) + for_each_pipe(display, pipe) num_active_planes += bw_state->num_active_planes[pipe]; return num_active_planes; } -static unsigned int intel_bw_data_rate(struct drm_i915_private *dev_priv, +static unsigned int intel_bw_data_rate(struct intel_display *display, const struct intel_bw_state *bw_state) { unsigned int data_rate = 0; enum pipe pipe; - for_each_pipe(dev_priv, pipe) + for_each_pipe(display, pipe) data_rate += bw_state->data_rate[pipe]; - if (DISPLAY_VER(dev_priv) >= 13 && intel_vtd_active()) - data_rate = data_rate * 105 / 100; + if (DISPLAY_VER(display) >= 13 && intel_display_vtd_active(display)) + data_rate = DIV_ROUND_UP(data_rate * 105, 100); return data_rate; } +struct intel_bw_state *to_intel_bw_state(struct intel_global_state *obj_state) +{ + return container_of(obj_state, struct intel_bw_state, base); +} + struct intel_bw_state * intel_atomic_get_old_bw_state(struct intel_atomic_state *state) { - struct drm_i915_private *dev_priv = to_i915(state->base.dev); + struct intel_display *display = to_intel_display(state); struct intel_global_state *bw_state; - bw_state = intel_atomic_get_old_global_obj_state(state, &dev_priv->bw_obj); + bw_state = intel_atomic_get_old_global_obj_state(state, &display->bw.obj); return to_intel_bw_state(bw_state); } @@ -364,10 +887,10 @@ intel_atomic_get_old_bw_state(struct intel_atomic_state *state) struct intel_bw_state * intel_atomic_get_new_bw_state(struct intel_atomic_state *state) { - struct drm_i915_private *dev_priv = to_i915(state->base.dev); + struct intel_display *display = to_intel_display(state); struct intel_global_state *bw_state; - bw_state = intel_atomic_get_new_global_obj_state(state, &dev_priv->bw_obj); + bw_state = intel_atomic_get_new_global_obj_state(state, &display->bw.obj); return to_intel_bw_state(bw_state); } @@ -375,145 +898,306 @@ intel_atomic_get_new_bw_state(struct intel_atomic_state *state) struct intel_bw_state * intel_atomic_get_bw_state(struct intel_atomic_state *state) { - struct drm_i915_private *dev_priv = to_i915(state->base.dev); + struct intel_display *display = to_intel_display(state); struct intel_global_state *bw_state; - bw_state = intel_atomic_get_global_obj_state(state, &dev_priv->bw_obj); + bw_state = intel_atomic_get_global_obj_state(state, &display->bw.obj); if (IS_ERR(bw_state)) return ERR_CAST(bw_state); return to_intel_bw_state(bw_state); } -int skl_bw_calc_min_cdclk(struct intel_atomic_state *state) +static unsigned int icl_max_bw_qgv_point_mask(struct intel_display *display, + int num_active_planes) { - struct drm_i915_private *dev_priv = to_i915(state->base.dev); - struct intel_bw_state *new_bw_state = NULL; - struct intel_bw_state *old_bw_state = NULL; - const struct intel_crtc_state *crtc_state; - struct intel_crtc *crtc; - int max_bw = 0; - enum pipe pipe; + unsigned int num_qgv_points = display->bw.max[0].num_qgv_points; + unsigned int max_bw_point = 0; + unsigned int max_bw = 0; int i; - for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) { - enum plane_id plane_id; - struct intel_dbuf_bw *crtc_bw; - - new_bw_state = intel_atomic_get_bw_state(state); - if (IS_ERR(new_bw_state)) - return PTR_ERR(new_bw_state); - - old_bw_state = intel_atomic_get_old_bw_state(state); + for (i = 0; i < num_qgv_points; i++) { + unsigned int max_data_rate = + icl_qgv_bw(display, num_active_planes, i); - crtc_bw = &new_bw_state->dbuf_bw[crtc->pipe]; + /* + * We need to know which qgv point gives us + * maximum bandwidth in order to disable SAGV + * if we find that we exceed SAGV block time + * with watermarks. By that moment we already + * have those, as it is calculated earlier in + * intel_atomic_check, + */ + if (max_data_rate > max_bw) { + max_bw_point = BIT(i); + max_bw = max_data_rate; + } + } - memset(&crtc_bw->used_bw, 0, sizeof(crtc_bw->used_bw)); + return max_bw_point; +} - if (!crtc_state->hw.active) - continue; +static u16 icl_prepare_qgv_points_mask(struct intel_display *display, + unsigned int qgv_points, + unsigned int psf_points) +{ + return ~(ICL_PCODE_REQ_QGV_PT(qgv_points) | + ADLS_PCODE_REQ_PSF_PT(psf_points)) & icl_qgv_points_mask(display); +} - for_each_plane_id_on_crtc(crtc, plane_id) { - const struct skl_ddb_entry *plane_alloc = - &crtc_state->wm.skl.plane_ddb_y[plane_id]; - const struct skl_ddb_entry *uv_plane_alloc = - &crtc_state->wm.skl.plane_ddb_uv[plane_id]; - unsigned int data_rate = crtc_state->data_rate[plane_id]; - unsigned int dbuf_mask = 0; - enum dbuf_slice slice; +static unsigned int icl_max_bw_psf_gv_point_mask(struct intel_display *display) +{ + unsigned int num_psf_gv_points = display->bw.max[0].num_psf_gv_points; + unsigned int max_bw_point_mask = 0; + unsigned int max_bw = 0; + int i; - dbuf_mask |= skl_ddb_dbuf_slice_mask(dev_priv, plane_alloc); - dbuf_mask |= skl_ddb_dbuf_slice_mask(dev_priv, uv_plane_alloc); + for (i = 0; i < num_psf_gv_points; i++) { + unsigned int max_data_rate = adl_psf_bw(display, i); - /* - * FIXME: To calculate that more properly we probably - * need to to split per plane data_rate into data_rate_y - * and data_rate_uv for multiplanar formats in order not - * to get accounted those twice if they happen to reside - * on different slices. - * However for pre-icl this would work anyway because - * we have only single slice and for icl+ uv plane has - * non-zero data rate. - * So in worst case those calculation are a bit - * pessimistic, which shouldn't pose any significant - * problem anyway. - */ - for_each_dbuf_slice_in_mask(dev_priv, slice, dbuf_mask) - crtc_bw->used_bw[slice] += data_rate; + if (max_data_rate > max_bw) { + max_bw_point_mask = BIT(i); + max_bw = max_data_rate; + } else if (max_data_rate == max_bw) { + max_bw_point_mask |= BIT(i); } } - if (!old_bw_state) + return max_bw_point_mask; +} + +static void icl_force_disable_sagv(struct intel_display *display, + struct intel_bw_state *bw_state) +{ + unsigned int qgv_points = icl_max_bw_qgv_point_mask(display, 0); + unsigned int psf_points = icl_max_bw_psf_gv_point_mask(display); + + bw_state->qgv_points_mask = icl_prepare_qgv_points_mask(display, + qgv_points, + psf_points); + + drm_dbg_kms(display->drm, "Forcing SAGV disable: mask 0x%x\n", + bw_state->qgv_points_mask); + + icl_pcode_restrict_qgv_points(display, bw_state->qgv_points_mask); +} + +void icl_sagv_pre_plane_update(struct intel_atomic_state *state) +{ + struct intel_display *display = to_intel_display(state); + const struct intel_bw_state *old_bw_state = + intel_atomic_get_old_bw_state(state); + const struct intel_bw_state *new_bw_state = + intel_atomic_get_new_bw_state(state); + u16 old_mask, new_mask; + + if (!new_bw_state) + return; + + old_mask = old_bw_state->qgv_points_mask; + new_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask; + + if (old_mask == new_mask) + return; + + WARN_ON(!new_bw_state->base.changed); + + drm_dbg_kms(display->drm, "Restricting QGV points: 0x%x -> 0x%x\n", + old_mask, new_mask); + + /* + * Restrict required qgv points before updating the configuration. + * According to BSpec we can't mask and unmask qgv points at the same + * time. Also masking should be done before updating the configuration + * and unmasking afterwards. + */ + icl_pcode_restrict_qgv_points(display, new_mask); +} + +void icl_sagv_post_plane_update(struct intel_atomic_state *state) +{ + struct intel_display *display = to_intel_display(state); + const struct intel_bw_state *old_bw_state = + intel_atomic_get_old_bw_state(state); + const struct intel_bw_state *new_bw_state = + intel_atomic_get_new_bw_state(state); + u16 old_mask, new_mask; + + if (!new_bw_state) + return; + + old_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask; + new_mask = new_bw_state->qgv_points_mask; + + if (old_mask == new_mask) + return; + + WARN_ON(!new_bw_state->base.changed); + + drm_dbg_kms(display->drm, "Relaxing QGV points: 0x%x -> 0x%x\n", + old_mask, new_mask); + + /* + * Allow required qgv points after updating the configuration. + * According to BSpec we can't mask and unmask qgv points at the same + * time. Also masking should be done before updating the configuration + * and unmasking afterwards. + */ + icl_pcode_restrict_qgv_points(display, new_mask); +} + +static int mtl_find_qgv_points(struct intel_display *display, + unsigned int data_rate, + unsigned int num_active_planes, + struct intel_bw_state *new_bw_state) +{ + unsigned int best_rate = UINT_MAX; + unsigned int num_qgv_points = display->bw.max[0].num_qgv_points; + unsigned int qgv_peak_bw = 0; + int i; + int ret; + + ret = intel_atomic_lock_global_state(&new_bw_state->base); + if (ret) + return ret; + + /* + * If SAGV cannot be enabled, disable the pcode SAGV by passing all 1's + * for qgv peak bw in PM Demand request. So assign UINT_MAX if SAGV is + * not enabled. PM Demand code will clamp the value for the register + */ + if (!intel_bw_can_enable_sagv(display, new_bw_state)) { + new_bw_state->qgv_point_peakbw = U16_MAX; + drm_dbg_kms(display->drm, "No SAGV, use UINT_MAX as peak bw."); return 0; + } - for_each_pipe(dev_priv, pipe) { - struct intel_dbuf_bw *crtc_bw; - enum dbuf_slice slice; + /* + * Find the best QGV point by comparing the data_rate with max data rate + * offered per plane group + */ + for (i = 0; i < num_qgv_points; i++) { + unsigned int bw_index = + tgl_max_bw_index(display, num_active_planes, i); + unsigned int max_data_rate; - crtc_bw = &new_bw_state->dbuf_bw[pipe]; + if (bw_index >= ARRAY_SIZE(display->bw.max)) + continue; - for_each_dbuf_slice(dev_priv, slice) { - /* - * Current experimental observations show that contrary - * to BSpec we get underruns once we exceed 64 * CDCLK - * for slices in total. - * As a temporary measure in order not to keep CDCLK - * bumped up all the time we calculate CDCLK according - * to this formula for overall bw consumed by slices. - */ - max_bw += crtc_bw->used_bw[slice]; + max_data_rate = display->bw.max[bw_index].deratedbw[i]; + + if (max_data_rate < data_rate) + continue; + + if (max_data_rate - data_rate < best_rate) { + best_rate = max_data_rate - data_rate; + qgv_peak_bw = display->bw.max[bw_index].peakbw[i]; } - } - new_bw_state->min_cdclk = max_bw / 64; + drm_dbg_kms(display->drm, "QGV point %d: max bw %d required %d qgv_peak_bw: %d\n", + i, max_data_rate, data_rate, qgv_peak_bw); + } - if (new_bw_state->min_cdclk != old_bw_state->min_cdclk) { - int ret = intel_atomic_lock_global_state(&new_bw_state->base); + drm_dbg_kms(display->drm, "Matching peaks QGV bw: %d for required data rate: %d\n", + qgv_peak_bw, data_rate); - if (ret) - return ret; + /* + * The display configuration cannot be supported if no QGV point + * satisfying the required data rate is found + */ + if (qgv_peak_bw == 0) { + drm_dbg_kms(display->drm, "No QGV points for bw %d for display configuration(%d active planes).\n", + data_rate, num_active_planes); + return -EINVAL; } + /* MTL PM DEMAND expects QGV BW parameter in multiples of 100 mbps */ + new_bw_state->qgv_point_peakbw = DIV_ROUND_CLOSEST(qgv_peak_bw, 100); + return 0; } -int intel_bw_calc_min_cdclk(struct intel_atomic_state *state) +static int icl_find_qgv_points(struct intel_display *display, + unsigned int data_rate, + unsigned int num_active_planes, + const struct intel_bw_state *old_bw_state, + struct intel_bw_state *new_bw_state) { - struct drm_i915_private *dev_priv = to_i915(state->base.dev); - struct intel_bw_state *new_bw_state = NULL; - struct intel_bw_state *old_bw_state = NULL; - const struct intel_crtc_state *crtc_state; - struct intel_crtc *crtc; - int min_cdclk = 0; - enum pipe pipe; + unsigned int num_psf_gv_points = display->bw.max[0].num_psf_gv_points; + unsigned int num_qgv_points = display->bw.max[0].num_qgv_points; + u16 psf_points = 0; + u16 qgv_points = 0; int i; + int ret; - for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) { - new_bw_state = intel_atomic_get_bw_state(state); - if (IS_ERR(new_bw_state)) - return PTR_ERR(new_bw_state); + ret = intel_atomic_lock_global_state(&new_bw_state->base); + if (ret) + return ret; - old_bw_state = intel_atomic_get_old_bw_state(state); + for (i = 0; i < num_qgv_points; i++) { + unsigned int max_data_rate = icl_qgv_bw(display, + num_active_planes, i); + if (max_data_rate >= data_rate) + qgv_points |= BIT(i); + + drm_dbg_kms(display->drm, "QGV point %d: max bw %d required %d\n", + i, max_data_rate, data_rate); } - if (!old_bw_state) - return 0; + for (i = 0; i < num_psf_gv_points; i++) { + unsigned int max_data_rate = adl_psf_bw(display, i); - for_each_pipe(dev_priv, pipe) { - struct intel_cdclk_state *cdclk_state; + if (max_data_rate >= data_rate) + psf_points |= BIT(i); - cdclk_state = intel_atomic_get_new_cdclk_state(state); - if (!cdclk_state) - return 0; + drm_dbg_kms(display->drm, "PSF GV point %d: max bw %d" + " required %d\n", + i, max_data_rate, data_rate); + } - min_cdclk = max(cdclk_state->min_cdclk[pipe], min_cdclk); + /* + * BSpec states that we always should have at least one allowed point + * left, so if we couldn't - simply reject the configuration for obvious + * reasons. + */ + if (qgv_points == 0) { + drm_dbg_kms(display->drm, "No QGV points provide sufficient memory" + " bandwidth %d for display configuration(%d active planes).\n", + data_rate, num_active_planes); + return -EINVAL; } - new_bw_state->min_cdclk = min_cdclk; + if (num_psf_gv_points > 0 && psf_points == 0) { + drm_dbg_kms(display->drm, "No PSF GV points provide sufficient memory" + " bandwidth %d for display configuration(%d active planes).\n", + data_rate, num_active_planes); + return -EINVAL; + } - if (new_bw_state->min_cdclk != old_bw_state->min_cdclk) { - int ret = intel_atomic_lock_global_state(&new_bw_state->base); + /* + * Leave only single point with highest bandwidth, if + * we can't enable SAGV due to the increased memory latency it may + * cause. + */ + if (!intel_bw_can_enable_sagv(display, new_bw_state)) { + qgv_points = icl_max_bw_qgv_point_mask(display, num_active_planes); + drm_dbg_kms(display->drm, "No SAGV, using single QGV point mask 0x%x\n", + qgv_points); + } + /* + * We store the ones which need to be masked as that is what PCode + * actually accepts as a parameter. + */ + new_bw_state->qgv_points_mask = icl_prepare_qgv_points_mask(display, + qgv_points, + psf_points); + /* + * If the actual mask had changed we need to make sure that + * the commits are serialized(in case this is a nomodeset, nonblocking) + */ + if (new_bw_state->qgv_points_mask != old_bw_state->qgv_points_mask) { + ret = intel_atomic_serialize_global_state(&new_bw_state->base); if (ret) return ret; } @@ -521,35 +1205,42 @@ int intel_bw_calc_min_cdclk(struct intel_atomic_state *state) return 0; } -int intel_bw_atomic_check(struct intel_atomic_state *state) +static int intel_bw_check_qgv_points(struct intel_display *display, + const struct intel_bw_state *old_bw_state, + struct intel_bw_state *new_bw_state) { - struct drm_i915_private *dev_priv = to_i915(state->base.dev); - struct intel_crtc_state *new_crtc_state, *old_crtc_state; - struct intel_bw_state *new_bw_state = NULL; - const struct intel_bw_state *old_bw_state = NULL; - unsigned int data_rate; - unsigned int num_active_planes; - struct intel_crtc *crtc; - int i, ret; - u32 allowed_points = 0; - unsigned int max_bw_point = 0, max_bw = 0; - unsigned int num_qgv_points = dev_priv->max_bw[0].num_qgv_points; - u32 mask = (1 << num_qgv_points) - 1; + unsigned int data_rate = intel_bw_data_rate(display, new_bw_state); + unsigned int num_active_planes = + intel_bw_num_active_planes(display, new_bw_state); - /* FIXME earlier gens need some checks too */ - if (DISPLAY_VER(dev_priv) < 11) - return 0; + data_rate = DIV_ROUND_UP(data_rate, 1000); + + if (DISPLAY_VER(display) >= 14) + return mtl_find_qgv_points(display, data_rate, num_active_planes, + new_bw_state); + else + return icl_find_qgv_points(display, data_rate, num_active_planes, + old_bw_state, new_bw_state); +} + +static int intel_bw_check_data_rate(struct intel_atomic_state *state, bool *changed) +{ + struct intel_display *display = to_intel_display(state); + const struct intel_crtc_state *new_crtc_state, *old_crtc_state; + struct intel_crtc *crtc; + int i; for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) { unsigned int old_data_rate = - intel_bw_crtc_data_rate(old_crtc_state); + intel_crtc_bw_data_rate(old_crtc_state); unsigned int new_data_rate = - intel_bw_crtc_data_rate(new_crtc_state); + intel_crtc_bw_data_rate(new_crtc_state); unsigned int old_active_planes = - intel_bw_crtc_num_active_planes(old_crtc_state); + intel_crtc_bw_num_active_planes(old_crtc_state); unsigned int new_active_planes = - intel_bw_crtc_num_active_planes(new_crtc_state); + intel_crtc_bw_num_active_planes(new_crtc_state); + struct intel_bw_state *new_bw_state; /* * Avoid locking the bw state when @@ -566,89 +1257,196 @@ int intel_bw_atomic_check(struct intel_atomic_state *state) new_bw_state->data_rate[crtc->pipe] = new_data_rate; new_bw_state->num_active_planes[crtc->pipe] = new_active_planes; - drm_dbg_kms(&dev_priv->drm, - "pipe %c data rate %u num active planes %u\n", - pipe_name(crtc->pipe), + *changed = true; + + drm_dbg_kms(display->drm, + "[CRTC:%d:%s] data rate %u num active planes %u\n", + crtc->base.base.id, crtc->base.name, new_bw_state->data_rate[crtc->pipe], new_bw_state->num_active_planes[crtc->pipe]); } - if (!new_bw_state) + return 0; +} + +static int intel_bw_modeset_checks(struct intel_atomic_state *state) +{ + const struct intel_bw_state *old_bw_state; + struct intel_bw_state *new_bw_state; + int ret; + + if (!intel_any_crtc_active_changed(state)) return 0; + new_bw_state = intel_atomic_get_bw_state(state); + if (IS_ERR(new_bw_state)) + return PTR_ERR(new_bw_state); + + old_bw_state = intel_atomic_get_old_bw_state(state); + + new_bw_state->active_pipes = + intel_calc_active_pipes(state, old_bw_state->active_pipes); + ret = intel_atomic_lock_global_state(&new_bw_state->base); if (ret) return ret; - data_rate = intel_bw_data_rate(dev_priv, new_bw_state); - data_rate = DIV_ROUND_UP(data_rate, 1000); + return 0; +} - num_active_planes = intel_bw_num_active_planes(dev_priv, new_bw_state); +static int intel_bw_check_sagv_mask(struct intel_atomic_state *state) +{ + struct intel_display *display = to_intel_display(state); + const struct intel_crtc_state *old_crtc_state; + const struct intel_crtc_state *new_crtc_state; + const struct intel_bw_state *old_bw_state = NULL; + struct intel_bw_state *new_bw_state = NULL; + struct intel_crtc *crtc; + int ret, i; - for (i = 0; i < num_qgv_points; i++) { - unsigned int max_data_rate; + for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, + new_crtc_state, i) { + if (intel_crtc_can_enable_sagv(old_crtc_state) == + intel_crtc_can_enable_sagv(new_crtc_state)) + continue; - max_data_rate = icl_max_bw(dev_priv, num_active_planes, i); - /* - * We need to know which qgv point gives us - * maximum bandwidth in order to disable SAGV - * if we find that we exceed SAGV block time - * with watermarks. By that moment we already - * have those, as it is calculated earlier in - * intel_atomic_check, - */ - if (max_data_rate > max_bw) { - max_bw_point = i; - max_bw = max_data_rate; - } - if (max_data_rate >= data_rate) - allowed_points |= BIT(i); - drm_dbg_kms(&dev_priv->drm, "QGV point %d: max bw %d required %d\n", - i, max_data_rate, data_rate); - } + new_bw_state = intel_atomic_get_bw_state(state); + if (IS_ERR(new_bw_state)) + return PTR_ERR(new_bw_state); - /* - * BSpec states that we always should have at least one allowed point - * left, so if we couldn't - simply reject the configuration for obvious - * reasons. - */ - if (allowed_points == 0) { - drm_dbg_kms(&dev_priv->drm, "No QGV points provide sufficient memory" - " bandwidth %d for display configuration(%d active planes).\n", - data_rate, num_active_planes); - return -EINVAL; - } + old_bw_state = intel_atomic_get_old_bw_state(state); - /* - * Leave only single point with highest bandwidth, if - * we can't enable SAGV due to the increased memory latency it may - * cause. - */ - if (!intel_can_enable_sagv(dev_priv, new_bw_state)) { - allowed_points = BIT(max_bw_point); - drm_dbg_kms(&dev_priv->drm, "No SAGV, using single QGV point %d\n", - max_bw_point); + if (intel_crtc_can_enable_sagv(new_crtc_state)) + new_bw_state->pipe_sagv_reject &= ~BIT(crtc->pipe); + else + new_bw_state->pipe_sagv_reject |= BIT(crtc->pipe); } - /* - * We store the ones which need to be masked as that is what PCode - * actually accepts as a parameter. - */ - new_bw_state->qgv_points_mask = ~allowed_points & mask; - old_bw_state = intel_atomic_get_old_bw_state(state); - /* - * If the actual mask had changed we need to make sure that - * the commits are serialized(in case this is a nomodeset, nonblocking) - */ - if (new_bw_state->qgv_points_mask != old_bw_state->qgv_points_mask) { + if (!new_bw_state) + return 0; + + if (intel_bw_can_enable_sagv(display, new_bw_state) != + intel_bw_can_enable_sagv(display, old_bw_state)) { ret = intel_atomic_serialize_global_state(&new_bw_state->base); if (ret) return ret; + } else if (new_bw_state->pipe_sagv_reject != old_bw_state->pipe_sagv_reject) { + ret = intel_atomic_lock_global_state(&new_bw_state->base); + if (ret) + return ret; } return 0; } +int intel_bw_atomic_check(struct intel_atomic_state *state) +{ + struct intel_display *display = to_intel_display(state); + bool changed = false; + struct intel_bw_state *new_bw_state; + const struct intel_bw_state *old_bw_state; + int ret; + + if (DISPLAY_VER(display) < 9) + return 0; + + ret = intel_bw_modeset_checks(state); + if (ret) + return ret; + + ret = intel_bw_check_sagv_mask(state); + if (ret) + return ret; + + /* FIXME earlier gens need some checks too */ + if (DISPLAY_VER(display) < 11) + return 0; + + ret = intel_bw_check_data_rate(state, &changed); + if (ret) + return ret; + + old_bw_state = intel_atomic_get_old_bw_state(state); + new_bw_state = intel_atomic_get_new_bw_state(state); + + if (new_bw_state && + intel_bw_can_enable_sagv(display, old_bw_state) != + intel_bw_can_enable_sagv(display, new_bw_state)) + changed = true; + + /* + * If none of our inputs (data rates, number of active + * planes, SAGV yes/no) changed then nothing to do here. + */ + if (!changed) + return 0; + + ret = intel_bw_check_qgv_points(display, old_bw_state, new_bw_state); + if (ret) + return ret; + + return 0; +} + +static void intel_bw_crtc_update(struct intel_bw_state *bw_state, + const struct intel_crtc_state *crtc_state) +{ + struct intel_display *display = to_intel_display(crtc_state); + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + + bw_state->data_rate[crtc->pipe] = + intel_crtc_bw_data_rate(crtc_state); + bw_state->num_active_planes[crtc->pipe] = + intel_crtc_bw_num_active_planes(crtc_state); + + drm_dbg_kms(display->drm, "pipe %c data rate %u num active planes %u\n", + pipe_name(crtc->pipe), + bw_state->data_rate[crtc->pipe], + bw_state->num_active_planes[crtc->pipe]); +} + +void intel_bw_update_hw_state(struct intel_display *display) +{ + struct intel_bw_state *bw_state = + to_intel_bw_state(display->bw.obj.state); + struct intel_crtc *crtc; + + if (DISPLAY_VER(display) < 9) + return; + + bw_state->active_pipes = 0; + bw_state->pipe_sagv_reject = 0; + + for_each_intel_crtc(display->drm, crtc) { + const struct intel_crtc_state *crtc_state = + to_intel_crtc_state(crtc->base.state); + enum pipe pipe = crtc->pipe; + + if (crtc_state->hw.active) + bw_state->active_pipes |= BIT(pipe); + + if (DISPLAY_VER(display) >= 11) + intel_bw_crtc_update(bw_state, crtc_state); + + /* initially SAGV has been forced off */ + bw_state->pipe_sagv_reject |= BIT(pipe); + } +} + +void intel_bw_crtc_disable_noatomic(struct intel_crtc *crtc) +{ + struct intel_display *display = to_intel_display(crtc); + struct intel_bw_state *bw_state = + to_intel_bw_state(display->bw.obj.state); + enum pipe pipe = crtc->pipe; + + if (DISPLAY_VER(display) < 9) + return; + + bw_state->data_rate[pipe] = 0; + bw_state->num_active_planes[pipe] = 0; +} + static struct intel_global_state * intel_bw_duplicate_state(struct intel_global_obj *obj) { @@ -672,7 +1470,7 @@ static const struct intel_global_state_funcs intel_bw_funcs = { .atomic_destroy_state = intel_bw_destroy_state, }; -int intel_bw_init(struct drm_i915_private *dev_priv) +int intel_bw_init(struct intel_display *display) { struct intel_bw_state *state; @@ -680,8 +1478,44 @@ int intel_bw_init(struct drm_i915_private *dev_priv) if (!state) return -ENOMEM; - intel_atomic_global_obj_init(dev_priv, &dev_priv->bw_obj, + intel_atomic_global_obj_init(display, &display->bw.obj, &state->base, &intel_bw_funcs); + /* + * Limit this only if we have SAGV. And for Display version 14 onwards + * sagv is handled though pmdemand requests + */ + if (intel_has_sagv(display) && IS_DISPLAY_VER(display, 11, 13)) + icl_force_disable_sagv(display, state); + return 0; } + +bool intel_bw_pmdemand_needs_update(struct intel_atomic_state *state) +{ + const struct intel_bw_state *new_bw_state, *old_bw_state; + + new_bw_state = intel_atomic_get_new_bw_state(state); + old_bw_state = intel_atomic_get_old_bw_state(state); + + if (new_bw_state && + new_bw_state->qgv_point_peakbw != old_bw_state->qgv_point_peakbw) + return true; + + return false; +} + +bool intel_bw_can_enable_sagv(struct intel_display *display, + const struct intel_bw_state *bw_state) +{ + if (DISPLAY_VER(display) < 11 && + bw_state->active_pipes && !is_power_of_2(bw_state->active_pipes)) + return false; + + return bw_state->pipe_sagv_reject == 0; +} + +int intel_bw_qgv_point_peakbw(const struct intel_bw_state *bw_state) +{ + return bw_state->qgv_point_peakbw; +} |