1 files changed, 520 insertions, 127 deletions

diff --git a/drivers/gpu/drm/i915/display/intel_dsb.c b/drivers/gpu/drm/i915/display/intel_dsb.c
index d62e050185e7..481488d1fe67 100644
--- a/drivers/gpu/drm/i915/display/intel_dsb.c
+++ b/drivers/gpu/drm/i915/display/intel_dsb.c
@@ -4,11 +4,15 @@
  *
  */
 
-#include "i915_drv.h"
+#include <drm/drm_print.h>
+#include <drm/drm_vblank.h>
+
 #include "i915_irq.h"
 #include "i915_reg.h"
+#include "i915_utils.h"
 #include "intel_crtc.h"
 #include "intel_de.h"
+#include "intel_display_rpm.h"
 #include "intel_display_types.h"
 #include "intel_dsb.h"
 #include "intel_dsb_buffer.h"
@@ -19,16 +23,8 @@
 
 #define CACHELINE_BYTES 64
 
-enum dsb_id {
-	INVALID_DSB = -1,
-	DSB1,
-	DSB2,
-	DSB3,
-	MAX_DSB_PER_PIPE
-};
-
 struct intel_dsb {
-	enum dsb_id id;
+	enum intel_dsb_id id;
 
 	struct intel_dsb_buffer dsb_buf;
 	struct intel_crtc *crtc;
@@ -45,13 +41,21 @@ struct intel_dsb {
 	unsigned int free_pos;
 
 	/*
-	 * ins_start_offset will help to store start dword of the dsb
-	 * instuction and help in identifying the batch of auto-increment
-	 * register.
+	 * Previously emitted DSB instruction. Used to
+	 * identify/adjust the instruction for indexed
+	 * register writes.
+	 */
+	u32 ins[2];
+
+	/*
+	 * Start of the previously emitted DSB instruction.
+	 * Used to adjust the instruction for indexed
+	 * register writes.
 	 */
 	unsigned int ins_start_offset;
 
-	int dewake_scanline;
+	u32 chicken;
+	int hw_dewake_scanline;
 };
 
 /**
@@ -91,13 +95,112 @@ struct intel_dsb {
 #define DSB_OPCODE_POLL			0xA
 /* see DSB_REG_VALUE_MASK */
 
+static bool pre_commit_is_vrr_active(struct intel_atomic_state *state,
+				     struct intel_crtc *crtc)
+{
+	const struct intel_crtc_state *old_crtc_state =
+		intel_atomic_get_old_crtc_state(state, crtc);
+	const struct intel_crtc_state *new_crtc_state =
+		intel_atomic_get_new_crtc_state(state, crtc);
+
+	/* VRR will be enabled afterwards, if necessary */
+	if (intel_crtc_needs_modeset(new_crtc_state))
+		return false;
+
+	/* VRR will have been disabled during intel_pre_plane_update() */
+	return old_crtc_state->vrr.enable && !intel_crtc_vrr_disabling(state, crtc);
+}
+
+static int dsb_vblank_delay(struct intel_atomic_state *state,
+			    struct intel_crtc *crtc)
+{
+	const struct intel_crtc_state *crtc_state =
+		intel_pre_commit_crtc_state(state, crtc);
+
+	if (pre_commit_is_vrr_active(state, crtc))
+		/*
+		 * When the push is sent during vblank it will trigger
+		 * on the next scanline, hence we have up to one extra
+		 * scanline until the delayed vblank occurs after
+		 * TRANS_PUSH has been written.
+		 */
+		return intel_vrr_vblank_delay(crtc_state) + 1;
+	else
+		return intel_mode_vblank_delay(&crtc_state->hw.adjusted_mode);
+}
+
+static int dsb_vtotal(struct intel_atomic_state *state,
+		      struct intel_crtc *crtc)
+{
+	const struct intel_crtc_state *crtc_state =
+		intel_pre_commit_crtc_state(state, crtc);
+
+	if (pre_commit_is_vrr_active(state, crtc))
+		return intel_vrr_vmax_vtotal(crtc_state);
+	else
+		return intel_mode_vtotal(&crtc_state->hw.adjusted_mode);
+}
+
+static int dsb_dewake_scanline_start(struct intel_atomic_state *state,
+				     struct intel_crtc *crtc)
+{
+	struct intel_display *display = to_intel_display(state);
+	const struct intel_crtc_state *crtc_state =
+		intel_pre_commit_crtc_state(state, crtc);
+	unsigned int latency = skl_watermark_max_latency(display, 0);
+
+	return intel_mode_vdisplay(&crtc_state->hw.adjusted_mode) -
+		intel_usecs_to_scanlines(&crtc_state->hw.adjusted_mode, latency);
+}
+
+static int dsb_dewake_scanline_end(struct intel_atomic_state *state,
+				   struct intel_crtc *crtc)
+{
+	const struct intel_crtc_state *crtc_state =
+		intel_pre_commit_crtc_state(state, crtc);
+
+	return intel_mode_vdisplay(&crtc_state->hw.adjusted_mode);
+}
+
+static int dsb_scanline_to_hw(struct intel_atomic_state *state,
+			      struct intel_crtc *crtc, int scanline)
+{
+	const struct intel_crtc_state *crtc_state =
+		intel_pre_commit_crtc_state(state, crtc);
+	int vtotal = dsb_vtotal(state, crtc);
+
+	return (scanline + vtotal - intel_crtc_scanline_offset(crtc_state)) % vtotal;
+}
+
+/*
+ * Bspec suggests that we should always set DSB_SKIP_WAITS_EN. We have approach
+ * different from what is explained in Bspec on how flip is considered being
+ * complete. We are waiting for vblank in DSB and generate interrupt when it
+ * happens and this interrupt is considered as indication of completion -> we
+ * definitely do not want to skip vblank wait. We also have concern what comes
+ * to skipping vblank evasion. I.e. arming registers are latched before we have
+ * managed writing them. Due to these reasons we are not setting
+ * DSB_SKIP_WAITS_EN.
+ */
+static u32 dsb_chicken(struct intel_atomic_state *state,
+		       struct intel_crtc *crtc)
+{
+	if (pre_commit_is_vrr_active(state, crtc))
+		return DSB_CTRL_WAIT_SAFE_WINDOW |
+			DSB_CTRL_NO_WAIT_VBLANK |
+			DSB_INST_WAIT_SAFE_WINDOW |
+			DSB_INST_NO_WAIT_VBLANK;
+	else
+		return 0;
+}
+
 static bool assert_dsb_has_room(struct intel_dsb *dsb)
 {
 	struct intel_crtc *crtc = dsb->crtc;
-	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
+	struct intel_display *display = to_intel_display(crtc->base.dev);
 
 	/* each instruction is 2 dwords */
-	return !drm_WARN(&i915->drm, dsb->free_pos > dsb->size - 2,
+	return !drm_WARN(display->drm, dsb->free_pos > dsb->size - 2,
 			 "[CRTC:%d:%s] DSB %d buffer overflow\n",
 			 crtc->base.base.id, crtc->base.name, dsb->id);
 }
@@ -105,25 +208,25 @@ static bool assert_dsb_has_room(struct intel_dsb *dsb)
 static void intel_dsb_dump(struct intel_dsb *dsb)
 {
 	struct intel_crtc *crtc = dsb->crtc;
-	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
+	struct intel_display *display = to_intel_display(crtc->base.dev);
 	int i;
 
-	drm_dbg_kms(&i915->drm, "[CRTC:%d:%s] DSB %d commands {\n",
+	drm_dbg_kms(display->drm, "[CRTC:%d:%s] DSB %d commands {\n",
 		    crtc->base.base.id, crtc->base.name, dsb->id);
 	for (i = 0; i < ALIGN(dsb->free_pos, 64 / 4); i += 4)
-		drm_dbg_kms(&i915->drm,
+		drm_dbg_kms(display->drm,
 			    " 0x%08x: 0x%08x 0x%08x 0x%08x 0x%08x\n", i * 4,
 			    intel_dsb_buffer_read(&dsb->dsb_buf, i),
 			    intel_dsb_buffer_read(&dsb->dsb_buf, i + 1),
 			    intel_dsb_buffer_read(&dsb->dsb_buf, i + 2),
 			    intel_dsb_buffer_read(&dsb->dsb_buf, i + 3));
-	drm_dbg_kms(&i915->drm, "}\n");
+	drm_dbg_kms(display->drm, "}\n");
 }
 
-static bool is_dsb_busy(struct drm_i915_private *i915, enum pipe pipe,
-			enum dsb_id id)
+static bool is_dsb_busy(struct intel_display *display, enum pipe pipe,
+			enum intel_dsb_id dsb_id)
 {
-	return intel_de_read_fw(i915, DSB_CTRL(pipe, id)) & DSB_STATUS_BUSY;
+	return intel_de_read_fw(display, DSB_CTRL(pipe, dsb_id)) & DSB_STATUS_BUSY;
 }
 
 static void intel_dsb_emit(struct intel_dsb *dsb, u32 ldw, u32 udw)
@@ -135,9 +238,11 @@ static void intel_dsb_emit(struct intel_dsb *dsb, u32 ldw, u32 udw)
 	dsb->free_pos = ALIGN(dsb->free_pos, 2);
 
 	dsb->ins_start_offset = dsb->free_pos;
+	dsb->ins[0] = ldw;
+	dsb->ins[1] = udw;
 
-	intel_dsb_buffer_write(&dsb->dsb_buf, dsb->free_pos++, ldw);
-	intel_dsb_buffer_write(&dsb->dsb_buf, dsb->free_pos++, udw);
+	intel_dsb_buffer_write(&dsb->dsb_buf, dsb->free_pos++, dsb->ins[0]);
+	intel_dsb_buffer_write(&dsb->dsb_buf, dsb->free_pos++, dsb->ins[1]);
 }
 
 static bool intel_dsb_prev_ins_is_write(struct intel_dsb *dsb,
@@ -153,23 +258,12 @@ static bool intel_dsb_prev_ins_is_write(struct intel_dsb *dsb,
 	if (dsb->free_pos == 0)
 		return false;
 
-	prev_opcode = intel_dsb_buffer_read(&dsb->dsb_buf,
-					    dsb->ins_start_offset + 1) & ~DSB_REG_VALUE_MASK;
-	prev_reg =  intel_dsb_buffer_read(&dsb->dsb_buf,
-					  dsb->ins_start_offset + 1) & DSB_REG_VALUE_MASK;
+	prev_opcode = dsb->ins[1] & ~DSB_REG_VALUE_MASK;
+	prev_reg =  dsb->ins[1] & DSB_REG_VALUE_MASK;
 
 	return prev_opcode == opcode && prev_reg == i915_mmio_reg_offset(reg);
 }
 
-static bool intel_dsb_prev_ins_is_mmio_write(struct intel_dsb *dsb, i915_reg_t reg)
-{
-	/* only full byte-enables can be converted to indexed writes */
-	return intel_dsb_prev_ins_is_write(dsb,
-					   DSB_OPCODE_MMIO_WRITE << DSB_OPCODE_SHIFT |
-					   DSB_BYTE_EN << DSB_BYTE_EN_SHIFT,
-					   reg);
-}
-
 static bool intel_dsb_prev_ins_is_indexed_write(struct intel_dsb *dsb, i915_reg_t reg)
 {
 	return intel_dsb_prev_ins_is_write(dsb,
@@ -178,19 +272,21 @@ static bool intel_dsb_prev_ins_is_indexed_write(struct intel_dsb *dsb, i915_reg_
 }
 
 /**
- * intel_dsb_reg_write() - Emit register wriite to the DSB context
+ * intel_dsb_reg_write_indexed() - Emit indexed register write to the DSB context
  * @dsb: DSB context
  * @reg: register address.
  * @val: value.
  *
  * This function is used for writing register-value pair in command
  * buffer of DSB.
+ *
+ * Note that indexed writes are slower than normal MMIO writes
+ * for a small number (less than 5 or so) of writes to the same
+ * register.
  */
-void intel_dsb_reg_write(struct intel_dsb *dsb,
-			 i915_reg_t reg, u32 val)
+void intel_dsb_reg_write_indexed(struct intel_dsb *dsb,
+				 i915_reg_t reg, u32 val)
 {
-	u32 old_val;
-
 	/*
 	 * For example the buffer will look like below for 3 dwords for auto
 	 * increment register:
@@ -207,40 +303,32 @@ void intel_dsb_reg_write(struct intel_dsb *dsb,
 	 * we are writing odd no of dwords, Zeros will be added in the end for
 	 * padding.
 	 */
-	if (!intel_dsb_prev_ins_is_mmio_write(dsb, reg) &&
-	    !intel_dsb_prev_ins_is_indexed_write(dsb, reg)) {
-		intel_dsb_emit(dsb, val,
-			       (DSB_OPCODE_MMIO_WRITE << DSB_OPCODE_SHIFT) |
-			       (DSB_BYTE_EN << DSB_BYTE_EN_SHIFT) |
+	if (!intel_dsb_prev_ins_is_indexed_write(dsb, reg))
+		intel_dsb_emit(dsb, 0, /* count */
+			       (DSB_OPCODE_INDEXED_WRITE << DSB_OPCODE_SHIFT) |
 			       i915_mmio_reg_offset(reg));
-	} else {
-		if (!assert_dsb_has_room(dsb))
-			return;
-
-		/* convert to indexed write? */
-		if (intel_dsb_prev_ins_is_mmio_write(dsb, reg)) {
-			u32 prev_val = intel_dsb_buffer_read(&dsb->dsb_buf,
-							     dsb->ins_start_offset + 0);
-
-			intel_dsb_buffer_write(&dsb->dsb_buf,
-					       dsb->ins_start_offset + 0, 1); /* count */
-			intel_dsb_buffer_write(&dsb->dsb_buf, dsb->ins_start_offset + 1,
-					       (DSB_OPCODE_INDEXED_WRITE << DSB_OPCODE_SHIFT) |
-					       i915_mmio_reg_offset(reg));
-			intel_dsb_buffer_write(&dsb->dsb_buf, dsb->ins_start_offset + 2, prev_val);
-
-			dsb->free_pos++;
-		}
 
-		intel_dsb_buffer_write(&dsb->dsb_buf, dsb->free_pos++, val);
-		/* Update the count */
-		old_val = intel_dsb_buffer_read(&dsb->dsb_buf, dsb->ins_start_offset);
-		intel_dsb_buffer_write(&dsb->dsb_buf, dsb->ins_start_offset, old_val + 1);
+	if (!assert_dsb_has_room(dsb))
+		return;
 
-		/* if number of data words is odd, then the last dword should be 0.*/
-		if (dsb->free_pos & 0x1)
-			intel_dsb_buffer_write(&dsb->dsb_buf, dsb->free_pos, 0);
-	}
+	/* Update the count */
+	dsb->ins[0]++;
+	intel_dsb_buffer_write(&dsb->dsb_buf, dsb->ins_start_offset + 0,
+			       dsb->ins[0]);
+
+	intel_dsb_buffer_write(&dsb->dsb_buf, dsb->free_pos++, val);
+	/* if number of data words is odd, then the last dword should be 0.*/
+	if (dsb->free_pos & 0x1)
+		intel_dsb_buffer_write(&dsb->dsb_buf, dsb->free_pos, 0);
+}
+
+void intel_dsb_reg_write(struct intel_dsb *dsb,
+			 i915_reg_t reg, u32 val)
+{
+	intel_dsb_emit(dsb, val,
+		       (DSB_OPCODE_MMIO_WRITE << DSB_OPCODE_SHIFT) |
+		       (DSB_BYTE_EN << DSB_BYTE_EN_SHIFT) |
+		       i915_mmio_reg_offset(reg));
 }
 
 static u32 intel_dsb_mask_to_byte_en(u32 mask)
@@ -290,6 +378,117 @@ void intel_dsb_nonpost_end(struct intel_dsb *dsb)
 	intel_dsb_noop(dsb, 4);
 }
 
+void intel_dsb_interrupt(struct intel_dsb *dsb)
+{
+	intel_dsb_emit(dsb, 0,
+		       DSB_OPCODE_INTERRUPT << DSB_OPCODE_SHIFT);
+}
+
+void intel_dsb_wait_usec(struct intel_dsb *dsb, int count)
+{
+	/* +1 to make sure we never wait less time than asked for */
+	intel_dsb_emit(dsb, count + 1,
+		       DSB_OPCODE_WAIT_USEC << DSB_OPCODE_SHIFT);
+}
+
+void intel_dsb_wait_vblanks(struct intel_dsb *dsb, int count)
+{
+	intel_dsb_emit(dsb, count,
+		       DSB_OPCODE_WAIT_VBLANKS << DSB_OPCODE_SHIFT);
+}
+
+static void intel_dsb_emit_wait_dsl(struct intel_dsb *dsb,
+				    u32 opcode, int lower, int upper)
+{
+	u64 window = ((u64)upper << DSB_SCANLINE_UPPER_SHIFT) |
+		((u64)lower << DSB_SCANLINE_LOWER_SHIFT);
+
+	intel_dsb_emit(dsb, lower_32_bits(window),
+		       (opcode << DSB_OPCODE_SHIFT) |
+		       upper_32_bits(window));
+}
+
+static void intel_dsb_wait_dsl(struct intel_atomic_state *state,
+			       struct intel_dsb *dsb,
+			       int lower_in, int upper_in,
+			       int lower_out, int upper_out)
+{
+	struct intel_crtc *crtc = dsb->crtc;
+
+	lower_in = dsb_scanline_to_hw(state, crtc, lower_in);
+	upper_in = dsb_scanline_to_hw(state, crtc, upper_in);
+
+	lower_out = dsb_scanline_to_hw(state, crtc, lower_out);
+	upper_out = dsb_scanline_to_hw(state, crtc, upper_out);
+
+	if (upper_in >= lower_in)
+		intel_dsb_emit_wait_dsl(dsb, DSB_OPCODE_WAIT_DSL_IN,
+					lower_in, upper_in);
+	else if (upper_out >= lower_out)
+		intel_dsb_emit_wait_dsl(dsb, DSB_OPCODE_WAIT_DSL_OUT,
+					lower_out, upper_out);
+	else
+		drm_WARN_ON(crtc->base.dev, 1); /* assert_dsl_ok() should have caught it already */
+}
+
+static void assert_dsl_ok(struct intel_atomic_state *state,
+			  struct intel_dsb *dsb,
+			  int start, int end)
+{
+	struct intel_crtc *crtc = dsb->crtc;
+	int vtotal = dsb_vtotal(state, crtc);
+
+	/*
+	 * Waiting for the entire frame doesn't make sense,
+	 * (IN==don't wait, OUT=wait forever).
+	 */
+	drm_WARN(crtc->base.dev, (end - start + vtotal) % vtotal == vtotal - 1,
+		 "[CRTC:%d:%s] DSB %d bad scanline window wait: %d-%d (vt=%d)\n",
+		 crtc->base.base.id, crtc->base.name, dsb->id,
+		 start, end, vtotal);
+}
+
+void intel_dsb_wait_scanline_in(struct intel_atomic_state *state,
+				struct intel_dsb *dsb,
+				int start, int end)
+{
+	assert_dsl_ok(state, dsb, start, end);
+
+	intel_dsb_wait_dsl(state, dsb,
+			   start, end,
+			   end + 1, start - 1);
+}
+
+void intel_dsb_wait_scanline_out(struct intel_atomic_state *state,
+				 struct intel_dsb *dsb,
+				 int start, int end)
+{
+	assert_dsl_ok(state, dsb, start, end);
+
+	intel_dsb_wait_dsl(state, dsb,
+			   end + 1, start - 1,
+			   start, end);
+}
+
+void intel_dsb_poll(struct intel_dsb *dsb,
+		    i915_reg_t reg, u32 mask, u32 val,
+		    int wait_us, int count)
+{
+	struct intel_crtc *crtc = dsb->crtc;
+	enum pipe pipe = crtc->pipe;
+
+	intel_dsb_reg_write(dsb, DSB_POLLMASK(pipe, dsb->id), mask);
+	intel_dsb_reg_write(dsb, DSB_POLLFUNC(pipe, dsb->id),
+			    DSB_POLL_ENABLE |
+			    DSB_POLL_WAIT(wait_us) | DSB_POLL_COUNT(count));
+
+	intel_dsb_noop(dsb, 5);
+
+	intel_dsb_emit(dsb, val,
+		       (DSB_OPCODE_POLL << DSB_OPCODE_SHIFT) |
+		       i915_mmio_reg_offset(reg));
+}
+
 static void intel_dsb_align_tail(struct intel_dsb *dsb)
 {
 	u32 aligned_tail, tail;
@@ -311,8 +510,10 @@ void intel_dsb_finish(struct intel_dsb *dsb)
 	/*
 	 * DSB_FORCE_DEWAKE remains active even after DSB is
 	 * disabled, so make sure to clear it (if set during
-	 * intel_dsb_commit()).
+	 * intel_dsb_commit()). And clear DSB_ENABLE_DEWAKE as
+	 * well for good measure.
 	 */
+	intel_dsb_reg_write(dsb, DSB_PMCTRL(crtc->pipe, dsb->id), 0);
 	intel_dsb_reg_write_masked(dsb, DSB_PMCTRL_2(crtc->pipe, dsb->id),
 				   DSB_FORCE_DEWAKE, 0);
 
@@ -321,55 +522,195 @@ void intel_dsb_finish(struct intel_dsb *dsb)
 	intel_dsb_buffer_flush_map(&dsb->dsb_buf);
 }
 
-static int intel_dsb_dewake_scanline(const struct intel_crtc_state *crtc_state)
+static u32 dsb_error_int_status(struct intel_display *display)
+{
+	u32 errors;
+
+	errors = DSB_GTT_FAULT_INT_STATUS |
+		DSB_RSPTIMEOUT_INT_STATUS |
+		DSB_POLL_ERR_INT_STATUS;
+
+	/*
+	 * All the non-existing status bits operate as
+	 * normal r/w bits, so any attempt to clear them
+	 * will just end up setting them. Never do that so
+	 * we won't mistake them for actual error interrupts.
+	 */
+	if (DISPLAY_VER(display) >= 14)
+		errors |= DSB_ATS_FAULT_INT_STATUS;
+
+	return errors;
+}
+
+static u32 dsb_error_int_en(struct intel_display *display)
 {
-	struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
-	const struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode;
-	unsigned int latency = skl_watermark_max_latency(i915, 0);
-	int vblank_start;
+	u32 errors;
+
+	errors = DSB_GTT_FAULT_INT_EN |
+		DSB_RSPTIMEOUT_INT_EN |
+		DSB_POLL_ERR_INT_EN;
+
+	if (DISPLAY_VER(display) >= 14)
+		errors |= DSB_ATS_FAULT_INT_EN;
+
+	return errors;
+}
+
+void intel_dsb_vblank_evade(struct intel_atomic_state *state,
+			    struct intel_dsb *dsb)
+{
+	struct intel_crtc *crtc = dsb->crtc;
+	const struct intel_crtc_state *crtc_state =
+		intel_pre_commit_crtc_state(state, crtc);
+	/* FIXME calibrate sensibly */
+	int latency = intel_usecs_to_scanlines(&crtc_state->hw.adjusted_mode, 20);
+	int start, end;
 
-	if (crtc_state->vrr.enable) {
-		vblank_start = intel_vrr_vmin_vblank_start(crtc_state);
+	/*
+	 * PIPEDSL is reading as 0 when in SRDENT(PSR1) or DEEP_SLEEP(PSR2). On
+	 * wake-up scanline counting starts from vblank_start - 1. We don't know
+	 * if wake-up is already ongoing when evasion starts. In worst case
+	 * PIPEDSL could start reading valid value right after checking the
+	 * scanline. In this scenario we wouldn't have enough time to write all
+	 * registers. To tackle this evade scanline 0 as well. As a drawback we
+	 * have 1 frame delay in flip when waking up.
+	 */
+	if (crtc_state->has_psr)
+		intel_dsb_emit_wait_dsl(dsb, DSB_OPCODE_WAIT_DSL_OUT, 0, 0);
+
+	if (pre_commit_is_vrr_active(state, crtc)) {
+		int vblank_delay = intel_vrr_vblank_delay(crtc_state);
+
+		end = intel_vrr_vmin_vblank_start(crtc_state);
+		start = end - vblank_delay - latency;
+		intel_dsb_wait_scanline_out(state, dsb, start, end);
+
+		end = intel_vrr_vmax_vblank_start(crtc_state);
+		start = end - vblank_delay - latency;
+		intel_dsb_wait_scanline_out(state, dsb, start, end);
 	} else {
-		vblank_start = adjusted_mode->crtc_vblank_start;
+		int vblank_delay = intel_mode_vblank_delay(&crtc_state->hw.adjusted_mode);
+
+		end = intel_mode_vblank_start(&crtc_state->hw.adjusted_mode);
+		start = end - vblank_delay - latency;
+		intel_dsb_wait_scanline_out(state, dsb, start, end);
+	}
+}
+
+static void _intel_dsb_chain(struct intel_atomic_state *state,
+			     struct intel_dsb *dsb,
+			     struct intel_dsb *chained_dsb,
+			     u32 ctrl)
+{
+	struct intel_display *display = to_intel_display(state->base.dev);
+	struct intel_crtc *crtc = dsb->crtc;
+	enum pipe pipe = crtc->pipe;
+	u32 tail;
+
+	if (drm_WARN_ON(display->drm, dsb->id == chained_dsb->id))
+		return;
+
+	tail = chained_dsb->free_pos * 4;
+	if (drm_WARN_ON(display->drm, !IS_ALIGNED(tail, CACHELINE_BYTES)))
+		return;
+
+	intel_dsb_reg_write(dsb, DSB_CTRL(pipe, chained_dsb->id),
+			    ctrl | DSB_ENABLE);
+
+	intel_dsb_reg_write(dsb, DSB_CHICKEN(pipe, chained_dsb->id),
+			    dsb_chicken(state, crtc));
+
+	intel_dsb_reg_write(dsb, DSB_INTERRUPT(pipe, chained_dsb->id),
+			    dsb_error_int_status(display) | DSB_PROG_INT_STATUS |
+			    dsb_error_int_en(display) | DSB_PROG_INT_EN);
+
+	if (ctrl & DSB_WAIT_FOR_VBLANK) {
+		int dewake_scanline = dsb_dewake_scanline_start(state, crtc);
+		int hw_dewake_scanline = dsb_scanline_to_hw(state, crtc, dewake_scanline);
+
+		intel_dsb_reg_write(dsb, DSB_PMCTRL(pipe, chained_dsb->id),
+				    DSB_ENABLE_DEWAKE |
+				    DSB_SCANLINE_FOR_DEWAKE(hw_dewake_scanline));
+	}
+
+	intel_dsb_reg_write(dsb, DSB_HEAD(pipe, chained_dsb->id),
+			    intel_dsb_buffer_ggtt_offset(&chained_dsb->dsb_buf));
+
+	intel_dsb_reg_write(dsb, DSB_TAIL(pipe, chained_dsb->id),
+			    intel_dsb_buffer_ggtt_offset(&chained_dsb->dsb_buf) + tail);
+
+	if (ctrl & DSB_WAIT_FOR_VBLANK) {
+		/*
+		 * Keep DEwake alive via the first DSB, in
+		 * case we're already past dewake_scanline,
+		 * and thus DSB_ENABLE_DEWAKE on the second
+		 * DSB won't do its job.
+		 */
+		intel_dsb_reg_write_masked(dsb, DSB_PMCTRL_2(pipe, dsb->id),
+					   DSB_FORCE_DEWAKE, DSB_FORCE_DEWAKE);
 
-		if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE)
-			vblank_start = DIV_ROUND_UP(vblank_start, 2);
+		intel_dsb_wait_scanline_out(state, dsb,
+					    dsb_dewake_scanline_start(state, crtc),
+					    dsb_dewake_scanline_end(state, crtc));
 	}
+}
+
+void intel_dsb_chain(struct intel_atomic_state *state,
+		     struct intel_dsb *dsb,
+		     struct intel_dsb *chained_dsb,
+		     bool wait_for_vblank)
+{
+	_intel_dsb_chain(state, dsb, chained_dsb,
+			 wait_for_vblank ? DSB_WAIT_FOR_VBLANK : 0);
+}
+
+void intel_dsb_wait_vblank_delay(struct intel_atomic_state *state,
+				 struct intel_dsb *dsb)
+{
+	struct intel_crtc *crtc = dsb->crtc;
+	const struct intel_crtc_state *crtc_state =
+		intel_pre_commit_crtc_state(state, crtc);
+	int usecs = intel_scanlines_to_usecs(&crtc_state->hw.adjusted_mode,
+					     dsb_vblank_delay(state, crtc));
 
-	return max(0, vblank_start - intel_usecs_to_scanlines(adjusted_mode, latency));
+	intel_dsb_wait_usec(dsb, usecs);
 }
 
 static void _intel_dsb_commit(struct intel_dsb *dsb, u32 ctrl,
-			      int dewake_scanline)
+			      int hw_dewake_scanline)
 {
 	struct intel_crtc *crtc = dsb->crtc;
-	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
+	struct intel_display *display = to_intel_display(crtc->base.dev);
 	enum pipe pipe = crtc->pipe;
 	u32 tail;
 
 	tail = dsb->free_pos * 4;
-	if (drm_WARN_ON(&dev_priv->drm, !IS_ALIGNED(tail, CACHELINE_BYTES)))
+	if (drm_WARN_ON(display->drm, !IS_ALIGNED(tail, CACHELINE_BYTES)))
 		return;
 
-	if (is_dsb_busy(dev_priv, pipe, dsb->id)) {
-		drm_err(&dev_priv->drm, "[CRTC:%d:%s] DSB %d is busy\n",
+	if (is_dsb_busy(display, pipe, dsb->id)) {
+		drm_err(display->drm, "[CRTC:%d:%s] DSB %d is busy\n",
 			crtc->base.base.id, crtc->base.name, dsb->id);
 		return;
 	}
 
-	intel_de_write_fw(dev_priv, DSB_CTRL(pipe, dsb->id),
+	intel_de_write_fw(display, DSB_CTRL(pipe, dsb->id),
 			  ctrl | DSB_ENABLE);
 
-	intel_de_write_fw(dev_priv, DSB_HEAD(pipe, dsb->id),
-			  intel_dsb_buffer_ggtt_offset(&dsb->dsb_buf));
+	intel_de_write_fw(display, DSB_CHICKEN(pipe, dsb->id),
+			  dsb->chicken);
 
-	if (dewake_scanline >= 0) {
-		int diff, hw_dewake_scanline;
+	intel_de_write_fw(display, DSB_INTERRUPT(pipe, dsb->id),
+			  dsb_error_int_status(display) | DSB_PROG_INT_STATUS |
+			  dsb_error_int_en(display) | DSB_PROG_INT_EN);
 
-		hw_dewake_scanline = intel_crtc_scanline_to_hw(crtc, dewake_scanline);
+	intel_de_write_fw(display, DSB_HEAD(pipe, dsb->id),
+			  intel_dsb_buffer_ggtt_offset(&dsb->dsb_buf));
+
+	if (hw_dewake_scanline >= 0) {
+		int diff, position;
 
-		intel_de_write_fw(dev_priv, DSB_PMCTRL(pipe, dsb->id),
+		intel_de_write_fw(display, DSB_PMCTRL(pipe, dsb->id),
 				  DSB_ENABLE_DEWAKE |
 				  DSB_SCANLINE_FOR_DEWAKE(hw_dewake_scanline));
 
@@ -377,13 +718,15 @@ static void _intel_dsb_commit(struct intel_dsb *dsb, u32 ctrl,
 		 * Force DEwake immediately if we're already past
 		 * or close to racing past the target scanline.
 		 */
-		diff = dewake_scanline - intel_get_crtc_scanline(crtc);
-		intel_de_write_fw(dev_priv, DSB_PMCTRL_2(pipe, dsb->id),
+		position = intel_de_read_fw(display, PIPEDSL(display, pipe)) & PIPEDSL_LINE_MASK;
+
+		diff = hw_dewake_scanline - position;
+		intel_de_write_fw(display, DSB_PMCTRL_2(pipe, dsb->id),
 				  (diff >= 0 && diff < 5 ? DSB_FORCE_DEWAKE : 0) |
 				  DSB_BLOCK_DEWAKE_EXTENSION);
 	}
 
-	intel_de_write_fw(dev_priv, DSB_TAIL(pipe, dsb->id),
+	intel_de_write_fw(display, DSB_TAIL(pipe, dsb->id),
 			  intel_dsb_buffer_ggtt_offset(&dsb->dsb_buf) + tail);
 }
 
@@ -399,27 +742,27 @@ void intel_dsb_commit(struct intel_dsb *dsb,
 {
 	_intel_dsb_commit(dsb,
 			  wait_for_vblank ? DSB_WAIT_FOR_VBLANK : 0,
-			  wait_for_vblank ? dsb->dewake_scanline : -1);
+			  wait_for_vblank ? dsb->hw_dewake_scanline : -1);
 }
 
 void intel_dsb_wait(struct intel_dsb *dsb)
 {
 	struct intel_crtc *crtc = dsb->crtc;
-	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
+	struct intel_display *display = to_intel_display(crtc->base.dev);
 	enum pipe pipe = crtc->pipe;
 
-	if (wait_for(!is_dsb_busy(dev_priv, pipe, dsb->id), 1)) {
+	if (wait_for(!is_dsb_busy(display, pipe, dsb->id), 1)) {
 		u32 offset = intel_dsb_buffer_ggtt_offset(&dsb->dsb_buf);
 
-		intel_de_write_fw(dev_priv, DSB_CTRL(pipe, dsb->id),
+		intel_de_write_fw(display, DSB_CTRL(pipe, dsb->id),
 				  DSB_ENABLE | DSB_HALT);
 
-		drm_err(&dev_priv->drm,
+		drm_err(display->drm,
 			"[CRTC:%d:%s] DSB %d timed out waiting for idle (current head=0x%x, head=0x%x, tail=0x%x)\n",
 			crtc->base.base.id, crtc->base.name, dsb->id,
-			intel_de_read_fw(dev_priv, DSB_CURRENT_HEAD(pipe, dsb->id)) - offset,
-			intel_de_read_fw(dev_priv, DSB_HEAD(pipe, dsb->id)) - offset,
-			intel_de_read_fw(dev_priv, DSB_TAIL(pipe, dsb->id)) - offset);
+			intel_de_read_fw(display, DSB_CURRENT_HEAD(pipe, dsb->id)) - offset,
+			intel_de_read_fw(display, DSB_HEAD(pipe, dsb->id)) - offset,
+			intel_de_read_fw(display, DSB_TAIL(pipe, dsb->id)) - offset);
 
 		intel_dsb_dump(dsb);
 	}
@@ -427,12 +770,20 @@ void intel_dsb_wait(struct intel_dsb *dsb)
 	/* Attempt to reset it */
 	dsb->free_pos = 0;
 	dsb->ins_start_offset = 0;
-	intel_de_write_fw(dev_priv, DSB_CTRL(pipe, dsb->id), 0);
+	dsb->ins[0] = 0;
+	dsb->ins[1] = 0;
+
+	intel_de_write_fw(display, DSB_CTRL(pipe, dsb->id), 0);
+
+	intel_de_write_fw(display, DSB_INTERRUPT(pipe, dsb->id),
+			  dsb_error_int_status(display) | DSB_PROG_INT_STATUS);
 }
 
 /**
  * intel_dsb_prepare() - Allocate, pin and map the DSB command buffer.
- * @crtc_state: the CRTC state
+ * @state: the atomic state
+ * @crtc: the CRTC
+ * @dsb_id: the DSB engine to use
  * @max_cmds: number of commands we need to fit into command buffer
  *
  * This function prepare the command buffer which is used to store dsb
@@ -441,27 +792,27 @@ void intel_dsb_wait(struct intel_dsb *dsb)
  * Returns:
  * DSB context, NULL on failure
  */
-struct intel_dsb *intel_dsb_prepare(const struct intel_crtc_state *crtc_state,
+struct intel_dsb *intel_dsb_prepare(struct intel_atomic_state *state,
+				    struct intel_crtc *crtc,
+				    enum intel_dsb_id dsb_id,
 				    unsigned int max_cmds)
 {
-	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
-	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
-	intel_wakeref_t wakeref;
+	struct intel_display *display = to_intel_display(state);
+	struct ref_tracker *wakeref;
 	struct intel_dsb *dsb;
 	unsigned int size;
 
-	if (!HAS_DSB(i915))
+	if (!HAS_DSB(display))
 		return NULL;
 
-	/* TODO: DSB is broken in Xe KMD, so disabling it until fixed */
-	if (!IS_ENABLED(I915))
+	if (!display->params.enable_dsb)
 		return NULL;
 
 	dsb = kzalloc(sizeof(*dsb), GFP_KERNEL);
 	if (!dsb)
 		goto out;
 
-	wakeref = intel_runtime_pm_get(&i915->runtime_pm);
+	wakeref = intel_display_rpm_get(display);
 
 	/* ~1 qword per instruction, full cachelines */
 	size = ALIGN(max_cmds * 8, CACHELINE_BYTES);
@@ -469,24 +820,25 @@ struct intel_dsb *intel_dsb_prepare(const struct intel_crtc_state *crtc_state,
 	if (!intel_dsb_buffer_create(crtc, &dsb->dsb_buf, size))
 		goto out_put_rpm;
 
-	intel_runtime_pm_put(&i915->runtime_pm, wakeref);
+	intel_display_rpm_put(display, wakeref);
 
-	dsb->id = DSB1;
+	dsb->id = dsb_id;
 	dsb->crtc = crtc;
 	dsb->size = size / 4; /* in dwords */
-	dsb->free_pos = 0;
-	dsb->ins_start_offset = 0;
-	dsb->dewake_scanline = intel_dsb_dewake_scanline(crtc_state);
+
+	dsb->chicken = dsb_chicken(state, crtc);
+	dsb->hw_dewake_scanline =
+		dsb_scanline_to_hw(state, crtc, dsb_dewake_scanline_start(state, crtc));
 
 	return dsb;
 
 out_put_rpm:
-	intel_runtime_pm_put(&i915->runtime_pm, wakeref);
+	intel_display_rpm_put(display, wakeref);
 	kfree(dsb);
 out:
-	drm_info_once(&i915->drm,
+	drm_info_once(display->drm,
 		      "[CRTC:%d:%s] DSB %d queue setup failed, will fallback to MMIO for display HW programming\n",
-		      crtc->base.base.id, crtc->base.name, DSB1);
+		      crtc->base.base.id, crtc->base.name, dsb_id);
 
 	return NULL;
 }
@@ -503,3 +855,44 @@ void intel_dsb_cleanup(struct intel_dsb *dsb)
 	intel_dsb_buffer_cleanup(&dsb->dsb_buf);
 	kfree(dsb);
 }
+
+void intel_dsb_irq_handler(struct intel_display *display,
+			   enum pipe pipe, enum intel_dsb_id dsb_id)
+{
+	struct intel_crtc *crtc = intel_crtc_for_pipe(display, pipe);
+	u32 tmp, errors;
+
+	tmp = intel_de_read_fw(display, DSB_INTERRUPT(pipe, dsb_id));
+	intel_de_write_fw(display, DSB_INTERRUPT(pipe, dsb_id), tmp);
+
+	if (tmp & DSB_PROG_INT_STATUS) {
+		spin_lock(&display->drm->event_lock);
+
+		if (crtc->dsb_event) {
+			/*
+			 * Update vblank counter/timestamp in case it
+			 * hasn't been done yet for this frame.
+			 */
+			drm_crtc_accurate_vblank_count(&crtc->base);
+
+			drm_crtc_send_vblank_event(&crtc->base, crtc->dsb_event);
+			crtc->dsb_event = NULL;
+		}
+
+		spin_unlock(&display->drm->event_lock);
+	}
+
+	errors = tmp & dsb_error_int_status(display);
+	if (errors & DSB_ATS_FAULT_INT_STATUS)
+		drm_err(display->drm, "[CRTC:%d:%s] DSB %d ATS fault\n",
+			crtc->base.base.id, crtc->base.name, dsb_id);
+	if (errors & DSB_GTT_FAULT_INT_STATUS)
+		drm_err(display->drm, "[CRTC:%d:%s] DSB %d GTT fault\n",
+			crtc->base.base.id, crtc->base.name, dsb_id);
+	if (errors & DSB_RSPTIMEOUT_INT_STATUS)
+		drm_err(display->drm, "[CRTC:%d:%s] DSB %d response timeout\n",
+			crtc->base.base.id, crtc->base.name, dsb_id);
+	if (errors & DSB_POLL_ERR_INT_STATUS)
+		drm_err(display->drm, "[CRTC:%d:%s] DSB %d poll error\n",
+			crtc->base.base.id, crtc->base.name, dsb_id);
+}