Merge drm/drm-next into drm-misc-next

Now that 4.19-rc1 is cut, backmerge it into -misc-next.

Signed-off-by: Sean Paul <seanpaul@chromium.org>

702 files changed, 65132 insertions, 15246 deletions

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index 71b9b861f60e..447c4c7a36d6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -73,6 +73,8 @@
 #include "amdgpu_virt.h"
 #include "amdgpu_gart.h"
 #include "amdgpu_debugfs.h"
+#include "amdgpu_job.h"
+#include "amdgpu_bo_list.h"
 
 /*
  * Modules parameters.
@@ -105,11 +107,8 @@ extern int amdgpu_vm_fault_stop;
 extern int amdgpu_vm_debug;
 extern int amdgpu_vm_update_mode;
 extern int amdgpu_dc;
-extern int amdgpu_dc_log;
 extern int amdgpu_sched_jobs;
 extern int amdgpu_sched_hw_submission;
-extern int amdgpu_no_evict;
-extern int amdgpu_direct_gma_size;
 extern uint amdgpu_pcie_gen_cap;
 extern uint amdgpu_pcie_lane_cap;
 extern uint amdgpu_cg_mask;
@@ -190,6 +189,7 @@ struct amdgpu_job;
 struct amdgpu_irq_src;
 struct amdgpu_fpriv;
 struct amdgpu_bo_va_mapping;
+struct amdgpu_atif;
 
 enum amdgpu_cp_irq {
 	AMDGPU_CP_IRQ_GFX_EOP = 0,
@@ -599,17 +599,6 @@ struct amdgpu_ib {
 
 extern const struct drm_sched_backend_ops amdgpu_sched_ops;
 
-int amdgpu_job_alloc(struct amdgpu_device *adev, unsigned num_ibs,
-		     struct amdgpu_job **job, struct amdgpu_vm *vm);
-int amdgpu_job_alloc_with_ib(struct amdgpu_device *adev, unsigned size,
-			     struct amdgpu_job **job);
-
-void amdgpu_job_free_resources(struct amdgpu_job *job);
-void amdgpu_job_free(struct amdgpu_job *job);
-int amdgpu_job_submit(struct amdgpu_job *job, struct amdgpu_ring *ring,
-		      struct drm_sched_entity *entity, void *owner,
-		      struct dma_fence **f);
-
 /*
  * Queue manager
  */
@@ -683,8 +672,8 @@ int amdgpu_ctx_ioctl(struct drm_device *dev, void *data,
 int amdgpu_ctx_wait_prev_fence(struct amdgpu_ctx *ctx, unsigned ring_id);
 
 void amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr);
-void amdgpu_ctx_mgr_entity_cleanup(struct amdgpu_ctx_mgr *mgr);
 void amdgpu_ctx_mgr_entity_fini(struct amdgpu_ctx_mgr *mgr);
+void amdgpu_ctx_mgr_entity_flush(struct amdgpu_ctx_mgr *mgr);
 void amdgpu_ctx_mgr_fini(struct amdgpu_ctx_mgr *mgr);
 
 
@@ -702,37 +691,6 @@ struct amdgpu_fpriv {
 };
 
 /*
- * residency list
- */
-struct amdgpu_bo_list_entry {
-	struct amdgpu_bo		*robj;
-	struct ttm_validate_buffer	tv;
-	struct amdgpu_bo_va		*bo_va;
-	uint32_t			priority;
-	struct page			**user_pages;
-	int				user_invalidated;
-};
-
-struct amdgpu_bo_list {
-	struct mutex lock;
-	struct rcu_head rhead;
-	struct kref refcount;
-	struct amdgpu_bo *gds_obj;
-	struct amdgpu_bo *gws_obj;
-	struct amdgpu_bo *oa_obj;
-	unsigned first_userptr;
-	unsigned num_entries;
-	struct amdgpu_bo_list_entry *array;
-};
-
-struct amdgpu_bo_list *
-amdgpu_bo_list_get(struct amdgpu_fpriv *fpriv, int id);
-void amdgpu_bo_list_get_list(struct amdgpu_bo_list *list,
-			     struct list_head *validated);
-void amdgpu_bo_list_put(struct amdgpu_bo_list *list);
-void amdgpu_bo_list_free(struct amdgpu_bo_list *list);
-
-/*
  * GFX stuff
  */
 #include "clearstate_defs.h"
@@ -930,6 +888,11 @@ struct amdgpu_ngg {
 	bool			init;
 };
 
+struct sq_work {
+	struct work_struct	work;
+	unsigned ih_data;
+};
+
 struct amdgpu_gfx {
 	struct mutex			gpu_clock_mutex;
 	struct amdgpu_gfx_config	config;
@@ -970,6 +933,8 @@ struct amdgpu_gfx {
 	struct amdgpu_irq_src		priv_inst_irq;
 	struct amdgpu_irq_src		cp_ecc_error_irq;
 	struct amdgpu_irq_src		sq_irq;
+	struct sq_work			sq_work;
+
 	/* gfx status */
 	uint32_t			gfx_current_status;
 	/* ce ram size*/
@@ -1021,6 +986,7 @@ struct amdgpu_cs_parser {
 
 	/* scheduler job object */
 	struct amdgpu_job	*job;
+	struct amdgpu_ring	*ring;
 
 	/* buffer objects */
 	struct ww_acquire_ctx		ticket;
@@ -1042,40 +1008,6 @@ struct amdgpu_cs_parser {
 	struct drm_syncobj **post_dep_syncobjs;
 };
 
-#define AMDGPU_PREAMBLE_IB_PRESENT          (1 << 0) /* bit set means command submit involves a preamble IB */
-#define AMDGPU_PREAMBLE_IB_PRESENT_FIRST    (1 << 1) /* bit set means preamble IB is first presented in belonging context */
-#define AMDGPU_HAVE_CTX_SWITCH              (1 << 2) /* bit set means context switch occured */
-
-struct amdgpu_job {
-	struct drm_sched_job    base;
-	struct amdgpu_device	*adev;
-	struct amdgpu_vm	*vm;
-	struct amdgpu_ring	*ring;
-	struct amdgpu_sync	sync;
-	struct amdgpu_sync	sched_sync;
-	struct amdgpu_ib	*ibs;
-	struct dma_fence	*fence; /* the hw fence */
-	uint32_t		preamble_status;
-	uint32_t		num_ibs;
-	void			*owner;
-	uint64_t		fence_ctx; /* the fence_context this job uses */
-	bool                    vm_needs_flush;
-	uint64_t		vm_pd_addr;
-	unsigned		vmid;
-	unsigned		pasid;
-	uint32_t		gds_base, gds_size;
-	uint32_t		gws_base, gws_size;
-	uint32_t		oa_base, oa_size;
-	uint32_t		vram_lost_counter;
-
-	/* user fence handling */
-	uint64_t		uf_addr;
-	uint64_t		uf_sequence;
-
-};
-#define to_amdgpu_job(sched_job)		\
-		container_of((sched_job), struct amdgpu_job, base)
-
 static inline u32 amdgpu_get_ib_value(struct amdgpu_cs_parser *p,
 				      uint32_t ib_idx, int idx)
 {
@@ -1271,43 +1203,6 @@ struct amdgpu_vram_scratch {
 /*
  * ACPI
  */
-struct amdgpu_atif_notification_cfg {
-	bool enabled;
-	int command_code;
-};
-
-struct amdgpu_atif_notifications {
-	bool display_switch;
-	bool expansion_mode_change;
-	bool thermal_state;
-	bool forced_power_state;
-	bool system_power_state;
-	bool display_conf_change;
-	bool px_gfx_switch;
-	bool brightness_change;
-	bool dgpu_display_event;
-};
-
-struct amdgpu_atif_functions {
-	bool system_params;
-	bool sbios_requests;
-	bool select_active_disp;
-	bool lid_state;
-	bool get_tv_standard;
-	bool set_tv_standard;
-	bool get_panel_expansion_mode;
-	bool set_panel_expansion_mode;
-	bool temperature_change;
-	bool graphics_device_types;
-};
-
-struct amdgpu_atif {
-	struct amdgpu_atif_notifications notifications;
-	struct amdgpu_atif_functions functions;
-	struct amdgpu_atif_notification_cfg notification_cfg;
-	struct amdgpu_encoder *encoder_for_bl;
-};
-
 struct amdgpu_atcs_functions {
 	bool get_ext_state;
 	bool pcie_perf_req;
@@ -1427,6 +1322,7 @@ enum amd_hw_ip_block_type {
 	PWR_HWIP,
 	NBIF_HWIP,
 	THM_HWIP,
+	CLK_HWIP,
 	MAX_HWIP
 };
 
@@ -1468,7 +1364,7 @@ struct amdgpu_device {
 #if defined(CONFIG_DEBUG_FS)
 	struct dentry			*debugfs_regs[AMDGPU_DEBUGFS_MAX_COMPONENTS];
 #endif
-	struct amdgpu_atif		atif;
+	struct amdgpu_atif		*atif;
 	struct amdgpu_atcs		atcs;
 	struct mutex			srbm_mutex;
 	/* GRBM index mutex. Protects concurrent access to GRBM index */
@@ -1617,9 +1513,9 @@ struct amdgpu_device {
 	DECLARE_HASHTABLE(mn_hash, 7);
 
 	/* tracking pinned memory */
-	u64 vram_pin_size;
-	u64 invisible_pin_size;
-	u64 gart_pin_size;
+	atomic64_t vram_pin_size;
+	atomic64_t visible_pin_size;
+	atomic64_t gart_pin_size;
 
 	/* amdkfd interface */
 	struct kfd_dev          *kfd;
@@ -1814,6 +1710,7 @@ amdgpu_get_sdma_instance(struct amdgpu_ring *ring)
 #define amdgpu_vm_write_pte(adev, ib, pe, value, count, incr) ((adev)->vm_manager.vm_pte_funcs->write_pte((ib), (pe), (value), (count), (incr)))
 #define amdgpu_vm_set_pte_pde(adev, ib, pe, addr, count, incr, flags) ((adev)->vm_manager.vm_pte_funcs->set_pte_pde((ib), (pe), (addr), (count), (incr), (flags)))
 #define amdgpu_ring_parse_cs(r, p, ib) ((r)->funcs->parse_cs((p), (ib)))
+#define amdgpu_ring_patch_cs_in_place(r, p, ib) ((r)->funcs->patch_cs_in_place((p), (ib)))
 #define amdgpu_ring_test_ring(r) (r)->funcs->test_ring((r))
 #define amdgpu_ring_test_ib(r, t) (r)->funcs->test_ib((r), (t))
 #define amdgpu_ring_get_rptr(r) (r)->funcs->get_rptr((r))
@@ -1867,8 +1764,6 @@ void amdgpu_display_update_priority(struct amdgpu_device *adev);
 
 void amdgpu_cs_report_moved_bytes(struct amdgpu_device *adev, u64 num_bytes,
 				  u64 num_vis_bytes);
-void amdgpu_ttm_placement_from_domain(struct amdgpu_bo *abo, u32 domain);
-bool amdgpu_ttm_bo_is_amdgpu_bo(struct ttm_buffer_object *bo);
 void amdgpu_device_vram_location(struct amdgpu_device *adev,
 				 struct amdgpu_gmc *mc, u64 base);
 void amdgpu_device_gart_location(struct amdgpu_device *adev,
@@ -1896,6 +1791,12 @@ static inline bool amdgpu_atpx_dgpu_req_power_for_displays(void) { return false;
 static inline bool amdgpu_has_atpx(void) { return false; }
 #endif
 
+#if defined(CONFIG_VGA_SWITCHEROO) && defined(CONFIG_ACPI)
+void *amdgpu_atpx_get_dhandle(void);
+#else
+static inline void *amdgpu_atpx_get_dhandle(void) { return NULL; }
+#endif
+
 /*
  * KMS
  */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c
index f4c474a95875..71efcf38f11b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c
@@ -57,6 +57,10 @@
 #define ACP_I2S_COMP2_CAP_REG_OFFSET		0xa8
 #define ACP_I2S_COMP1_PLAY_REG_OFFSET		0x6c
 #define ACP_I2S_COMP2_PLAY_REG_OFFSET		0x68
+#define ACP_BT_PLAY_REGS_START			0x14970
+#define ACP_BT_PLAY_REGS_END			0x14a24
+#define ACP_BT_COMP1_REG_OFFSET			0xac
+#define ACP_BT_COMP2_REG_OFFSET			0xa8
 
 #define mmACP_PGFSM_RETAIN_REG			0x51c9
 #define mmACP_PGFSM_CONFIG_REG			0x51ca
@@ -77,7 +81,7 @@
 #define ACP_SOFT_RESET_DONE_TIME_OUT_VALUE	0x000000FF
 
 #define ACP_TIMEOUT_LOOP			0x000000FF
-#define ACP_DEVS				3
+#define ACP_DEVS				4
 #define ACP_SRC_ID				162
 
 enum {
@@ -316,14 +320,13 @@ static int acp_hw_init(void *handle)
 	if (adev->acp.acp_cell == NULL)
 		return -ENOMEM;
 
-	adev->acp.acp_res = kcalloc(4, sizeof(struct resource), GFP_KERNEL);
-
+	adev->acp.acp_res = kcalloc(5, sizeof(struct resource), GFP_KERNEL);
 	if (adev->acp.acp_res == NULL) {
 		kfree(adev->acp.acp_cell);
 		return -ENOMEM;
 	}
 
-	i2s_pdata = kcalloc(2, sizeof(struct i2s_platform_data), GFP_KERNEL);
+	i2s_pdata = kcalloc(3, sizeof(struct i2s_platform_data), GFP_KERNEL);
 	if (i2s_pdata == NULL) {
 		kfree(adev->acp.acp_res);
 		kfree(adev->acp.acp_cell);
@@ -358,6 +361,20 @@ static int acp_hw_init(void *handle)
 	i2s_pdata[1].i2s_reg_comp1 = ACP_I2S_COMP1_CAP_REG_OFFSET;
 	i2s_pdata[1].i2s_reg_comp2 = ACP_I2S_COMP2_CAP_REG_OFFSET;
 
+	i2s_pdata[2].quirks = DW_I2S_QUIRK_COMP_REG_OFFSET;
+	switch (adev->asic_type) {
+	case CHIP_STONEY:
+		i2s_pdata[2].quirks |= DW_I2S_QUIRK_16BIT_IDX_OVERRIDE;
+		break;
+	default:
+		break;
+	}
+
+	i2s_pdata[2].cap = DWC_I2S_PLAY | DWC_I2S_RECORD;
+	i2s_pdata[2].snd_rates = SNDRV_PCM_RATE_8000_96000;
+	i2s_pdata[2].i2s_reg_comp1 = ACP_BT_COMP1_REG_OFFSET;
+	i2s_pdata[2].i2s_reg_comp2 = ACP_BT_COMP2_REG_OFFSET;
+
 	adev->acp.acp_res[0].name = "acp2x_dma";
 	adev->acp.acp_res[0].flags = IORESOURCE_MEM;
 	adev->acp.acp_res[0].start = acp_base;
@@ -373,13 +390,18 @@ static int acp_hw_init(void *handle)
 	adev->acp.acp_res[2].start = acp_base + ACP_I2S_CAP_REGS_START;
 	adev->acp.acp_res[2].end = acp_base + ACP_I2S_CAP_REGS_END;
 
-	adev->acp.acp_res[3].name = "acp2x_dma_irq";
-	adev->acp.acp_res[3].flags = IORESOURCE_IRQ;
-	adev->acp.acp_res[3].start = amdgpu_irq_create_mapping(adev, 162);
-	adev->acp.acp_res[3].end = adev->acp.acp_res[3].start;
+	adev->acp.acp_res[3].name = "acp2x_dw_bt_i2s_play_cap";
+	adev->acp.acp_res[3].flags = IORESOURCE_MEM;
+	adev->acp.acp_res[3].start = acp_base + ACP_BT_PLAY_REGS_START;
+	adev->acp.acp_res[3].end = acp_base + ACP_BT_PLAY_REGS_END;
+
+	adev->acp.acp_res[4].name = "acp2x_dma_irq";
+	adev->acp.acp_res[4].flags = IORESOURCE_IRQ;
+	adev->acp.acp_res[4].start = amdgpu_irq_create_mapping(adev, 162);
+	adev->acp.acp_res[4].end = adev->acp.acp_res[4].start;
 
 	adev->acp.acp_cell[0].name = "acp_audio_dma";
-	adev->acp.acp_cell[0].num_resources = 4;
+	adev->acp.acp_cell[0].num_resources = 5;
 	adev->acp.acp_cell[0].resources = &adev->acp.acp_res[0];
 	adev->acp.acp_cell[0].platform_data = &adev->asic_type;
 	adev->acp.acp_cell[0].pdata_size = sizeof(adev->asic_type);
@@ -396,6 +418,12 @@ static int acp_hw_init(void *handle)
 	adev->acp.acp_cell[2].platform_data = &i2s_pdata[1];
 	adev->acp.acp_cell[2].pdata_size = sizeof(struct i2s_platform_data);
 
+	adev->acp.acp_cell[3].name = "designware-i2s";
+	adev->acp.acp_cell[3].num_resources = 1;
+	adev->acp.acp_cell[3].resources = &adev->acp.acp_res[3];
+	adev->acp.acp_cell[3].platform_data = &i2s_pdata[2];
+	adev->acp.acp_cell[3].pdata_size = sizeof(struct i2s_platform_data);
+
 	r = mfd_add_hotplug_devices(adev->acp.parent, adev->acp.acp_cell,
 								ACP_DEVS);
 	if (r)
@@ -451,7 +479,6 @@ static int acp_hw_init(void *handle)
 	val = cgs_read_register(adev->acp.cgs_device, mmACP_SOFT_RESET);
 	val &= ~ACP_SOFT_RESET__SoftResetAud_MASK;
 	cgs_write_register(adev->acp.cgs_device, mmACP_SOFT_RESET, val);
-
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c
index 8fa850a070e0..353993218f21 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c
@@ -34,6 +34,45 @@
 #include "amd_acpi.h"
 #include "atom.h"
 
+struct amdgpu_atif_notification_cfg {
+	bool enabled;
+	int command_code;
+};
+
+struct amdgpu_atif_notifications {
+	bool display_switch;
+	bool expansion_mode_change;
+	bool thermal_state;
+	bool forced_power_state;
+	bool system_power_state;
+	bool display_conf_change;
+	bool px_gfx_switch;
+	bool brightness_change;
+	bool dgpu_display_event;
+};
+
+struct amdgpu_atif_functions {
+	bool system_params;
+	bool sbios_requests;
+	bool select_active_disp;
+	bool lid_state;
+	bool get_tv_standard;
+	bool set_tv_standard;
+	bool get_panel_expansion_mode;
+	bool set_panel_expansion_mode;
+	bool temperature_change;
+	bool graphics_device_types;
+};
+
+struct amdgpu_atif {
+	acpi_handle handle;
+
+	struct amdgpu_atif_notifications notifications;
+	struct amdgpu_atif_functions functions;
+	struct amdgpu_atif_notification_cfg notification_cfg;
+	struct amdgpu_encoder *encoder_for_bl;
+};
+
 /* Call the ATIF method
  */
 /**
@@ -46,8 +85,9 @@
  * Executes the requested ATIF function (all asics).
  * Returns a pointer to the acpi output buffer.
  */
-static union acpi_object *amdgpu_atif_call(acpi_handle handle, int function,
-		struct acpi_buffer *params)
+static union acpi_object *amdgpu_atif_call(struct amdgpu_atif *atif,
+					   int function,
+					   struct acpi_buffer *params)
 {
 	acpi_status status;
 	union acpi_object atif_arg_elements[2];
@@ -70,7 +110,8 @@ static union acpi_object *amdgpu_atif_call(acpi_handle handle, int function,
 		atif_arg_elements[1].integer.value = 0;
 	}
 
-	status = acpi_evaluate_object(handle, "ATIF", &atif_arg, &buffer);
+	status = acpi_evaluate_object(atif->handle, NULL, &atif_arg,
+				      &buffer);
 
 	/* Fail only if calling the method fails and ATIF is supported */
 	if (ACPI_FAILURE(status) && status != AE_NOT_FOUND) {
@@ -141,15 +182,14 @@ static void amdgpu_atif_parse_functions(struct amdgpu_atif_functions *f, u32 mas
  * (all asics).
  * returns 0 on success, error on failure.
  */
-static int amdgpu_atif_verify_interface(acpi_handle handle,
-		struct amdgpu_atif *atif)
+static int amdgpu_atif_verify_interface(struct amdgpu_atif *atif)
 {
 	union acpi_object *info;
 	struct atif_verify_interface output;
 	size_t size;
 	int err = 0;
 
-	info = amdgpu_atif_call(handle, ATIF_FUNCTION_VERIFY_INTERFACE, NULL);
+	info = amdgpu_atif_call(atif, ATIF_FUNCTION_VERIFY_INTERFACE, NULL);
 	if (!info)
 		return -EIO;
 
@@ -176,6 +216,35 @@ out:
 	return err;
 }
 
+static acpi_handle amdgpu_atif_probe_handle(acpi_handle dhandle)
+{
+	acpi_handle handle = NULL;
+	char acpi_method_name[255] = { 0 };
+	struct acpi_buffer buffer = { sizeof(acpi_method_name), acpi_method_name };
+	acpi_status status;
+
+	/* For PX/HG systems, ATIF and ATPX are in the iGPU's namespace, on dGPU only
+	 * systems, ATIF is in the dGPU's namespace.
+	 */
+	status = acpi_get_handle(dhandle, "ATIF", &handle);
+	if (ACPI_SUCCESS(status))
+		goto out;
+
+	if (amdgpu_has_atpx()) {
+		status = acpi_get_handle(amdgpu_atpx_get_dhandle(), "ATIF",
+					 &handle);
+		if (ACPI_SUCCESS(status))
+			goto out;
+	}
+
+	DRM_DEBUG_DRIVER("No ATIF handle found\n");
+	return NULL;
+out:
+	acpi_get_name(handle, ACPI_FULL_PATHNAME, &buffer);
+	DRM_DEBUG_DRIVER("Found ATIF handle %s\n", acpi_method_name);
+	return handle;
+}
+
 /**
  * amdgpu_atif_get_notification_params - determine notify configuration
  *
@@ -188,15 +257,16 @@ out:
  * where n is specified in the result if a notifier is used.
  * Returns 0 on success, error on failure.
  */
-static int amdgpu_atif_get_notification_params(acpi_handle handle,
-		struct amdgpu_atif_notification_cfg *n)
+static int amdgpu_atif_get_notification_params(struct amdgpu_atif *atif)
 {
 	union acpi_object *info;
+	struct amdgpu_atif_notification_cfg *n = &atif->notification_cfg;
 	struct atif_system_params params;
 	size_t size;
 	int err = 0;
 
-	info = amdgpu_atif_call(handle, ATIF_FUNCTION_GET_SYSTEM_PARAMETERS, NULL);
+	info = amdgpu_atif_call(atif, ATIF_FUNCTION_GET_SYSTEM_PARAMETERS,
+				NULL);
 	if (!info) {
 		err = -EIO;
 		goto out;
@@ -250,14 +320,15 @@ out:
  * (all asics).
  * Returns 0 on success, error on failure.
  */
-static int amdgpu_atif_get_sbios_requests(acpi_handle handle,
-		struct atif_sbios_requests *req)
+static int amdgpu_atif_get_sbios_requests(struct amdgpu_atif *atif,
+					  struct atif_sbios_requests *req)
 {
 	union acpi_object *info;
 	size_t size;
 	int count = 0;
 
-	info = amdgpu_atif_call(handle, ATIF_FUNCTION_GET_SYSTEM_BIOS_REQUESTS, NULL);
+	info = amdgpu_atif_call(atif, ATIF_FUNCTION_GET_SYSTEM_BIOS_REQUESTS,
+				NULL);
 	if (!info)
 		return -EIO;
 
@@ -290,11 +361,9 @@ out:
  * Returns NOTIFY code
  */
 static int amdgpu_atif_handler(struct amdgpu_device *adev,
-			struct acpi_bus_event *event)
+			       struct acpi_bus_event *event)
 {
-	struct amdgpu_atif *atif = &adev->atif;
-	struct atif_sbios_requests req;
-	acpi_handle handle;
+	struct amdgpu_atif *atif = adev->atif;
 	int count;
 
 	DRM_DEBUG_DRIVER("event, device_class = %s, type = %#x\n",
@@ -303,48 +372,54 @@ static int amdgpu_atif_handler(struct amdgpu_device *adev,
 	if (strcmp(event->device_class, ACPI_VIDEO_CLASS) != 0)
 		return NOTIFY_DONE;
 
-	if (!atif->notification_cfg.enabled ||
+	if (!atif ||
+	    !atif->notification_cfg.enabled ||
 	    event->type != atif->notification_cfg.command_code)
 		/* Not our event */
 		return NOTIFY_DONE;
 
-	/* Check pending SBIOS requests */
-	handle = ACPI_HANDLE(&adev->pdev->dev);
-	count = amdgpu_atif_get_sbios_requests(handle, &req);
+	if (atif->functions.sbios_requests) {
+		struct atif_sbios_requests req;
 
-	if (count <= 0)
-		return NOTIFY_DONE;
+		/* Check pending SBIOS requests */
+		count = amdgpu_atif_get_sbios_requests(atif, &req);
 
-	DRM_DEBUG_DRIVER("ATIF: %d pending SBIOS requests\n", count);
+		if (count <= 0)
+			return NOTIFY_DONE;
 
-	if (req.pending & ATIF_PANEL_BRIGHTNESS_CHANGE_REQUEST) {
-		struct amdgpu_encoder *enc = atif->encoder_for_bl;
+		DRM_DEBUG_DRIVER("ATIF: %d pending SBIOS requests\n", count);
 
-		if (enc) {
-			struct amdgpu_encoder_atom_dig *dig = enc->enc_priv;
+		/* todo: add DC handling */
+		if ((req.pending & ATIF_PANEL_BRIGHTNESS_CHANGE_REQUEST) &&
+		    !amdgpu_device_has_dc_support(adev)) {
+			struct amdgpu_encoder *enc = atif->encoder_for_bl;
 
-			DRM_DEBUG_DRIVER("Changing brightness to %d\n",
-					req.backlight_level);
+			if (enc) {
+				struct amdgpu_encoder_atom_dig *dig = enc->enc_priv;
+
+				DRM_DEBUG_DRIVER("Changing brightness to %d\n",
+						 req.backlight_level);
 
-			amdgpu_display_backlight_set_level(adev, enc, req.backlight_level);
+				amdgpu_display_backlight_set_level(adev, enc, req.backlight_level);
 
 #if defined(CONFIG_BACKLIGHT_CLASS_DEVICE) || defined(CONFIG_BACKLIGHT_CLASS_DEVICE_MODULE)
-			backlight_force_update(dig->bl_dev,
-					       BACKLIGHT_UPDATE_HOTKEY);
+				backlight_force_update(dig->bl_dev,
+						       BACKLIGHT_UPDATE_HOTKEY);
 #endif
+			}
 		}
-	}
-	if (req.pending & ATIF_DGPU_DISPLAY_EVENT) {
-		if ((adev->flags & AMD_IS_PX) &&
-		    amdgpu_atpx_dgpu_req_power_for_displays()) {
-			pm_runtime_get_sync(adev->ddev->dev);
-			/* Just fire off a uevent and let userspace tell us what to do */
-			drm_helper_hpd_irq_event(adev->ddev);
-			pm_runtime_mark_last_busy(adev->ddev->dev);
-			pm_runtime_put_autosuspend(adev->ddev->dev);
+		if (req.pending & ATIF_DGPU_DISPLAY_EVENT) {
+			if ((adev->flags & AMD_IS_PX) &&
+			    amdgpu_atpx_dgpu_req_power_for_displays()) {
+				pm_runtime_get_sync(adev->ddev->dev);
+				/* Just fire off a uevent and let userspace tell us what to do */
+				drm_helper_hpd_irq_event(adev->ddev);
+				pm_runtime_mark_last_busy(adev->ddev->dev);
+				pm_runtime_put_autosuspend(adev->ddev->dev);
+			}
 		}
+		/* TODO: check other events */
 	}
-	/* TODO: check other events */
 
 	/* We've handled the event, stop the notifier chain. The ACPI interface
 	 * overloads ACPI_VIDEO_NOTIFY_PROBE, we don't want to send that to
@@ -641,8 +716,8 @@ static int amdgpu_acpi_event(struct notifier_block *nb,
  */
 int amdgpu_acpi_init(struct amdgpu_device *adev)
 {
-	acpi_handle handle;
-	struct amdgpu_atif *atif = &adev->atif;
+	acpi_handle handle, atif_handle;
+	struct amdgpu_atif *atif;
 	struct amdgpu_atcs *atcs = &adev->atcs;
 	int ret;
 
@@ -658,12 +733,26 @@ int amdgpu_acpi_init(struct amdgpu_device *adev)
 		DRM_DEBUG_DRIVER("Call to ATCS verify_interface failed: %d\n", ret);
 	}
 
+	/* Probe for ATIF, and initialize it if found */
+	atif_handle = amdgpu_atif_probe_handle(handle);
+	if (!atif_handle)
+		goto out;
+
+	atif = kzalloc(sizeof(*atif), GFP_KERNEL);
+	if (!atif) {
+		DRM_WARN("Not enough memory to initialize ATIF\n");
+		goto out;
+	}
+	atif->handle = atif_handle;
+
 	/* Call the ATIF method */
-	ret = amdgpu_atif_verify_interface(handle, atif);
+	ret = amdgpu_atif_verify_interface(atif);
 	if (ret) {
 		DRM_DEBUG_DRIVER("Call to ATIF verify_interface failed: %d\n", ret);
+		kfree(atif);
 		goto out;
 	}
+	adev->atif = atif;
 
 	if (atif->notifications.brightness_change) {
 		struct drm_encoder *tmp;
@@ -693,8 +782,7 @@ int amdgpu_acpi_init(struct amdgpu_device *adev)
 	}
 
 	if (atif->functions.system_params) {
-		ret = amdgpu_atif_get_notification_params(handle,
-				&atif->notification_cfg);
+		ret = amdgpu_atif_get_notification_params(atif);
 		if (ret) {
 			DRM_DEBUG_DRIVER("Call to GET_SYSTEM_PARAMS failed: %d\n",
 					ret);
@@ -720,4 +808,6 @@ out:
 void amdgpu_acpi_fini(struct amdgpu_device *adev)
 {
 	unregister_acpi_notifier(&adev->acpi_nb);
+	if (adev->atif)
+		kfree(adev->atif);
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
index 305143fcc1ce..f8bbbb3a9504 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
@@ -243,6 +243,33 @@ int amdgpu_amdkfd_resume(struct amdgpu_device *adev)
 	return r;
 }
 
+int amdgpu_amdkfd_pre_reset(struct amdgpu_device *adev)
+{
+	int r = 0;
+
+	if (adev->kfd)
+		r = kgd2kfd->pre_reset(adev->kfd);
+
+	return r;
+}
+
+int amdgpu_amdkfd_post_reset(struct amdgpu_device *adev)
+{
+	int r = 0;
+
+	if (adev->kfd)
+		r = kgd2kfd->post_reset(adev->kfd);
+
+	return r;
+}
+
+void amdgpu_amdkfd_gpu_reset(struct kgd_dev *kgd)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
+
+	amdgpu_device_gpu_recover(adev, NULL, false);
+}
+
 int alloc_gtt_mem(struct kgd_dev *kgd, size_t size,
 			void **mem_obj, uint64_t *gpu_addr,
 			void **cpu_ptr)
@@ -251,7 +278,6 @@ int alloc_gtt_mem(struct kgd_dev *kgd, size_t size,
 	struct amdgpu_bo *bo = NULL;
 	struct amdgpu_bo_param bp;
 	int r;
-	uint64_t gpu_addr_tmp = 0;
 	void *cpu_ptr_tmp = NULL;
 
 	memset(&bp, 0, sizeof(bp));
@@ -275,13 +301,18 @@ int alloc_gtt_mem(struct kgd_dev *kgd, size_t size,
 		goto allocate_mem_reserve_bo_failed;
 	}
 
-	r = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT,
-				&gpu_addr_tmp);
+	r = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT);
 	if (r) {
 		dev_err(adev->dev, "(%d) failed to pin bo for amdkfd\n", r);
 		goto allocate_mem_pin_bo_failed;
 	}
 
+	r = amdgpu_ttm_alloc_gart(&bo->tbo);
+	if (r) {
+		dev_err(adev->dev, "%p bind failed\n", bo);
+		goto allocate_mem_kmap_bo_failed;
+	}
+
 	r = amdgpu_bo_kmap(bo, &cpu_ptr_tmp);
 	if (r) {
 		dev_err(adev->dev,
@@ -290,7 +321,7 @@ int alloc_gtt_mem(struct kgd_dev *kgd, size_t size,
 	}
 
 	*mem_obj = bo;
-	*gpu_addr = gpu_addr_tmp;
+	*gpu_addr = amdgpu_bo_gpu_offset(bo);
 	*cpu_ptr = cpu_ptr_tmp;
 
 	amdgpu_bo_unreserve(bo);
@@ -457,6 +488,14 @@ err:
 	return ret;
 }
 
+void amdgpu_amdkfd_set_compute_idle(struct kgd_dev *kgd, bool idle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
+
+	amdgpu_dpm_switch_power_profile(adev,
+					PP_SMC_POWER_PROFILE_COMPUTE, !idle);
+}
+
 bool amdgpu_amdkfd_is_kfd_vmid(struct amdgpu_device *adev, u32 vmid)
 {
 	if (adev->kfd) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
index a8418a3f4e9d..2f379c183ed2 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
@@ -119,6 +119,7 @@ int amdgpu_amdkfd_evict_userptr(struct kgd_mem *mem, struct mm_struct *mm);
 int amdgpu_amdkfd_submit_ib(struct kgd_dev *kgd, enum kgd_engine_type engine,
 				uint32_t vmid, uint64_t gpu_addr,
 				uint32_t *ib_cmd, uint32_t ib_len);
+void amdgpu_amdkfd_set_compute_idle(struct kgd_dev *kgd, bool idle);
 
 struct kfd2kgd_calls *amdgpu_amdkfd_gfx_7_get_functions(void);
 struct kfd2kgd_calls *amdgpu_amdkfd_gfx_8_0_get_functions(void);
@@ -126,6 +127,12 @@ struct kfd2kgd_calls *amdgpu_amdkfd_gfx_9_0_get_functions(void);
 
 bool amdgpu_amdkfd_is_kfd_vmid(struct amdgpu_device *adev, u32 vmid);
 
+int amdgpu_amdkfd_pre_reset(struct amdgpu_device *adev);
+
+int amdgpu_amdkfd_post_reset(struct amdgpu_device *adev);
+
+void amdgpu_amdkfd_gpu_reset(struct kgd_dev *kgd);
+
 /* Shared API */
 int alloc_gtt_mem(struct kgd_dev *kgd, size_t size,
 			void **mem_obj, uint64_t *gpu_addr,
@@ -183,6 +190,9 @@ int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct kgd_dev *kgd,
 int amdgpu_amdkfd_gpuvm_restore_process_bos(void *process_info,
 					    struct dma_fence **ef);
 
+int amdgpu_amdkfd_gpuvm_get_vm_fault_info(struct kgd_dev *kgd,
+					      struct kfd_vm_fault_info *info);
+
 void amdgpu_amdkfd_gpuvm_init_mem_limits(void);
 void amdgpu_amdkfd_unreserve_system_memory_limit(struct amdgpu_bo *bo);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c
index ea79908dac4c..ea3f698aef5e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c
@@ -145,6 +145,7 @@ static void set_vm_context_page_table_base(struct kgd_dev *kgd, uint32_t vmid,
 		uint32_t page_table_base);
 static int invalidate_tlbs(struct kgd_dev *kgd, uint16_t pasid);
 static int invalidate_tlbs_vmid(struct kgd_dev *kgd, uint16_t vmid);
+static uint32_t read_vmid_from_vmfault_reg(struct kgd_dev *kgd);
 
 /* Because of REG_GET_FIELD() being used, we put this function in the
  * asic specific file.
@@ -216,6 +217,10 @@ static const struct kfd2kgd_calls kfd2kgd = {
 	.invalidate_tlbs = invalidate_tlbs,
 	.invalidate_tlbs_vmid = invalidate_tlbs_vmid,
 	.submit_ib = amdgpu_amdkfd_submit_ib,
+	.get_vm_fault_info = amdgpu_amdkfd_gpuvm_get_vm_fault_info,
+	.read_vmid_from_vmfault_reg = read_vmid_from_vmfault_reg,
+	.gpu_recover = amdgpu_amdkfd_gpu_reset,
+	.set_compute_idle = amdgpu_amdkfd_set_compute_idle
 };
 
 struct kfd2kgd_calls *amdgpu_amdkfd_gfx_7_get_functions(void)
@@ -571,6 +576,9 @@ static int kgd_hqd_destroy(struct kgd_dev *kgd, void *mqd,
 	unsigned long flags, end_jiffies;
 	int retry;
 
+	if (adev->in_gpu_reset)
+		return -EIO;
+
 	acquire_queue(kgd, pipe_id, queue_id);
 	WREG32(mmCP_HQD_PQ_DOORBELL_CONTROL, 0);
 
@@ -882,6 +890,9 @@ static int invalidate_tlbs(struct kgd_dev *kgd, uint16_t pasid)
 	int vmid;
 	unsigned int tmp;
 
+	if (adev->in_gpu_reset)
+		return -EIO;
+
 	for (vmid = 0; vmid < 16; vmid++) {
 		if (!amdgpu_amdkfd_is_kfd_vmid(adev, vmid))
 			continue;
@@ -911,3 +922,19 @@ static int invalidate_tlbs_vmid(struct kgd_dev *kgd, uint16_t vmid)
 	RREG32(mmVM_INVALIDATE_RESPONSE);
 	return 0;
 }
+
+ /**
+  * read_vmid_from_vmfault_reg - read vmid from register
+  *
+  * adev: amdgpu_device pointer
+  * @vmid: vmid pointer
+  * read vmid from register (CIK).
+  */
+static uint32_t read_vmid_from_vmfault_reg(struct kgd_dev *kgd)
+{
+	struct amdgpu_device *adev = get_amdgpu_device(kgd);
+
+	uint32_t status = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_STATUS);
+
+	return REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS, VMID);
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c
index 19dd665e7307..f6e53e9352bd 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c
@@ -176,6 +176,9 @@ static const struct kfd2kgd_calls kfd2kgd = {
 	.invalidate_tlbs = invalidate_tlbs,
 	.invalidate_tlbs_vmid = invalidate_tlbs_vmid,
 	.submit_ib = amdgpu_amdkfd_submit_ib,
+	.get_vm_fault_info = amdgpu_amdkfd_gpuvm_get_vm_fault_info,
+	.gpu_recover = amdgpu_amdkfd_gpu_reset,
+	.set_compute_idle = amdgpu_amdkfd_set_compute_idle
 };
 
 struct kfd2kgd_calls *amdgpu_amdkfd_gfx_8_0_get_functions(void)
@@ -568,6 +571,9 @@ static int kgd_hqd_destroy(struct kgd_dev *kgd, void *mqd,
 	int retry;
 	struct vi_mqd *m = get_mqd(mqd);
 
+	if (adev->in_gpu_reset)
+		return -EIO;
+
 	acquire_queue(kgd, pipe_id, queue_id);
 
 	if (m->cp_hqd_vmid == 0)
@@ -844,6 +850,9 @@ static int invalidate_tlbs(struct kgd_dev *kgd, uint16_t pasid)
 	int vmid;
 	unsigned int tmp;
 
+	if (adev->in_gpu_reset)
+		return -EIO;
+
 	for (vmid = 0; vmid < 16; vmid++) {
 		if (!amdgpu_amdkfd_is_kfd_vmid(adev, vmid))
 			continue;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
index 1db60aa5b7f0..8efedfcb9dfc 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
@@ -213,6 +213,8 @@ static const struct kfd2kgd_calls kfd2kgd = {
 	.invalidate_tlbs = invalidate_tlbs,
 	.invalidate_tlbs_vmid = invalidate_tlbs_vmid,
 	.submit_ib = amdgpu_amdkfd_submit_ib,
+	.gpu_recover = amdgpu_amdkfd_gpu_reset,
+	.set_compute_idle = amdgpu_amdkfd_set_compute_idle
 };
 
 struct kfd2kgd_calls *amdgpu_amdkfd_gfx_9_0_get_functions(void)
@@ -679,6 +681,9 @@ static int kgd_hqd_destroy(struct kgd_dev *kgd, void *mqd,
 	uint32_t temp;
 	struct v9_mqd *m = get_mqd(mqd);
 
+	if (adev->in_gpu_reset)
+		return -EIO;
+
 	acquire_queue(kgd, pipe_id, queue_id);
 
 	if (m->cp_hqd_vmid == 0)
@@ -866,6 +871,9 @@ static int invalidate_tlbs(struct kgd_dev *kgd, uint16_t pasid)
 	int vmid;
 	struct amdgpu_ring *ring = &adev->gfx.kiq.ring;
 
+	if (adev->in_gpu_reset)
+		return -EIO;
+
 	if (ring->ready)
 		return invalidate_tlbs_with_kiq(adev, pasid);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index ff8fd75f7ca5..f92597c292fe 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -206,11 +206,9 @@ static int amdgpu_amdkfd_remove_eviction_fence(struct amdgpu_bo *bo,
 					struct amdgpu_amdkfd_fence ***ef_list,
 					unsigned int *ef_count)
 {
-	struct reservation_object_list *fobj;
-	struct reservation_object *resv;
-	unsigned int i = 0, j = 0, k = 0, shared_count;
-	unsigned int count = 0;
-	struct amdgpu_amdkfd_fence **fence_list;
+	struct reservation_object *resv = bo->tbo.resv;
+	struct reservation_object_list *old, *new;
+	unsigned int i, j, k;
 
 	if (!ef && !ef_list)
 		return -EINVAL;
@@ -220,76 +218,67 @@ static int amdgpu_amdkfd_remove_eviction_fence(struct amdgpu_bo *bo,
 		*ef_count = 0;
 	}
 
-	resv = bo->tbo.resv;
-	fobj = reservation_object_get_list(resv);
-
-	if (!fobj)
+	old = reservation_object_get_list(resv);
+	if (!old)
 		return 0;
 
-	preempt_disable();
-	write_seqcount_begin(&resv->seq);
+	new = kmalloc(offsetof(typeof(*new), shared[old->shared_max]),
+		      GFP_KERNEL);
+	if (!new)
+		return -ENOMEM;
 
-	/* Go through all the shared fences in the resevation object. If
-	 * ef is specified and it exists in the list, remove it and reduce the
-	 * count. If ef is not specified, then get the count of eviction fences
-	 * present.
+	/* Go through all the shared fences in the resevation object and sort
+	 * the interesting ones to the end of the list.
 	 */
-	shared_count = fobj->shared_count;
-	for (i = 0; i < shared_count; ++i) {
+	for (i = 0, j = old->shared_count, k = 0; i < old->shared_count; ++i) {
 		struct dma_fence *f;
 
-		f = rcu_dereference_protected(fobj->shared[i],
+		f = rcu_dereference_protected(old->shared[i],
 					      reservation_object_held(resv));
 
-		if (ef) {
-			if (f->context == ef->base.context) {
-				dma_fence_put(f);
-				fobj->shared_count--;
-			} else {
-				RCU_INIT_POINTER(fobj->shared[j++], f);
-			}
-		} else if (to_amdgpu_amdkfd_fence(f))
-			count++;
+		if ((ef && f->context == ef->base.context) ||
+		    (!ef && to_amdgpu_amdkfd_fence(f)))
+			RCU_INIT_POINTER(new->shared[--j], f);
+		else
+			RCU_INIT_POINTER(new->shared[k++], f);
 	}
-	write_seqcount_end(&resv->seq);
-	preempt_enable();
-
-	if (ef || !count)
-		return 0;
-
-	/* Alloc memory for count number of eviction fence pointers. Fill the
-	 * ef_list array and ef_count
-	 */
-	fence_list = kcalloc(count, sizeof(struct amdgpu_amdkfd_fence *),
-			     GFP_KERNEL);
-	if (!fence_list)
-		return -ENOMEM;
-
-	preempt_disable();
-	write_seqcount_begin(&resv->seq);
+	new->shared_max = old->shared_max;
+	new->shared_count = k;
 
-	j = 0;
-	for (i = 0; i < shared_count; ++i) {
-		struct dma_fence *f;
-		struct amdgpu_amdkfd_fence *efence;
+	if (!ef) {
+		unsigned int count = old->shared_count - j;
 
-		f = rcu_dereference_protected(fobj->shared[i],
-			reservation_object_held(resv));
+		/* Alloc memory for count number of eviction fence pointers.
+		 * Fill the ef_list array and ef_count
+		 */
+		*ef_list = kcalloc(count, sizeof(**ef_list), GFP_KERNEL);
+		*ef_count = count;
 
-		efence = to_amdgpu_amdkfd_fence(f);
-		if (efence) {
-			fence_list[k++] = efence;
-			fobj->shared_count--;
-		} else {
-			RCU_INIT_POINTER(fobj->shared[j++], f);
+		if (!*ef_list) {
+			kfree(new);
+			return -ENOMEM;
 		}
 	}
 
+	/* Install the new fence list, seqcount provides the barriers */
+	preempt_disable();
+	write_seqcount_begin(&resv->seq);
+	RCU_INIT_POINTER(resv->fence, new);
 	write_seqcount_end(&resv->seq);
 	preempt_enable();
 
-	*ef_list = fence_list;
-	*ef_count = k;
+	/* Drop the references to the removed fences or move them to ef_list */
+	for (i = j, k = 0; i < old->shared_count; ++i) {
+		struct dma_fence *f;
+
+		f = rcu_dereference_protected(new->shared[i],
+					      reservation_object_held(resv));
+		if (!ef)
+			(*ef_list)[k++] = to_amdgpu_amdkfd_fence(f);
+		else
+			dma_fence_put(f);
+	}
+	kfree_rcu(old, rcu);
 
 	return 0;
 }
@@ -334,7 +323,7 @@ static int amdgpu_amdkfd_bo_validate(struct amdgpu_bo *bo, uint32_t domain,
 		 "Called with userptr BO"))
 		return -EINVAL;
 
-	amdgpu_ttm_placement_from_domain(bo, domain);
+	amdgpu_bo_placement_from_domain(bo, domain);
 
 	ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
 	if (ret)
@@ -622,7 +611,7 @@ static int init_user_pages(struct kgd_mem *mem, struct mm_struct *mm,
 		pr_err("%s: Failed to reserve BO\n", __func__);
 		goto release_out;
 	}
-	amdgpu_ttm_placement_from_domain(bo, mem->domain);
+	amdgpu_bo_placement_from_domain(bo, mem->domain);
 	ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
 	if (ret)
 		pr_err("%s: failed to validate BO\n", __func__);
@@ -1587,7 +1576,7 @@ int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct kgd_dev *kgd,
 		goto bo_reserve_failed;
 	}
 
-	ret = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT, NULL);
+	ret = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT);
 	if (ret) {
 		pr_err("Failed to pin bo. ret %d\n", ret);
 		goto pin_failed;
@@ -1621,6 +1610,20 @@ bo_reserve_failed:
 	return ret;
 }
 
+int amdgpu_amdkfd_gpuvm_get_vm_fault_info(struct kgd_dev *kgd,
+					      struct kfd_vm_fault_info *mem)
+{
+	struct amdgpu_device *adev;
+
+	adev = (struct amdgpu_device *)kgd;
+	if (atomic_read(&adev->gmc.vm_fault_info_updated) == 1) {
+		*mem = *adev->gmc.vm_fault_info;
+		mb();
+		atomic_set(&adev->gmc.vm_fault_info_updated, 0);
+	}
+	return 0;
+}
+
 /* Evict a userptr BO by stopping the queues if necessary
  *
  * Runs in MMU notifier, may be in RECLAIM_FS context. This means it
@@ -1680,7 +1683,7 @@ static int update_invalid_user_pages(struct amdkfd_process_info *process_info,
 
 		if (amdgpu_bo_reserve(bo, true))
 			return -EAGAIN;
-		amdgpu_ttm_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);
+		amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);
 		ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
 		amdgpu_bo_unreserve(bo);
 		if (ret) {
@@ -1824,7 +1827,7 @@ static int validate_invalid_user_pages(struct amdkfd_process_info *process_info)
 		if (mem->user_pages[0]) {
 			amdgpu_ttm_tt_set_user_pages(bo->tbo.ttm,
 						     mem->user_pages);
-			amdgpu_ttm_placement_from_domain(bo, mem->domain);
+			amdgpu_bo_placement_from_domain(bo, mem->domain);
 			ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
 			if (ret) {
 				pr_err("%s: failed to validate BO\n", __func__);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c
index daa06e7c5bb7..a028661d9e20 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c
@@ -32,7 +32,7 @@ struct amdgpu_atpx_functions {
 	bool switch_start;
 	bool switch_end;
 	bool disp_connectors_mapping;
-	bool disp_detetion_ports;
+	bool disp_detection_ports;
 };
 
 struct amdgpu_atpx {
@@ -90,6 +90,12 @@ bool amdgpu_atpx_dgpu_req_power_for_displays(void) {
 	return amdgpu_atpx_priv.atpx.dgpu_req_power_for_displays;
 }
 
+#if defined(CONFIG_ACPI)
+void *amdgpu_atpx_get_dhandle(void) {
+	return amdgpu_atpx_priv.dhandle;
+}
+#endif
+
 /**
  * amdgpu_atpx_call - call an ATPX method
  *
@@ -156,7 +162,7 @@ static void amdgpu_atpx_parse_functions(struct amdgpu_atpx_functions *f, u32 mas
 	f->switch_start = mask & ATPX_GRAPHICS_DEVICE_SWITCH_START_NOTIFICATION_SUPPORTED;
 	f->switch_end = mask & ATPX_GRAPHICS_DEVICE_SWITCH_END_NOTIFICATION_SUPPORTED;
 	f->disp_connectors_mapping = mask & ATPX_GET_DISPLAY_CONNECTORS_MAPPING_SUPPORTED;
-	f->disp_detetion_ports = mask & ATPX_GET_DISPLAY_DETECTION_PORTS_SUPPORTED;
+	f->disp_detection_ports = mask & ATPX_GET_DISPLAY_DETECTION_PORTS_SUPPORTED;
 }
 
 /**
@@ -569,6 +575,7 @@ static const struct amdgpu_px_quirk amdgpu_px_quirk_list[] = {
 	{ 0x1002, 0x6900, 0x1002, 0x0124, AMDGPU_PX_QUIRK_FORCE_ATPX },
 	{ 0x1002, 0x6900, 0x1028, 0x0812, AMDGPU_PX_QUIRK_FORCE_ATPX },
 	{ 0x1002, 0x6900, 0x1028, 0x0813, AMDGPU_PX_QUIRK_FORCE_ATPX },
+	{ 0x1002, 0x6900, 0x1025, 0x125A, AMDGPU_PX_QUIRK_FORCE_ATPX },
 	{ 0, 0, 0, 0, 0 },
 };
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c
index 19cfff31f2e1..3079ea8523c5 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c
@@ -95,11 +95,17 @@ static void amdgpu_benchmark_move(struct amdgpu_device *adev, unsigned size,
 	r = amdgpu_bo_reserve(sobj, false);
 	if (unlikely(r != 0))
 		goto out_cleanup;
-	r = amdgpu_bo_pin(sobj, sdomain, &saddr);
+	r = amdgpu_bo_pin(sobj, sdomain);
+	if (r) {
+		amdgpu_bo_unreserve(sobj);
+		goto out_cleanup;
+	}
+	r = amdgpu_ttm_alloc_gart(&sobj->tbo);
 	amdgpu_bo_unreserve(sobj);
 	if (r) {
 		goto out_cleanup;
 	}
+	saddr = amdgpu_bo_gpu_offset(sobj);
 	bp.domain = ddomain;
 	r = amdgpu_bo_create(adev, &bp, &dobj);
 	if (r) {
@@ -108,11 +114,17 @@ static void amdgpu_benchmark_move(struct amdgpu_device *adev, unsigned size,
 	r = amdgpu_bo_reserve(dobj, false);
 	if (unlikely(r != 0))
 		goto out_cleanup;
-	r = amdgpu_bo_pin(dobj, ddomain, &daddr);
+	r = amdgpu_bo_pin(dobj, ddomain);
+	if (r) {
+		amdgpu_bo_unreserve(sobj);
+		goto out_cleanup;
+	}
+	r = amdgpu_ttm_alloc_gart(&dobj->tbo);
 	amdgpu_bo_unreserve(dobj);
 	if (r) {
 		goto out_cleanup;
 	}
+	daddr = amdgpu_bo_gpu_offset(dobj);
 
 	if (adev->mman.buffer_funcs) {
 		time = amdgpu_benchmark_do_move(adev, size, saddr, daddr, n);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c
index 92be7f6de197..d472a2c8399f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c
@@ -35,92 +35,53 @@
 #define AMDGPU_BO_LIST_MAX_PRIORITY	32u
 #define AMDGPU_BO_LIST_NUM_BUCKETS	(AMDGPU_BO_LIST_MAX_PRIORITY + 1)
 
-static int amdgpu_bo_list_set(struct amdgpu_device *adev,
-				     struct drm_file *filp,
-				     struct amdgpu_bo_list *list,
-				     struct drm_amdgpu_bo_list_entry *info,
-				     unsigned num_entries);
+static void amdgpu_bo_list_free_rcu(struct rcu_head *rcu)
+{
+	struct amdgpu_bo_list *list = container_of(rcu, struct amdgpu_bo_list,
+						   rhead);
+
+	kvfree(list);
+}
 
-static void amdgpu_bo_list_release_rcu(struct kref *ref)
+static void amdgpu_bo_list_free(struct kref *ref)
 {
-	unsigned i;
 	struct amdgpu_bo_list *list = container_of(ref, struct amdgpu_bo_list,
 						   refcount);
+	struct amdgpu_bo_list_entry *e;
 
-	for (i = 0; i < list->num_entries; ++i)
-		amdgpu_bo_unref(&list->array[i].robj);
+	amdgpu_bo_list_for_each_entry(e, list)
+		amdgpu_bo_unref(&e->robj);
 
-	mutex_destroy(&list->lock);
-	kvfree(list->array);
-	kfree_rcu(list, rhead);
+	call_rcu(&list->rhead, amdgpu_bo_list_free_rcu);
 }
 
-static int amdgpu_bo_list_create(struct amdgpu_device *adev,
-				 struct drm_file *filp,
-				 struct drm_amdgpu_bo_list_entry *info,
-				 unsigned num_entries,
-				 int *id)
+int amdgpu_bo_list_create(struct amdgpu_device *adev, struct drm_file *filp,
+			  struct drm_amdgpu_bo_list_entry *info,
+			  unsigned num_entries, struct amdgpu_bo_list **result)
 {
-	int r;
-	struct amdgpu_fpriv *fpriv = filp->driver_priv;
+	unsigned last_entry = 0, first_userptr = num_entries;
+	struct amdgpu_bo_list_entry *array;
 	struct amdgpu_bo_list *list;
+	uint64_t total_size = 0;
+	size_t size;
+	unsigned i;
+	int r;
+
+	if (num_entries > SIZE_MAX / sizeof(struct amdgpu_bo_list_entry))
+		return -EINVAL;
 
-	list = kzalloc(sizeof(struct amdgpu_bo_list), GFP_KERNEL);
+	size = sizeof(struct amdgpu_bo_list);
+	size += num_entries * sizeof(struct amdgpu_bo_list_entry);
+	list = kvmalloc(size, GFP_KERNEL);
 	if (!list)
 		return -ENOMEM;
 
-	/* initialize bo list*/
-	mutex_init(&list->lock);
 	kref_init(&list->refcount);
-	r = amdgpu_bo_list_set(adev, filp, list, info, num_entries);
-	if (r) {
-		kfree(list);
-		return r;
-	}
-
-	/* idr alloc should be called only after initialization of bo list. */
-	mutex_lock(&fpriv->bo_list_lock);
-	r = idr_alloc(&fpriv->bo_list_handles, list, 1, 0, GFP_KERNEL);
-	mutex_unlock(&fpriv->bo_list_lock);
-	if (r < 0) {
-		amdgpu_bo_list_free(list);
-		return r;
-	}
-	*id = r;
-
-	return 0;
-}
-
-static void amdgpu_bo_list_destroy(struct amdgpu_fpriv *fpriv, int id)
-{
-	struct amdgpu_bo_list *list;
-
-	mutex_lock(&fpriv->bo_list_lock);
-	list = idr_remove(&fpriv->bo_list_handles, id);
-	mutex_unlock(&fpriv->bo_list_lock);
-	if (list)
-		kref_put(&list->refcount, amdgpu_bo_list_release_rcu);
-}
-
-static int amdgpu_bo_list_set(struct amdgpu_device *adev,
-				     struct drm_file *filp,
-				     struct amdgpu_bo_list *list,
-				     struct drm_amdgpu_bo_list_entry *info,
-				     unsigned num_entries)
-{
-	struct amdgpu_bo_list_entry *array;
-	struct amdgpu_bo *gds_obj = adev->gds.gds_gfx_bo;
-	struct amdgpu_bo *gws_obj = adev->gds.gws_gfx_bo;
-	struct amdgpu_bo *oa_obj = adev->gds.oa_gfx_bo;
-
-	unsigned last_entry = 0, first_userptr = num_entries;
-	unsigned i;
-	int r;
-	unsigned long total_size = 0;
+	list->gds_obj = adev->gds.gds_gfx_bo;
+	list->gws_obj = adev->gds.gws_gfx_bo;
+	list->oa_obj = adev->gds.oa_gfx_bo;
 
-	array = kvmalloc_array(num_entries, sizeof(struct amdgpu_bo_list_entry), GFP_KERNEL);
-	if (!array)
-		return -ENOMEM;
+	array = amdgpu_bo_list_array_entry(list, 0);
 	memset(array, 0, num_entries * sizeof(struct amdgpu_bo_list_entry));
 
 	for (i = 0; i < num_entries; ++i) {
@@ -157,59 +118,56 @@ static int amdgpu_bo_list_set(struct amdgpu_device *adev,
 		entry->tv.shared = !entry->robj->prime_shared_count;
 
 		if (entry->robj->preferred_domains == AMDGPU_GEM_DOMAIN_GDS)
-			gds_obj = entry->robj;
+			list->gds_obj = entry->robj;
 		if (entry->robj->preferred_domains == AMDGPU_GEM_DOMAIN_GWS)
-			gws_obj = entry->robj;
+			list->gws_obj = entry->robj;
 		if (entry->robj->preferred_domains == AMDGPU_GEM_DOMAIN_OA)
-			oa_obj = entry->robj;
+			list->oa_obj = entry->robj;
 
 		total_size += amdgpu_bo_size(entry->robj);
 		trace_amdgpu_bo_list_set(list, entry->robj);
 	}
 
-	for (i = 0; i < list->num_entries; ++i)
-		amdgpu_bo_unref(&list->array[i].robj);
-
-	kvfree(list->array);
-
-	list->gds_obj = gds_obj;
-	list->gws_obj = gws_obj;
-	list->oa_obj = oa_obj;
 	list->first_userptr = first_userptr;
-	list->array = array;
 	list->num_entries = num_entries;
 
 	trace_amdgpu_cs_bo_status(list->num_entries, total_size);
+
+	*result = list;
 	return 0;
 
 error_free:
 	while (i--)
 		amdgpu_bo_unref(&array[i].robj);
-	kvfree(array);
+	kvfree(list);
 	return r;
+
 }
 
-struct amdgpu_bo_list *
-amdgpu_bo_list_get(struct amdgpu_fpriv *fpriv, int id)
+static void amdgpu_bo_list_destroy(struct amdgpu_fpriv *fpriv, int id)
 {
-	struct amdgpu_bo_list *result;
+	struct amdgpu_bo_list *list;
+
+	mutex_lock(&fpriv->bo_list_lock);
+	list = idr_remove(&fpriv->bo_list_handles, id);
+	mutex_unlock(&fpriv->bo_list_lock);
+	if (list)
+		kref_put(&list->refcount, amdgpu_bo_list_free);
+}
 
+int amdgpu_bo_list_get(struct amdgpu_fpriv *fpriv, int id,
+		       struct amdgpu_bo_list **result)
+{
 	rcu_read_lock();
-	result = idr_find(&fpriv->bo_list_handles, id);
+	*result = idr_find(&fpriv->bo_list_handles, id);
 
-	if (result) {
-		if (kref_get_unless_zero(&result->refcount)) {
-			rcu_read_unlock();
-			mutex_lock(&result->lock);
-		} else {
-			rcu_read_unlock();
-			result = NULL;
-		}
-	} else {
+	if (*result && kref_get_unless_zero(&(*result)->refcount)) {
 		rcu_read_unlock();
+		return 0;
 	}
 
-	return result;
+	rcu_read_unlock();
+	return -ENOENT;
 }
 
 void amdgpu_bo_list_get_list(struct amdgpu_bo_list *list,
@@ -220,6 +178,7 @@ void amdgpu_bo_list_get_list(struct amdgpu_bo_list *list,
 	 * concatenated in descending order.
 	 */
 	struct list_head bucket[AMDGPU_BO_LIST_NUM_BUCKETS];
+	struct amdgpu_bo_list_entry *e;
 	unsigned i;
 
 	for (i = 0; i < AMDGPU_BO_LIST_NUM_BUCKETS; i++)
@@ -230,14 +189,13 @@ void amdgpu_bo_list_get_list(struct amdgpu_bo_list *list,
 	 * in the list, the sort mustn't change the ordering of buffers
 	 * with the same priority, i.e. it must be stable.
 	 */
-	for (i = 0; i < list->num_entries; i++) {
-		unsigned priority = list->array[i].priority;
+	amdgpu_bo_list_for_each_entry(e, list) {
+		unsigned priority = e->priority;
 
-		if (!list->array[i].robj->parent)
-			list_add_tail(&list->array[i].tv.head,
-				      &bucket[priority]);
+		if (!e->robj->parent)
+			list_add_tail(&e->tv.head, &bucket[priority]);
 
-		list->array[i].user_pages = NULL;
+		e->user_pages = NULL;
 	}
 
 	/* Connect the sorted buckets in the output list. */
@@ -247,71 +205,82 @@ void amdgpu_bo_list_get_list(struct amdgpu_bo_list *list,
 
 void amdgpu_bo_list_put(struct amdgpu_bo_list *list)
 {
-	mutex_unlock(&list->lock);
-	kref_put(&list->refcount, amdgpu_bo_list_release_rcu);
-}
-
-void amdgpu_bo_list_free(struct amdgpu_bo_list *list)
-{
-	unsigned i;
-
-	for (i = 0; i < list->num_entries; ++i)
-		amdgpu_bo_unref(&list->array[i].robj);
-
-	mutex_destroy(&list->lock);
-	kvfree(list->array);
-	kfree(list);
+	kref_put(&list->refcount, amdgpu_bo_list_free);
 }
 
-int amdgpu_bo_list_ioctl(struct drm_device *dev, void *data,
-				struct drm_file *filp)
+int amdgpu_bo_create_list_entry_array(struct drm_amdgpu_bo_list_in *in,
+				      struct drm_amdgpu_bo_list_entry **info_param)
 {
+	const void __user *uptr = u64_to_user_ptr(in->bo_info_ptr);
 	const uint32_t info_size = sizeof(struct drm_amdgpu_bo_list_entry);
-
-	struct amdgpu_device *adev = dev->dev_private;
-	struct amdgpu_fpriv *fpriv = filp->driver_priv;
-	union drm_amdgpu_bo_list *args = data;
-	uint32_t handle = args->in.list_handle;
-	const void __user *uptr = u64_to_user_ptr(args->in.bo_info_ptr);
-
 	struct drm_amdgpu_bo_list_entry *info;
-	struct amdgpu_bo_list *list;
-
 	int r;
 
-	info = kvmalloc_array(args->in.bo_number,
-			     sizeof(struct drm_amdgpu_bo_list_entry), GFP_KERNEL);
+	info = kvmalloc_array(in->bo_number, info_size, GFP_KERNEL);
 	if (!info)
 		return -ENOMEM;
 
 	/* copy the handle array from userspace to a kernel buffer */
 	r = -EFAULT;
-	if (likely(info_size == args->in.bo_info_size)) {
-		unsigned long bytes = args->in.bo_number *
-			args->in.bo_info_size;
+	if (likely(info_size == in->bo_info_size)) {
+		unsigned long bytes = in->bo_number *
+			in->bo_info_size;
 
 		if (copy_from_user(info, uptr, bytes))
 			goto error_free;
 
 	} else {
-		unsigned long bytes = min(args->in.bo_info_size, info_size);
+		unsigned long bytes = min(in->bo_info_size, info_size);
 		unsigned i;
 
-		memset(info, 0, args->in.bo_number * info_size);
-		for (i = 0; i < args->in.bo_number; ++i) {
+		memset(info, 0, in->bo_number * info_size);
+		for (i = 0; i < in->bo_number; ++i) {
 			if (copy_from_user(&info[i], uptr, bytes))
 				goto error_free;
 
-			uptr += args->in.bo_info_size;
+			uptr += in->bo_info_size;
 		}
 	}
 
+	*info_param = info;
+	return 0;
+
+error_free:
+	kvfree(info);
+	return r;
+}
+
+int amdgpu_bo_list_ioctl(struct drm_device *dev, void *data,
+				struct drm_file *filp)
+{
+	struct amdgpu_device *adev = dev->dev_private;
+	struct amdgpu_fpriv *fpriv = filp->driver_priv;
+	union drm_amdgpu_bo_list *args = data;
+	uint32_t handle = args->in.list_handle;
+	struct drm_amdgpu_bo_list_entry *info = NULL;
+	struct amdgpu_bo_list *list, *old;
+	int r;
+
+	r = amdgpu_bo_create_list_entry_array(&args->in, &info);
+	if (r)
+		goto error_free;
+
 	switch (args->in.operation) {
 	case AMDGPU_BO_LIST_OP_CREATE:
 		r = amdgpu_bo_list_create(adev, filp, info, args->in.bo_number,
-					  &handle);
+					  &list);
 		if (r)
 			goto error_free;
+
+		mutex_lock(&fpriv->bo_list_lock);
+		r = idr_alloc(&fpriv->bo_list_handles, list, 1, 0, GFP_KERNEL);
+		mutex_unlock(&fpriv->bo_list_lock);
+		if (r < 0) {
+			amdgpu_bo_list_put(list);
+			return r;
+		}
+
+		handle = r;
 		break;
 
 	case AMDGPU_BO_LIST_OP_DESTROY:
@@ -320,17 +289,22 @@ int amdgpu_bo_list_ioctl(struct drm_device *dev, void *data,
 		break;
 
 	case AMDGPU_BO_LIST_OP_UPDATE:
-		r = -ENOENT;
-		list = amdgpu_bo_list_get(fpriv, handle);
-		if (!list)
+		r = amdgpu_bo_list_create(adev, filp, info, args->in.bo_number,
+					  &list);
+		if (r)
 			goto error_free;
 
-		r = amdgpu_bo_list_set(adev, filp, list, info,
-					      args->in.bo_number);
-		amdgpu_bo_list_put(list);
-		if (r)
+		mutex_lock(&fpriv->bo_list_lock);
+		old = idr_replace(&fpriv->bo_list_handles, list, handle);
+		mutex_unlock(&fpriv->bo_list_lock);
+
+		if (IS_ERR(old)) {
+			amdgpu_bo_list_put(list);
+			r = PTR_ERR(old);
 			goto error_free;
+		}
 
+		amdgpu_bo_list_put(old);
 		break;
 
 	default:
@@ -345,6 +319,7 @@ int amdgpu_bo_list_ioctl(struct drm_device *dev, void *data,
 	return 0;
 
 error_free:
-	kvfree(info);
+	if (info)
+		kvfree(info);
 	return r;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.h
new file mode 100644
index 000000000000..61b089768e1c
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.h
@@ -0,0 +1,85 @@
+/*
+ * Copyright 2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef __AMDGPU_BO_LIST_H__
+#define __AMDGPU_BO_LIST_H__
+
+#include <drm/ttm/ttm_execbuf_util.h>
+#include <drm/amdgpu_drm.h>
+
+struct amdgpu_device;
+struct amdgpu_bo;
+struct amdgpu_bo_va;
+struct amdgpu_fpriv;
+
+struct amdgpu_bo_list_entry {
+	struct amdgpu_bo		*robj;
+	struct ttm_validate_buffer	tv;
+	struct amdgpu_bo_va		*bo_va;
+	uint32_t			priority;
+	struct page			**user_pages;
+	int				user_invalidated;
+};
+
+struct amdgpu_bo_list {
+	struct rcu_head rhead;
+	struct kref refcount;
+	struct amdgpu_bo *gds_obj;
+	struct amdgpu_bo *gws_obj;
+	struct amdgpu_bo *oa_obj;
+	unsigned first_userptr;
+	unsigned num_entries;
+};
+
+int amdgpu_bo_list_get(struct amdgpu_fpriv *fpriv, int id,
+		       struct amdgpu_bo_list **result);
+void amdgpu_bo_list_get_list(struct amdgpu_bo_list *list,
+			     struct list_head *validated);
+void amdgpu_bo_list_put(struct amdgpu_bo_list *list);
+int amdgpu_bo_create_list_entry_array(struct drm_amdgpu_bo_list_in *in,
+				      struct drm_amdgpu_bo_list_entry **info_param);
+
+int amdgpu_bo_list_create(struct amdgpu_device *adev,
+				 struct drm_file *filp,
+				 struct drm_amdgpu_bo_list_entry *info,
+				 unsigned num_entries,
+				 struct amdgpu_bo_list **list);
+
+static inline struct amdgpu_bo_list_entry *
+amdgpu_bo_list_array_entry(struct amdgpu_bo_list *list, unsigned index)
+{
+	struct amdgpu_bo_list_entry *array = (void *)&list[1];
+
+	return &array[index];
+}
+
+#define amdgpu_bo_list_for_each_entry(e, list) \
+	for (e = amdgpu_bo_list_array_entry(list, 0); \
+	     e != amdgpu_bo_list_array_entry(list, (list)->num_entries); \
+	     ++e)
+
+#define amdgpu_bo_list_for_each_userptr_entry(e, list) \
+	for (e = amdgpu_bo_list_array_entry(list, (list)->first_userptr); \
+	     e != amdgpu_bo_list_array_entry(list, (list)->num_entries); \
+	     ++e)
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c
index e950730f1933..693ec5ea4950 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c
@@ -314,17 +314,17 @@ static int amdgpu_cgs_get_firmware_info(struct cgs_device *cgs_device,
 					(adev->pdev->revision == 0x81) ||
 					(adev->pdev->device == 0x665f)) {
 					info->is_kicker = true;
-					strcpy(fw_name, "radeon/bonaire_k_smc.bin");
+					strcpy(fw_name, "amdgpu/bonaire_k_smc.bin");
 				} else {
-					strcpy(fw_name, "radeon/bonaire_smc.bin");
+					strcpy(fw_name, "amdgpu/bonaire_smc.bin");
 				}
 				break;
 			case CHIP_HAWAII:
 				if (adev->pdev->revision == 0x80) {
 					info->is_kicker = true;
-					strcpy(fw_name, "radeon/hawaii_k_smc.bin");
+					strcpy(fw_name, "amdgpu/hawaii_k_smc.bin");
 				} else {
-					strcpy(fw_name, "radeon/hawaii_smc.bin");
+					strcpy(fw_name, "amdgpu/hawaii_smc.bin");
 				}
 				break;
 			case CHIP_TOPAZ:
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
index 7a625f3989a0..502b94fb116a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
@@ -66,11 +66,35 @@ static int amdgpu_cs_user_fence_chunk(struct amdgpu_cs_parser *p,
 	return 0;
 }
 
-static int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, void *data)
+static int amdgpu_cs_bo_handles_chunk(struct amdgpu_cs_parser *p,
+				      struct drm_amdgpu_bo_list_in *data)
+{
+	int r;
+	struct drm_amdgpu_bo_list_entry *info = NULL;
+
+	r = amdgpu_bo_create_list_entry_array(data, &info);
+	if (r)
+		return r;
+
+	r = amdgpu_bo_list_create(p->adev, p->filp, info, data->bo_number,
+				  &p->bo_list);
+	if (r)
+		goto error_free;
+
+	kvfree(info);
+	return 0;
+
+error_free:
+	if (info)
+		kvfree(info);
+
+	return r;
+}
+
+static int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, union drm_amdgpu_cs *cs)
 {
 	struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
 	struct amdgpu_vm *vm = &fpriv->vm;
-	union drm_amdgpu_cs *cs = data;
 	uint64_t *chunk_array_user;
 	uint64_t *chunk_array;
 	unsigned size, num_ibs = 0;
@@ -164,6 +188,19 @@ static int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, void *data)
 
 			break;
 
+		case AMDGPU_CHUNK_ID_BO_HANDLES:
+			size = sizeof(struct drm_amdgpu_bo_list_in);
+			if (p->chunks[i].length_dw * sizeof(uint32_t) < size) {
+				ret = -EINVAL;
+				goto free_partial_kdata;
+			}
+
+			ret = amdgpu_cs_bo_handles_chunk(p, p->chunks[i].kdata);
+			if (ret)
+				goto free_partial_kdata;
+
+			break;
+
 		case AMDGPU_CHUNK_ID_DEPENDENCIES:
 		case AMDGPU_CHUNK_ID_SYNCOBJ_IN:
 		case AMDGPU_CHUNK_ID_SYNCOBJ_OUT:
@@ -187,6 +224,10 @@ static int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, void *data)
 	if (p->uf_entry.robj)
 		p->job->uf_addr = uf_offset;
 	kfree(chunk_array);
+
+	/* Use this opportunity to fill in task info for the vm */
+	amdgpu_vm_set_task_info(vm);
+
 	return 0;
 
 free_all_kdata:
@@ -258,7 +299,7 @@ static void amdgpu_cs_get_threshold_for_moves(struct amdgpu_device *adev,
 		return;
 	}
 
-	total_vram = adev->gmc.real_vram_size - adev->vram_pin_size;
+	total_vram = adev->gmc.real_vram_size - atomic64_read(&adev->vram_pin_size);
 	used_vram = amdgpu_vram_mgr_usage(&adev->mman.bdev.man[TTM_PL_VRAM]);
 	free_vram = used_vram >= total_vram ? 0 : total_vram - used_vram;
 
@@ -378,7 +419,7 @@ static int amdgpu_cs_bo_validate(struct amdgpu_cs_parser *p,
 	}
 
 retry:
-	amdgpu_ttm_placement_from_domain(bo, domain);
+	amdgpu_bo_placement_from_domain(bo, domain);
 	r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
 
 	p->bytes_moved += ctx.bytes_moved;
@@ -437,7 +478,7 @@ static bool amdgpu_cs_try_evict(struct amdgpu_cs_parser *p,
 		update_bytes_moved_vis =
 				!amdgpu_gmc_vram_full_visible(&adev->gmc) &&
 				amdgpu_bo_in_cpu_visible_vram(bo);
-		amdgpu_ttm_placement_from_domain(bo, other);
+		amdgpu_bo_placement_from_domain(bo, other);
 		r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
 		p->bytes_moved += ctx.bytes_moved;
 		if (update_bytes_moved_vis)
@@ -491,8 +532,8 @@ static int amdgpu_cs_list_validate(struct amdgpu_cs_parser *p,
 		/* Check if we have user pages and nobody bound the BO already */
 		if (amdgpu_ttm_tt_userptr_needs_pages(bo->tbo.ttm) &&
 		    lobj->user_pages) {
-			amdgpu_ttm_placement_from_domain(bo,
-							 AMDGPU_GEM_DOMAIN_CPU);
+			amdgpu_bo_placement_from_domain(bo,
+							AMDGPU_GEM_DOMAIN_CPU);
 			r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
 			if (r)
 				return r;
@@ -520,23 +561,38 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
 				union drm_amdgpu_cs *cs)
 {
 	struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
+	struct amdgpu_vm *vm = &fpriv->vm;
 	struct amdgpu_bo_list_entry *e;
 	struct list_head duplicates;
-	unsigned i, tries = 10;
 	struct amdgpu_bo *gds;
 	struct amdgpu_bo *gws;
 	struct amdgpu_bo *oa;
+	unsigned tries = 10;
 	int r;
 
 	INIT_LIST_HEAD(&p->validated);
 
-	p->bo_list = amdgpu_bo_list_get(fpriv, cs->in.bo_list_handle);
-	if (p->bo_list) {
-		amdgpu_bo_list_get_list(p->bo_list, &p->validated);
-		if (p->bo_list->first_userptr != p->bo_list->num_entries)
-			p->mn = amdgpu_mn_get(p->adev, AMDGPU_MN_TYPE_GFX);
+	/* p->bo_list could already be assigned if AMDGPU_CHUNK_ID_BO_HANDLES is present */
+	if (cs->in.bo_list_handle) {
+		if (p->bo_list)
+			return -EINVAL;
+
+		r = amdgpu_bo_list_get(fpriv, cs->in.bo_list_handle,
+				       &p->bo_list);
+		if (r)
+			return r;
+	} else if (!p->bo_list) {
+		/* Create a empty bo_list when no handle is provided */
+		r = amdgpu_bo_list_create(p->adev, p->filp, NULL, 0,
+					  &p->bo_list);
+		if (r)
+			return r;
 	}
 
+	amdgpu_bo_list_get_list(p->bo_list, &p->validated);
+	if (p->bo_list->first_userptr != p->bo_list->num_entries)
+		p->mn = amdgpu_mn_get(p->adev, AMDGPU_MN_TYPE_GFX);
+
 	INIT_LIST_HEAD(&duplicates);
 	amdgpu_vm_get_pd_bo(&fpriv->vm, &p->validated, &p->vm_pd);
 
@@ -545,7 +601,6 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
 
 	while (1) {
 		struct list_head need_pages;
-		unsigned i;
 
 		r = ttm_eu_reserve_buffers(&p->ticket, &p->validated, true,
 					   &duplicates);
@@ -555,17 +610,9 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
 			goto error_free_pages;
 		}
 
-		/* Without a BO list we don't have userptr BOs */
-		if (!p->bo_list)
-			break;
-
 		INIT_LIST_HEAD(&need_pages);
-		for (i = p->bo_list->first_userptr;
-		     i < p->bo_list->num_entries; ++i) {
-			struct amdgpu_bo *bo;
-
-			e = &p->bo_list->array[i];
-			bo = e->robj;
+		amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) {
+			struct amdgpu_bo *bo = e->robj;
 
 			if (amdgpu_ttm_tt_userptr_invalidated(bo->tbo.ttm,
 				 &e->user_invalidated) && e->user_pages) {
@@ -657,23 +704,12 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
 	amdgpu_cs_report_moved_bytes(p->adev, p->bytes_moved,
 				     p->bytes_moved_vis);
 
-	if (p->bo_list) {
-		struct amdgpu_vm *vm = &fpriv->vm;
-		unsigned i;
-
-		gds = p->bo_list->gds_obj;
-		gws = p->bo_list->gws_obj;
-		oa = p->bo_list->oa_obj;
-		for (i = 0; i < p->bo_list->num_entries; i++) {
-			struct amdgpu_bo *bo = p->bo_list->array[i].robj;
+	gds = p->bo_list->gds_obj;
+	gws = p->bo_list->gws_obj;
+	oa = p->bo_list->oa_obj;
 
-			p->bo_list->array[i].bo_va = amdgpu_vm_bo_find(vm, bo);
-		}
-	} else {
-		gds = p->adev->gds.gds_gfx_bo;
-		gws = p->adev->gds.gws_gfx_bo;
-		oa = p->adev->gds.oa_gfx_bo;
-	}
+	amdgpu_bo_list_for_each_entry(e, p->bo_list)
+		e->bo_va = amdgpu_vm_bo_find(vm, e->robj);
 
 	if (gds) {
 		p->job->gds_base = amdgpu_bo_gpu_offset(gds);
@@ -701,18 +737,13 @@ error_validate:
 
 error_free_pages:
 
-	if (p->bo_list) {
-		for (i = p->bo_list->first_userptr;
-		     i < p->bo_list->num_entries; ++i) {
-			e = &p->bo_list->array[i];
-
-			if (!e->user_pages)
-				continue;
+	amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) {
+		if (!e->user_pages)
+			continue;
 
-			release_pages(e->user_pages,
-				      e->robj->tbo.ttm->num_pages);
-			kvfree(e->user_pages);
-		}
+		release_pages(e->user_pages,
+			      e->robj->tbo.ttm->num_pages);
+		kvfree(e->user_pages);
 	}
 
 	return r;
@@ -774,12 +805,13 @@ static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser *parser, int error,
 
 static int amdgpu_bo_vm_update_pte(struct amdgpu_cs_parser *p)
 {
-	struct amdgpu_device *adev = p->adev;
 	struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
+	struct amdgpu_device *adev = p->adev;
 	struct amdgpu_vm *vm = &fpriv->vm;
+	struct amdgpu_bo_list_entry *e;
 	struct amdgpu_bo_va *bo_va;
 	struct amdgpu_bo *bo;
-	int i, r;
+	int r;
 
 	r = amdgpu_vm_clear_freed(adev, vm, NULL);
 	if (r)
@@ -809,29 +841,26 @@ static int amdgpu_bo_vm_update_pte(struct amdgpu_cs_parser *p)
 			return r;
 	}
 
-	if (p->bo_list) {
-		for (i = 0; i < p->bo_list->num_entries; i++) {
-			struct dma_fence *f;
-
-			/* ignore duplicates */
-			bo = p->bo_list->array[i].robj;
-			if (!bo)
-				continue;
+	amdgpu_bo_list_for_each_entry(e, p->bo_list) {
+		struct dma_fence *f;
 
-			bo_va = p->bo_list->array[i].bo_va;
-			if (bo_va == NULL)
-				continue;
+		/* ignore duplicates */
+		bo = e->robj;
+		if (!bo)
+			continue;
 
-			r = amdgpu_vm_bo_update(adev, bo_va, false);
-			if (r)
-				return r;
+		bo_va = e->bo_va;
+		if (bo_va == NULL)
+			continue;
 
-			f = bo_va->last_pt_update;
-			r = amdgpu_sync_fence(adev, &p->job->sync, f, false);
-			if (r)
-				return r;
-		}
+		r = amdgpu_vm_bo_update(adev, bo_va, false);
+		if (r)
+			return r;
 
+		f = bo_va->last_pt_update;
+		r = amdgpu_sync_fence(adev, &p->job->sync, f, false);
+		if (r)
+			return r;
 	}
 
 	r = amdgpu_vm_handle_moved(adev, vm);
@@ -846,15 +875,14 @@ static int amdgpu_bo_vm_update_pte(struct amdgpu_cs_parser *p)
 	if (r)
 		return r;
 
-	if (amdgpu_vm_debug && p->bo_list) {
+	if (amdgpu_vm_debug) {
 		/* Invalidate all BOs to test for userspace bugs */
-		for (i = 0; i < p->bo_list->num_entries; i++) {
+		amdgpu_bo_list_for_each_entry(e, p->bo_list) {
 			/* ignore duplicates */
-			bo = p->bo_list->array[i].robj;
-			if (!bo)
+			if (!e->robj)
 				continue;
 
-			amdgpu_vm_bo_invalidate(adev, bo, false);
+			amdgpu_vm_bo_invalidate(adev, e->robj, false);
 		}
 	}
 
@@ -866,11 +894,11 @@ static int amdgpu_cs_ib_vm_chunk(struct amdgpu_device *adev,
 {
 	struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
 	struct amdgpu_vm *vm = &fpriv->vm;
-	struct amdgpu_ring *ring = p->job->ring;
+	struct amdgpu_ring *ring = p->ring;
 	int r;
 
 	/* Only for UVD/VCE VM emulation */
-	if (p->job->ring->funcs->parse_cs) {
+	if (p->ring->funcs->parse_cs || p->ring->funcs->patch_cs_in_place) {
 		unsigned i, j;
 
 		for (i = 0, j = 0; i < p->nchunks && j < p->job->num_ibs; i++) {
@@ -911,12 +939,20 @@ static int amdgpu_cs_ib_vm_chunk(struct amdgpu_device *adev,
 			offset = m->start * AMDGPU_GPU_PAGE_SIZE;
 			kptr += va_start - offset;
 
-			memcpy(ib->ptr, kptr, chunk_ib->ib_bytes);
-			amdgpu_bo_kunmap(aobj);
-
-			r = amdgpu_ring_parse_cs(ring, p, j);
-			if (r)
-				return r;
+			if (p->ring->funcs->parse_cs) {
+				memcpy(ib->ptr, kptr, chunk_ib->ib_bytes);
+				amdgpu_bo_kunmap(aobj);
+
+				r = amdgpu_ring_parse_cs(ring, p, j);
+				if (r)
+					return r;
+			} else {
+				ib->ptr = (uint32_t *)kptr;
+				r = amdgpu_ring_patch_cs_in_place(ring, p, j);
+				amdgpu_bo_kunmap(aobj);
+				if (r)
+					return r;
+			}
 
 			j++;
 		}
@@ -928,6 +964,10 @@ static int amdgpu_cs_ib_vm_chunk(struct amdgpu_device *adev,
 		r = amdgpu_bo_vm_update_pte(p);
 		if (r)
 			return r;
+
+		r = reservation_object_reserve_shared(vm->root.base.bo->tbo.resv);
+		if (r)
+			return r;
 	}
 
 	return amdgpu_cs_sync_rings(p);
@@ -980,10 +1020,10 @@ static int amdgpu_cs_ib_fill(struct amdgpu_device *adev,
 			}
 		}
 
-		if (parser->job->ring && parser->job->ring != ring)
+		if (parser->ring && parser->ring != ring)
 			return -EINVAL;
 
-		parser->job->ring = ring;
+		parser->ring = ring;
 
 		r =  amdgpu_ib_get(adev, vm,
 					ring->funcs->parse_cs ? chunk_ib->ib_bytes : 0,
@@ -1002,11 +1042,11 @@ static int amdgpu_cs_ib_fill(struct amdgpu_device *adev,
 
 	/* UVD & VCE fw doesn't support user fences */
 	if (parser->job->uf_addr && (
-	    parser->job->ring->funcs->type == AMDGPU_RING_TYPE_UVD ||
-	    parser->job->ring->funcs->type == AMDGPU_RING_TYPE_VCE))
+	    parser->ring->funcs->type == AMDGPU_RING_TYPE_UVD ||
+	    parser->ring->funcs->type == AMDGPU_RING_TYPE_VCE))
 		return -EINVAL;
 
-	return amdgpu_ctx_wait_prev_fence(parser->ctx, parser->job->ring->idx);
+	return amdgpu_ctx_wait_prev_fence(parser->ctx, parser->ring->idx);
 }
 
 static int amdgpu_cs_process_fence_dep(struct amdgpu_cs_parser *p,
@@ -1157,31 +1197,30 @@ static void amdgpu_cs_post_dependencies(struct amdgpu_cs_parser *p)
 static int amdgpu_cs_submit(struct amdgpu_cs_parser *p,
 			    union drm_amdgpu_cs *cs)
 {
-	struct amdgpu_ring *ring = p->job->ring;
+	struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
+	struct amdgpu_ring *ring = p->ring;
 	struct drm_sched_entity *entity = &p->ctx->rings[ring->idx].entity;
+	enum drm_sched_priority priority;
+	struct amdgpu_bo_list_entry *e;
 	struct amdgpu_job *job;
-	unsigned i;
 	uint64_t seq;
 
 	int r;
 
 	amdgpu_mn_lock(p->mn);
-	if (p->bo_list) {
-		for (i = p->bo_list->first_userptr;
-		     i < p->bo_list->num_entries; ++i) {
-			struct amdgpu_bo *bo = p->bo_list->array[i].robj;
-
-			if (amdgpu_ttm_tt_userptr_needs_pages(bo->tbo.ttm)) {
-				amdgpu_mn_unlock(p->mn);
-				return -ERESTARTSYS;
-			}
+	amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) {
+		struct amdgpu_bo *bo = e->robj;
+
+		if (amdgpu_ttm_tt_userptr_needs_pages(bo->tbo.ttm)) {
+			amdgpu_mn_unlock(p->mn);
+			return -ERESTARTSYS;
 		}
 	}
 
 	job = p->job;
 	p->job = NULL;
 
-	r = drm_sched_job_init(&job->base, &ring->sched, entity, p->filp);
+	r = drm_sched_job_init(&job->base, entity, p->filp);
 	if (r) {
 		amdgpu_job_free(job);
 		amdgpu_mn_unlock(p->mn);
@@ -1189,7 +1228,6 @@ static int amdgpu_cs_submit(struct amdgpu_cs_parser *p,
 	}
 
 	job->owner = p->filp;
-	job->fence_ctx = entity->fence_context;
 	p->fence = dma_fence_get(&job->base.s_fence->finished);
 
 	r = amdgpu_ctx_add_fence(p->ctx, ring, p->fence, &seq);
@@ -1207,11 +1245,15 @@ static int amdgpu_cs_submit(struct amdgpu_cs_parser *p,
 	job->uf_sequence = seq;
 
 	amdgpu_job_free_resources(job);
-	amdgpu_ring_priority_get(job->ring, job->base.s_priority);
 
 	trace_amdgpu_cs_ioctl(job);
+	amdgpu_vm_bo_trace_cs(&fpriv->vm, &p->ticket);
+	priority = job->base.s_priority;
 	drm_sched_entity_push_job(&job->base, entity);
 
+	ring = to_amdgpu_ring(entity->rq->sched);
+	amdgpu_ring_priority_get(ring, priority);
+
 	ttm_eu_fence_buffer_objects(&p->ticket, &p->validated, p->fence);
 	amdgpu_mn_unlock(p->mn);
 
@@ -1602,7 +1644,7 @@ int amdgpu_cs_find_mapping(struct amdgpu_cs_parser *parser,
 
 	if (!((*bo)->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)) {
 		(*bo)->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
-		amdgpu_ttm_placement_from_domain(*bo, (*bo)->allowed_domains);
+		amdgpu_bo_placement_from_domain(*bo, (*bo)->allowed_domains);
 		r = ttm_bo_validate(&(*bo)->tbo, &(*bo)->placement, &ctx);
 		if (r)
 			return r;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
index 64b3a1ed04dc..df6965761046 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
@@ -90,8 +90,8 @@ static int amdgpu_ctx_init(struct amdgpu_device *adev,
 		if (ring == &adev->gfx.kiq.ring)
 			continue;
 
-		r = drm_sched_entity_init(&ring->sched, &ctx->rings[i].entity,
-					  rq, &ctx->guilty);
+		r = drm_sched_entity_init(&ctx->rings[i].entity,
+					  &rq, 1, &ctx->guilty);
 		if (r)
 			goto failed;
 	}
@@ -104,8 +104,7 @@ static int amdgpu_ctx_init(struct amdgpu_device *adev,
 
 failed:
 	for (j = 0; j < i; j++)
-		drm_sched_entity_fini(&adev->rings[j]->sched,
-				      &ctx->rings[j].entity);
+		drm_sched_entity_destroy(&ctx->rings[j].entity);
 	kfree(ctx->fences);
 	ctx->fences = NULL;
 	return r;
@@ -178,8 +177,7 @@ static void amdgpu_ctx_do_release(struct kref *ref)
 		if (ctx->adev->rings[i] == &ctx->adev->gfx.kiq.ring)
 			continue;
 
-		drm_sched_entity_fini(&ctx->adev->rings[i]->sched,
-			&ctx->rings[i].entity);
+		drm_sched_entity_destroy(&ctx->rings[i].entity);
 	}
 
 	amdgpu_ctx_fini(ref);
@@ -444,7 +442,7 @@ void amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr)
 	idr_init(&mgr->ctx_handles);
 }
 
-void amdgpu_ctx_mgr_entity_fini(struct amdgpu_ctx_mgr *mgr)
+void amdgpu_ctx_mgr_entity_flush(struct amdgpu_ctx_mgr *mgr)
 {
 	struct amdgpu_ctx *ctx;
 	struct idr *idp;
@@ -466,14 +464,14 @@ void amdgpu_ctx_mgr_entity_fini(struct amdgpu_ctx_mgr *mgr)
 			if (ctx->adev->rings[i] == &ctx->adev->gfx.kiq.ring)
 				continue;
 
-			max_wait = drm_sched_entity_do_release(&ctx->adev->rings[i]->sched,
-					  &ctx->rings[i].entity, max_wait);
+			max_wait = drm_sched_entity_flush(&ctx->rings[i].entity,
+							  max_wait);
 		}
 	}
 	mutex_unlock(&mgr->lock);
 }
 
-void amdgpu_ctx_mgr_entity_cleanup(struct amdgpu_ctx_mgr *mgr)
+void amdgpu_ctx_mgr_entity_fini(struct amdgpu_ctx_mgr *mgr)
 {
 	struct amdgpu_ctx *ctx;
 	struct idr *idp;
@@ -492,8 +490,7 @@ void amdgpu_ctx_mgr_entity_cleanup(struct amdgpu_ctx_mgr *mgr)
 				continue;
 
 			if (kref_read(&ctx->refcount) == 1)
-				drm_sched_entity_cleanup(&ctx->adev->rings[i]->sched,
-					&ctx->rings[i].entity);
+				drm_sched_entity_fini(&ctx->rings[i].entity);
 			else
 				DRM_ERROR("ctx %p is still alive\n", ctx);
 		}
@@ -506,7 +503,7 @@ void amdgpu_ctx_mgr_fini(struct amdgpu_ctx_mgr *mgr)
 	struct idr *idp;
 	uint32_t id;
 
-	amdgpu_ctx_mgr_entity_cleanup(mgr);
+	amdgpu_ctx_mgr_entity_fini(mgr);
 
 	idp = &mgr->ctx_handles;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index d82d0d314285..8ab5ccbc14ac 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -1076,7 +1076,7 @@ static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = {
 /**
  * amdgpu_device_ip_set_clockgating_state - set the CG state
  *
- * @adev: amdgpu_device pointer
+ * @dev: amdgpu_device pointer
  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
  * @state: clockgating state (gate or ungate)
  *
@@ -1110,7 +1110,7 @@ int amdgpu_device_ip_set_clockgating_state(void *dev,
 /**
  * amdgpu_device_ip_set_powergating_state - set the PG state
  *
- * @adev: amdgpu_device pointer
+ * @dev: amdgpu_device pointer
  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
  * @state: powergating state (gate or ungate)
  *
@@ -1221,7 +1221,7 @@ bool amdgpu_device_ip_is_idle(struct amdgpu_device *adev,
  * amdgpu_device_ip_get_ip_block - get a hw IP pointer
  *
  * @adev: amdgpu_device pointer
- * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
+ * @type: Type of hardware IP (SMU, GFX, UVD, etc.)
  *
  * Returns a pointer to the hardware IP block structure
  * if it exists for the asic, otherwise NULL.
@@ -1707,10 +1707,6 @@ static int amdgpu_device_ip_late_set_cg_state(struct amdgpu_device *adev)
 	if (amdgpu_emu_mode == 1)
 		return 0;
 
-	r = amdgpu_ib_ring_tests(adev);
-	if (r)
-		DRM_ERROR("ib ring test failed (%d).\n", r);
-
 	for (i = 0; i < adev->num_ip_blocks; i++) {
 		if (!adev->ip_blocks[i].status.valid)
 			continue;
@@ -1730,17 +1726,34 @@ static int amdgpu_device_ip_late_set_cg_state(struct amdgpu_device *adev)
 		}
 	}
 
-	if (adev->powerplay.pp_feature & PP_GFXOFF_MASK) {
-		/* enable gfx powergating */
-		amdgpu_device_ip_set_powergating_state(adev,
-						       AMD_IP_BLOCK_TYPE_GFX,
-						       AMD_PG_STATE_GATE);
-		/* enable gfxoff */
-		amdgpu_device_ip_set_powergating_state(adev,
-						       AMD_IP_BLOCK_TYPE_SMC,
-						       AMD_PG_STATE_GATE);
-	}
+	return 0;
+}
 
+static int amdgpu_device_ip_late_set_pg_state(struct amdgpu_device *adev)
+{
+	int i = 0, r;
+
+	if (amdgpu_emu_mode == 1)
+		return 0;
+
+	for (i = 0; i < adev->num_ip_blocks; i++) {
+		if (!adev->ip_blocks[i].status.valid)
+			continue;
+		/* skip CG for VCE/UVD, it's handled specially */
+		if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
+		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
+		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
+		    adev->ip_blocks[i].version->funcs->set_powergating_state) {
+			/* enable powergating to save power */
+			r = adev->ip_blocks[i].version->funcs->set_powergating_state((void *)adev,
+										     AMD_PG_STATE_GATE);
+			if (r) {
+				DRM_ERROR("set_powergating_state(gate) of IP block <%s> failed %d\n",
+					  adev->ip_blocks[i].version->funcs->name, r);
+				return r;
+			}
+		}
+	}
 	return 0;
 }
 
@@ -1774,6 +1787,9 @@ static int amdgpu_device_ip_late_init(struct amdgpu_device *adev)
 		}
 	}
 
+	amdgpu_device_ip_late_set_cg_state(adev);
+	amdgpu_device_ip_late_set_pg_state(adev);
+
 	queue_delayed_work(system_wq, &adev->late_init_work,
 			   msecs_to_jiffies(AMDGPU_RESUME_MS));
 
@@ -1812,6 +1828,8 @@ static int amdgpu_device_ip_fini(struct amdgpu_device *adev)
 					  adev->ip_blocks[i].version->funcs->name, r);
 				return r;
 			}
+			if (adev->powerplay.pp_funcs->set_powergating_by_smu)
+				amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, false);
 			r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
 			/* XXX handle errors */
 			if (r) {
@@ -1900,11 +1918,15 @@ static void amdgpu_device_ip_late_init_func_handler(struct work_struct *work)
 {
 	struct amdgpu_device *adev =
 		container_of(work, struct amdgpu_device, late_init_work.work);
-	amdgpu_device_ip_late_set_cg_state(adev);
+	int r;
+
+	r = amdgpu_ib_ring_tests(adev);
+	if (r)
+		DRM_ERROR("ib ring test failed (%d).\n", r);
 }
 
 /**
- * amdgpu_device_ip_suspend - run suspend for hardware IPs
+ * amdgpu_device_ip_suspend_phase1 - run suspend for hardware IPs (phase 1)
  *
  * @adev: amdgpu_device pointer
  *
@@ -1914,18 +1936,60 @@ static void amdgpu_device_ip_late_init_func_handler(struct work_struct *work)
  * in each IP into a state suitable for suspend.
  * Returns 0 on success, negative error code on failure.
  */
-int amdgpu_device_ip_suspend(struct amdgpu_device *adev)
+static int amdgpu_device_ip_suspend_phase1(struct amdgpu_device *adev)
 {
 	int i, r;
 
 	if (amdgpu_sriov_vf(adev))
 		amdgpu_virt_request_full_gpu(adev, false);
 
-	/* ungate SMC block powergating */
-	if (adev->powerplay.pp_feature & PP_GFXOFF_MASK)
-		amdgpu_device_ip_set_powergating_state(adev,
-						       AMD_IP_BLOCK_TYPE_SMC,
-						       AMD_PG_STATE_UNGATE);
+	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
+		if (!adev->ip_blocks[i].status.valid)
+			continue;
+		/* displays are handled separately */
+		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) {
+			/* ungate blocks so that suspend can properly shut them down */
+			if (adev->ip_blocks[i].version->funcs->set_clockgating_state) {
+				r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
+											     AMD_CG_STATE_UNGATE);
+				if (r) {
+					DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n",
+						  adev->ip_blocks[i].version->funcs->name, r);
+				}
+			}
+			/* XXX handle errors */
+			r = adev->ip_blocks[i].version->funcs->suspend(adev);
+			/* XXX handle errors */
+			if (r) {
+				DRM_ERROR("suspend of IP block <%s> failed %d\n",
+					  adev->ip_blocks[i].version->funcs->name, r);
+			}
+		}
+	}
+
+	if (amdgpu_sriov_vf(adev))
+		amdgpu_virt_release_full_gpu(adev, false);
+
+	return 0;
+}
+
+/**
+ * amdgpu_device_ip_suspend_phase2 - run suspend for hardware IPs (phase 2)
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Main suspend function for hardware IPs.  The list of all the hardware
+ * IPs that make up the asic is walked, clockgating is disabled and the
+ * suspend callbacks are run.  suspend puts the hardware and software state
+ * in each IP into a state suitable for suspend.
+ * Returns 0 on success, negative error code on failure.
+ */
+static int amdgpu_device_ip_suspend_phase2(struct amdgpu_device *adev)
+{
+	int i, r;
+
+	if (amdgpu_sriov_vf(adev))
+		amdgpu_virt_request_full_gpu(adev, false);
 
 	/* ungate SMC block first */
 	r = amdgpu_device_ip_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_SMC,
@@ -1934,9 +1998,16 @@ int amdgpu_device_ip_suspend(struct amdgpu_device *adev)
 		DRM_ERROR("set_clockgating_state(ungate) SMC failed %d\n", r);
 	}
 
+	/* call smu to disable gfx off feature first when suspend */
+	if (adev->powerplay.pp_funcs->set_powergating_by_smu)
+		amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, false);
+
 	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
 		if (!adev->ip_blocks[i].status.valid)
 			continue;
+		/* displays are handled in phase1 */
+		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE)
+			continue;
 		/* ungate blocks so that suspend can properly shut them down */
 		if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_SMC &&
 			adev->ip_blocks[i].version->funcs->set_clockgating_state) {
@@ -1962,6 +2033,29 @@ int amdgpu_device_ip_suspend(struct amdgpu_device *adev)
 	return 0;
 }
 
+/**
+ * amdgpu_device_ip_suspend - run suspend for hardware IPs
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Main suspend function for hardware IPs.  The list of all the hardware
+ * IPs that make up the asic is walked, clockgating is disabled and the
+ * suspend callbacks are run.  suspend puts the hardware and software state
+ * in each IP into a state suitable for suspend.
+ * Returns 0 on success, negative error code on failure.
+ */
+int amdgpu_device_ip_suspend(struct amdgpu_device *adev)
+{
+	int r;
+
+	r = amdgpu_device_ip_suspend_phase1(adev);
+	if (r)
+		return r;
+	r = amdgpu_device_ip_suspend_phase2(adev);
+
+	return r;
+}
+
 static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev)
 {
 	int i, r;
@@ -1984,7 +2078,7 @@ static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev)
 				continue;
 
 			r = block->version->funcs->hw_init(adev);
-			DRM_INFO("RE-INIT: %s %s\n", block->version->funcs->name, r?"failed":"successed");
+			DRM_INFO("RE-INIT: %s %s\n", block->version->funcs->name, r?"failed":"succeeded");
 			if (r)
 				return r;
 		}
@@ -2019,7 +2113,7 @@ static int amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device *adev)
 				continue;
 
 			r = block->version->funcs->hw_init(adev);
-			DRM_INFO("RE-INIT: %s %s\n", block->version->funcs->name, r?"failed":"successed");
+			DRM_INFO("RE-INIT: %s %s\n", block->version->funcs->name, r?"failed":"succeeded");
 			if (r)
 				return r;
 		}
@@ -2209,7 +2303,7 @@ bool amdgpu_device_has_dc_support(struct amdgpu_device *adev)
  * amdgpu_device_init - initialize the driver
  *
  * @adev: amdgpu_device pointer
- * @pdev: drm dev pointer
+ * @ddev: drm dev pointer
  * @pdev: pci dev pointer
  * @flags: driver flags
  *
@@ -2582,8 +2676,9 @@ void amdgpu_device_fini(struct amdgpu_device *adev)
 /**
  * amdgpu_device_suspend - initiate device suspend
  *
- * @pdev: drm dev pointer
- * @state: suspend state
+ * @dev: drm dev pointer
+ * @suspend: suspend state
+ * @fbcon : notify the fbdev of suspend
  *
  * Puts the hw in the suspend state (all asics).
  * Returns 0 for success or an error on failure.
@@ -2607,6 +2702,9 @@ int amdgpu_device_suspend(struct drm_device *dev, bool suspend, bool fbcon)
 
 	drm_kms_helper_poll_disable(dev);
 
+	if (fbcon)
+		amdgpu_fbdev_set_suspend(adev, 1);
+
 	if (!amdgpu_device_has_dc_support(adev)) {
 		/* turn off display hw */
 		drm_modeset_lock_all(dev);
@@ -2614,44 +2712,46 @@ int amdgpu_device_suspend(struct drm_device *dev, bool suspend, bool fbcon)
 			drm_helper_connector_dpms(connector, DRM_MODE_DPMS_OFF);
 		}
 		drm_modeset_unlock_all(dev);
-	}
-
-	amdgpu_amdkfd_suspend(adev);
-
-	/* unpin the front buffers and cursors */
-	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
-		struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
-		struct drm_framebuffer *fb = crtc->primary->fb;
-		struct amdgpu_bo *robj;
-
-		if (amdgpu_crtc->cursor_bo) {
-			struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
-			r = amdgpu_bo_reserve(aobj, true);
-			if (r == 0) {
-				amdgpu_bo_unpin(aobj);
-				amdgpu_bo_unreserve(aobj);
+			/* unpin the front buffers and cursors */
+		list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
+			struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
+			struct drm_framebuffer *fb = crtc->primary->fb;
+			struct amdgpu_bo *robj;
+
+			if (amdgpu_crtc->cursor_bo) {
+				struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
+				r = amdgpu_bo_reserve(aobj, true);
+				if (r == 0) {
+					amdgpu_bo_unpin(aobj);
+					amdgpu_bo_unreserve(aobj);
+				}
 			}
-		}
 
-		if (fb == NULL || fb->obj[0] == NULL) {
-			continue;
-		}
-		robj = gem_to_amdgpu_bo(fb->obj[0]);
-		/* don't unpin kernel fb objects */
-		if (!amdgpu_fbdev_robj_is_fb(adev, robj)) {
-			r = amdgpu_bo_reserve(robj, true);
-			if (r == 0) {
-				amdgpu_bo_unpin(robj);
-				amdgpu_bo_unreserve(robj);
+			if (fb == NULL || fb->obj[0] == NULL) {
+				continue;
+			}
+			robj = gem_to_amdgpu_bo(fb->obj[0]);
+			/* don't unpin kernel fb objects */
+			if (!amdgpu_fbdev_robj_is_fb(adev, robj)) {
+				r = amdgpu_bo_reserve(robj, true);
+				if (r == 0) {
+					amdgpu_bo_unpin(robj);
+					amdgpu_bo_unreserve(robj);
+				}
 			}
 		}
 	}
+
+	amdgpu_amdkfd_suspend(adev);
+
+	r = amdgpu_device_ip_suspend_phase1(adev);
+
 	/* evict vram memory */
 	amdgpu_bo_evict_vram(adev);
 
 	amdgpu_fence_driver_suspend(adev);
 
-	r = amdgpu_device_ip_suspend(adev);
+	r = amdgpu_device_ip_suspend_phase2(adev);
 
 	/* evict remaining vram memory
 	 * This second call to evict vram is to evict the gart page table
@@ -2670,18 +2770,15 @@ int amdgpu_device_suspend(struct drm_device *dev, bool suspend, bool fbcon)
 			DRM_ERROR("amdgpu asic reset failed\n");
 	}
 
-	if (fbcon) {
-		console_lock();
-		amdgpu_fbdev_set_suspend(adev, 1);
-		console_unlock();
-	}
 	return 0;
 }
 
 /**
  * amdgpu_device_resume - initiate device resume
  *
- * @pdev: drm dev pointer
+ * @dev: drm dev pointer
+ * @resume: resume state
+ * @fbcon : notify the fbdev of resume
  *
  * Bring the hw back to operating state (all asics).
  * Returns 0 for success or an error on failure.
@@ -2697,15 +2794,12 @@ int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon)
 	if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
 		return 0;
 
-	if (fbcon)
-		console_lock();
-
 	if (resume) {
 		pci_set_power_state(dev->pdev, PCI_D0);
 		pci_restore_state(dev->pdev);
 		r = pci_enable_device(dev->pdev);
 		if (r)
-			goto unlock;
+			return r;
 	}
 
 	/* post card */
@@ -2718,29 +2812,30 @@ int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon)
 	r = amdgpu_device_ip_resume(adev);
 	if (r) {
 		DRM_ERROR("amdgpu_device_ip_resume failed (%d).\n", r);
-		goto unlock;
+		return r;
 	}
 	amdgpu_fence_driver_resume(adev);
 
 
 	r = amdgpu_device_ip_late_init(adev);
 	if (r)
-		goto unlock;
-
-	/* pin cursors */
-	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
-		struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
-
-		if (amdgpu_crtc->cursor_bo) {
-			struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
-			r = amdgpu_bo_reserve(aobj, true);
-			if (r == 0) {
-				r = amdgpu_bo_pin(aobj,
-						  AMDGPU_GEM_DOMAIN_VRAM,
-						  &amdgpu_crtc->cursor_addr);
-				if (r != 0)
-					DRM_ERROR("Failed to pin cursor BO (%d)\n", r);
-				amdgpu_bo_unreserve(aobj);
+		return r;
+
+	if (!amdgpu_device_has_dc_support(adev)) {
+		/* pin cursors */
+		list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
+			struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
+
+			if (amdgpu_crtc->cursor_bo) {
+				struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
+				r = amdgpu_bo_reserve(aobj, true);
+				if (r == 0) {
+					r = amdgpu_bo_pin(aobj, AMDGPU_GEM_DOMAIN_VRAM);
+					if (r != 0)
+						DRM_ERROR("Failed to pin cursor BO (%d)\n", r);
+					amdgpu_crtc->cursor_addr = amdgpu_bo_gpu_offset(aobj);
+					amdgpu_bo_unreserve(aobj);
+				}
 			}
 		}
 	}
@@ -2748,6 +2843,9 @@ int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon)
 	if (r)
 		return r;
 
+	/* Make sure IB tests flushed */
+	flush_delayed_work(&adev->late_init_work);
+
 	/* blat the mode back in */
 	if (fbcon) {
 		if (!amdgpu_device_has_dc_support(adev)) {
@@ -2761,6 +2859,7 @@ int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon)
 			}
 			drm_modeset_unlock_all(dev);
 		}
+		amdgpu_fbdev_set_suspend(adev, 0);
 	}
 
 	drm_kms_helper_poll_enable(dev);
@@ -2784,15 +2883,7 @@ int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon)
 #ifdef CONFIG_PM
 	dev->dev->power.disable_depth--;
 #endif
-
-	if (fbcon)
-		amdgpu_fbdev_set_suspend(adev, 0);
-
-unlock:
-	if (fbcon)
-		console_unlock();
-
-	return r;
+	return 0;
 }
 
 /**
@@ -3017,7 +3108,7 @@ static int amdgpu_device_handle_vram_lost(struct amdgpu_device *adev)
 	long tmo;
 
 	if (amdgpu_sriov_runtime(adev))
-		tmo = msecs_to_jiffies(amdgpu_lockup_timeout);
+		tmo = msecs_to_jiffies(8000);
 	else
 		tmo = msecs_to_jiffies(100);
 
@@ -3069,7 +3160,7 @@ static int amdgpu_device_handle_vram_lost(struct amdgpu_device *adev)
  * @adev: amdgpu device pointer
  *
  * attempt to do soft-reset or full-reset and reinitialize Asic
- * return 0 means successed otherwise failed
+ * return 0 means succeeded otherwise failed
  */
 static int amdgpu_device_reset(struct amdgpu_device *adev)
 {
@@ -3144,9 +3235,10 @@ out:
  * amdgpu_device_reset_sriov - reset ASIC for SR-IOV vf
  *
  * @adev: amdgpu device pointer
+ * @from_hypervisor: request from hypervisor
  *
  * do VF FLR and reinitialize Asic
- * return 0 means successed otherwise failed
+ * return 0 means succeeded otherwise failed
  */
 static int amdgpu_device_reset_sriov(struct amdgpu_device *adev,
 				     bool from_hypervisor)
@@ -3191,7 +3283,7 @@ error:
  *
  * @adev: amdgpu device pointer
  * @job: which job trigger hang
- * @force forces reset regardless of amdgpu_gpu_recovery
+ * @force: forces reset regardless of amdgpu_gpu_recovery
  *
  * Attempt to reset the GPU if it has hung (all asics).
  * Returns 0 for success or an error on failure.
@@ -3218,6 +3310,9 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
 	atomic_inc(&adev->gpu_reset_counter);
 	adev->in_gpu_reset = 1;
 
+	/* Block kfd */
+	amdgpu_amdkfd_pre_reset(adev);
+
 	/* block TTM */
 	resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
 
@@ -3230,10 +3325,10 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
 
 		kthread_park(ring->sched.thread);
 
-		if (job && job->ring->idx != i)
+		if (job && job->base.sched == &ring->sched)
 			continue;
 
-		drm_sched_hw_job_reset(&ring->sched, &job->base);
+		drm_sched_hw_job_reset(&ring->sched, job ? &job->base : NULL);
 
 		/* after all hw jobs are reset, hw fence is meaningless, so force_completion */
 		amdgpu_fence_driver_force_completion(ring);
@@ -3254,7 +3349,7 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
 		 * or all rings (in the case @job is NULL)
 		 * after above amdgpu_reset accomplished
 		 */
-		if ((!job || job->ring->idx == i) && !r)
+		if ((!job || job->base.sched == &ring->sched) && !r)
 			drm_sched_job_recovery(&ring->sched);
 
 		kthread_unpark(ring->sched.thread);
@@ -3271,9 +3366,11 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
 		dev_info(adev->dev, "GPU reset(%d) failed\n", atomic_read(&adev->gpu_reset_counter));
 		amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, r);
 	} else {
-		dev_info(adev->dev, "GPU reset(%d) successed!\n",atomic_read(&adev->gpu_reset_counter));
+		dev_info(adev->dev, "GPU reset(%d) succeeded!\n",atomic_read(&adev->gpu_reset_counter));
 	}
 
+	/*unlock kfd */
+	amdgpu_amdkfd_post_reset(adev);
 	amdgpu_vf_error_trans_all(adev);
 	adev->in_gpu_reset = 0;
 	mutex_unlock(&adev->lock_reset);
@@ -3291,8 +3388,9 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
  */
 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
 {
-	u32 mask;
-	int ret;
+	struct pci_dev *pdev;
+	enum pci_bus_speed speed_cap;
+	enum pcie_link_width link_width;
 
 	if (amdgpu_pcie_gen_cap)
 		adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap;
@@ -3310,27 +3408,61 @@ static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
 	}
 
 	if (adev->pm.pcie_gen_mask == 0) {
-		ret = drm_pcie_get_speed_cap_mask(adev->ddev, &mask);
-		if (!ret) {
-			adev->pm.pcie_gen_mask = (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
+		/* asic caps */
+		pdev = adev->pdev;
+		speed_cap = pcie_get_speed_cap(pdev);
+		if (speed_cap == PCI_SPEED_UNKNOWN) {
+			adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
 						  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
 						  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
-
-			if (mask & DRM_PCIE_SPEED_25)
-				adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1;
-			if (mask & DRM_PCIE_SPEED_50)
-				adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2;
-			if (mask & DRM_PCIE_SPEED_80)
-				adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3;
 		} else {
-			adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
+			if (speed_cap == PCIE_SPEED_16_0GT)
+				adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
+							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
+							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
+							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4);
+			else if (speed_cap == PCIE_SPEED_8_0GT)
+				adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
+							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
+							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
+			else if (speed_cap == PCIE_SPEED_5_0GT)
+				adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
+							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2);
+			else
+				adev->pm.pcie_gen_mask |= CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1;
+		}
+		/* platform caps */
+		pdev = adev->ddev->pdev->bus->self;
+		speed_cap = pcie_get_speed_cap(pdev);
+		if (speed_cap == PCI_SPEED_UNKNOWN) {
+			adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
+						   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
+		} else {
+			if (speed_cap == PCIE_SPEED_16_0GT)
+				adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
+							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
+							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
+							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4);
+			else if (speed_cap == PCIE_SPEED_8_0GT)
+				adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
+							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
+							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3);
+			else if (speed_cap == PCIE_SPEED_5_0GT)
+				adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
+							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
+			else
+				adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1;
+
 		}
 	}
 	if (adev->pm.pcie_mlw_mask == 0) {
-		ret = drm_pcie_get_max_link_width(adev->ddev, &mask);
-		if (!ret) {
-			switch (mask) {
-			case 32:
+		pdev = adev->ddev->pdev->bus->self;
+		link_width = pcie_get_width_cap(pdev);
+		if (link_width == PCIE_LNK_WIDTH_UNKNOWN) {
+			adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_PCIE_MLW_MASK;
+		} else {
+			switch (link_width) {
+			case PCIE_LNK_X32:
 				adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 |
 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
@@ -3339,7 +3471,7 @@ static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
 				break;
-			case 16:
+			case PCIE_LNK_X16:
 				adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
@@ -3347,36 +3479,34 @@ static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
 				break;
-			case 12:
+			case PCIE_LNK_X12:
 				adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
 				break;
-			case 8:
+			case PCIE_LNK_X8:
 				adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
 				break;
-			case 4:
+			case PCIE_LNK_X4:
 				adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
 				break;
-			case 2:
+			case PCIE_LNK_X2:
 				adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
 				break;
-			case 1:
+			case PCIE_LNK_X1:
 				adev->pm.pcie_mlw_mask = CAIL_PCIE_LINK_WIDTH_SUPPORT_X1;
 				break;
 			default:
 				break;
 			}
-		} else {
-			adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
 		}
 	}
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c
index 76ee8e04ff11..6748cd7fc129 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c
@@ -157,7 +157,6 @@ int amdgpu_display_crtc_page_flip_target(struct drm_crtc *crtc,
 	struct amdgpu_bo *new_abo;
 	unsigned long flags;
 	u64 tiling_flags;
-	u64 base;
 	int i, r;
 
 	work = kzalloc(sizeof *work, GFP_KERNEL);
@@ -189,12 +188,18 @@ int amdgpu_display_crtc_page_flip_target(struct drm_crtc *crtc,
 		goto cleanup;
 	}
 
-	r = amdgpu_bo_pin(new_abo, amdgpu_display_supported_domains(adev), &base);
+	r = amdgpu_bo_pin(new_abo, amdgpu_display_supported_domains(adev));
 	if (unlikely(r != 0)) {
 		DRM_ERROR("failed to pin new abo buffer before flip\n");
 		goto unreserve;
 	}
 
+	r = amdgpu_ttm_alloc_gart(&new_abo->tbo);
+	if (unlikely(r != 0)) {
+		DRM_ERROR("%p bind failed\n", new_abo);
+		goto unpin;
+	}
+
 	r = reservation_object_get_fences_rcu(new_abo->tbo.resv, &work->excl,
 					      &work->shared_count,
 					      &work->shared);
@@ -206,7 +211,7 @@ int amdgpu_display_crtc_page_flip_target(struct drm_crtc *crtc,
 	amdgpu_bo_get_tiling_flags(new_abo, &tiling_flags);
 	amdgpu_bo_unreserve(new_abo);
 
-	work->base = base;
+	work->base = amdgpu_bo_gpu_offset(new_abo);
 	work->target_vblank = target - (uint32_t)drm_crtc_vblank_count(crtc) +
 		amdgpu_get_vblank_counter_kms(dev, work->crtc_id);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.c
index 77ad59ade85c..1c4595562f8f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.c
@@ -28,6 +28,7 @@
 #include "amdgpu_i2c.h"
 #include "amdgpu_dpm.h"
 #include "atom.h"
+#include "amd_pcie.h"
 
 void amdgpu_dpm_print_class_info(u32 class, u32 class2)
 {
@@ -936,9 +937,11 @@ enum amdgpu_pcie_gen amdgpu_get_pcie_gen_support(struct amdgpu_device *adev,
 	case AMDGPU_PCIE_GEN3:
 		return AMDGPU_PCIE_GEN3;
 	default:
-		if ((sys_mask & DRM_PCIE_SPEED_80) && (default_gen == AMDGPU_PCIE_GEN3))
+		if ((sys_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) &&
+		    (default_gen == AMDGPU_PCIE_GEN3))
 			return AMDGPU_PCIE_GEN3;
-		else if ((sys_mask & DRM_PCIE_SPEED_50) && (default_gen == AMDGPU_PCIE_GEN2))
+		else if ((sys_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2) &&
+			 (default_gen == AMDGPU_PCIE_GEN2))
 			return AMDGPU_PCIE_GEN2;
 		else
 			return AMDGPU_PCIE_GEN1;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h
index 9acfbee91c40..ff24e1cc5b65 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h
@@ -287,12 +287,6 @@ enum amdgpu_pcie_gen {
 #define amdgpu_dpm_force_performance_level(adev, l) \
 		((adev)->powerplay.pp_funcs->force_performance_level((adev)->powerplay.pp_handle, (l)))
 
-#define amdgpu_dpm_powergate_uvd(adev, g) \
-		((adev)->powerplay.pp_funcs->powergate_uvd((adev)->powerplay.pp_handle, (g)))
-
-#define amdgpu_dpm_powergate_vce(adev, g) \
-		((adev)->powerplay.pp_funcs->powergate_vce((adev)->powerplay.pp_handle, (g)))
-
 #define amdgpu_dpm_get_current_power_state(adev) \
 		((adev)->powerplay.pp_funcs->get_current_power_state((adev)->powerplay.pp_handle))
 
@@ -347,6 +341,10 @@ enum amdgpu_pcie_gen {
 		((adev)->powerplay.pp_funcs->set_clockgating_by_smu(\
 			(adev)->powerplay.pp_handle, msg_id))
 
+#define amdgpu_dpm_set_powergating_by_smu(adev, block_type, gate) \
+		((adev)->powerplay.pp_funcs->set_powergating_by_smu(\
+			(adev)->powerplay.pp_handle, block_type, gate))
+
 #define amdgpu_dpm_get_power_profile_mode(adev, buf) \
 		((adev)->powerplay.pp_funcs->get_power_profile_mode(\
 			(adev)->powerplay.pp_handle, buf))
@@ -359,10 +357,6 @@ enum amdgpu_pcie_gen {
 		((adev)->powerplay.pp_funcs->odn_edit_dpm_table(\
 			(adev)->powerplay.pp_handle, type, parameter, size))
 
-#define amdgpu_dpm_set_mmhub_powergating_by_smu(adev) \
-		((adev)->powerplay.pp_funcs->set_mmhub_powergating_by_smu( \
-		(adev)->powerplay.pp_handle))
-
 struct amdgpu_dpm {
 	struct amdgpu_ps        *ps;
 	/* number of valid power states */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index a549483032b0..8843a06360fa 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -1,10 +1,3 @@
-/**
- * \file amdgpu_drv.c
- * AMD Amdgpu driver
- *
- * \author Gareth Hughes <gareth@valinux.com>
- */
-
 /*
  * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California.
  * All Rights Reserved.
@@ -76,9 +69,10 @@
  * - 3.24.0 - Add high priority compute support for gfx9
  * - 3.25.0 - Add support for sensor query info (stable pstate sclk/mclk).
  * - 3.26.0 - GFX9: Process AMDGPU_IB_FLAG_TC_WB_NOT_INVALIDATE.
+ * - 3.27.0 - Add new chunk to to AMDGPU_CS to enable BO_LIST creation.
  */
 #define KMS_DRIVER_MAJOR	3
-#define KMS_DRIVER_MINOR	26
+#define KMS_DRIVER_MINOR	27
 #define KMS_DRIVER_PATCHLEVEL	0
 
 int amdgpu_vram_limit = 0;
@@ -110,11 +104,8 @@ int amdgpu_vram_page_split = 512;
 int amdgpu_vm_update_mode = -1;
 int amdgpu_exp_hw_support = 0;
 int amdgpu_dc = -1;
-int amdgpu_dc_log = 0;
 int amdgpu_sched_jobs = 32;
 int amdgpu_sched_hw_submission = 2;
-int amdgpu_no_evict = 0;
-int amdgpu_direct_gma_size = 0;
 uint amdgpu_pcie_gen_cap = 0;
 uint amdgpu_pcie_lane_cap = 0;
 uint amdgpu_cg_mask = 0xffffffff;
@@ -122,7 +113,8 @@ uint amdgpu_pg_mask = 0xffffffff;
 uint amdgpu_sdma_phase_quantum = 32;
 char *amdgpu_disable_cu = NULL;
 char *amdgpu_virtual_display = NULL;
-uint amdgpu_pp_feature_mask = 0xffff3fff; /* gfxoff (bit 15) disabled by default */
+/* OverDrive(bit 14),gfxoff(bit 15),stutter mode(bit 17) disabled by default*/
+uint amdgpu_pp_feature_mask = 0xfffd3fff;
 int amdgpu_ngg = 0;
 int amdgpu_prim_buf_per_se = 0;
 int amdgpu_pos_buf_per_se = 0;
@@ -135,163 +127,368 @@ int amdgpu_gpu_recovery = -1; /* auto */
 int amdgpu_emu_mode = 0;
 uint amdgpu_smu_memory_pool_size = 0;
 
+/**
+ * DOC: vramlimit (int)
+ * Restrict the total amount of VRAM in MiB for testing.  The default is 0 (Use full VRAM).
+ */
 MODULE_PARM_DESC(vramlimit, "Restrict VRAM for testing, in megabytes");
 module_param_named(vramlimit, amdgpu_vram_limit, int, 0600);
 
+/**
+ * DOC: vis_vramlimit (int)
+ * Restrict the amount of CPU visible VRAM in MiB for testing.  The default is 0 (Use full CPU visible VRAM).
+ */
 MODULE_PARM_DESC(vis_vramlimit, "Restrict visible VRAM for testing, in megabytes");
 module_param_named(vis_vramlimit, amdgpu_vis_vram_limit, int, 0444);
 
+/**
+ * DOC: gartsize (uint)
+ * Restrict the size of GART in Mib (32, 64, etc.) for testing. The default is -1 (The size depends on asic).
+ */
 MODULE_PARM_DESC(gartsize, "Size of GART to setup in megabytes (32, 64, etc., -1=auto)");
 module_param_named(gartsize, amdgpu_gart_size, uint, 0600);
 
+/**
+ * DOC: gttsize (int)
+ * Restrict the size of GTT domain in MiB for testing. The default is -1 (It's VRAM size if 3GB < VRAM < 3/4 RAM,
+ * otherwise 3/4 RAM size).
+ */
 MODULE_PARM_DESC(gttsize, "Size of the GTT domain in megabytes (-1 = auto)");
 module_param_named(gttsize, amdgpu_gtt_size, int, 0600);
 
+/**
+ * DOC: moverate (int)
+ * Set maximum buffer migration rate in MB/s. The default is -1 (8 MB/s).
+ */
 MODULE_PARM_DESC(moverate, "Maximum buffer migration rate in MB/s. (32, 64, etc., -1=auto, 0=1=disabled)");
 module_param_named(moverate, amdgpu_moverate, int, 0600);
 
+/**
+ * DOC: benchmark (int)
+ * Run benchmarks. The default is 0 (Skip benchmarks).
+ */
 MODULE_PARM_DESC(benchmark, "Run benchmark");
 module_param_named(benchmark, amdgpu_benchmarking, int, 0444);
 
+/**
+ * DOC: test (int)
+ * Test BO GTT->VRAM and VRAM->GTT GPU copies. The default is 0 (Skip test, only set 1 to run test).
+ */
 MODULE_PARM_DESC(test, "Run tests");
 module_param_named(test, amdgpu_testing, int, 0444);
 
+/**
+ * DOC: audio (int)
+ * Set HDMI/DPAudio. Only affects non-DC display handling. The default is -1 (Enabled), set 0 to disabled it.
+ */
 MODULE_PARM_DESC(audio, "Audio enable (-1 = auto, 0 = disable, 1 = enable)");
 module_param_named(audio, amdgpu_audio, int, 0444);
 
+/**
+ * DOC: disp_priority (int)
+ * Set display Priority (1 = normal, 2 = high). Only affects non-DC display handling. The default is 0 (auto).
+ */
 MODULE_PARM_DESC(disp_priority, "Display Priority (0 = auto, 1 = normal, 2 = high)");
 module_param_named(disp_priority, amdgpu_disp_priority, int, 0444);
 
+/**
+ * DOC: hw_i2c (int)
+ * To enable hw i2c engine. Only affects non-DC display handling. The default is 0 (Disabled).
+ */
 MODULE_PARM_DESC(hw_i2c, "hw i2c engine enable (0 = disable)");
 module_param_named(hw_i2c, amdgpu_hw_i2c, int, 0444);
 
+/**
+ * DOC: pcie_gen2 (int)
+ * To disable PCIE Gen2/3 mode (0 = disable, 1 = enable). The default is -1 (auto, enabled).
+ */
 MODULE_PARM_DESC(pcie_gen2, "PCIE Gen2 mode (-1 = auto, 0 = disable, 1 = enable)");
 module_param_named(pcie_gen2, amdgpu_pcie_gen2, int, 0444);
 
+/**
+ * DOC: msi (int)
+ * To disable Message Signaled Interrupts (MSI) functionality (1 = enable, 0 = disable). The default is -1 (auto, enabled).
+ */
 MODULE_PARM_DESC(msi, "MSI support (1 = enable, 0 = disable, -1 = auto)");
 module_param_named(msi, amdgpu_msi, int, 0444);
 
+/**
+ * DOC: lockup_timeout (int)
+ * Set GPU scheduler timeout value in ms. Value 0 is invalidated, will be adjusted to 10000.
+ * Negative values mean 'infinite timeout' (MAX_JIFFY_OFFSET). The default is 10000.
+ */
 MODULE_PARM_DESC(lockup_timeout, "GPU lockup timeout in ms > 0 (default 10000)");
 module_param_named(lockup_timeout, amdgpu_lockup_timeout, int, 0444);
 
+/**
+ * DOC: dpm (int)
+ * Override for dynamic power management setting (1 = enable, 0 = disable). The default is -1 (auto).
+ */
 MODULE_PARM_DESC(dpm, "DPM support (1 = enable, 0 = disable, -1 = auto)");
 module_param_named(dpm, amdgpu_dpm, int, 0444);
 
+/**
+ * DOC: fw_load_type (int)
+ * Set different firmware loading type for debugging (0 = direct, 1 = SMU, 2 = PSP). The default is -1 (auto).
+ */
 MODULE_PARM_DESC(fw_load_type, "firmware loading type (0 = direct, 1 = SMU, 2 = PSP, -1 = auto)");
 module_param_named(fw_load_type, amdgpu_fw_load_type, int, 0444);
 
+/**
+ * DOC: aspm (int)
+ * To disable ASPM (1 = enable, 0 = disable). The default is -1 (auto, enabled).
+ */
 MODULE_PARM_DESC(aspm, "ASPM support (1 = enable, 0 = disable, -1 = auto)");
 module_param_named(aspm, amdgpu_aspm, int, 0444);
 
+/**
+ * DOC: runpm (int)
+ * Override for runtime power management control for dGPUs in PX/HG laptops. The amdgpu driver can dynamically power down
+ * the dGPU on PX/HG laptops when it is idle. The default is -1 (auto enable). Setting the value to 0 disables this functionality.
+ */
 MODULE_PARM_DESC(runpm, "PX runtime pm (1 = force enable, 0 = disable, -1 = PX only default)");
 module_param_named(runpm, amdgpu_runtime_pm, int, 0444);
 
+/**
+ * DOC: ip_block_mask (uint)
+ * Override what IP blocks are enabled on the GPU. Each GPU is a collection of IP blocks (gfx, display, video, etc.).
+ * Use this parameter to disable specific blocks. Note that the IP blocks do not have a fixed index. Some asics may not have
+ * some IPs or may include multiple instances of an IP so the ordering various from asic to asic. See the driver output in
+ * the kernel log for the list of IPs on the asic. The default is 0xffffffff (enable all blocks on a device).
+ */
 MODULE_PARM_DESC(ip_block_mask, "IP Block Mask (all blocks enabled (default))");
 module_param_named(ip_block_mask, amdgpu_ip_block_mask, uint, 0444);
 
+/**
+ * DOC: bapm (int)
+ * Bidirectional Application Power Management (BAPM) used to dynamically share TDP between CPU and GPU. Set value 0 to disable it.
+ * The default -1 (auto, enabled)
+ */
 MODULE_PARM_DESC(bapm, "BAPM support (1 = enable, 0 = disable, -1 = auto)");
 module_param_named(bapm, amdgpu_bapm, int, 0444);
 
+/**
+ * DOC: deep_color (int)
+ * Set 1 to enable Deep Color support. Only affects non-DC display handling. The default is 0 (disabled).
+ */
 MODULE_PARM_DESC(deep_color, "Deep Color support (1 = enable, 0 = disable (default))");
 module_param_named(deep_color, amdgpu_deep_color, int, 0444);
 
+/**
+ * DOC: vm_size (int)
+ * Override the size of the GPU's per client virtual address space in GiB.  The default is -1 (automatic for each asic).
+ */
 MODULE_PARM_DESC(vm_size, "VM address space size in gigabytes (default 64GB)");
 module_param_named(vm_size, amdgpu_vm_size, int, 0444);
 
+/**
+ * DOC: vm_fragment_size (int)
+ * Override VM fragment size in bits (4, 5, etc. 4 = 64K, 9 = 2M). The default is -1 (automatic for each asic).
+ */
 MODULE_PARM_DESC(vm_fragment_size, "VM fragment size in bits (4, 5, etc. 4 = 64K (default), Max 9 = 2M)");
 module_param_named(vm_fragment_size, amdgpu_vm_fragment_size, int, 0444);
 
+/**
+ * DOC: vm_block_size (int)
+ * Override VM page table size in bits (default depending on vm_size and hw setup). The default is -1 (automatic for each asic).
+ */
 MODULE_PARM_DESC(vm_block_size, "VM page table size in bits (default depending on vm_size)");
 module_param_named(vm_block_size, amdgpu_vm_block_size, int, 0444);
 
+/**
+ * DOC: vm_fault_stop (int)
+ * Stop on VM fault for debugging (0 = never, 1 = print first, 2 = always). The default is 0 (No stop).
+ */
 MODULE_PARM_DESC(vm_fault_stop, "Stop on VM fault (0 = never (default), 1 = print first, 2 = always)");
 module_param_named(vm_fault_stop, amdgpu_vm_fault_stop, int, 0444);
 
+/**
+ * DOC: vm_debug (int)
+ * Debug VM handling (0 = disabled, 1 = enabled). The default is 0 (Disabled).
+ */
 MODULE_PARM_DESC(vm_debug, "Debug VM handling (0 = disabled (default), 1 = enabled)");
 module_param_named(vm_debug, amdgpu_vm_debug, int, 0644);
 
+/**
+ * DOC: vm_update_mode (int)
+ * Override VM update mode. VM updated by using CPU (0 = never, 1 = Graphics only, 2 = Compute only, 3 = Both). The default
+ * is -1 (Only in large BAR(LB) systems Compute VM tables will be updated by CPU, otherwise 0, never).
+ */
 MODULE_PARM_DESC(vm_update_mode, "VM update using CPU (0 = never (default except for large BAR(LB)), 1 = Graphics only, 2 = Compute only (default for LB), 3 = Both");
 module_param_named(vm_update_mode, amdgpu_vm_update_mode, int, 0444);
 
+/**
+ * DOC: vram_page_split (int)
+ * Override the number of pages after we split VRAM allocations (default 512, -1 = disable). The default is 512.
+ */
 MODULE_PARM_DESC(vram_page_split, "Number of pages after we split VRAM allocations (default 512, -1 = disable)");
 module_param_named(vram_page_split, amdgpu_vram_page_split, int, 0444);
 
+/**
+ * DOC: exp_hw_support (int)
+ * Enable experimental hw support (1 = enable). The default is 0 (disabled).
+ */
 MODULE_PARM_DESC(exp_hw_support, "experimental hw support (1 = enable, 0 = disable (default))");
 module_param_named(exp_hw_support, amdgpu_exp_hw_support, int, 0444);
 
+/**
+ * DOC: dc (int)
+ * Disable/Enable Display Core driver for debugging (1 = enable, 0 = disable). The default is -1 (automatic for each asic).
+ */
 MODULE_PARM_DESC(dc, "Display Core driver (1 = enable, 0 = disable, -1 = auto (default))");
 module_param_named(dc, amdgpu_dc, int, 0444);
 
-MODULE_PARM_DESC(dc_log, "Display Core Log Level (0 = minimal (default), 1 = chatty");
-module_param_named(dc_log, amdgpu_dc_log, int, 0444);
-
+/**
+ * DOC: sched_jobs (int)
+ * Override the max number of jobs supported in the sw queue. The default is 32.
+ */
 MODULE_PARM_DESC(sched_jobs, "the max number of jobs supported in the sw queue (default 32)");
 module_param_named(sched_jobs, amdgpu_sched_jobs, int, 0444);
 
+/**
+ * DOC: sched_hw_submission (int)
+ * Override the max number of HW submissions. The default is 2.
+ */
 MODULE_PARM_DESC(sched_hw_submission, "the max number of HW submissions (default 2)");
 module_param_named(sched_hw_submission, amdgpu_sched_hw_submission, int, 0444);
 
+/**
+ * DOC: ppfeaturemask (uint)
+ * Override power features enabled. See enum PP_FEATURE_MASK in drivers/gpu/drm/amd/include/amd_shared.h.
+ * The default is the current set of stable power features.
+ */
 MODULE_PARM_DESC(ppfeaturemask, "all power features enabled (default))");
 module_param_named(ppfeaturemask, amdgpu_pp_feature_mask, uint, 0444);
 
-MODULE_PARM_DESC(no_evict, "Support pinning request from user space (1 = enable, 0 = disable (default))");
-module_param_named(no_evict, amdgpu_no_evict, int, 0444);
-
-MODULE_PARM_DESC(direct_gma_size, "Direct GMA size in megabytes (max 96MB)");
-module_param_named(direct_gma_size, amdgpu_direct_gma_size, int, 0444);
-
+/**
+ * DOC: pcie_gen_cap (uint)
+ * Override PCIE gen speed capabilities. See the CAIL flags in drivers/gpu/drm/amd/include/amd_pcie.h.
+ * The default is 0 (automatic for each asic).
+ */
 MODULE_PARM_DESC(pcie_gen_cap, "PCIE Gen Caps (0: autodetect (default))");
 module_param_named(pcie_gen_cap, amdgpu_pcie_gen_cap, uint, 0444);
 
+/**
+ * DOC: pcie_lane_cap (uint)
+ * Override PCIE lanes capabilities. See the CAIL flags in drivers/gpu/drm/amd/include/amd_pcie.h.
+ * The default is 0 (automatic for each asic).
+ */
 MODULE_PARM_DESC(pcie_lane_cap, "PCIE Lane Caps (0: autodetect (default))");
 module_param_named(pcie_lane_cap, amdgpu_pcie_lane_cap, uint, 0444);
 
+/**
+ * DOC: cg_mask (uint)
+ * Override Clockgating features enabled on GPU (0 = disable clock gating). See the AMD_CG_SUPPORT flags in
+ * drivers/gpu/drm/amd/include/amd_shared.h. The default is 0xffffffff (all enabled).
+ */
 MODULE_PARM_DESC(cg_mask, "Clockgating flags mask (0 = disable clock gating)");
 module_param_named(cg_mask, amdgpu_cg_mask, uint, 0444);
 
+/**
+ * DOC: pg_mask (uint)
+ * Override Powergating features enabled on GPU (0 = disable power gating). See the AMD_PG_SUPPORT flags in
+ * drivers/gpu/drm/amd/include/amd_shared.h. The default is 0xffffffff (all enabled).
+ */
 MODULE_PARM_DESC(pg_mask, "Powergating flags mask (0 = disable power gating)");
 module_param_named(pg_mask, amdgpu_pg_mask, uint, 0444);
 
+/**
+ * DOC: sdma_phase_quantum (uint)
+ * Override SDMA context switch phase quantum (x 1K GPU clock cycles, 0 = no change). The default is 32.
+ */
 MODULE_PARM_DESC(sdma_phase_quantum, "SDMA context switch phase quantum (x 1K GPU clock cycles, 0 = no change (default 32))");
 module_param_named(sdma_phase_quantum, amdgpu_sdma_phase_quantum, uint, 0444);
 
+/**
+ * DOC: disable_cu (charp)
+ * Set to disable CUs (It's set like se.sh.cu,...). The default is NULL.
+ */
 MODULE_PARM_DESC(disable_cu, "Disable CUs (se.sh.cu,...)");
 module_param_named(disable_cu, amdgpu_disable_cu, charp, 0444);
 
+/**
+ * DOC: virtual_display (charp)
+ * Set to enable virtual display feature. This feature provides a virtual display hardware on headless boards
+ * or in virtualized environments. It will be set like xxxx:xx:xx.x,x;xxxx:xx:xx.x,x. It's the pci address of
+ * the device, plus the number of crtcs to expose. E.g., 0000:26:00.0,4 would enable 4 virtual crtcs on the pci
+ * device at 26:00.0. The default is NULL.
+ */
 MODULE_PARM_DESC(virtual_display,
 		 "Enable virtual display feature (the virtual_display will be set like xxxx:xx:xx.x,x;xxxx:xx:xx.x,x)");
 module_param_named(virtual_display, amdgpu_virtual_display, charp, 0444);
 
+/**
+ * DOC: ngg (int)
+ * Set to enable Next Generation Graphics (1 = enable). The default is 0 (disabled).
+ */
 MODULE_PARM_DESC(ngg, "Next Generation Graphics (1 = enable, 0 = disable(default depending on gfx))");
 module_param_named(ngg, amdgpu_ngg, int, 0444);
 
+/**
+ * DOC: prim_buf_per_se (int)
+ * Override the size of Primitive Buffer per Shader Engine in Byte. The default is 0 (depending on gfx).
+ */
 MODULE_PARM_DESC(prim_buf_per_se, "the size of Primitive Buffer per Shader Engine (default depending on gfx)");
 module_param_named(prim_buf_per_se, amdgpu_prim_buf_per_se, int, 0444);
 
+/**
+ * DOC: pos_buf_per_se (int)
+ * Override the size of Position Buffer per Shader Engine in Byte. The default is 0 (depending on gfx).
+ */
 MODULE_PARM_DESC(pos_buf_per_se, "the size of Position Buffer per Shader Engine (default depending on gfx)");
 module_param_named(pos_buf_per_se, amdgpu_pos_buf_per_se, int, 0444);
 
+/**
+ * DOC: cntl_sb_buf_per_se (int)
+ * Override the size of Control Sideband per Shader Engine in Byte. The default is 0 (depending on gfx).
+ */
 MODULE_PARM_DESC(cntl_sb_buf_per_se, "the size of Control Sideband per Shader Engine (default depending on gfx)");
 module_param_named(cntl_sb_buf_per_se, amdgpu_cntl_sb_buf_per_se, int, 0444);
 
+/**
+ * DOC: param_buf_per_se (int)
+ * Override the size of Off-Chip Pramater Cache per Shader Engine in Byte. The default is 0 (depending on gfx).
+ */
 MODULE_PARM_DESC(param_buf_per_se, "the size of Off-Chip Pramater Cache per Shader Engine (default depending on gfx)");
 module_param_named(param_buf_per_se, amdgpu_param_buf_per_se, int, 0444);
 
+/**
+ * DOC: job_hang_limit (int)
+ * Set how much time allow a job hang and not drop it. The default is 0.
+ */
 MODULE_PARM_DESC(job_hang_limit, "how much time allow a job hang and not drop it (default 0)");
 module_param_named(job_hang_limit, amdgpu_job_hang_limit, int ,0444);
 
+/**
+ * DOC: lbpw (int)
+ * Override Load Balancing Per Watt (LBPW) support (1 = enable, 0 = disable). The default is -1 (auto, enabled).
+ */
 MODULE_PARM_DESC(lbpw, "Load Balancing Per Watt (LBPW) support (1 = enable, 0 = disable, -1 = auto)");
 module_param_named(lbpw, amdgpu_lbpw, int, 0444);
 
 MODULE_PARM_DESC(compute_multipipe, "Force compute queues to be spread across pipes (1 = enable, 0 = disable, -1 = auto)");
 module_param_named(compute_multipipe, amdgpu_compute_multipipe, int, 0444);
 
+/**
+ * DOC: gpu_recovery (int)
+ * Set to enable GPU recovery mechanism (1 = enable, 0 = disable). The default is -1 (auto, disabled except SRIOV).
+ */
 MODULE_PARM_DESC(gpu_recovery, "Enable GPU recovery mechanism, (1 = enable, 0 = disable, -1 = auto)");
 module_param_named(gpu_recovery, amdgpu_gpu_recovery, int, 0444);
 
+/**
+ * DOC: emu_mode (int)
+ * Set value 1 to enable emulation mode. This is only needed when running on an emulator. The default is 0 (disabled).
+ */
 MODULE_PARM_DESC(emu_mode, "Emulation mode, (1 = enable, 0 = disable)");
 module_param_named(emu_mode, amdgpu_emu_mode, int, 0444);
 
+/**
+ * DOC: si_support (int)
+ * Set SI support driver. This parameter works after set config CONFIG_DRM_AMDGPU_SI. For SI asic, when radeon driver is enabled,
+ * set value 0 to use radeon driver, while set value 1 to use amdgpu driver. The default is using radeon driver when it available,
+ * otherwise using amdgpu driver.
+ */
 #ifdef CONFIG_DRM_AMDGPU_SI
 
 #if defined(CONFIG_DRM_RADEON) || defined(CONFIG_DRM_RADEON_MODULE)
@@ -305,6 +502,12 @@ MODULE_PARM_DESC(si_support, "SI support (1 = enabled (default), 0 = disabled)")
 module_param_named(si_support, amdgpu_si_support, int, 0444);
 #endif
 
+/**
+ * DOC: cik_support (int)
+ * Set CIK support driver. This parameter works after set config CONFIG_DRM_AMDGPU_CIK. For CIK asic, when radeon driver is enabled,
+ * set value 0 to use radeon driver, while set value 1 to use amdgpu driver. The default is using radeon driver when it available,
+ * otherwise using amdgpu driver.
+ */
 #ifdef CONFIG_DRM_AMDGPU_CIK
 
 #if defined(CONFIG_DRM_RADEON) || defined(CONFIG_DRM_RADEON_MODULE)
@@ -318,6 +521,11 @@ MODULE_PARM_DESC(cik_support, "CIK support (1 = enabled (default), 0 = disabled)
 module_param_named(cik_support, amdgpu_cik_support, int, 0444);
 #endif
 
+/**
+ * DOC: smu_memory_pool_size (uint)
+ * It is used to reserve gtt for smu debug usage, setting value 0 to disable it. The actual size is value * 256MiB.
+ * E.g. 0x1 = 256Mbyte, 0x2 = 512Mbyte, 0x4 = 1 Gbyte, 0x8 = 2GByte. The default is 0 (disabled).
+ */
 MODULE_PARM_DESC(smu_memory_pool_size,
 	"reserve gtt for smu debug usage, 0 = disable,"
 		"0x1 = 256Mbyte, 0x2 = 512Mbyte, 0x4 = 1 Gbyte, 0x8 = 2GByte");
@@ -664,7 +872,7 @@ retry_init:
 err_pci:
 	pci_disable_device(pdev);
 err_free:
-	drm_dev_unref(dev);
+	drm_dev_put(dev);
 	return ret;
 }
 
@@ -674,7 +882,7 @@ amdgpu_pci_remove(struct pci_dev *pdev)
 	struct drm_device *dev = pci_get_drvdata(pdev);
 
 	drm_dev_unregister(dev);
-	drm_dev_unref(dev);
+	drm_dev_put(dev);
 	pci_disable_device(pdev);
 	pci_set_drvdata(pdev, NULL);
 }
@@ -860,7 +1068,7 @@ static int amdgpu_flush(struct file *f, fl_owner_t id)
 	struct drm_file *file_priv = f->private_data;
 	struct amdgpu_fpriv *fpriv = file_priv->driver_priv;
 
-	amdgpu_ctx_mgr_entity_fini(&fpriv->ctx_mgr);
+	amdgpu_ctx_mgr_entity_flush(&fpriv->ctx_mgr);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c
index bc5fd8ebab5d..69c5d22f29bd 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c
@@ -146,7 +146,7 @@ static int amdgpufb_create_pinned_object(struct amdgpu_fbdev *rfbdev,
 				       AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
 				       AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS |
 				       AMDGPU_GEM_CREATE_VRAM_CLEARED,
-				       true, NULL, &gobj);
+				       ttm_bo_type_kernel, NULL, &gobj);
 	if (ret) {
 		pr_err("failed to allocate framebuffer (%d)\n", aligned_size);
 		return -ENOMEM;
@@ -168,11 +168,19 @@ static int amdgpufb_create_pinned_object(struct amdgpu_fbdev *rfbdev,
 	}
 
 
-	ret = amdgpu_bo_pin(abo, domain, NULL);
+	ret = amdgpu_bo_pin(abo, domain);
 	if (ret) {
 		amdgpu_bo_unreserve(abo);
 		goto out_unref;
 	}
+
+	ret = amdgpu_ttm_alloc_gart(&abo->tbo);
+	if (ret) {
+		amdgpu_bo_unreserve(abo);
+		dev_err(adev->dev, "%p bind failed\n", abo);
+		goto out_unref;
+	}
+
 	ret = amdgpu_bo_kmap(abo, NULL);
 	amdgpu_bo_unreserve(abo);
 	if (ret) {
@@ -365,8 +373,8 @@ void amdgpu_fbdev_fini(struct amdgpu_device *adev)
 void amdgpu_fbdev_set_suspend(struct amdgpu_device *adev, int state)
 {
 	if (adev->mode_info.rfbdev)
-		drm_fb_helper_set_suspend(&adev->mode_info.rfbdev->helper,
-			state);
+		drm_fb_helper_set_suspend_unlocked(&adev->mode_info.rfbdev->helper,
+						   state);
 }
 
 int amdgpu_fbdev_total_size(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c
index dd11b7313ca0..a54d5655a191 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c
@@ -143,14 +143,12 @@ int amdgpu_gart_table_vram_alloc(struct amdgpu_device *adev)
  */
 int amdgpu_gart_table_vram_pin(struct amdgpu_device *adev)
 {
-	uint64_t gpu_addr;
 	int r;
 
 	r = amdgpu_bo_reserve(adev->gart.robj, false);
 	if (unlikely(r != 0))
 		return r;
-	r = amdgpu_bo_pin(adev->gart.robj,
-				AMDGPU_GEM_DOMAIN_VRAM, &gpu_addr);
+	r = amdgpu_bo_pin(adev->gart.robj, AMDGPU_GEM_DOMAIN_VRAM);
 	if (r) {
 		amdgpu_bo_unreserve(adev->gart.robj);
 		return r;
@@ -159,7 +157,7 @@ int amdgpu_gart_table_vram_pin(struct amdgpu_device *adev)
 	if (r)
 		amdgpu_bo_unpin(adev->gart.robj);
 	amdgpu_bo_unreserve(adev->gart.robj);
-	adev->gart.table_addr = gpu_addr;
+	adev->gart.table_addr = amdgpu_bo_gpu_offset(adev->gart.robj);
 	return r;
 }
 
@@ -234,7 +232,7 @@ int amdgpu_gart_unbind(struct amdgpu_device *adev, uint64_t offset,
 	}
 
 	t = offset / AMDGPU_GPU_PAGE_SIZE;
-	p = t / (PAGE_SIZE / AMDGPU_GPU_PAGE_SIZE);
+	p = t / AMDGPU_GPU_PAGES_IN_CPU_PAGE;
 	for (i = 0; i < pages; i++, p++) {
 #ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS
 		adev->gart.pages[p] = NULL;
@@ -243,7 +241,7 @@ int amdgpu_gart_unbind(struct amdgpu_device *adev, uint64_t offset,
 		if (!adev->gart.ptr)
 			continue;
 
-		for (j = 0; j < (PAGE_SIZE / AMDGPU_GPU_PAGE_SIZE); j++, t++) {
+		for (j = 0; j < AMDGPU_GPU_PAGES_IN_CPU_PAGE; j++, t++) {
 			amdgpu_gmc_set_pte_pde(adev, adev->gart.ptr,
 					       t, page_base, flags);
 			page_base += AMDGPU_GPU_PAGE_SIZE;
@@ -282,7 +280,7 @@ int amdgpu_gart_map(struct amdgpu_device *adev, uint64_t offset,
 
 	for (i = 0; i < pages; i++) {
 		page_base = dma_addr[i];
-		for (j = 0; j < (PAGE_SIZE / AMDGPU_GPU_PAGE_SIZE); j++, t++) {
+		for (j = 0; j < AMDGPU_GPU_PAGES_IN_CPU_PAGE; j++, t++) {
 			amdgpu_gmc_set_pte_pde(adev, dst, t, page_base, flags);
 			page_base += AMDGPU_GPU_PAGE_SIZE;
 		}
@@ -319,7 +317,7 @@ int amdgpu_gart_bind(struct amdgpu_device *adev, uint64_t offset,
 
 #ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS
 	t = offset / AMDGPU_GPU_PAGE_SIZE;
-	p = t / (PAGE_SIZE / AMDGPU_GPU_PAGE_SIZE);
+	p = t / AMDGPU_GPU_PAGES_IN_CPU_PAGE;
 	for (i = 0; i < pages; i++, p++)
 		adev->gart.pages[p] = pagelist ? pagelist[i] : NULL;
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.h
index 456295c00291..9f9e9dc87da1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.h
@@ -37,6 +37,8 @@ struct amdgpu_bo;
 #define AMDGPU_GPU_PAGE_SHIFT 12
 #define AMDGPU_GPU_PAGE_ALIGN(a) (((a) + AMDGPU_GPU_PAGE_MASK) & ~AMDGPU_GPU_PAGE_MASK)
 
+#define AMDGPU_GPU_PAGES_IN_CPU_PAGE (PAGE_SIZE / AMDGPU_GPU_PAGE_SIZE)
+
 struct amdgpu_gart {
 	u64				table_addr;
 	struct amdgpu_bo		*robj;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
index 89743cdc1c2c..71792d820ae0 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
@@ -265,7 +265,7 @@ int amdgpu_gem_create_ioctl(struct drm_device *dev, void *data,
 
 	r = amdgpu_gem_object_create(adev, size, args->in.alignment,
 				     (u32)(0xffffffff & args->in.domains),
-				     flags, false, resv, &gobj);
+				     flags, ttm_bo_type_device, resv, &gobj);
 	if (flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) {
 		if (!r) {
 			struct amdgpu_bo *abo = gem_to_amdgpu_bo(gobj);
@@ -317,7 +317,7 @@ int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data,
 
 	/* create a gem object to contain this object in */
 	r = amdgpu_gem_object_create(adev, args->size, 0, AMDGPU_GEM_DOMAIN_CPU,
-				     0, 0, NULL, &gobj);
+				     0, ttm_bo_type_device, NULL, &gobj);
 	if (r)
 		return r;
 
@@ -344,7 +344,7 @@ int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data,
 		if (r)
 			goto free_pages;
 
-		amdgpu_ttm_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);
+		amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);
 		r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
 		amdgpu_bo_unreserve(bo);
 		if (r)
@@ -766,7 +766,7 @@ int amdgpu_mode_dumb_create(struct drm_file *file_priv,
 				amdgpu_display_supported_domains(adev));
 	r = amdgpu_gem_object_create(adev, args->size, 0, domain,
 				     AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
-				     false, NULL, &gobj);
+				     ttm_bo_type_device, NULL, &gobj);
 	if (r)
 		return -ENOMEM;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
index 6cb4948233cb..bb5a47a45790 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
@@ -105,6 +105,8 @@ struct amdgpu_gmc {
 	/* protects concurrent invalidation */
 	spinlock_t		invalidate_lock;
 	bool			translate_further;
+	struct kfd_vm_fault_info *vm_fault_info;
+	atomic_t		vm_fault_info_updated;
 
 	const struct amdgpu_gmc_funcs	*gmc_funcs;
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
index 31f8170313b4..5518e623fed2 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
@@ -139,7 +139,7 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
 	/* ring tests don't use a job */
 	if (job) {
 		vm = job->vm;
-		fence_ctx = job->fence_ctx;
+		fence_ctx = job->base.s_fence->scheduled.context;
 	} else {
 		vm = NULL;
 		fence_ctx = 0;
@@ -231,6 +231,12 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
 	if (ib->flags & AMDGPU_IB_FLAG_TC_WB_NOT_INVALIDATE)
 		fence_flags |= AMDGPU_FENCE_FLAG_TC_WB_ONLY;
 
+	/* wrap the last IB with fence */
+	if (job && job->uf_addr) {
+		amdgpu_ring_emit_fence(ring, job->uf_addr, job->uf_sequence,
+				       fence_flags | AMDGPU_FENCE_FLAG_64BIT);
+	}
+
 	r = amdgpu_fence_emit(ring, f, fence_flags);
 	if (r) {
 		dev_err(adev->dev, "failed to emit fence (%d)\n", r);
@@ -243,12 +249,6 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
 	if (ring->funcs->insert_end)
 		ring->funcs->insert_end(ring);
 
-	/* wrap the last IB with fence */
-	if (job && job->uf_addr) {
-		amdgpu_ring_emit_fence(ring, job->uf_addr, job->uf_sequence,
-				       fence_flags | AMDGPU_FENCE_FLAG_64BIT);
-	}
-
 	if (patch_offset != ~0 && ring->funcs->patch_cond_exec)
 		amdgpu_ring_patch_cond_exec(ring, patch_offset);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c
index a1c78f90eadf..3a072a7a39f0 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c
@@ -578,11 +578,6 @@ void amdgpu_vmid_mgr_init(struct amdgpu_device *adev)
 			list_add_tail(&id_mgr->ids[j].list, &id_mgr->ids_lru);
 		}
 	}
-
-	adev->vm_manager.fence_context =
-		dma_fence_context_alloc(AMDGPU_MAX_RINGS);
-	for (i = 0; i < AMDGPU_MAX_RINGS; ++i)
-		adev->vm_manager.seqno[i] = 0;
 }
 
 /**
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
index 2bd56760c744..391e2f7c03aa 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
@@ -30,14 +30,14 @@
 
 static void amdgpu_job_timedout(struct drm_sched_job *s_job)
 {
-	struct amdgpu_job *job = container_of(s_job, struct amdgpu_job, base);
+	struct amdgpu_ring *ring = to_amdgpu_ring(s_job->sched);
+	struct amdgpu_job *job = to_amdgpu_job(s_job);
 
-	DRM_ERROR("ring %s timeout, last signaled seq=%u, last emitted seq=%u\n",
-		  job->base.sched->name,
-		  atomic_read(&job->ring->fence_drv.last_seq),
-		  job->ring->fence_drv.sync_seq);
+	DRM_ERROR("ring %s timeout, signaled seq=%u, emitted seq=%u\n",
+		  job->base.sched->name, atomic_read(&ring->fence_drv.last_seq),
+		  ring->fence_drv.sync_seq);
 
-	amdgpu_device_gpu_recover(job->adev, job, false);
+	amdgpu_device_gpu_recover(ring->adev, job, false);
 }
 
 int amdgpu_job_alloc(struct amdgpu_device *adev, unsigned num_ibs,
@@ -54,7 +54,11 @@ int amdgpu_job_alloc(struct amdgpu_device *adev, unsigned num_ibs,
 	if (!*job)
 		return -ENOMEM;
 
-	(*job)->adev = adev;
+	/*
+	 * Initialize the scheduler to at least some ring so that we always
+	 * have a pointer to adev.
+	 */
+	(*job)->base.sched = &adev->rings[0]->sched;
 	(*job)->vm = vm;
 	(*job)->ibs = (void *)&(*job)[1];
 	(*job)->num_ibs = num_ibs;
@@ -86,6 +90,7 @@ int amdgpu_job_alloc_with_ib(struct amdgpu_device *adev, unsigned size,
 
 void amdgpu_job_free_resources(struct amdgpu_job *job)
 {
+	struct amdgpu_ring *ring = to_amdgpu_ring(job->base.sched);
 	struct dma_fence *f;
 	unsigned i;
 
@@ -93,14 +98,15 @@ void amdgpu_job_free_resources(struct amdgpu_job *job)
 	f = job->base.s_fence ? &job->base.s_fence->finished : job->fence;
 
 	for (i = 0; i < job->num_ibs; ++i)
-		amdgpu_ib_free(job->adev, &job->ibs[i], f);
+		amdgpu_ib_free(ring->adev, &job->ibs[i], f);
 }
 
 static void amdgpu_job_free_cb(struct drm_sched_job *s_job)
 {
-	struct amdgpu_job *job = container_of(s_job, struct amdgpu_job, base);
+	struct amdgpu_ring *ring = to_amdgpu_ring(s_job->sched);
+	struct amdgpu_job *job = to_amdgpu_job(s_job);
 
-	amdgpu_ring_priority_put(job->ring, s_job->s_priority);
+	amdgpu_ring_priority_put(ring, s_job->s_priority);
 	dma_fence_put(job->fence);
 	amdgpu_sync_free(&job->sync);
 	amdgpu_sync_free(&job->sched_sync);
@@ -117,50 +123,68 @@ void amdgpu_job_free(struct amdgpu_job *job)
 	kfree(job);
 }
 
-int amdgpu_job_submit(struct amdgpu_job *job, struct amdgpu_ring *ring,
-		      struct drm_sched_entity *entity, void *owner,
-		      struct dma_fence **f)
+int amdgpu_job_submit(struct amdgpu_job *job, struct drm_sched_entity *entity,
+		      void *owner, struct dma_fence **f)
 {
+	enum drm_sched_priority priority;
+	struct amdgpu_ring *ring;
 	int r;
-	job->ring = ring;
 
 	if (!f)
 		return -EINVAL;
 
-	r = drm_sched_job_init(&job->base, &ring->sched, entity, owner);
+	r = drm_sched_job_init(&job->base, entity, owner);
 	if (r)
 		return r;
 
 	job->owner = owner;
-	job->fence_ctx = entity->fence_context;
 	*f = dma_fence_get(&job->base.s_fence->finished);
 	amdgpu_job_free_resources(job);
-	amdgpu_ring_priority_get(job->ring, job->base.s_priority);
+	priority = job->base.s_priority;
 	drm_sched_entity_push_job(&job->base, entity);
 
+	ring = to_amdgpu_ring(entity->rq->sched);
+	amdgpu_ring_priority_get(ring, priority);
+
+	return 0;
+}
+
+int amdgpu_job_submit_direct(struct amdgpu_job *job, struct amdgpu_ring *ring,
+			     struct dma_fence **fence)
+{
+	int r;
+
+	job->base.sched = &ring->sched;
+	r = amdgpu_ib_schedule(ring, job->num_ibs, job->ibs, NULL, fence);
+	job->fence = dma_fence_get(*fence);
+	if (r)
+		return r;
+
+	amdgpu_job_free(job);
 	return 0;
 }
 
 static struct dma_fence *amdgpu_job_dependency(struct drm_sched_job *sched_job,
 					       struct drm_sched_entity *s_entity)
 {
+	struct amdgpu_ring *ring = to_amdgpu_ring(s_entity->rq->sched);
 	struct amdgpu_job *job = to_amdgpu_job(sched_job);
 	struct amdgpu_vm *vm = job->vm;
+	struct dma_fence *fence;
 	bool explicit = false;
 	int r;
-	struct dma_fence *fence = amdgpu_sync_get_fence(&job->sync, &explicit);
 
+	fence = amdgpu_sync_get_fence(&job->sync, &explicit);
 	if (fence && explicit) {
 		if (drm_sched_dependency_optimized(fence, s_entity)) {
-			r = amdgpu_sync_fence(job->adev, &job->sched_sync, fence, false);
+			r = amdgpu_sync_fence(ring->adev, &job->sched_sync,
+					      fence, false);
 			if (r)
-				DRM_ERROR("Error adding fence to sync (%d)\n", r);
+				DRM_ERROR("Error adding fence (%d)\n", r);
 		}
 	}
 
 	while (fence == NULL && vm && !job->vmid) {
-		struct amdgpu_ring *ring = job->ring;
-
 		r = amdgpu_vmid_grab(vm, ring, &job->sync,
 				     &job->base.s_fence->finished,
 				     job);
@@ -175,30 +199,25 @@ static struct dma_fence *amdgpu_job_dependency(struct drm_sched_job *sched_job,
 
 static struct dma_fence *amdgpu_job_run(struct drm_sched_job *sched_job)
 {
+	struct amdgpu_ring *ring = to_amdgpu_ring(sched_job->sched);
 	struct dma_fence *fence = NULL, *finished;
-	struct amdgpu_device *adev;
 	struct amdgpu_job *job;
 	int r;
 
-	if (!sched_job) {
-		DRM_ERROR("job is null\n");
-		return NULL;
-	}
 	job = to_amdgpu_job(sched_job);
 	finished = &job->base.s_fence->finished;
-	adev = job->adev;
 
 	BUG_ON(amdgpu_sync_peek_fence(&job->sync, NULL));
 
 	trace_amdgpu_sched_run_job(job);
 
-	if (job->vram_lost_counter != atomic_read(&adev->vram_lost_counter))
+	if (job->vram_lost_counter != atomic_read(&ring->adev->vram_lost_counter))
 		dma_fence_set_error(finished, -ECANCELED);/* skip IB as well if VRAM lost */
 
 	if (finished->error < 0) {
 		DRM_INFO("Skip scheduling IBs!\n");
 	} else {
-		r = amdgpu_ib_schedule(job->ring, job->num_ibs, job->ibs, job,
+		r = amdgpu_ib_schedule(ring, job->num_ibs, job->ibs, job,
 				       &fence);
 		if (r)
 			DRM_ERROR("Error scheduling IBs (%d)\n", r);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
new file mode 100644
index 000000000000..57cfe78a262b
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
@@ -0,0 +1,74 @@
+/*
+ * Copyright 2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef __AMDGPU_JOB_H__
+#define __AMDGPU_JOB_H__
+
+/* bit set means command submit involves a preamble IB */
+#define AMDGPU_PREAMBLE_IB_PRESENT          (1 << 0)
+/* bit set means preamble IB is first presented in belonging context */
+#define AMDGPU_PREAMBLE_IB_PRESENT_FIRST    (1 << 1)
+/* bit set means context switch occured */
+#define AMDGPU_HAVE_CTX_SWITCH              (1 << 2)
+
+#define to_amdgpu_job(sched_job)		\
+		container_of((sched_job), struct amdgpu_job, base)
+
+struct amdgpu_fence;
+
+struct amdgpu_job {
+	struct drm_sched_job    base;
+	struct amdgpu_vm	*vm;
+	struct amdgpu_sync	sync;
+	struct amdgpu_sync	sched_sync;
+	struct amdgpu_ib	*ibs;
+	struct dma_fence	*fence; /* the hw fence */
+	uint32_t		preamble_status;
+	uint32_t		num_ibs;
+	void			*owner;
+	bool                    vm_needs_flush;
+	uint64_t		vm_pd_addr;
+	unsigned		vmid;
+	unsigned		pasid;
+	uint32_t		gds_base, gds_size;
+	uint32_t		gws_base, gws_size;
+	uint32_t		oa_base, oa_size;
+	uint32_t		vram_lost_counter;
+
+	/* user fence handling */
+	uint64_t		uf_addr;
+	uint64_t		uf_sequence;
+
+};
+
+int amdgpu_job_alloc(struct amdgpu_device *adev, unsigned num_ibs,
+		     struct amdgpu_job **job, struct amdgpu_vm *vm);
+int amdgpu_job_alloc_with_ib(struct amdgpu_device *adev, unsigned size,
+			     struct amdgpu_job **job);
+
+void amdgpu_job_free_resources(struct amdgpu_job *job);
+void amdgpu_job_free(struct amdgpu_job *job);
+int amdgpu_job_submit(struct amdgpu_job *job, struct drm_sched_entity *entity,
+		      void *owner, struct dma_fence **f);
+int amdgpu_job_submit_direct(struct amdgpu_job *job, struct amdgpu_ring *ring,
+			     struct dma_fence **fence);
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
index 2060f208e60b..bd98cc5fb97b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
@@ -328,64 +328,68 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file
 		case AMDGPU_HW_IP_GFX:
 			type = AMD_IP_BLOCK_TYPE_GFX;
 			for (i = 0; i < adev->gfx.num_gfx_rings; i++)
-				ring_mask |= ((adev->gfx.gfx_ring[i].ready ? 1 : 0) << i);
+				ring_mask |= adev->gfx.gfx_ring[i].ready << i;
 			ib_start_alignment = 32;
 			ib_size_alignment = 32;
 			break;
 		case AMDGPU_HW_IP_COMPUTE:
 			type = AMD_IP_BLOCK_TYPE_GFX;
 			for (i = 0; i < adev->gfx.num_compute_rings; i++)
-				ring_mask |= ((adev->gfx.compute_ring[i].ready ? 1 : 0) << i);
+				ring_mask |= adev->gfx.compute_ring[i].ready << i;
 			ib_start_alignment = 32;
 			ib_size_alignment = 32;
 			break;
 		case AMDGPU_HW_IP_DMA:
 			type = AMD_IP_BLOCK_TYPE_SDMA;
 			for (i = 0; i < adev->sdma.num_instances; i++)
-				ring_mask |= ((adev->sdma.instance[i].ring.ready ? 1 : 0) << i);
+				ring_mask |= adev->sdma.instance[i].ring.ready << i;
 			ib_start_alignment = 256;
 			ib_size_alignment = 4;
 			break;
 		case AMDGPU_HW_IP_UVD:
 			type = AMD_IP_BLOCK_TYPE_UVD;
-			for (i = 0; i < adev->uvd.num_uvd_inst; i++)
-				ring_mask |= ((adev->uvd.inst[i].ring.ready ? 1 : 0) << i);
+			for (i = 0; i < adev->uvd.num_uvd_inst; i++) {
+				if (adev->uvd.harvest_config & (1 << i))
+					continue;
+				ring_mask |= adev->uvd.inst[i].ring.ready;
+			}
 			ib_start_alignment = 64;
 			ib_size_alignment = 64;
 			break;
 		case AMDGPU_HW_IP_VCE:
 			type = AMD_IP_BLOCK_TYPE_VCE;
 			for (i = 0; i < adev->vce.num_rings; i++)
-				ring_mask |= ((adev->vce.ring[i].ready ? 1 : 0) << i);
+				ring_mask |= adev->vce.ring[i].ready << i;
 			ib_start_alignment = 4;
 			ib_size_alignment = 1;
 			break;
 		case AMDGPU_HW_IP_UVD_ENC:
 			type = AMD_IP_BLOCK_TYPE_UVD;
-			for (i = 0; i < adev->uvd.num_uvd_inst; i++)
+			for (i = 0; i < adev->uvd.num_uvd_inst; i++) {
+				if (adev->uvd.harvest_config & (1 << i))
+					continue;
 				for (j = 0; j < adev->uvd.num_enc_rings; j++)
-					ring_mask |=
-					((adev->uvd.inst[i].ring_enc[j].ready ? 1 : 0) <<
-					(j + i * adev->uvd.num_enc_rings));
+					ring_mask |= adev->uvd.inst[i].ring_enc[j].ready << j;
+			}
 			ib_start_alignment = 64;
 			ib_size_alignment = 64;
 			break;
 		case AMDGPU_HW_IP_VCN_DEC:
 			type = AMD_IP_BLOCK_TYPE_VCN;
-			ring_mask = adev->vcn.ring_dec.ready ? 1 : 0;
+			ring_mask = adev->vcn.ring_dec.ready;
 			ib_start_alignment = 16;
 			ib_size_alignment = 16;
 			break;
 		case AMDGPU_HW_IP_VCN_ENC:
 			type = AMD_IP_BLOCK_TYPE_VCN;
 			for (i = 0; i < adev->vcn.num_enc_rings; i++)
-				ring_mask |= ((adev->vcn.ring_enc[i].ready ? 1 : 0) << i);
+				ring_mask |= adev->vcn.ring_enc[i].ready << i;
 			ib_start_alignment = 64;
 			ib_size_alignment = 1;
 			break;
 		case AMDGPU_HW_IP_VCN_JPEG:
 			type = AMD_IP_BLOCK_TYPE_VCN;
-			ring_mask = adev->vcn.ring_jpeg.ready ? 1 : 0;
+			ring_mask = adev->vcn.ring_jpeg.ready;
 			ib_start_alignment = 16;
 			ib_size_alignment = 16;
 			break;
@@ -501,13 +505,13 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file
 	case AMDGPU_INFO_VRAM_GTT: {
 		struct drm_amdgpu_info_vram_gtt vram_gtt;
 
-		vram_gtt.vram_size = adev->gmc.real_vram_size;
-		vram_gtt.vram_size -= adev->vram_pin_size;
-		vram_gtt.vram_cpu_accessible_size = adev->gmc.visible_vram_size;
-		vram_gtt.vram_cpu_accessible_size -= (adev->vram_pin_size - adev->invisible_pin_size);
+		vram_gtt.vram_size = adev->gmc.real_vram_size -
+			atomic64_read(&adev->vram_pin_size);
+		vram_gtt.vram_cpu_accessible_size = adev->gmc.visible_vram_size -
+			atomic64_read(&adev->visible_pin_size);
 		vram_gtt.gtt_size = adev->mman.bdev.man[TTM_PL_TT].size;
 		vram_gtt.gtt_size *= PAGE_SIZE;
-		vram_gtt.gtt_size -= adev->gart_pin_size;
+		vram_gtt.gtt_size -= atomic64_read(&adev->gart_pin_size);
 		return copy_to_user(out, &vram_gtt,
 				    min((size_t)size, sizeof(vram_gtt))) ? -EFAULT : 0;
 	}
@@ -516,17 +520,16 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file
 
 		memset(&mem, 0, sizeof(mem));
 		mem.vram.total_heap_size = adev->gmc.real_vram_size;
-		mem.vram.usable_heap_size =
-			adev->gmc.real_vram_size - adev->vram_pin_size;
+		mem.vram.usable_heap_size = adev->gmc.real_vram_size -
+			atomic64_read(&adev->vram_pin_size);
 		mem.vram.heap_usage =
 			amdgpu_vram_mgr_usage(&adev->mman.bdev.man[TTM_PL_VRAM]);
 		mem.vram.max_allocation = mem.vram.usable_heap_size * 3 / 4;
 
 		mem.cpu_accessible_vram.total_heap_size =
 			adev->gmc.visible_vram_size;
-		mem.cpu_accessible_vram.usable_heap_size =
-			adev->gmc.visible_vram_size -
-			(adev->vram_pin_size - adev->invisible_pin_size);
+		mem.cpu_accessible_vram.usable_heap_size = adev->gmc.visible_vram_size -
+			atomic64_read(&adev->visible_pin_size);
 		mem.cpu_accessible_vram.heap_usage =
 			amdgpu_vram_mgr_vis_usage(&adev->mman.bdev.man[TTM_PL_VRAM]);
 		mem.cpu_accessible_vram.max_allocation =
@@ -534,8 +537,8 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file
 
 		mem.gtt.total_heap_size = adev->mman.bdev.man[TTM_PL_TT].size;
 		mem.gtt.total_heap_size *= PAGE_SIZE;
-		mem.gtt.usable_heap_size = mem.gtt.total_heap_size
-			- adev->gart_pin_size;
+		mem.gtt.usable_heap_size = mem.gtt.total_heap_size -
+			atomic64_read(&adev->gart_pin_size);
 		mem.gtt.heap_usage =
 			amdgpu_gtt_mgr_usage(&adev->mman.bdev.man[TTM_PL_TT]);
 		mem.gtt.max_allocation = mem.gtt.usable_heap_size * 3 / 4;
@@ -964,7 +967,7 @@ void amdgpu_driver_postclose_kms(struct drm_device *dev,
 	amdgpu_bo_unref(&pd);
 
 	idr_for_each_entry(&fpriv->bo_list_handles, list, handle)
-		amdgpu_bo_list_free(list);
+		amdgpu_bo_list_put(list);
 
 	idr_destroy(&fpriv->bo_list_handles);
 	mutex_destroy(&fpriv->bo_list_lock);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c
index 72a3e8c68876..e55508b39496 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c
@@ -58,7 +58,8 @@
  *
  * @adev: amdgpu device pointer
  * @mm: process address space
- * @mn: MMU notifier structur
+ * @mn: MMU notifier structure
+ * @type: type of MMU notifier
  * @work: destruction work item
  * @node: hash table node to find structure by adev and mn
  * @lock: rw semaphore protecting the notifier nodes
@@ -177,12 +178,18 @@ void amdgpu_mn_unlock(struct amdgpu_mn *mn)
  *
  * @amn: our notifier
  */
-static void amdgpu_mn_read_lock(struct amdgpu_mn *amn)
+static int amdgpu_mn_read_lock(struct amdgpu_mn *amn, bool blockable)
 {
-	mutex_lock(&amn->read_lock);
+	if (blockable)
+		mutex_lock(&amn->read_lock);
+	else if (!mutex_trylock(&amn->read_lock))
+		return -EAGAIN;
+
 	if (atomic_inc_return(&amn->recursion) == 1)
 		down_read_non_owner(&amn->lock);
 	mutex_unlock(&amn->read_lock);
+
+	return 0;
 }
 
 /**
@@ -238,10 +245,11 @@ static void amdgpu_mn_invalidate_node(struct amdgpu_mn_node *node,
  * Block for operations on BOs to finish and mark pages as accessed and
  * potentially dirty.
  */
-static void amdgpu_mn_invalidate_range_start_gfx(struct mmu_notifier *mn,
+static int amdgpu_mn_invalidate_range_start_gfx(struct mmu_notifier *mn,
 						 struct mm_struct *mm,
 						 unsigned long start,
-						 unsigned long end)
+						 unsigned long end,
+						 bool blockable)
 {
 	struct amdgpu_mn *amn = container_of(mn, struct amdgpu_mn, mn);
 	struct interval_tree_node *it;
@@ -249,24 +257,35 @@ static void amdgpu_mn_invalidate_range_start_gfx(struct mmu_notifier *mn,
 	/* notification is exclusive, but interval is inclusive */
 	end -= 1;
 
-	amdgpu_mn_read_lock(amn);
+	/* TODO we should be able to split locking for interval tree and
+	 * amdgpu_mn_invalidate_node
+	 */
+	if (amdgpu_mn_read_lock(amn, blockable))
+		return -EAGAIN;
 
 	it = interval_tree_iter_first(&amn->objects, start, end);
 	while (it) {
 		struct amdgpu_mn_node *node;
 
+		if (!blockable) {
+			amdgpu_mn_read_unlock(amn);
+			return -EAGAIN;
+		}
+
 		node = container_of(it, struct amdgpu_mn_node, it);
 		it = interval_tree_iter_next(it, start, end);
 
 		amdgpu_mn_invalidate_node(node, start, end);
 	}
+
+	return 0;
 }
 
 /**
  * amdgpu_mn_invalidate_range_start_hsa - callback to notify about mm change
  *
  * @mn: our notifier
- * @mn: the mm this callback is about
+ * @mm: the mm this callback is about
  * @start: start of updated range
  * @end: end of updated range
  *
@@ -274,10 +293,11 @@ static void amdgpu_mn_invalidate_range_start_gfx(struct mmu_notifier *mn,
  * necessitates evicting all user-mode queues of the process. The BOs
  * are restorted in amdgpu_mn_invalidate_range_end_hsa.
  */
-static void amdgpu_mn_invalidate_range_start_hsa(struct mmu_notifier *mn,
+static int amdgpu_mn_invalidate_range_start_hsa(struct mmu_notifier *mn,
 						 struct mm_struct *mm,
 						 unsigned long start,
-						 unsigned long end)
+						 unsigned long end,
+						 bool blockable)
 {
 	struct amdgpu_mn *amn = container_of(mn, struct amdgpu_mn, mn);
 	struct interval_tree_node *it;
@@ -285,13 +305,19 @@ static void amdgpu_mn_invalidate_range_start_hsa(struct mmu_notifier *mn,
 	/* notification is exclusive, but interval is inclusive */
 	end -= 1;
 
-	amdgpu_mn_read_lock(amn);
+	if (amdgpu_mn_read_lock(amn, blockable))
+		return -EAGAIN;
 
 	it = interval_tree_iter_first(&amn->objects, start, end);
 	while (it) {
 		struct amdgpu_mn_node *node;
 		struct amdgpu_bo *bo;
 
+		if (!blockable) {
+			amdgpu_mn_read_unlock(amn);
+			return -EAGAIN;
+		}
+
 		node = container_of(it, struct amdgpu_mn_node, it);
 		it = interval_tree_iter_next(it, start, end);
 
@@ -303,6 +329,8 @@ static void amdgpu_mn_invalidate_range_start_hsa(struct mmu_notifier *mn,
 				amdgpu_amdkfd_evict_userptr(mem, mm);
 		}
 	}
+
+	return 0;
 }
 
 /**
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
index 512f59836436..b0e14a3d54ef 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
@@ -51,7 +51,7 @@
  *
  */
 
-static bool amdgpu_need_backup(struct amdgpu_device *adev)
+static bool amdgpu_bo_need_backup(struct amdgpu_device *adev)
 {
 	if (adev->flags & AMD_IS_APU)
 		return false;
@@ -63,11 +63,35 @@ static bool amdgpu_need_backup(struct amdgpu_device *adev)
 	return true;
 }
 
-static void amdgpu_ttm_bo_destroy(struct ttm_buffer_object *tbo)
+/**
+ * amdgpu_bo_subtract_pin_size - Remove BO from pin_size accounting
+ *
+ * @bo: &amdgpu_bo buffer object
+ *
+ * This function is called when a BO stops being pinned, and updates the
+ * &amdgpu_device pin_size values accordingly.
+ */
+static void amdgpu_bo_subtract_pin_size(struct amdgpu_bo *bo)
+{
+	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
+
+	if (bo->tbo.mem.mem_type == TTM_PL_VRAM) {
+		atomic64_sub(amdgpu_bo_size(bo), &adev->vram_pin_size);
+		atomic64_sub(amdgpu_vram_mgr_bo_visible_size(bo),
+			     &adev->visible_pin_size);
+	} else if (bo->tbo.mem.mem_type == TTM_PL_TT) {
+		atomic64_sub(amdgpu_bo_size(bo), &adev->gart_pin_size);
+	}
+}
+
+static void amdgpu_bo_destroy(struct ttm_buffer_object *tbo)
 {
 	struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev);
 	struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo);
 
+	if (bo->pin_count > 0)
+		amdgpu_bo_subtract_pin_size(bo);
+
 	if (bo->kfd_bo)
 		amdgpu_amdkfd_unreserve_system_memory_limit(bo);
 
@@ -87,7 +111,7 @@ static void amdgpu_ttm_bo_destroy(struct ttm_buffer_object *tbo)
 }
 
 /**
- * amdgpu_ttm_bo_is_amdgpu_bo - check if the buffer object is an &amdgpu_bo
+ * amdgpu_bo_is_amdgpu_bo - check if the buffer object is an &amdgpu_bo
  * @bo: buffer object to be checked
  *
  * Uses destroy function associated with the object to determine if this is
@@ -96,22 +120,22 @@ static void amdgpu_ttm_bo_destroy(struct ttm_buffer_object *tbo)
  * Returns:
  * true if the object belongs to &amdgpu_bo, false if not.
  */
-bool amdgpu_ttm_bo_is_amdgpu_bo(struct ttm_buffer_object *bo)
+bool amdgpu_bo_is_amdgpu_bo(struct ttm_buffer_object *bo)
 {
-	if (bo->destroy == &amdgpu_ttm_bo_destroy)
+	if (bo->destroy == &amdgpu_bo_destroy)
 		return true;
 	return false;
 }
 
 /**
- * amdgpu_ttm_placement_from_domain - set buffer's placement
+ * amdgpu_bo_placement_from_domain - set buffer's placement
  * @abo: &amdgpu_bo buffer object whose placement is to be set
  * @domain: requested domain
  *
  * Sets buffer's placement according to requested domain and the buffer's
  * flags.
  */
-void amdgpu_ttm_placement_from_domain(struct amdgpu_bo *abo, u32 domain)
+void amdgpu_bo_placement_from_domain(struct amdgpu_bo *abo, u32 domain)
 {
 	struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev);
 	struct ttm_placement *placement = &abo->placement;
@@ -192,6 +216,8 @@ void amdgpu_ttm_placement_from_domain(struct amdgpu_bo *abo, u32 domain)
 		c++;
 	}
 
+	BUG_ON(c >= AMDGPU_BO_MAX_PLACEMENTS);
+
 	placement->num_placement = c;
 	placement->placement = places;
 
@@ -252,22 +278,33 @@ int amdgpu_bo_create_reserved(struct amdgpu_device *adev,
 		goto error_free;
 	}
 
-	r = amdgpu_bo_pin(*bo_ptr, domain, gpu_addr);
+	r = amdgpu_bo_pin(*bo_ptr, domain);
 	if (r) {
 		dev_err(adev->dev, "(%d) kernel bo pin failed\n", r);
 		goto error_unreserve;
 	}
 
+	r = amdgpu_ttm_alloc_gart(&(*bo_ptr)->tbo);
+	if (r) {
+		dev_err(adev->dev, "%p bind failed\n", *bo_ptr);
+		goto error_unpin;
+	}
+
+	if (gpu_addr)
+		*gpu_addr = amdgpu_bo_gpu_offset(*bo_ptr);
+
 	if (cpu_addr) {
 		r = amdgpu_bo_kmap(*bo_ptr, cpu_addr);
 		if (r) {
 			dev_err(adev->dev, "(%d) kernel bo map failed\n", r);
-			goto error_unreserve;
+			goto error_unpin;
 		}
 	}
 
 	return 0;
 
+error_unpin:
+	amdgpu_bo_unpin(*bo_ptr);
 error_unreserve:
 	amdgpu_bo_unreserve(*bo_ptr);
 
@@ -453,13 +490,13 @@ static int amdgpu_bo_do_create(struct amdgpu_device *adev,
 #endif
 
 	bo->tbo.bdev = &adev->mman.bdev;
-	amdgpu_ttm_placement_from_domain(bo, bp->domain);
+	amdgpu_bo_placement_from_domain(bo, bp->domain);
 	if (bp->type == ttm_bo_type_kernel)
 		bo->tbo.priority = 1;
 
 	r = ttm_bo_init_reserved(&adev->mman.bdev, &bo->tbo, size, bp->type,
 				 &bo->placement, page_align, &ctx, acc_size,
-				 NULL, bp->resv, &amdgpu_ttm_bo_destroy);
+				 NULL, bp->resv, &amdgpu_bo_destroy);
 	if (unlikely(r != 0))
 		return r;
 
@@ -559,7 +596,7 @@ int amdgpu_bo_create(struct amdgpu_device *adev,
 	if (r)
 		return r;
 
-	if ((flags & AMDGPU_GEM_CREATE_SHADOW) && amdgpu_need_backup(adev)) {
+	if ((flags & AMDGPU_GEM_CREATE_SHADOW) && amdgpu_bo_need_backup(adev)) {
 		if (!bp->resv)
 			WARN_ON(reservation_object_lock((*bo_ptr)->tbo.resv,
 							NULL));
@@ -647,7 +684,7 @@ int amdgpu_bo_validate(struct amdgpu_bo *bo)
 	domain = bo->preferred_domains;
 
 retry:
-	amdgpu_ttm_placement_from_domain(bo, domain);
+	amdgpu_bo_placement_from_domain(bo, domain);
 	r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
 	if (unlikely(r == -ENOMEM) && domain != bo->allowed_domains) {
 		domain = bo->allowed_domains;
@@ -788,7 +825,7 @@ struct amdgpu_bo *amdgpu_bo_ref(struct amdgpu_bo *bo)
 	if (bo == NULL)
 		return NULL;
 
-	ttm_bo_reference(&bo->tbo);
+	ttm_bo_get(&bo->tbo);
 	return bo;
 }
 
@@ -806,9 +843,8 @@ void amdgpu_bo_unref(struct amdgpu_bo **bo)
 		return;
 
 	tbo = &((*bo)->tbo);
-	ttm_bo_unref(&tbo);
-	if (tbo == NULL)
-		*bo = NULL;
+	ttm_bo_put(tbo);
+	*bo = NULL;
 }
 
 /**
@@ -817,7 +853,6 @@ void amdgpu_bo_unref(struct amdgpu_bo **bo)
  * @domain: domain to be pinned to
  * @min_offset: the start of requested address range
  * @max_offset: the end of requested address range
- * @gpu_addr: GPU offset of the &amdgpu_bo buffer object
  *
  * Pins the buffer object according to requested domain and address range. If
  * the memory is unbound gart memory, binds the pages into gart table. Adjusts
@@ -835,8 +870,7 @@ void amdgpu_bo_unref(struct amdgpu_bo **bo)
  * 0 for success or a negative error code on failure.
  */
 int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain,
-			     u64 min_offset, u64 max_offset,
-			     u64 *gpu_addr)
+			     u64 min_offset, u64 max_offset)
 {
 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
 	struct ttm_operation_ctx ctx = { false, false };
@@ -868,8 +902,6 @@ int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain,
 			return -EINVAL;
 
 		bo->pin_count++;
-		if (gpu_addr)
-			*gpu_addr = amdgpu_bo_gpu_offset(bo);
 
 		if (max_offset != 0) {
 			u64 domain_start = bo->tbo.bdev->man[mem_type].gpu_offset;
@@ -884,7 +916,7 @@ int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain,
 	/* force to pin into visible video ram */
 	if (!(bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS))
 		bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
-	amdgpu_ttm_placement_from_domain(bo, domain);
+	amdgpu_bo_placement_from_domain(bo, domain);
 	for (i = 0; i < bo->placement.num_placement; i++) {
 		unsigned fpfn, lpfn;
 
@@ -905,22 +937,15 @@ int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain,
 		goto error;
 	}
 
-	r = amdgpu_ttm_alloc_gart(&bo->tbo);
-	if (unlikely(r)) {
-		dev_err(adev->dev, "%p bind failed\n", bo);
-		goto error;
-	}
-
 	bo->pin_count = 1;
-	if (gpu_addr != NULL)
-		*gpu_addr = amdgpu_bo_gpu_offset(bo);
 
 	domain = amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type);
 	if (domain == AMDGPU_GEM_DOMAIN_VRAM) {
-		adev->vram_pin_size += amdgpu_bo_size(bo);
-		adev->invisible_pin_size += amdgpu_vram_mgr_bo_invisible_size(bo);
+		atomic64_add(amdgpu_bo_size(bo), &adev->vram_pin_size);
+		atomic64_add(amdgpu_vram_mgr_bo_visible_size(bo),
+			     &adev->visible_pin_size);
 	} else if (domain == AMDGPU_GEM_DOMAIN_GTT) {
-		adev->gart_pin_size += amdgpu_bo_size(bo);
+		atomic64_add(amdgpu_bo_size(bo), &adev->gart_pin_size);
 	}
 
 error:
@@ -931,7 +956,6 @@ error:
  * amdgpu_bo_pin - pin an &amdgpu_bo buffer object
  * @bo: &amdgpu_bo buffer object to be pinned
  * @domain: domain to be pinned to
- * @gpu_addr: GPU offset of the &amdgpu_bo buffer object
  *
  * A simple wrapper to amdgpu_bo_pin_restricted().
  * Provides a simpler API for buffers that do not have any strict restrictions
@@ -940,9 +964,9 @@ error:
  * Returns:
  * 0 for success or a negative error code on failure.
  */
-int amdgpu_bo_pin(struct amdgpu_bo *bo, u32 domain, u64 *gpu_addr)
+int amdgpu_bo_pin(struct amdgpu_bo *bo, u32 domain)
 {
-	return amdgpu_bo_pin_restricted(bo, domain, 0, 0, gpu_addr);
+	return amdgpu_bo_pin_restricted(bo, domain, 0, 0);
 }
 
 /**
@@ -969,12 +993,7 @@ int amdgpu_bo_unpin(struct amdgpu_bo *bo)
 	if (bo->pin_count)
 		return 0;
 
-	if (bo->tbo.mem.mem_type == TTM_PL_VRAM) {
-		adev->vram_pin_size -= amdgpu_bo_size(bo);
-		adev->invisible_pin_size -= amdgpu_vram_mgr_bo_invisible_size(bo);
-	} else if (bo->tbo.mem.mem_type == TTM_PL_TT) {
-		adev->gart_pin_size -= amdgpu_bo_size(bo);
-	}
+	amdgpu_bo_subtract_pin_size(bo);
 
 	for (i = 0; i < bo->placement.num_placement; i++) {
 		bo->placements[i].lpfn = 0;
@@ -1228,7 +1247,7 @@ void amdgpu_bo_move_notify(struct ttm_buffer_object *bo,
 	struct amdgpu_bo *abo;
 	struct ttm_mem_reg *old_mem = &bo->mem;
 
-	if (!amdgpu_ttm_bo_is_amdgpu_bo(bo))
+	if (!amdgpu_bo_is_amdgpu_bo(bo))
 		return;
 
 	abo = ttm_to_amdgpu_bo(bo);
@@ -1245,7 +1264,7 @@ void amdgpu_bo_move_notify(struct ttm_buffer_object *bo,
 		return;
 
 	/* move_notify is called before move happens */
-	trace_amdgpu_ttm_bo_move(abo, new_mem->mem_type, old_mem->mem_type);
+	trace_amdgpu_bo_move(abo, new_mem->mem_type, old_mem->mem_type);
 }
 
 /**
@@ -1267,7 +1286,7 @@ int amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo)
 	unsigned long offset, size;
 	int r;
 
-	if (!amdgpu_ttm_bo_is_amdgpu_bo(bo))
+	if (!amdgpu_bo_is_amdgpu_bo(bo))
 		return 0;
 
 	abo = ttm_to_amdgpu_bo(bo);
@@ -1289,8 +1308,8 @@ int amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo)
 
 	/* hurrah the memory is not visible ! */
 	atomic64_inc(&adev->num_vram_cpu_page_faults);
-	amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM |
-					 AMDGPU_GEM_DOMAIN_GTT);
+	amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM |
+					AMDGPU_GEM_DOMAIN_GTT);
 
 	/* Avoid costly evictions; only set GTT as a busy placement */
 	abo->placement.num_busy_placement = 1;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
index 731748033878..18945dd6982d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
@@ -32,6 +32,7 @@
 #include "amdgpu.h"
 
 #define AMDGPU_BO_INVALID_OFFSET	LONG_MAX
+#define AMDGPU_BO_MAX_PLACEMENTS	3
 
 struct amdgpu_bo_param {
 	unsigned long			size;
@@ -77,7 +78,7 @@ struct amdgpu_bo {
 	/* Protected by tbo.reserved */
 	u32				preferred_domains;
 	u32				allowed_domains;
-	struct ttm_place		placements[AMDGPU_GEM_DOMAIN_MAX + 1];
+	struct ttm_place		placements[AMDGPU_BO_MAX_PLACEMENTS];
 	struct ttm_placement		placement;
 	struct ttm_buffer_object	tbo;
 	struct ttm_bo_kmap_obj		kmap;
@@ -234,6 +235,9 @@ static inline bool amdgpu_bo_explicit_sync(struct amdgpu_bo *bo)
 	return bo->flags & AMDGPU_GEM_CREATE_EXPLICIT_SYNC;
 }
 
+bool amdgpu_bo_is_amdgpu_bo(struct ttm_buffer_object *bo);
+void amdgpu_bo_placement_from_domain(struct amdgpu_bo *abo, u32 domain);
+
 int amdgpu_bo_create(struct amdgpu_device *adev,
 		     struct amdgpu_bo_param *bp,
 		     struct amdgpu_bo **bo_ptr);
@@ -252,10 +256,9 @@ void *amdgpu_bo_kptr(struct amdgpu_bo *bo);
 void amdgpu_bo_kunmap(struct amdgpu_bo *bo);
 struct amdgpu_bo *amdgpu_bo_ref(struct amdgpu_bo *bo);
 void amdgpu_bo_unref(struct amdgpu_bo **bo);
-int amdgpu_bo_pin(struct amdgpu_bo *bo, u32 domain, u64 *gpu_addr);
+int amdgpu_bo_pin(struct amdgpu_bo *bo, u32 domain);
 int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain,
-			     u64 min_offset, u64 max_offset,
-			     u64 *gpu_addr);
+			     u64 min_offset, u64 max_offset);
 int amdgpu_bo_unpin(struct amdgpu_bo *bo);
 int amdgpu_bo_evict_vram(struct amdgpu_device *adev);
 int amdgpu_bo_init(struct amdgpu_device *adev);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c
index 113edffb5960..8f98629fbe59 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c
@@ -31,7 +31,7 @@
 #include <linux/power_supply.h>
 #include <linux/hwmon.h>
 #include <linux/hwmon-sysfs.h>
-
+#include <linux/nospec.h>
 
 static int amdgpu_debugfs_pm_init(struct amdgpu_device *adev);
 
@@ -403,6 +403,7 @@ static ssize_t amdgpu_set_pp_force_state(struct device *dev,
 			count = -EINVAL;
 			goto fail;
 		}
+		idx = array_index_nospec(idx, ARRAY_SIZE(data.states));
 
 		amdgpu_dpm_get_pp_num_states(adev, &data);
 		state = data.states[idx];
@@ -606,40 +607,59 @@ static ssize_t amdgpu_get_pp_dpm_sclk(struct device *dev,
 		return snprintf(buf, PAGE_SIZE, "\n");
 }
 
-static ssize_t amdgpu_set_pp_dpm_sclk(struct device *dev,
-		struct device_attribute *attr,
-		const char *buf,
-		size_t count)
+/*
+ * Worst case: 32 bits individually specified, in octal at 12 characters
+ * per line (+1 for \n).
+ */
+#define AMDGPU_MASK_BUF_MAX	(32 * 13)
+
+static ssize_t amdgpu_read_mask(const char *buf, size_t count, uint32_t *mask)
 {
-	struct drm_device *ddev = dev_get_drvdata(dev);
-	struct amdgpu_device *adev = ddev->dev_private;
 	int ret;
 	long level;
-	uint32_t mask = 0;
 	char *sub_str = NULL;
 	char *tmp;
-	char buf_cpy[count];
+	char buf_cpy[AMDGPU_MASK_BUF_MAX + 1];
 	const char delimiter[3] = {' ', '\n', '\0'};
+	size_t bytes;
 
-	memcpy(buf_cpy, buf, count+1);
+	*mask = 0;
+
+	bytes = min(count, sizeof(buf_cpy) - 1);
+	memcpy(buf_cpy, buf, bytes);
+	buf_cpy[bytes] = '\0';
 	tmp = buf_cpy;
 	while (tmp[0]) {
-		sub_str =  strsep(&tmp, delimiter);
+		sub_str = strsep(&tmp, delimiter);
 		if (strlen(sub_str)) {
 			ret = kstrtol(sub_str, 0, &level);
-
-			if (ret) {
-				count = -EINVAL;
-				goto fail;
-			}
-			mask |= 1 << level;
+			if (ret)
+				return -EINVAL;
+			*mask |= 1 << level;
 		} else
 			break;
 	}
+
+	return 0;
+}
+
+static ssize_t amdgpu_set_pp_dpm_sclk(struct device *dev,
+		struct device_attribute *attr,
+		const char *buf,
+		size_t count)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+	struct amdgpu_device *adev = ddev->dev_private;
+	int ret;
+	uint32_t mask = 0;
+
+	ret = amdgpu_read_mask(buf, count, &mask);
+	if (ret)
+		return ret;
+
 	if (adev->powerplay.pp_funcs->force_clock_level)
 		amdgpu_dpm_force_clock_level(adev, PP_SCLK, mask);
 
-fail:
 	return count;
 }
 
@@ -664,32 +684,15 @@ static ssize_t amdgpu_set_pp_dpm_mclk(struct device *dev,
 	struct drm_device *ddev = dev_get_drvdata(dev);
 	struct amdgpu_device *adev = ddev->dev_private;
 	int ret;
-	long level;
 	uint32_t mask = 0;
-	char *sub_str = NULL;
-	char *tmp;
-	char buf_cpy[count];
-	const char delimiter[3] = {' ', '\n', '\0'};
 
-	memcpy(buf_cpy, buf, count+1);
-	tmp = buf_cpy;
-	while (tmp[0]) {
-		sub_str =  strsep(&tmp, delimiter);
-		if (strlen(sub_str)) {
-			ret = kstrtol(sub_str, 0, &level);
+	ret = amdgpu_read_mask(buf, count, &mask);
+	if (ret)
+		return ret;
 
-			if (ret) {
-				count = -EINVAL;
-				goto fail;
-			}
-			mask |= 1 << level;
-		} else
-			break;
-	}
 	if (adev->powerplay.pp_funcs->force_clock_level)
 		amdgpu_dpm_force_clock_level(adev, PP_MCLK, mask);
 
-fail:
 	return count;
 }
 
@@ -714,33 +717,15 @@ static ssize_t amdgpu_set_pp_dpm_pcie(struct device *dev,
 	struct drm_device *ddev = dev_get_drvdata(dev);
 	struct amdgpu_device *adev = ddev->dev_private;
 	int ret;
-	long level;
 	uint32_t mask = 0;
-	char *sub_str = NULL;
-	char *tmp;
-	char buf_cpy[count];
-	const char delimiter[3] = {' ', '\n', '\0'};
-
-	memcpy(buf_cpy, buf, count+1);
-	tmp = buf_cpy;
 
-	while (tmp[0]) {
-		sub_str =  strsep(&tmp, delimiter);
-		if (strlen(sub_str)) {
-			ret = kstrtol(sub_str, 0, &level);
+	ret = amdgpu_read_mask(buf, count, &mask);
+	if (ret)
+		return ret;
 
-			if (ret) {
-				count = -EINVAL;
-				goto fail;
-			}
-			mask |= 1 << level;
-		} else
-			break;
-	}
 	if (adev->powerplay.pp_funcs->force_clock_level)
 		amdgpu_dpm_force_clock_level(adev, PP_PCIE, mask);
 
-fail:
 	return count;
 }
 
@@ -918,6 +903,36 @@ fail:
 	return -EINVAL;
 }
 
+/**
+ * DOC: busy_percent
+ *
+ * The amdgpu driver provides a sysfs API for reading how busy the GPU
+ * is as a percentage.  The file gpu_busy_percent is used for this.
+ * The SMU firmware computes a percentage of load based on the
+ * aggregate activity level in the IP cores.
+ */
+static ssize_t amdgpu_get_busy_percent(struct device *dev,
+		struct device_attribute *attr,
+		char *buf)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+	struct amdgpu_device *adev = ddev->dev_private;
+	int r, value, size = sizeof(value);
+
+	/* sanity check PP is enabled */
+	if (!(adev->powerplay.pp_funcs &&
+	      adev->powerplay.pp_funcs->read_sensor))
+		return -EINVAL;
+
+	/* read the IP busy sensor */
+	r = amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_GPU_LOAD,
+				   (void *)&value, &size);
+	if (r)
+		return r;
+
+	return snprintf(buf, PAGE_SIZE, "%d\n", value);
+}
+
 static DEVICE_ATTR(power_dpm_state, S_IRUGO | S_IWUSR, amdgpu_get_dpm_state, amdgpu_set_dpm_state);
 static DEVICE_ATTR(power_dpm_force_performance_level, S_IRUGO | S_IWUSR,
 		   amdgpu_get_dpm_forced_performance_level,
@@ -951,6 +966,8 @@ static DEVICE_ATTR(pp_power_profile_mode, S_IRUGO | S_IWUSR,
 static DEVICE_ATTR(pp_od_clk_voltage, S_IRUGO | S_IWUSR,
 		amdgpu_get_pp_od_clk_voltage,
 		amdgpu_set_pp_od_clk_voltage);
+static DEVICE_ATTR(gpu_busy_percent, S_IRUGO,
+		amdgpu_get_busy_percent, NULL);
 
 static ssize_t amdgpu_hwmon_show_temp(struct device *dev,
 				      struct device_attribute *attr,
@@ -1169,7 +1186,7 @@ static ssize_t amdgpu_hwmon_show_vddnb(struct device *dev,
 	int r, size = sizeof(vddnb);
 
 	/* only APUs have vddnb */
-	if  (adev->flags & AMD_IS_APU)
+	if  (!(adev->flags & AMD_IS_APU))
 		return -EINVAL;
 
 	/* Can't get voltage when the card is off */
@@ -1697,10 +1714,10 @@ static void amdgpu_dpm_change_power_state_locked(struct amdgpu_device *adev)
 
 void amdgpu_dpm_enable_uvd(struct amdgpu_device *adev, bool enable)
 {
-	if (adev->powerplay.pp_funcs->powergate_uvd) {
+	if (adev->powerplay.pp_funcs->set_powergating_by_smu) {
 		/* enable/disable UVD */
 		mutex_lock(&adev->pm.mutex);
-		amdgpu_dpm_powergate_uvd(adev, !enable);
+		amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_UVD, !enable);
 		mutex_unlock(&adev->pm.mutex);
 	} else {
 		if (enable) {
@@ -1719,10 +1736,10 @@ void amdgpu_dpm_enable_uvd(struct amdgpu_device *adev, bool enable)
 
 void amdgpu_dpm_enable_vce(struct amdgpu_device *adev, bool enable)
 {
-	if (adev->powerplay.pp_funcs->powergate_vce) {
+	if (adev->powerplay.pp_funcs->set_powergating_by_smu) {
 		/* enable/disable VCE */
 		mutex_lock(&adev->pm.mutex);
-		amdgpu_dpm_powergate_vce(adev, !enable);
+		amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_VCE, !enable);
 		mutex_unlock(&adev->pm.mutex);
 	} else {
 		if (enable) {
@@ -1854,6 +1871,13 @@ int amdgpu_pm_sysfs_init(struct amdgpu_device *adev)
 				"pp_od_clk_voltage\n");
 		return ret;
 	}
+	ret = device_create_file(adev->dev,
+			&dev_attr_gpu_busy_percent);
+	if (ret) {
+		DRM_ERROR("failed to create device file	"
+				"gpu_busy_level\n");
+		return ret;
+	}
 	ret = amdgpu_debugfs_pm_init(adev);
 	if (ret) {
 		DRM_ERROR("Failed to register debugfs file for dpm!\n");
@@ -1889,6 +1913,7 @@ void amdgpu_pm_sysfs_fini(struct amdgpu_device *adev)
 			&dev_attr_pp_power_profile_mode);
 	device_remove_file(adev->dev,
 			&dev_attr_pp_od_clk_voltage);
+	device_remove_file(adev->dev, &dev_attr_gpu_busy_percent);
 }
 
 void amdgpu_pm_compute_clocks(struct amdgpu_device *adev)
@@ -1919,7 +1944,7 @@ void amdgpu_pm_compute_clocks(struct amdgpu_device *adev)
 		if (!amdgpu_device_has_dc_support(adev)) {
 			mutex_lock(&adev->pm.mutex);
 			amdgpu_dpm_get_active_displays(adev);
-			adev->pm.pm_display_cfg.num_display = adev->pm.dpm.new_active_crtcs;
+			adev->pm.pm_display_cfg.num_display = adev->pm.dpm.new_active_crtc_count;
 			adev->pm.pm_display_cfg.vrefresh = amdgpu_dpm_get_vrefresh(adev);
 			adev->pm.pm_display_cfg.min_vblank_time = amdgpu_dpm_get_vblank_time(adev);
 			/* we have issues with mclk switching with refresh rates over 120 hz on the non-DC code. */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_prime.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_prime.c
index b2286bc41aec..1c5d97f4b4dd 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_prime.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_prime.c
@@ -191,7 +191,6 @@ error:
 /**
  * amdgpu_gem_map_attach - &dma_buf_ops.attach implementation
  * @dma_buf: shared DMA buffer
- * @target_dev: target device
  * @attach: DMA-buf attachment
  *
  * Makes sure that the shared DMA buffer can be accessed by the target device.
@@ -233,7 +232,7 @@ static int amdgpu_gem_map_attach(struct dma_buf *dma_buf,
 	}
 
 	/* pin buffer into GTT */
-	r = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT, NULL);
+	r = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT);
 	if (r)
 		goto error_unreserve;
 
@@ -324,7 +323,7 @@ static int amdgpu_gem_begin_cpu_access(struct dma_buf *dma_buf,
 		return ret;
 
 	if (!bo->pin_count && (bo->allowed_domains & AMDGPU_GEM_DOMAIN_GTT)) {
-		amdgpu_ttm_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);
+		amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);
 		ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
 	}
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
index 9f1a5bd39ae8..5b39d1399630 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
@@ -131,6 +131,11 @@ psp_cmd_submit_buf(struct psp_context *psp,
 		msleep(1);
 	}
 
+	if (ucode) {
+		ucode->tmr_mc_addr_lo = psp->cmd_buf_mem->resp.fw_addr_lo;
+		ucode->tmr_mc_addr_hi = psp->cmd_buf_mem->resp.fw_addr_hi;
+	}
+
 	return ret;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_queue_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_queue_mgr.c
index ea9850c9224d..a172bba32b45 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_queue_mgr.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_queue_mgr.c
@@ -66,8 +66,6 @@ static int amdgpu_identity_map(struct amdgpu_device *adev,
 			       u32 ring,
 			       struct amdgpu_ring **out_ring)
 {
-	u32 instance;
-
 	switch (mapper->hw_ip) {
 	case AMDGPU_HW_IP_GFX:
 		*out_ring = &adev->gfx.gfx_ring[ring];
@@ -79,16 +77,13 @@ static int amdgpu_identity_map(struct amdgpu_device *adev,
 		*out_ring = &adev->sdma.instance[ring].ring;
 		break;
 	case AMDGPU_HW_IP_UVD:
-		instance = ring;
-		*out_ring = &adev->uvd.inst[instance].ring;
+		*out_ring = &adev->uvd.inst[0].ring;
 		break;
 	case AMDGPU_HW_IP_VCE:
 		*out_ring = &adev->vce.ring[ring];
 		break;
 	case AMDGPU_HW_IP_UVD_ENC:
-		instance = ring / adev->uvd.num_enc_rings;
-		*out_ring =
-		&adev->uvd.inst[instance].ring_enc[ring%adev->uvd.num_enc_rings];
+		*out_ring = &adev->uvd.inst[0].ring_enc[ring];
 		break;
 	case AMDGPU_HW_IP_VCN_DEC:
 		*out_ring = &adev->vcn.ring_dec;
@@ -219,7 +214,7 @@ int amdgpu_queue_mgr_map(struct amdgpu_device *adev,
 			 u32 hw_ip, u32 instance, u32 ring,
 			 struct amdgpu_ring **out_ring)
 {
-	int r, ip_num_rings;
+	int i, r, ip_num_rings = 0;
 	struct amdgpu_queue_mapper *mapper = &mgr->mapper[hw_ip];
 
 	if (!adev || !mgr || !out_ring)
@@ -248,14 +243,21 @@ int amdgpu_queue_mgr_map(struct amdgpu_device *adev,
 		ip_num_rings = adev->sdma.num_instances;
 		break;
 	case AMDGPU_HW_IP_UVD:
-		ip_num_rings = adev->uvd.num_uvd_inst;
+		for (i = 0; i < adev->uvd.num_uvd_inst; i++) {
+			if (!(adev->uvd.harvest_config & (1 << i)))
+				ip_num_rings++;
+		}
 		break;
 	case AMDGPU_HW_IP_VCE:
 		ip_num_rings = adev->vce.num_rings;
 		break;
 	case AMDGPU_HW_IP_UVD_ENC:
+		for (i = 0; i < adev->uvd.num_uvd_inst; i++) {
+			if (!(adev->uvd.harvest_config & (1 << i)))
+				ip_num_rings++;
+		}
 		ip_num_rings =
-			adev->uvd.num_enc_rings * adev->uvd.num_uvd_inst;
+			adev->uvd.num_enc_rings * ip_num_rings;
 		break;
 	case AMDGPU_HW_IP_VCN_DEC:
 		ip_num_rings = 1;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
index 19e45a3953e0..93794a85f83d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
@@ -211,7 +211,8 @@ void amdgpu_ring_priority_get(struct amdgpu_ring *ring,
 	if (!ring->funcs->set_priority)
 		return;
 
-	atomic_inc(&ring->num_jobs[priority]);
+	if (atomic_inc_return(&ring->num_jobs[priority]) <= 0)
+		return;
 
 	mutex_lock(&ring->priority_mutex);
 	if (priority <= ring->priority)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
index a293f4e6760d..d242b9a51e90 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
@@ -44,6 +44,8 @@
 #define AMDGPU_FENCE_FLAG_INT           (1 << 1)
 #define AMDGPU_FENCE_FLAG_TC_WB_ONLY    (1 << 2)
 
+#define to_amdgpu_ring(s) container_of((s), struct amdgpu_ring, sched)
+
 enum amdgpu_ring_type {
 	AMDGPU_RING_TYPE_GFX,
 	AMDGPU_RING_TYPE_COMPUTE,
@@ -121,6 +123,7 @@ struct amdgpu_ring_funcs {
 	void (*set_wptr)(struct amdgpu_ring *ring);
 	/* validating and patching of IBs */
 	int (*parse_cs)(struct amdgpu_cs_parser *p, uint32_t ib_idx);
+	int (*patch_cs_in_place)(struct amdgpu_cs_parser *p, uint32_t ib_idx);
 	/* constants to calculate how many DW are needed for an emit */
 	unsigned emit_frame_size;
 	unsigned emit_ib_size;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c
index 86a0715d9431..1cafe8d83a4d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c
@@ -53,9 +53,8 @@ static int amdgpu_sched_process_priority_override(struct amdgpu_device *adev,
 						  int fd,
 						  enum drm_sched_priority priority)
 {
-	struct file *filp = fcheck(fd);
+	struct file *filp = fget(fd);
 	struct drm_file *file;
-	struct pid *pid;
 	struct amdgpu_fpriv *fpriv;
 	struct amdgpu_ctx *ctx;
 	uint32_t id;
@@ -63,20 +62,12 @@ static int amdgpu_sched_process_priority_override(struct amdgpu_device *adev,
 	if (!filp)
 		return -EINVAL;
 
-	pid = get_pid(((struct drm_file *)filp->private_data)->pid);
+	file = filp->private_data;
+	fpriv = file->driver_priv;
+	idr_for_each_entry(&fpriv->ctx_mgr.ctx_handles, ctx, id)
+		amdgpu_ctx_priority_override(ctx, priority);
 
-	mutex_lock(&adev->ddev->filelist_mutex);
-	list_for_each_entry(file, &adev->ddev->filelist, lhead) {
-		if (file->pid != pid)
-			continue;
-
-		fpriv = file->driver_priv;
-		idr_for_each_entry(&fpriv->ctx_mgr.ctx_handles, ctx, id)
-				amdgpu_ctx_priority_override(ctx, priority);
-	}
-	mutex_unlock(&adev->ddev->filelist_mutex);
-
-	put_pid(pid);
+	fput(filp);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_test.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_test.c
index e3878256743a..8904e62dca7a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_test.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_test.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
 /*
  * Copyright 2009 VMware, Inc.
  *
@@ -75,11 +76,12 @@ static void amdgpu_do_test_moves(struct amdgpu_device *adev)
 	r = amdgpu_bo_reserve(vram_obj, false);
 	if (unlikely(r != 0))
 		goto out_unref;
-	r = amdgpu_bo_pin(vram_obj, AMDGPU_GEM_DOMAIN_VRAM, &vram_addr);
+	r = amdgpu_bo_pin(vram_obj, AMDGPU_GEM_DOMAIN_VRAM);
 	if (r) {
 		DRM_ERROR("Failed to pin VRAM object\n");
 		goto out_unres;
 	}
+	vram_addr = amdgpu_bo_gpu_offset(vram_obj);
 	for (i = 0; i < n; i++) {
 		void *gtt_map, *vram_map;
 		void **gart_start, **gart_end;
@@ -96,11 +98,17 @@ static void amdgpu_do_test_moves(struct amdgpu_device *adev)
 		r = amdgpu_bo_reserve(gtt_obj[i], false);
 		if (unlikely(r != 0))
 			goto out_lclean_unref;
-		r = amdgpu_bo_pin(gtt_obj[i], AMDGPU_GEM_DOMAIN_GTT, &gart_addr);
+		r = amdgpu_bo_pin(gtt_obj[i], AMDGPU_GEM_DOMAIN_GTT);
 		if (r) {
 			DRM_ERROR("Failed to pin GTT object %d\n", i);
 			goto out_lclean_unres;
 		}
+		r = amdgpu_ttm_alloc_gart(&gtt_obj[i]->tbo);
+		if (r) {
+			DRM_ERROR("%p bind failed\n", gtt_obj[i]);
+			goto out_lclean_unpin;
+		}
+		gart_addr = amdgpu_bo_gpu_offset(gtt_obj[i]);
 
 		r = amdgpu_bo_kmap(gtt_obj[i], &gtt_map);
 		if (r) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
index e96e26d3f3b0..7206a0025b17 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
@@ -150,10 +150,10 @@ TRACE_EVENT(amdgpu_cs,
 
 	    TP_fast_assign(
 			   __entry->bo_list = p->bo_list;
-			   __entry->ring = p->job->ring->idx;
+			   __entry->ring = p->ring->idx;
 			   __entry->dw = p->job->ibs[i].length_dw;
 			   __entry->fences = amdgpu_fence_count_emitted(
-				p->job->ring);
+				p->ring);
 			   ),
 	    TP_printk("bo_list=%p, ring=%u, dw=%u, fences=%u",
 		      __entry->bo_list, __entry->ring, __entry->dw,
@@ -178,7 +178,7 @@ TRACE_EVENT(amdgpu_cs_ioctl,
 			   __assign_str(timeline, AMDGPU_JOB_GET_TIMELINE_NAME(job))
 			   __entry->context = job->base.s_fence->finished.context;
 			   __entry->seqno = job->base.s_fence->finished.seqno;
-			   __entry->ring_name = job->ring->name;
+			   __entry->ring_name = to_amdgpu_ring(job->base.sched)->name;
 			   __entry->num_ibs = job->num_ibs;
 			   ),
 	    TP_printk("sched_job=%llu, timeline=%s, context=%u, seqno=%u, ring_name=%s, num_ibs=%u",
@@ -203,7 +203,7 @@ TRACE_EVENT(amdgpu_sched_run_job,
 			   __assign_str(timeline, AMDGPU_JOB_GET_TIMELINE_NAME(job))
 			   __entry->context = job->base.s_fence->finished.context;
 			   __entry->seqno = job->base.s_fence->finished.seqno;
-			   __entry->ring_name = job->ring->name;
+			   __entry->ring_name = to_amdgpu_ring(job->base.sched)->name;
 			   __entry->num_ibs = job->num_ibs;
 			   ),
 	    TP_printk("sched_job=%llu, timeline=%s, context=%u, seqno=%u, ring_name=%s, num_ibs=%u",
@@ -314,6 +314,11 @@ DEFINE_EVENT(amdgpu_vm_mapping, amdgpu_vm_bo_mapping,
 	    TP_ARGS(mapping)
 );
 
+DEFINE_EVENT(amdgpu_vm_mapping, amdgpu_vm_bo_cs,
+	    TP_PROTO(struct amdgpu_bo_va_mapping *mapping),
+	    TP_ARGS(mapping)
+);
+
 TRACE_EVENT(amdgpu_vm_set_ptes,
 	    TP_PROTO(uint64_t pe, uint64_t addr, unsigned count,
 		     uint32_t incr, uint64_t flags),
@@ -436,7 +441,7 @@ TRACE_EVENT(amdgpu_cs_bo_status,
 			__entry->total_bo, __entry->total_size)
 );
 
-TRACE_EVENT(amdgpu_ttm_bo_move,
+TRACE_EVENT(amdgpu_bo_move,
 	    TP_PROTO(struct amdgpu_bo* bo, uint32_t new_placement, uint32_t old_placement),
 	    TP_ARGS(bo, new_placement, old_placement),
 	    TP_STRUCT__entry(
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index 0c084d3d0865..fcf421263fd9 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -92,11 +92,9 @@ static void amdgpu_ttm_mem_global_release(struct drm_global_reference *ref)
 }
 
 /**
- * amdgpu_ttm_global_init - Initialize global TTM memory reference
- * 							structures.
+ * amdgpu_ttm_global_init - Initialize global TTM memory reference structures.
  *
- * @adev:  	AMDGPU device for which the global structures need to be
- *			registered.
+ * @adev: AMDGPU device for which the global structures need to be registered.
  *
  * This is called as part of the AMDGPU ttm init from amdgpu_ttm_init()
  * during bring up.
@@ -104,8 +102,6 @@ static void amdgpu_ttm_mem_global_release(struct drm_global_reference *ref)
 static int amdgpu_ttm_global_init(struct amdgpu_device *adev)
 {
 	struct drm_global_reference *global_ref;
-	struct amdgpu_ring *ring;
-	struct drm_sched_rq *rq;
 	int r;
 
 	/* ensure reference is false in case init fails */
@@ -138,21 +134,10 @@ static int amdgpu_ttm_global_init(struct amdgpu_device *adev)
 
 	mutex_init(&adev->mman.gtt_window_lock);
 
-	ring = adev->mman.buffer_funcs_ring;
-	rq = &ring->sched.sched_rq[DRM_SCHED_PRIORITY_KERNEL];
-	r = drm_sched_entity_init(&ring->sched, &adev->mman.entity,
-				  rq, NULL);
-	if (r) {
-		DRM_ERROR("Failed setting up TTM BO move run queue.\n");
-		goto error_entity;
-	}
-
 	adev->mman.mem_global_referenced = true;
 
 	return 0;
 
-error_entity:
-	drm_global_item_unref(&adev->mman.bo_global_ref.ref);
 error_bo:
 	drm_global_item_unref(&adev->mman.mem_global_ref);
 error_mem:
@@ -162,8 +147,6 @@ error_mem:
 static void amdgpu_ttm_global_fini(struct amdgpu_device *adev)
 {
 	if (adev->mman.mem_global_referenced) {
-		drm_sched_entity_fini(adev->mman.entity.sched,
-				      &adev->mman.entity);
 		mutex_destroy(&adev->mman.gtt_window_lock);
 		drm_global_item_unref(&adev->mman.bo_global_ref.ref);
 		drm_global_item_unref(&adev->mman.mem_global_ref);
@@ -177,13 +160,12 @@ static int amdgpu_invalidate_caches(struct ttm_bo_device *bdev, uint32_t flags)
 }
 
 /**
- * amdgpu_init_mem_type - 	Initialize a memory manager for a specific
- * 							type of memory request.
+ * amdgpu_init_mem_type - Initialize a memory manager for a specific type of
+ * memory request.
  *
- * @bdev:	The TTM BO device object (contains a reference to
- * 			amdgpu_device)
- * @type:	The type of memory requested
- * @man:
+ * @bdev: The TTM BO device object (contains a reference to amdgpu_device)
+ * @type: The type of memory requested
+ * @man: The memory type manager for each domain
  *
  * This is called by ttm_bo_init_mm() when a buffer object is being
  * initialized.
@@ -263,7 +245,7 @@ static void amdgpu_evict_flags(struct ttm_buffer_object *bo,
 	}
 
 	/* Object isn't an AMDGPU object so ignore */
-	if (!amdgpu_ttm_bo_is_amdgpu_bo(bo)) {
+	if (!amdgpu_bo_is_amdgpu_bo(bo)) {
 		placement->placement = &placements;
 		placement->busy_placement = &placements;
 		placement->num_placement = 1;
@@ -276,7 +258,7 @@ static void amdgpu_evict_flags(struct ttm_buffer_object *bo,
 	case TTM_PL_VRAM:
 		if (!adev->mman.buffer_funcs_enabled) {
 			/* Move to system memory */
-			amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_CPU);
+			amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_CPU);
 		} else if (!amdgpu_gmc_vram_full_visible(&adev->gmc) &&
 			   !(abo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) &&
 			   amdgpu_bo_in_cpu_visible_vram(abo)) {
@@ -286,7 +268,7 @@ static void amdgpu_evict_flags(struct ttm_buffer_object *bo,
 			 * BO will be evicted to GTT rather than causing other
 			 * BOs to be evicted from VRAM
 			 */
-			amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM |
+			amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM |
 							 AMDGPU_GEM_DOMAIN_GTT);
 			abo->placements[0].fpfn = adev->gmc.visible_vram_size >> PAGE_SHIFT;
 			abo->placements[0].lpfn = 0;
@@ -294,12 +276,12 @@ static void amdgpu_evict_flags(struct ttm_buffer_object *bo,
 			abo->placement.num_busy_placement = 1;
 		} else {
 			/* Move to GTT memory */
-			amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_GTT);
+			amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_GTT);
 		}
 		break;
 	case TTM_PL_TT:
 	default:
-		amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_CPU);
+		amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_CPU);
 	}
 	*placement = abo->placement;
 }
@@ -307,8 +289,8 @@ static void amdgpu_evict_flags(struct ttm_buffer_object *bo,
 /**
  * amdgpu_verify_access - Verify access for a mmap call
  *
- * @bo:		The buffer object to map
- * @filp:	The file pointer from the process performing the mmap
+ * @bo:	The buffer object to map
+ * @filp: The file pointer from the process performing the mmap
  *
  * This is called by ttm_bo_mmap() to verify whether a process
  * has the right to mmap a BO to their process space.
@@ -333,11 +315,10 @@ static int amdgpu_verify_access(struct ttm_buffer_object *bo, struct file *filp)
 /**
  * amdgpu_move_null - Register memory for a buffer object
  *
- * @bo:			The bo to assign the memory to
- * @new_mem:	The memory to be assigned.
+ * @bo: The bo to assign the memory to
+ * @new_mem: The memory to be assigned.
  *
- * Assign the memory from new_mem to the memory of the buffer object
- * bo.
+ * Assign the memory from new_mem to the memory of the buffer object bo.
  */
 static void amdgpu_move_null(struct ttm_buffer_object *bo,
 			     struct ttm_mem_reg *new_mem)
@@ -350,8 +331,12 @@ static void amdgpu_move_null(struct ttm_buffer_object *bo,
 }
 
 /**
- * amdgpu_mm_node_addr -	Compute the GPU relative offset of a GTT
- * 							buffer.
+ * amdgpu_mm_node_addr - Compute the GPU relative offset of a GTT buffer.
+ *
+ * @bo: The bo to assign the memory to.
+ * @mm_node: Memory manager node for drm allocator.
+ * @mem: The region where the bo resides.
+ *
  */
 static uint64_t amdgpu_mm_node_addr(struct ttm_buffer_object *bo,
 				    struct drm_mm_node *mm_node,
@@ -367,10 +352,12 @@ static uint64_t amdgpu_mm_node_addr(struct ttm_buffer_object *bo,
 }
 
 /**
- * amdgpu_find_mm_node -	Helper function finds the drm_mm_node
- *  						corresponding to @offset. It also modifies
- * 							the offset to be within the drm_mm_node
- * 							returned
+ * amdgpu_find_mm_node - Helper function finds the drm_mm_node corresponding to
+ * @offset. It also modifies the offset to be within the drm_mm_node returned
+ *
+ * @mem: The region where the bo resides.
+ * @offset: The offset that drm_mm_node is used for finding.
+ *
  */
 static struct drm_mm_node *amdgpu_find_mm_node(struct ttm_mem_reg *mem,
 					       unsigned long *offset)
@@ -512,8 +499,8 @@ error:
 /**
  * amdgpu_move_blit - Copy an entire buffer to another buffer
  *
- * This is a helper called by amdgpu_bo_move() and
- * amdgpu_move_vram_ram() to help move buffers to and from VRAM.
+ * This is a helper called by amdgpu_bo_move() and amdgpu_move_vram_ram() to
+ * help move buffers to and from VRAM.
  */
 static int amdgpu_move_blit(struct ttm_buffer_object *bo,
 			    bool evict, bool no_wait_gpu,
@@ -595,7 +582,7 @@ static int amdgpu_move_vram_ram(struct ttm_buffer_object *bo, bool evict,
 	}
 
 	/* blit VRAM to GTT */
-	r = amdgpu_move_blit(bo, true, ctx->no_wait_gpu, &tmp_mem, old_mem);
+	r = amdgpu_move_blit(bo, evict, ctx->no_wait_gpu, &tmp_mem, old_mem);
 	if (unlikely(r)) {
 		goto out_cleanup;
 	}
@@ -647,7 +634,7 @@ static int amdgpu_move_ram_vram(struct ttm_buffer_object *bo, bool evict,
 	}
 
 	/* copy to VRAM */
-	r = amdgpu_move_blit(bo, true, ctx->no_wait_gpu, new_mem, old_mem);
+	r = amdgpu_move_blit(bo, evict, ctx->no_wait_gpu, new_mem, old_mem);
 	if (unlikely(r)) {
 		goto out_cleanup;
 	}
@@ -809,8 +796,8 @@ struct amdgpu_ttm_tt {
 };
 
 /**
- * amdgpu_ttm_tt_get_user_pages - 	Pin pages of memory pointed to
- * 									by a USERPTR pointer to memory
+ * amdgpu_ttm_tt_get_user_pages - Pin pages of memory pointed to by a USERPTR
+ * pointer to memory
  *
  * Called by amdgpu_gem_userptr_ioctl() and amdgpu_cs_parser_bos().
  * This provides a wrapper around the get_user_pages() call to provide
@@ -833,8 +820,10 @@ int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages)
 	down_read(&mm->mmap_sem);
 
 	if (gtt->userflags & AMDGPU_GEM_USERPTR_ANONONLY) {
-		/* check that we only use anonymous memory
-		   to prevent problems with writeback */
+		/*
+		 * check that we only use anonymous memory to prevent problems
+		 * with writeback
+		 */
 		unsigned long end = gtt->userptr + ttm->num_pages * PAGE_SIZE;
 		struct vm_area_struct *vma;
 
@@ -885,10 +874,9 @@ release_pages:
 }
 
 /**
- * amdgpu_ttm_tt_set_user_pages - 	Copy pages in, putting old pages
- * 									as necessary.
+ * amdgpu_ttm_tt_set_user_pages - Copy pages in, putting old pages as necessary.
  *
- * Called by amdgpu_cs_list_validate().  This creates the page list
+ * Called by amdgpu_cs_list_validate(). This creates the page list
  * that backs user memory and will ultimately be mapped into the device
  * address space.
  */
@@ -930,8 +918,7 @@ void amdgpu_ttm_tt_mark_user_pages(struct ttm_tt *ttm)
 }
 
 /**
- * amdgpu_ttm_tt_pin_userptr - 	prepare the sg table with the
- * 								user pages
+ * amdgpu_ttm_tt_pin_userptr - 	prepare the sg table with the user pages
  *
  * Called by amdgpu_ttm_backend_bind()
  **/
@@ -1310,8 +1297,8 @@ static void amdgpu_ttm_tt_unpopulate(struct ttm_tt *ttm)
 }
 
 /**
- * amdgpu_ttm_tt_set_userptr -	Initialize userptr GTT ttm_tt
- * 								for the current task
+ * amdgpu_ttm_tt_set_userptr - Initialize userptr GTT ttm_tt for the current
+ * task
  *
  * @ttm: The ttm_tt object to bind this userptr object to
  * @addr:  The address in the current tasks VM space to use
@@ -1361,9 +1348,8 @@ struct mm_struct *amdgpu_ttm_tt_get_usermm(struct ttm_tt *ttm)
 }
 
 /**
- * amdgpu_ttm_tt_affect_userptr -	Determine if a ttm_tt object lays
- * 									inside an address range for the
- * 									current task.
+ * amdgpu_ttm_tt_affect_userptr - Determine if a ttm_tt object lays inside an
+ * address range for the current task.
  *
  */
 bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start,
@@ -1401,8 +1387,7 @@ bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start,
 }
 
 /**
- * amdgpu_ttm_tt_userptr_invalidated -	Has the ttm_tt object been
- * 										invalidated?
+ * amdgpu_ttm_tt_userptr_invalidated - Has the ttm_tt object been invalidated?
  */
 bool amdgpu_ttm_tt_userptr_invalidated(struct ttm_tt *ttm,
 				       int *last_invalidated)
@@ -1415,10 +1400,8 @@ bool amdgpu_ttm_tt_userptr_invalidated(struct ttm_tt *ttm,
 }
 
 /**
- * amdgpu_ttm_tt_userptr_needs_pages -	Have the pages backing this
- * 										ttm_tt object been invalidated
- * 										since the last time they've
- * 										been set?
+ * amdgpu_ttm_tt_userptr_needs_pages - Have the pages backing this ttm_tt object
+ * been invalidated since the last time they've been set?
  */
 bool amdgpu_ttm_tt_userptr_needs_pages(struct ttm_tt *ttm)
 {
@@ -1474,13 +1457,12 @@ uint64_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device *adev, struct ttm_tt *ttm,
 }
 
 /**
- * amdgpu_ttm_bo_eviction_valuable -	Check to see if we can evict
- * 										a buffer object.
+ * amdgpu_ttm_bo_eviction_valuable - Check to see if we can evict a buffer
+ * object.
  *
- * Return true if eviction is sensible.  Called by
- * ttm_mem_evict_first() on behalf of ttm_bo_mem_force_space()
- * which tries to evict buffer objects until it can find space
- * for a new object and by ttm_bo_force_list_clean() which is
+ * Return true if eviction is sensible. Called by ttm_mem_evict_first() on
+ * behalf of ttm_bo_mem_force_space() which tries to evict buffer objects until
+ * it can find space for a new object and by ttm_bo_force_list_clean() which is
  * used to clean out a memory space.
  */
 static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
@@ -1530,8 +1512,7 @@ static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
 }
 
 /**
- * amdgpu_ttm_access_memory -	Read or Write memory that backs a
- * 								buffer object.
+ * amdgpu_ttm_access_memory - Read or Write memory that backs a buffer object.
  *
  * @bo:  The buffer object to read/write
  * @offset:  Offset into buffer object
@@ -1695,7 +1676,7 @@ static int amdgpu_ttm_fw_reserve_vram_init(struct amdgpu_device *adev)
 			AMDGPU_GEM_DOMAIN_VRAM,
 			adev->fw_vram_usage.start_offset,
 			(adev->fw_vram_usage.start_offset +
-			adev->fw_vram_usage.size), NULL);
+			adev->fw_vram_usage.size));
 		if (r)
 			goto error_pin;
 		r = amdgpu_bo_kmap(adev->fw_vram_usage.reserved_bo,
@@ -1719,8 +1700,8 @@ error_create:
 	return r;
 }
 /**
- * amdgpu_ttm_init -	Init the memory management (ttm) as well as
- * 						various gtt/vram related fields.
+ * amdgpu_ttm_init - Init the memory management (ttm) as well as various
+ * gtt/vram related fields.
  *
  * This initializes all of the memory space pools that the TTM layer
  * will need such as the GTT space (system memory mapped to the device),
@@ -1871,8 +1852,7 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
 }
 
 /**
- * amdgpu_ttm_late_init -	Handle any late initialization for
- * 							amdgpu_ttm
+ * amdgpu_ttm_late_init - Handle any late initialization for amdgpu_ttm
  */
 void amdgpu_ttm_late_init(struct amdgpu_device *adev)
 {
@@ -1921,10 +1901,30 @@ void amdgpu_ttm_set_buffer_funcs_status(struct amdgpu_device *adev, bool enable)
 {
 	struct ttm_mem_type_manager *man = &adev->mman.bdev.man[TTM_PL_VRAM];
 	uint64_t size;
+	int r;
 
-	if (!adev->mman.initialized || adev->in_gpu_reset)
+	if (!adev->mman.initialized || adev->in_gpu_reset ||
+	    adev->mman.buffer_funcs_enabled == enable)
 		return;
 
+	if (enable) {
+		struct amdgpu_ring *ring;
+		struct drm_sched_rq *rq;
+
+		ring = adev->mman.buffer_funcs_ring;
+		rq = &ring->sched.sched_rq[DRM_SCHED_PRIORITY_KERNEL];
+		r = drm_sched_entity_init(&adev->mman.entity, &rq, 1, NULL);
+		if (r) {
+			DRM_ERROR("Failed setting up TTM BO move entity (%d)\n",
+				  r);
+			return;
+		}
+	} else {
+		drm_sched_entity_destroy(&adev->mman.entity);
+		dma_fence_put(man->move);
+		man->move = NULL;
+	}
+
 	/* this just adjusts TTM size idea, which sets lpfn to the correct value */
 	if (enable)
 		size = adev->gmc.real_vram_size;
@@ -2002,7 +2002,7 @@ static int amdgpu_map_buffer(struct ttm_buffer_object *bo,
 	if (r)
 		goto error_free;
 
-	r = amdgpu_job_submit(job, ring, &adev->mman.entity,
+	r = amdgpu_job_submit(job, &adev->mman.entity,
 			      AMDGPU_FENCE_OWNER_UNDEFINED, &fence);
 	if (r)
 		goto error_free;
@@ -2071,24 +2071,19 @@ int amdgpu_copy_buffer(struct amdgpu_ring *ring, uint64_t src_offset,
 
 	amdgpu_ring_pad_ib(ring, &job->ibs[0]);
 	WARN_ON(job->ibs[0].length_dw > num_dw);
-	if (direct_submit) {
-		r = amdgpu_ib_schedule(ring, job->num_ibs, job->ibs,
-				       NULL, fence);
-		job->fence = dma_fence_get(*fence);
-		if (r)
-			DRM_ERROR("Error scheduling IBs (%d)\n", r);
-		amdgpu_job_free(job);
-	} else {
-		r = amdgpu_job_submit(job, ring, &adev->mman.entity,
+	if (direct_submit)
+		r = amdgpu_job_submit_direct(job, ring, fence);
+	else
+		r = amdgpu_job_submit(job, &adev->mman.entity,
 				      AMDGPU_FENCE_OWNER_UNDEFINED, fence);
-		if (r)
-			goto error_free;
-	}
+	if (r)
+		goto error_free;
 
 	return r;
 
 error_free:
 	amdgpu_job_free(job);
+	DRM_ERROR("Error scheduling IBs (%d)\n", r);
 	return r;
 }
 
@@ -2171,7 +2166,7 @@ int amdgpu_fill_buffer(struct amdgpu_bo *bo,
 
 	amdgpu_ring_pad_ib(ring, &job->ibs[0]);
 	WARN_ON(job->ibs[0].length_dw > num_dw);
-	r = amdgpu_job_submit(job, ring, &adev->mman.entity,
+	r = amdgpu_job_submit(job, &adev->mman.entity,
 			      AMDGPU_FENCE_OWNER_UNDEFINED, fence);
 	if (r)
 		goto error_free;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
index e5da4654b630..8b3cc6687769 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
@@ -73,7 +73,7 @@ bool amdgpu_gtt_mgr_has_gart_addr(struct ttm_mem_reg *mem);
 uint64_t amdgpu_gtt_mgr_usage(struct ttm_mem_type_manager *man);
 int amdgpu_gtt_mgr_recover(struct ttm_mem_type_manager *man);
 
-u64 amdgpu_vram_mgr_bo_invisible_size(struct amdgpu_bo *bo);
+u64 amdgpu_vram_mgr_bo_visible_size(struct amdgpu_bo *bo);
 uint64_t amdgpu_vram_mgr_usage(struct ttm_mem_type_manager *man);
 uint64_t amdgpu_vram_mgr_vis_usage(struct ttm_mem_type_manager *man);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h
index 08e38579af24..bdc472b6e641 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h
@@ -194,6 +194,7 @@ enum AMDGPU_UCODE_ID {
 	AMDGPU_UCODE_ID_SMC,
 	AMDGPU_UCODE_ID_UVD,
 	AMDGPU_UCODE_ID_VCE,
+	AMDGPU_UCODE_ID_VCN,
 	AMDGPU_UCODE_ID_MAXIMUM,
 };
 
@@ -226,6 +227,9 @@ struct amdgpu_firmware_info {
 	void *kaddr;
 	/* ucode_size_bytes */
 	uint32_t ucode_size;
+	/* starting tmr mc address */
+	uint32_t tmr_mc_addr_lo;
+	uint32_t tmr_mc_addr_hi;
 };
 
 void amdgpu_ucode_print_mc_hdr(const struct common_firmware_header *hdr);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
index cc15d3230402..e5a6db6beab7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
@@ -53,11 +53,11 @@
 
 /* Firmware Names */
 #ifdef CONFIG_DRM_AMDGPU_CIK
-#define FIRMWARE_BONAIRE	"radeon/bonaire_uvd.bin"
-#define FIRMWARE_KABINI	"radeon/kabini_uvd.bin"
-#define FIRMWARE_KAVERI	"radeon/kaveri_uvd.bin"
-#define FIRMWARE_HAWAII	"radeon/hawaii_uvd.bin"
-#define FIRMWARE_MULLINS	"radeon/mullins_uvd.bin"
+#define FIRMWARE_BONAIRE	"amdgpu/bonaire_uvd.bin"
+#define FIRMWARE_KABINI	"amdgpu/kabini_uvd.bin"
+#define FIRMWARE_KAVERI	"amdgpu/kaveri_uvd.bin"
+#define FIRMWARE_HAWAII	"amdgpu/hawaii_uvd.bin"
+#define FIRMWARE_MULLINS	"amdgpu/mullins_uvd.bin"
 #endif
 #define FIRMWARE_TONGA		"amdgpu/tonga_uvd.bin"
 #define FIRMWARE_CARRIZO	"amdgpu/carrizo_uvd.bin"
@@ -122,8 +122,6 @@ static void amdgpu_uvd_idle_work_handler(struct work_struct *work);
 
 int amdgpu_uvd_sw_init(struct amdgpu_device *adev)
 {
-	struct amdgpu_ring *ring;
-	struct drm_sched_rq *rq;
 	unsigned long bo_size;
 	const char *fw_name;
 	const struct common_firmware_header *hdr;
@@ -255,7 +253,8 @@ int amdgpu_uvd_sw_init(struct amdgpu_device *adev)
 		bo_size += AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8);
 
 	for (j = 0; j < adev->uvd.num_uvd_inst; j++) {
-
+		if (adev->uvd.harvest_config & (1 << j))
+			continue;
 		r = amdgpu_bo_create_kernel(adev, bo_size, PAGE_SIZE,
 					    AMDGPU_GEM_DOMAIN_VRAM, &adev->uvd.inst[j].vcpu_bo,
 					    &adev->uvd.inst[j].gpu_addr, &adev->uvd.inst[j].cpu_addr);
@@ -263,21 +262,13 @@ int amdgpu_uvd_sw_init(struct amdgpu_device *adev)
 			dev_err(adev->dev, "(%d) failed to allocate UVD bo\n", r);
 			return r;
 		}
+	}
 
-		ring = &adev->uvd.inst[j].ring;
-		rq = &ring->sched.sched_rq[DRM_SCHED_PRIORITY_NORMAL];
-		r = drm_sched_entity_init(&ring->sched, &adev->uvd.inst[j].entity,
-					  rq, NULL);
-		if (r != 0) {
-			DRM_ERROR("Failed setting up UVD(%d) run queue.\n", j);
-			return r;
-		}
-
-		for (i = 0; i < adev->uvd.max_handles; ++i) {
-			atomic_set(&adev->uvd.inst[j].handles[i], 0);
-			adev->uvd.inst[j].filp[i] = NULL;
-		}
+	for (i = 0; i < adev->uvd.max_handles; ++i) {
+		atomic_set(&adev->uvd.handles[i], 0);
+		adev->uvd.filp[i] = NULL;
 	}
+
 	/* from uvd v5.0 HW addressing capacity increased to 64 bits */
 	if (!amdgpu_device_ip_block_version_cmp(adev, AMD_IP_BLOCK_TYPE_UVD, 5, 0))
 		adev->uvd.address_64_bit = true;
@@ -306,10 +297,12 @@ int amdgpu_uvd_sw_fini(struct amdgpu_device *adev)
 {
 	int i, j;
 
-	for (j = 0; j < adev->uvd.num_uvd_inst; ++j) {
-		kfree(adev->uvd.inst[j].saved_bo);
+	drm_sched_entity_destroy(&adev->uvd.entity);
 
-		drm_sched_entity_fini(&adev->uvd.inst[j].ring.sched, &adev->uvd.inst[j].entity);
+	for (j = 0; j < adev->uvd.num_uvd_inst; ++j) {
+		if (adev->uvd.harvest_config & (1 << j))
+			continue;
+		kvfree(adev->uvd.inst[j].saved_bo);
 
 		amdgpu_bo_free_kernel(&adev->uvd.inst[j].vcpu_bo,
 				      &adev->uvd.inst[j].gpu_addr,
@@ -325,6 +318,29 @@ int amdgpu_uvd_sw_fini(struct amdgpu_device *adev)
 	return 0;
 }
 
+/**
+ * amdgpu_uvd_entity_init - init entity
+ *
+ * @adev: amdgpu_device pointer
+ *
+ */
+int amdgpu_uvd_entity_init(struct amdgpu_device *adev)
+{
+	struct amdgpu_ring *ring;
+	struct drm_sched_rq *rq;
+	int r;
+
+	ring = &adev->uvd.inst[0].ring;
+	rq = &ring->sched.sched_rq[DRM_SCHED_PRIORITY_NORMAL];
+	r = drm_sched_entity_init(&adev->uvd.entity, &rq, 1, NULL);
+	if (r) {
+		DRM_ERROR("Failed setting up UVD kernel entity.\n");
+		return r;
+	}
+
+	return 0;
+}
+
 int amdgpu_uvd_suspend(struct amdgpu_device *adev)
 {
 	unsigned size;
@@ -333,24 +349,26 @@ int amdgpu_uvd_suspend(struct amdgpu_device *adev)
 
 	cancel_delayed_work_sync(&adev->uvd.idle_work);
 
+	/* only valid for physical mode */
+	if (adev->asic_type < CHIP_POLARIS10) {
+		for (i = 0; i < adev->uvd.max_handles; ++i)
+			if (atomic_read(&adev->uvd.handles[i]))
+				break;
+
+		if (i == adev->uvd.max_handles)
+			return 0;
+	}
+
 	for (j = 0; j < adev->uvd.num_uvd_inst; ++j) {
+		if (adev->uvd.harvest_config & (1 << j))
+			continue;
 		if (adev->uvd.inst[j].vcpu_bo == NULL)
 			continue;
 
-		/* only valid for physical mode */
-		if (adev->asic_type < CHIP_POLARIS10) {
-			for (i = 0; i < adev->uvd.max_handles; ++i)
-				if (atomic_read(&adev->uvd.inst[j].handles[i]))
-					break;
-
-			if (i == adev->uvd.max_handles)
-				continue;
-		}
-
 		size = amdgpu_bo_size(adev->uvd.inst[j].vcpu_bo);
 		ptr = adev->uvd.inst[j].cpu_addr;
 
-		adev->uvd.inst[j].saved_bo = kmalloc(size, GFP_KERNEL);
+		adev->uvd.inst[j].saved_bo = kvmalloc(size, GFP_KERNEL);
 		if (!adev->uvd.inst[j].saved_bo)
 			return -ENOMEM;
 
@@ -366,6 +384,8 @@ int amdgpu_uvd_resume(struct amdgpu_device *adev)
 	int i;
 
 	for (i = 0; i < adev->uvd.num_uvd_inst; i++) {
+		if (adev->uvd.harvest_config & (1 << i))
+			continue;
 		if (adev->uvd.inst[i].vcpu_bo == NULL)
 			return -EINVAL;
 
@@ -374,7 +394,7 @@ int amdgpu_uvd_resume(struct amdgpu_device *adev)
 
 		if (adev->uvd.inst[i].saved_bo != NULL) {
 			memcpy_toio(ptr, adev->uvd.inst[i].saved_bo, size);
-			kfree(adev->uvd.inst[i].saved_bo);
+			kvfree(adev->uvd.inst[i].saved_bo);
 			adev->uvd.inst[i].saved_bo = NULL;
 		} else {
 			const struct common_firmware_header *hdr;
@@ -398,30 +418,27 @@ int amdgpu_uvd_resume(struct amdgpu_device *adev)
 
 void amdgpu_uvd_free_handles(struct amdgpu_device *adev, struct drm_file *filp)
 {
-	struct amdgpu_ring *ring;
-	int i, j, r;
-
-	for (j = 0; j < adev->uvd.num_uvd_inst; j++) {
-		ring = &adev->uvd.inst[j].ring;
+	struct amdgpu_ring *ring = &adev->uvd.inst[0].ring;
+	int i, r;
 
-		for (i = 0; i < adev->uvd.max_handles; ++i) {
-			uint32_t handle = atomic_read(&adev->uvd.inst[j].handles[i]);
-			if (handle != 0 && adev->uvd.inst[j].filp[i] == filp) {
-				struct dma_fence *fence;
-
-				r = amdgpu_uvd_get_destroy_msg(ring, handle,
-							       false, &fence);
-				if (r) {
-					DRM_ERROR("Error destroying UVD(%d) %d!\n", j, r);
-					continue;
-				}
+	for (i = 0; i < adev->uvd.max_handles; ++i) {
+		uint32_t handle = atomic_read(&adev->uvd.handles[i]);
 
-				dma_fence_wait(fence, false);
-				dma_fence_put(fence);
+		if (handle != 0 && adev->uvd.filp[i] == filp) {
+			struct dma_fence *fence;
 
-				adev->uvd.inst[j].filp[i] = NULL;
-				atomic_set(&adev->uvd.inst[j].handles[i], 0);
+			r = amdgpu_uvd_get_destroy_msg(ring, handle, false,
+						       &fence);
+			if (r) {
+				DRM_ERROR("Error destroying UVD %d!\n", r);
+				continue;
 			}
+
+			dma_fence_wait(fence, false);
+			dma_fence_put(fence);
+
+			adev->uvd.filp[i] = NULL;
+			atomic_set(&adev->uvd.handles[i], 0);
 		}
 	}
 }
@@ -476,7 +493,7 @@ static int amdgpu_uvd_cs_pass1(struct amdgpu_uvd_cs_ctx *ctx)
 		if (cmd == 0x0 || cmd == 0x3) {
 			/* yes, force it into VRAM */
 			uint32_t domain = AMDGPU_GEM_DOMAIN_VRAM;
-			amdgpu_ttm_placement_from_domain(bo, domain);
+			amdgpu_bo_placement_from_domain(bo, domain);
 		}
 		amdgpu_uvd_force_into_uvd_segment(bo);
 
@@ -696,16 +713,15 @@ static int amdgpu_uvd_cs_msg(struct amdgpu_uvd_cs_ctx *ctx,
 	void *ptr;
 	long r;
 	int i;
-	uint32_t ip_instance = ctx->parser->job->ring->me;
 
 	if (offset & 0x3F) {
-		DRM_ERROR("UVD(%d) messages must be 64 byte aligned!\n", ip_instance);
+		DRM_ERROR("UVD messages must be 64 byte aligned!\n");
 		return -EINVAL;
 	}
 
 	r = amdgpu_bo_kmap(bo, &ptr);
 	if (r) {
-		DRM_ERROR("Failed mapping the UVD(%d) message (%ld)!\n", ip_instance, r);
+		DRM_ERROR("Failed mapping the UVD) message (%ld)!\n", r);
 		return r;
 	}
 
@@ -715,7 +731,7 @@ static int amdgpu_uvd_cs_msg(struct amdgpu_uvd_cs_ctx *ctx,
 	handle = msg[2];
 
 	if (handle == 0) {
-		DRM_ERROR("Invalid UVD(%d) handle!\n", ip_instance);
+		DRM_ERROR("Invalid UVD handle!\n");
 		return -EINVAL;
 	}
 
@@ -726,18 +742,19 @@ static int amdgpu_uvd_cs_msg(struct amdgpu_uvd_cs_ctx *ctx,
 
 		/* try to alloc a new handle */
 		for (i = 0; i < adev->uvd.max_handles; ++i) {
-			if (atomic_read(&adev->uvd.inst[ip_instance].handles[i]) == handle) {
-				DRM_ERROR("(%d)Handle 0x%x already in use!\n", ip_instance, handle);
+			if (atomic_read(&adev->uvd.handles[i]) == handle) {
+				DRM_ERROR(")Handle 0x%x already in use!\n",
+					  handle);
 				return -EINVAL;
 			}
 
-			if (!atomic_cmpxchg(&adev->uvd.inst[ip_instance].handles[i], 0, handle)) {
-				adev->uvd.inst[ip_instance].filp[i] = ctx->parser->filp;
+			if (!atomic_cmpxchg(&adev->uvd.handles[i], 0, handle)) {
+				adev->uvd.filp[i] = ctx->parser->filp;
 				return 0;
 			}
 		}
 
-		DRM_ERROR("No more free UVD(%d) handles!\n", ip_instance);
+		DRM_ERROR("No more free UVD handles!\n");
 		return -ENOSPC;
 
 	case 1:
@@ -749,27 +766,27 @@ static int amdgpu_uvd_cs_msg(struct amdgpu_uvd_cs_ctx *ctx,
 
 		/* validate the handle */
 		for (i = 0; i < adev->uvd.max_handles; ++i) {
-			if (atomic_read(&adev->uvd.inst[ip_instance].handles[i]) == handle) {
-				if (adev->uvd.inst[ip_instance].filp[i] != ctx->parser->filp) {
-					DRM_ERROR("UVD(%d) handle collision detected!\n", ip_instance);
+			if (atomic_read(&adev->uvd.handles[i]) == handle) {
+				if (adev->uvd.filp[i] != ctx->parser->filp) {
+					DRM_ERROR("UVD handle collision detected!\n");
 					return -EINVAL;
 				}
 				return 0;
 			}
 		}
 
-		DRM_ERROR("Invalid UVD(%d) handle 0x%x!\n", ip_instance, handle);
+		DRM_ERROR("Invalid UVD handle 0x%x!\n", handle);
 		return -ENOENT;
 
 	case 2:
 		/* it's a destroy msg, free the handle */
 		for (i = 0; i < adev->uvd.max_handles; ++i)
-			atomic_cmpxchg(&adev->uvd.inst[ip_instance].handles[i], handle, 0);
+			atomic_cmpxchg(&adev->uvd.handles[i], handle, 0);
 		amdgpu_bo_kunmap(bo);
 		return 0;
 
 	default:
-		DRM_ERROR("Illegal UVD(%d) message type (%d)!\n", ip_instance, msg_type);
+		DRM_ERROR("Illegal UVD message type (%d)!\n", msg_type);
 		return -EINVAL;
 	}
 	BUG();
@@ -1017,7 +1034,7 @@ static int amdgpu_uvd_send_msg(struct amdgpu_ring *ring, struct amdgpu_bo *bo,
 	if (!ring->adev->uvd.address_64_bit) {
 		struct ttm_operation_ctx ctx = { true, false };
 
-		amdgpu_ttm_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_VRAM);
+		amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_VRAM);
 		amdgpu_uvd_force_into_uvd_segment(bo);
 		r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
 		if (r)
@@ -1062,19 +1079,16 @@ static int amdgpu_uvd_send_msg(struct amdgpu_ring *ring, struct amdgpu_bo *bo,
 		if (r < 0)
 			goto err_free;
 
-		r = amdgpu_ib_schedule(ring, 1, ib, NULL, &f);
-		job->fence = dma_fence_get(f);
+		r = amdgpu_job_submit_direct(job, ring, &f);
 		if (r)
 			goto err_free;
-
-		amdgpu_job_free(job);
 	} else {
 		r = amdgpu_sync_resv(adev, &job->sync, bo->tbo.resv,
 				     AMDGPU_FENCE_OWNER_UNDEFINED, false);
 		if (r)
 			goto err_free;
 
-		r = amdgpu_job_submit(job, ring, &adev->uvd.inst[ring->me].entity,
+		r = amdgpu_job_submit(job, &adev->uvd.entity,
 				      AMDGPU_FENCE_OWNER_UNDEFINED, &f);
 		if (r)
 			goto err_free;
@@ -1166,6 +1180,8 @@ static void amdgpu_uvd_idle_work_handler(struct work_struct *work)
 	unsigned fences = 0, i, j;
 
 	for (i = 0; i < adev->uvd.num_uvd_inst; ++i) {
+		if (adev->uvd.harvest_config & (1 << i))
+			continue;
 		fences += amdgpu_fence_count_emitted(&adev->uvd.inst[i].ring);
 		for (j = 0; j < adev->uvd.num_enc_rings; ++j) {
 			fences += amdgpu_fence_count_emitted(&adev->uvd.inst[i].ring_enc[j]);
@@ -1276,7 +1292,7 @@ uint32_t amdgpu_uvd_used_handles(struct amdgpu_device *adev)
 		 * necessarily linear. So we need to count
 		 * all non-zero handles.
 		 */
-		if (atomic_read(&adev->uvd.inst->handles[i]))
+		if (atomic_read(&adev->uvd.handles[i]))
 			used_handles++;
 	}
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.h
index 8b23a1b00c76..a3ab1a41060f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.h
@@ -42,30 +42,34 @@ struct amdgpu_uvd_inst {
 	void			*cpu_addr;
 	uint64_t		gpu_addr;
 	void			*saved_bo;
-	atomic_t		handles[AMDGPU_MAX_UVD_HANDLES];
-	struct drm_file		*filp[AMDGPU_MAX_UVD_HANDLES];
 	struct amdgpu_ring	ring;
 	struct amdgpu_ring	ring_enc[AMDGPU_MAX_UVD_ENC_RINGS];
 	struct amdgpu_irq_src	irq;
-	struct drm_sched_entity entity;
-	struct drm_sched_entity entity_enc;
 	uint32_t                srbm_soft_reset;
 };
 
+#define AMDGPU_UVD_HARVEST_UVD0 (1 << 0)
+#define AMDGPU_UVD_HARVEST_UVD1 (1 << 1)
+
 struct amdgpu_uvd {
 	const struct firmware	*fw;	/* UVD firmware */
 	unsigned		fw_version;
 	unsigned		max_handles;
 	unsigned		num_enc_rings;
-	uint8_t		num_uvd_inst;
+	uint8_t			num_uvd_inst;
 	bool			address_64_bit;
 	bool			use_ctx_buf;
-	struct amdgpu_uvd_inst		inst[AMDGPU_MAX_UVD_INSTANCES];
+	struct amdgpu_uvd_inst	inst[AMDGPU_MAX_UVD_INSTANCES];
+	struct drm_file		*filp[AMDGPU_MAX_UVD_HANDLES];
+	atomic_t		handles[AMDGPU_MAX_UVD_HANDLES];
+	struct drm_sched_entity entity;
 	struct delayed_work	idle_work;
+	unsigned		harvest_config;
 };
 
 int amdgpu_uvd_sw_init(struct amdgpu_device *adev);
 int amdgpu_uvd_sw_fini(struct amdgpu_device *adev);
+int amdgpu_uvd_entity_init(struct amdgpu_device *adev);
 int amdgpu_uvd_suspend(struct amdgpu_device *adev);
 int amdgpu_uvd_resume(struct amdgpu_device *adev);
 int amdgpu_uvd_get_create_msg(struct amdgpu_ring *ring, uint32_t handle,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
index 23d960ec1cf2..0cc5190f4f36 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
@@ -40,11 +40,11 @@
 
 /* Firmware Names */
 #ifdef CONFIG_DRM_AMDGPU_CIK
-#define FIRMWARE_BONAIRE	"radeon/bonaire_vce.bin"
-#define FIRMWARE_KABINI	"radeon/kabini_vce.bin"
-#define FIRMWARE_KAVERI	"radeon/kaveri_vce.bin"
-#define FIRMWARE_HAWAII	"radeon/hawaii_vce.bin"
-#define FIRMWARE_MULLINS	"radeon/mullins_vce.bin"
+#define FIRMWARE_BONAIRE	"amdgpu/bonaire_vce.bin"
+#define FIRMWARE_KABINI	"amdgpu/kabini_vce.bin"
+#define FIRMWARE_KAVERI	"amdgpu/kaveri_vce.bin"
+#define FIRMWARE_HAWAII	"amdgpu/hawaii_vce.bin"
+#define FIRMWARE_MULLINS	"amdgpu/mullins_vce.bin"
 #endif
 #define FIRMWARE_TONGA		"amdgpu/tonga_vce.bin"
 #define FIRMWARE_CARRIZO	"amdgpu/carrizo_vce.bin"
@@ -90,8 +90,6 @@ static void amdgpu_vce_idle_work_handler(struct work_struct *work);
  */
 int amdgpu_vce_sw_init(struct amdgpu_device *adev, unsigned long size)
 {
-	struct amdgpu_ring *ring;
-	struct drm_sched_rq *rq;
 	const char *fw_name;
 	const struct common_firmware_header *hdr;
 	unsigned ucode_version, version_major, version_minor, binary_id;
@@ -188,15 +186,6 @@ int amdgpu_vce_sw_init(struct amdgpu_device *adev, unsigned long size)
 		return r;
 	}
 
-	ring = &adev->vce.ring[0];
-	rq = &ring->sched.sched_rq[DRM_SCHED_PRIORITY_NORMAL];
-	r = drm_sched_entity_init(&ring->sched, &adev->vce.entity,
-				  rq, NULL);
-	if (r != 0) {
-		DRM_ERROR("Failed setting up VCE run queue.\n");
-		return r;
-	}
-
 	for (i = 0; i < AMDGPU_MAX_VCE_HANDLES; ++i) {
 		atomic_set(&adev->vce.handles[i], 0);
 		adev->vce.filp[i] = NULL;
@@ -222,7 +211,7 @@ int amdgpu_vce_sw_fini(struct amdgpu_device *adev)
 	if (adev->vce.vcpu_bo == NULL)
 		return 0;
 
-	drm_sched_entity_fini(&adev->vce.ring[0].sched, &adev->vce.entity);
+	drm_sched_entity_destroy(&adev->vce.entity);
 
 	amdgpu_bo_free_kernel(&adev->vce.vcpu_bo, &adev->vce.gpu_addr,
 		(void **)&adev->vce.cpu_addr);
@@ -237,6 +226,29 @@ int amdgpu_vce_sw_fini(struct amdgpu_device *adev)
 }
 
 /**
+ * amdgpu_vce_entity_init - init entity
+ *
+ * @adev: amdgpu_device pointer
+ *
+ */
+int amdgpu_vce_entity_init(struct amdgpu_device *adev)
+{
+	struct amdgpu_ring *ring;
+	struct drm_sched_rq *rq;
+	int r;
+
+	ring = &adev->vce.ring[0];
+	rq = &ring->sched.sched_rq[DRM_SCHED_PRIORITY_NORMAL];
+	r = drm_sched_entity_init(&adev->vce.entity, &rq, 1, NULL);
+	if (r != 0) {
+		DRM_ERROR("Failed setting up VCE run queue.\n");
+		return r;
+	}
+
+	return 0;
+}
+
+/**
  * amdgpu_vce_suspend - unpin VCE fw memory
  *
  * @adev: amdgpu_device pointer
@@ -470,12 +482,10 @@ int amdgpu_vce_get_create_msg(struct amdgpu_ring *ring, uint32_t handle,
 	for (i = ib->length_dw; i < ib_size_dw; ++i)
 		ib->ptr[i] = 0x0;
 
-	r = amdgpu_ib_schedule(ring, 1, ib, NULL, &f);
-	job->fence = dma_fence_get(f);
+	r = amdgpu_job_submit_direct(job, ring, &f);
 	if (r)
 		goto err;
 
-	amdgpu_job_free(job);
 	if (fence)
 		*fence = dma_fence_get(f);
 	dma_fence_put(f);
@@ -532,19 +542,13 @@ int amdgpu_vce_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle,
 	for (i = ib->length_dw; i < ib_size_dw; ++i)
 		ib->ptr[i] = 0x0;
 
-	if (direct) {
-		r = amdgpu_ib_schedule(ring, 1, ib, NULL, &f);
-		job->fence = dma_fence_get(f);
-		if (r)
-			goto err;
-
-		amdgpu_job_free(job);
-	} else {
-		r = amdgpu_job_submit(job, ring, &ring->adev->vce.entity,
+	if (direct)
+		r = amdgpu_job_submit_direct(job, ring, &f);
+	else
+		r = amdgpu_job_submit(job, &ring->adev->vce.entity,
 				      AMDGPU_FENCE_OWNER_UNDEFINED, &f);
-		if (r)
-			goto err;
-	}
+	if (r)
+		goto err;
 
 	if (fence)
 		*fence = dma_fence_get(f);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.h
index 71781267ee4c..a1f209eed4c4 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.h
@@ -55,6 +55,7 @@ struct amdgpu_vce {
 
 int amdgpu_vce_sw_init(struct amdgpu_device *adev, unsigned long size);
 int amdgpu_vce_sw_fini(struct amdgpu_device *adev);
+int amdgpu_vce_entity_init(struct amdgpu_device *adev);
 int amdgpu_vce_suspend(struct amdgpu_device *adev);
 int amdgpu_vce_resume(struct amdgpu_device *adev);
 int amdgpu_vce_get_create_msg(struct amdgpu_ring *ring, uint32_t handle,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c
index a66cd521a875..fd654a4406db 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c
@@ -111,9 +111,10 @@ int amdgpu_vcn_sw_init(struct amdgpu_device *adev)
 			version_major, version_minor, family_id);
 	}
 
-	bo_size = AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8)
-		  +  AMDGPU_VCN_STACK_SIZE + AMDGPU_VCN_HEAP_SIZE
+	bo_size = AMDGPU_VCN_STACK_SIZE + AMDGPU_VCN_HEAP_SIZE
 		  +  AMDGPU_VCN_SESSION_SIZE * 40;
+	if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP)
+		bo_size += AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8);
 	r = amdgpu_bo_create_kernel(adev, bo_size, PAGE_SIZE,
 				    AMDGPU_GEM_DOMAIN_VRAM, &adev->vcn.vcpu_bo,
 				    &adev->vcn.gpu_addr, &adev->vcn.cpu_addr);
@@ -129,7 +130,7 @@ int amdgpu_vcn_sw_fini(struct amdgpu_device *adev)
 {
 	int i;
 
-	kfree(adev->vcn.saved_bo);
+	kvfree(adev->vcn.saved_bo);
 
 	amdgpu_bo_free_kernel(&adev->vcn.vcpu_bo,
 			      &adev->vcn.gpu_addr,
@@ -160,7 +161,7 @@ int amdgpu_vcn_suspend(struct amdgpu_device *adev)
 	size = amdgpu_bo_size(adev->vcn.vcpu_bo);
 	ptr = adev->vcn.cpu_addr;
 
-	adev->vcn.saved_bo = kmalloc(size, GFP_KERNEL);
+	adev->vcn.saved_bo = kvmalloc(size, GFP_KERNEL);
 	if (!adev->vcn.saved_bo)
 		return -ENOMEM;
 
@@ -182,18 +183,20 @@ int amdgpu_vcn_resume(struct amdgpu_device *adev)
 
 	if (adev->vcn.saved_bo != NULL) {
 		memcpy_toio(ptr, adev->vcn.saved_bo, size);
-		kfree(adev->vcn.saved_bo);
+		kvfree(adev->vcn.saved_bo);
 		adev->vcn.saved_bo = NULL;
 	} else {
 		const struct common_firmware_header *hdr;
 		unsigned offset;
 
 		hdr = (const struct common_firmware_header *)adev->vcn.fw->data;
-		offset = le32_to_cpu(hdr->ucode_array_offset_bytes);
-		memcpy_toio(adev->vcn.cpu_addr, adev->vcn.fw->data + offset,
-			    le32_to_cpu(hdr->ucode_size_bytes));
-		size -= le32_to_cpu(hdr->ucode_size_bytes);
-		ptr += le32_to_cpu(hdr->ucode_size_bytes);
+		if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
+			offset = le32_to_cpu(hdr->ucode_array_offset_bytes);
+			memcpy_toio(adev->vcn.cpu_addr, adev->vcn.fw->data + offset,
+				    le32_to_cpu(hdr->ucode_size_bytes));
+			size -= le32_to_cpu(hdr->ucode_size_bytes);
+			ptr += le32_to_cpu(hdr->ucode_size_bytes);
+		}
 		memset_io(ptr, 0, size);
 	}
 
@@ -211,6 +214,8 @@ static void amdgpu_vcn_idle_work_handler(struct work_struct *work)
 		fences += amdgpu_fence_count_emitted(&adev->vcn.ring_enc[i]);
 	}
 
+	fences += amdgpu_fence_count_emitted(&adev->vcn.ring_jpeg);
+
 	if (fences == 0) {
 		if (adev->pm.dpm_enabled)
 			amdgpu_dpm_enable_uvd(adev, false);
@@ -227,7 +232,7 @@ void amdgpu_vcn_ring_begin_use(struct amdgpu_ring *ring)
 	struct amdgpu_device *adev = ring->adev;
 	bool set_clocks = !cancel_delayed_work_sync(&adev->vcn.idle_work);
 
-	if (set_clocks && adev->pm.dpm_enabled) {
+	if (set_clocks) {
 		if (adev->pm.dpm_enabled)
 			amdgpu_dpm_enable_uvd(adev, true);
 		else
@@ -306,13 +311,10 @@ static int amdgpu_vcn_dec_send_msg(struct amdgpu_ring *ring,
 	}
 	ib->length_dw = 16;
 
-	r = amdgpu_ib_schedule(ring, 1, ib, NULL, &f);
-	job->fence = dma_fence_get(f);
+	r = amdgpu_job_submit_direct(job, ring, &f);
 	if (r)
 		goto err_free;
 
-	amdgpu_job_free(job);
-
 	amdgpu_bo_fence(bo, f, false);
 	amdgpu_bo_unreserve(bo);
 	amdgpu_bo_unref(&bo);
@@ -497,12 +499,10 @@ static int amdgpu_vcn_enc_get_create_msg(struct amdgpu_ring *ring, uint32_t hand
 	for (i = ib->length_dw; i < ib_size_dw; ++i)
 		ib->ptr[i] = 0x0;
 
-	r = amdgpu_ib_schedule(ring, 1, ib, NULL, &f);
-	job->fence = dma_fence_get(f);
+	r = amdgpu_job_submit_direct(job, ring, &f);
 	if (r)
 		goto err;
 
-	amdgpu_job_free(job);
 	if (fence)
 		*fence = dma_fence_get(f);
 	dma_fence_put(f);
@@ -551,12 +551,10 @@ static int amdgpu_vcn_enc_get_destroy_msg(struct amdgpu_ring *ring, uint32_t han
 	for (i = ib->length_dw; i < ib_size_dw; ++i)
 		ib->ptr[i] = 0x0;
 
-	r = amdgpu_ib_schedule(ring, 1, ib, NULL, &f);
-	job->fence = dma_fence_get(f);
+	r = amdgpu_job_submit_direct(job, ring, &f);
 	if (r)
 		goto err;
 
-	amdgpu_job_free(job);
 	if (fence)
 		*fence = dma_fence_get(f);
 	dma_fence_put(f);
@@ -664,12 +662,10 @@ static int amdgpu_vcn_jpeg_set_reg(struct amdgpu_ring *ring, uint32_t handle,
 	}
 	ib->length_dw = 16;
 
-	r = amdgpu_ib_schedule(ring, 1, ib, NULL, &f);
-	job->fence = dma_fence_get(f);
+	r = amdgpu_job_submit_direct(job, ring, &f);
 	if (r)
 		goto err;
 
-	amdgpu_job_free(job);
 	if (fence)
 		*fence = dma_fence_get(f);
 	dma_fence_put(f);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index 422d1a434db4..ece0ac703e27 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -156,6 +156,9 @@ static void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base,
 		return;
 	list_add_tail(&base->bo_list, &bo->va);
 
+	if (bo->tbo.type == ttm_bo_type_kernel)
+		list_move(&base->vm_status, &vm->relocated);
+
 	if (bo->tbo.resv != vm->root.base.bo->tbo.resv)
 		return;
 
@@ -384,7 +387,7 @@ static int amdgpu_vm_clear_bo(struct amdgpu_device *adev,
 		ats_entries = 0;
 	}
 
-	ring = container_of(vm->entity.sched, struct amdgpu_ring, sched);
+	ring = container_of(vm->entity.rq->sched, struct amdgpu_ring, sched);
 
 	r = reservation_object_reserve_shared(bo->tbo.resv);
 	if (r)
@@ -422,8 +425,8 @@ static int amdgpu_vm_clear_bo(struct amdgpu_device *adev,
 	if (r)
 		goto error_free;
 
-	r = amdgpu_job_submit(job, ring, &vm->entity,
-			      AMDGPU_FENCE_OWNER_UNDEFINED, &fence);
+	r = amdgpu_job_submit(job, &vm->entity, AMDGPU_FENCE_OWNER_UNDEFINED,
+			      &fence);
 	if (r)
 		goto error_free;
 
@@ -492,11 +495,12 @@ static int amdgpu_vm_alloc_levels(struct amdgpu_device *adev,
 	eaddr = eaddr & ((1 << shift) - 1);
 
 	flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
+	if (vm->root.base.bo->shadow)
+		flags |= AMDGPU_GEM_CREATE_SHADOW;
 	if (vm->use_cpu_for_update)
 		flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
 	else
-		flags |= (AMDGPU_GEM_CREATE_NO_CPU_ACCESS |
-				AMDGPU_GEM_CREATE_SHADOW);
+		flags |= AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
 
 	/* walk over the address space and allocate the page tables */
 	for (pt_idx = from; pt_idx <= to; ++pt_idx) {
@@ -540,7 +544,6 @@ static int amdgpu_vm_alloc_levels(struct amdgpu_device *adev,
 			pt->parent = amdgpu_bo_ref(parent->base.bo);
 
 			amdgpu_vm_bo_base_init(&entry->base, vm, pt);
-			list_move(&entry->base.vm_status, &vm->relocated);
 		}
 
 		if (level < AMDGPU_VM_PTB) {
@@ -1082,7 +1085,7 @@ restart:
 					   struct amdgpu_vm_bo_base,
 					   vm_status);
 		bo_base->moved = false;
-		list_move(&bo_base->vm_status, &vm->idle);
+		list_del_init(&bo_base->vm_status);
 
 		bo = bo_base->bo->parent;
 		if (!bo)
@@ -1111,15 +1114,15 @@ restart:
 		struct amdgpu_ring *ring;
 		struct dma_fence *fence;
 
-		ring = container_of(vm->entity.sched, struct amdgpu_ring,
+		ring = container_of(vm->entity.rq->sched, struct amdgpu_ring,
 				    sched);
 
 		amdgpu_ring_pad_ib(ring, params.ib);
 		amdgpu_sync_resv(adev, &job->sync, root->tbo.resv,
 				 AMDGPU_FENCE_OWNER_VM, false);
 		WARN_ON(params.ib->length_dw > ndw);
-		r = amdgpu_job_submit(job, ring, &vm->entity,
-				      AMDGPU_FENCE_OWNER_VM, &fence);
+		r = amdgpu_job_submit(job, &vm->entity, AMDGPU_FENCE_OWNER_VM,
+				      &fence);
 		if (r)
 			goto error;
 
@@ -1401,7 +1404,7 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
 					   addr, flags);
 	}
 
-	ring = container_of(vm->entity.sched, struct amdgpu_ring, sched);
+	ring = container_of(vm->entity.rq->sched, struct amdgpu_ring, sched);
 
 	nptes = last - start + 1;
 
@@ -1483,8 +1486,7 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
 
 	amdgpu_ring_pad_ib(ring, params.ib);
 	WARN_ON(params.ib->length_dw > ndw);
-	r = amdgpu_job_submit(job, ring, &vm->entity,
-			      AMDGPU_FENCE_OWNER_VM, &f);
+	r = amdgpu_job_submit(job, &vm->entity, AMDGPU_FENCE_OWNER_VM, &f);
 	if (r)
 		goto error_free;
 
@@ -1567,7 +1569,7 @@ static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev,
 		if (nodes) {
 			addr = nodes->start << PAGE_SHIFT;
 			max_entries = (nodes->size - pfn) *
-				(PAGE_SIZE / AMDGPU_GPU_PAGE_SIZE);
+				AMDGPU_GPU_PAGES_IN_CPU_PAGE;
 		} else {
 			addr = 0;
 			max_entries = S64_MAX;
@@ -1578,7 +1580,7 @@ static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev,
 
 			max_entries = min(max_entries, 16ull * 1024ull);
 			for (count = 1;
-			     count < max_entries / (PAGE_SIZE / AMDGPU_GPU_PAGE_SIZE);
+			     count < max_entries / AMDGPU_GPU_PAGES_IN_CPU_PAGE;
 			     ++count) {
 				uint64_t idx = pfn + count;
 
@@ -1592,7 +1594,7 @@ static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev,
 				dma_addr = pages_addr;
 			} else {
 				addr = pages_addr[pfn];
-				max_entries = count * (PAGE_SIZE / AMDGPU_GPU_PAGE_SIZE);
+				max_entries = count * AMDGPU_GPU_PAGES_IN_CPU_PAGE;
 			}
 
 		} else if (flags & AMDGPU_PTE_VALID) {
@@ -1607,7 +1609,7 @@ static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev,
 		if (r)
 			return r;
 
-		pfn += (last - start + 1) / (PAGE_SIZE / AMDGPU_GPU_PAGE_SIZE);
+		pfn += (last - start + 1) / AMDGPU_GPU_PAGES_IN_CPU_PAGE;
 		if (nodes && nodes->size == pfn) {
 			pfn = 0;
 			++nodes;
@@ -1645,18 +1647,17 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev,
 	uint64_t flags;
 	int r;
 
-	if (clear || !bo_va->base.bo) {
+	if (clear || !bo) {
 		mem = NULL;
 		nodes = NULL;
 		exclusive = NULL;
 	} else {
 		struct ttm_dma_tt *ttm;
 
-		mem = &bo_va->base.bo->tbo.mem;
+		mem = &bo->tbo.mem;
 		nodes = mem->mm_node;
 		if (mem->mem_type == TTM_PL_TT) {
-			ttm = container_of(bo_va->base.bo->tbo.ttm,
-					   struct ttm_dma_tt, ttm);
+			ttm = container_of(bo->tbo.ttm, struct ttm_dma_tt, ttm);
 			pages_addr = ttm->dma_address;
 		}
 		exclusive = reservation_object_get_excl(bo->tbo.resv);
@@ -2344,6 +2345,35 @@ struct amdgpu_bo_va_mapping *amdgpu_vm_bo_lookup_mapping(struct amdgpu_vm *vm,
 }
 
 /**
+ * amdgpu_vm_bo_trace_cs - trace all reserved mappings
+ *
+ * @vm: the requested vm
+ * @ticket: CS ticket
+ *
+ * Trace all mappings of BOs reserved during a command submission.
+ */
+void amdgpu_vm_bo_trace_cs(struct amdgpu_vm *vm, struct ww_acquire_ctx *ticket)
+{
+	struct amdgpu_bo_va_mapping *mapping;
+
+	if (!trace_amdgpu_vm_bo_cs_enabled())
+		return;
+
+	for (mapping = amdgpu_vm_it_iter_first(&vm->va, 0, U64_MAX); mapping;
+	     mapping = amdgpu_vm_it_iter_next(mapping, 0, U64_MAX)) {
+		if (mapping->bo_va && mapping->bo_va->base.bo) {
+			struct amdgpu_bo *bo;
+
+			bo = mapping->bo_va->base.bo;
+			if (READ_ONCE(bo->tbo.resv->lock.ctx) != ticket)
+				continue;
+		}
+
+		trace_amdgpu_vm_bo_cs(mapping);
+	}
+}
+
+/**
  * amdgpu_vm_bo_rmv - remove a bo to a specific vm
  *
  * @adev: amdgpu_device pointer
@@ -2562,8 +2592,7 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	ring_instance %= adev->vm_manager.vm_pte_num_rings;
 	ring = adev->vm_manager.vm_pte_rings[ring_instance];
 	rq = &ring->sched.sched_rq[DRM_SCHED_PRIORITY_KERNEL];
-	r = drm_sched_entity_init(&ring->sched, &vm->entity,
-				  rq, NULL);
+	r = drm_sched_entity_init(&vm->entity, &rq, 1, NULL);
 	if (r)
 		return r;
 
@@ -2588,7 +2617,7 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
 	if (vm->use_cpu_for_update)
 		flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
-	else
+	else if (vm_context != AMDGPU_VM_CONTEXT_COMPUTE)
 		flags |= AMDGPU_GEM_CREATE_SHADOW;
 
 	size = amdgpu_vm_bo_size(adev, adev->vm_manager.root_level);
@@ -2643,7 +2672,7 @@ error_free_root:
 	vm->root.base.bo = NULL;
 
 error_free_sched_entity:
-	drm_sched_entity_fini(&ring->sched, &vm->entity);
+	drm_sched_entity_destroy(&vm->entity);
 
 	return r;
 }
@@ -2663,8 +2692,7 @@ error_free_sched_entity:
  * - pasid (old PASID is released, because compute manages its own PASIDs)
  *
  * Reinitializes the page directory to reflect the changed ATS
- * setting. May leave behind an unused shadow BO for the page
- * directory when switching from SDMA updates to CPU updates.
+ * setting.
  *
  * Returns:
  * 0 for success, -errno for errors.
@@ -2714,6 +2742,9 @@ int amdgpu_vm_make_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm)
 		vm->pasid = 0;
 	}
 
+	/* Free the shadow bo for compute VM */
+	amdgpu_bo_unref(&vm->root.base.bo->shadow);
+
 error:
 	amdgpu_bo_unreserve(vm->root.base.bo);
 	return r;
@@ -2780,7 +2811,7 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
 		spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);
 	}
 
-	drm_sched_entity_fini(vm->entity.sched, &vm->entity);
+	drm_sched_entity_destroy(&vm->entity);
 
 	if (!RB_EMPTY_ROOT(&vm->va.rb_root)) {
 		dev_err(adev->dev, "still active bo inside vm\n");
@@ -2942,3 +2973,42 @@ int amdgpu_vm_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
 
 	return 0;
 }
+
+/**
+ * amdgpu_vm_get_task_info - Extracts task info for a PASID.
+ *
+ * @dev: drm device pointer
+ * @pasid: PASID identifier for VM
+ * @task_info: task_info to fill.
+ */
+void amdgpu_vm_get_task_info(struct amdgpu_device *adev, unsigned int pasid,
+			 struct amdgpu_task_info *task_info)
+{
+	struct amdgpu_vm *vm;
+
+	spin_lock(&adev->vm_manager.pasid_lock);
+
+	vm = idr_find(&adev->vm_manager.pasid_idr, pasid);
+	if (vm)
+		*task_info = vm->task_info;
+
+	spin_unlock(&adev->vm_manager.pasid_lock);
+}
+
+/**
+ * amdgpu_vm_set_task_info - Sets VMs task info.
+ *
+ * @vm: vm for which to set the info
+ */
+void amdgpu_vm_set_task_info(struct amdgpu_vm *vm)
+{
+	if (!vm->task_info.pid) {
+		vm->task_info.pid = current->pid;
+		get_task_comm(vm->task_info.task_name, current);
+
+		if (current->group_leader->mm == current->mm) {
+			vm->task_info.tgid = current->group_leader->pid;
+			get_task_comm(vm->task_info.process_name, current->group_leader);
+		}
+	}
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
index 061b99a18cb8..67a15d439ac0 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
@@ -164,6 +164,14 @@ struct amdgpu_vm_pt {
 #define AMDGPU_VM_FAULT_PASID(fault) ((u64)(fault) >> 48)
 #define AMDGPU_VM_FAULT_ADDR(fault)  ((u64)(fault) & 0xfffffffff000ULL)
 
+
+struct amdgpu_task_info {
+	char	process_name[TASK_COMM_LEN];
+	char	task_name[TASK_COMM_LEN];
+	pid_t	pid;
+	pid_t	tgid;
+};
+
 struct amdgpu_vm {
 	/* tree of virtual addresses mapped */
 	struct rb_root_cached	va;
@@ -215,6 +223,9 @@ struct amdgpu_vm {
 
 	/* Valid while the PD is reserved or fenced */
 	uint64_t		pd_phys_addr;
+
+	/* Some basic info about the task */
+	struct amdgpu_task_info task_info;
 };
 
 struct amdgpu_vm_manager {
@@ -307,6 +318,7 @@ int amdgpu_vm_bo_clear_mappings(struct amdgpu_device *adev,
 				uint64_t saddr, uint64_t size);
 struct amdgpu_bo_va_mapping *amdgpu_vm_bo_lookup_mapping(struct amdgpu_vm *vm,
 							 uint64_t addr);
+void amdgpu_vm_bo_trace_cs(struct amdgpu_vm *vm, struct ww_acquire_ctx *ticket);
 void amdgpu_vm_bo_rmv(struct amdgpu_device *adev,
 		      struct amdgpu_bo_va *bo_va);
 void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint32_t vm_size,
@@ -317,4 +329,9 @@ bool amdgpu_vm_need_pipeline_sync(struct amdgpu_ring *ring,
 				  struct amdgpu_job *job);
 void amdgpu_vm_check_compute_bug(struct amdgpu_device *adev);
 
+void amdgpu_vm_get_task_info(struct amdgpu_device *adev, unsigned int pasid,
+			 struct amdgpu_task_info *task_info);
+
+void amdgpu_vm_set_task_info(struct amdgpu_vm *vm);
+
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
index b6333f92ba45..9cfa8a9ada92 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
@@ -97,33 +97,29 @@ static u64 amdgpu_vram_mgr_vis_size(struct amdgpu_device *adev,
 }
 
 /**
- * amdgpu_vram_mgr_bo_invisible_size - CPU invisible BO size
+ * amdgpu_vram_mgr_bo_visible_size - CPU visible BO size
  *
  * @bo: &amdgpu_bo buffer object (must be in VRAM)
  *
  * Returns:
- * How much of the given &amdgpu_bo buffer object lies in CPU invisible VRAM.
+ * How much of the given &amdgpu_bo buffer object lies in CPU visible VRAM.
  */
-u64 amdgpu_vram_mgr_bo_invisible_size(struct amdgpu_bo *bo)
+u64 amdgpu_vram_mgr_bo_visible_size(struct amdgpu_bo *bo)
 {
 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
 	struct ttm_mem_reg *mem = &bo->tbo.mem;
 	struct drm_mm_node *nodes = mem->mm_node;
 	unsigned pages = mem->num_pages;
-	u64 usage = 0;
+	u64 usage;
 
-	if (adev->gmc.visible_vram_size == adev->gmc.real_vram_size)
-		return 0;
+	if (amdgpu_gmc_vram_full_visible(&adev->gmc))
+		return amdgpu_bo_size(bo);
 
 	if (mem->start >= adev->gmc.visible_vram_size >> PAGE_SHIFT)
-		return amdgpu_bo_size(bo);
+		return 0;
 
-	while (nodes && pages) {
-		usage += nodes->size << PAGE_SHIFT;
-		usage -= amdgpu_vram_mgr_vis_size(adev, nodes);
-		pages -= nodes->size;
-		++nodes;
-	}
+	for (usage = 0; nodes && pages; pages -= nodes->size, nodes++)
+		usage += amdgpu_vram_mgr_vis_size(adev, nodes);
 
 	return usage;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/atom.c b/drivers/gpu/drm/amd/amdgpu/atom.c
index b18c31a701e2..e9934de1b9cf 100644
--- a/drivers/gpu/drm/amd/amdgpu/atom.c
+++ b/drivers/gpu/drm/amd/amdgpu/atom.c
@@ -1221,7 +1221,7 @@ static int amdgpu_atom_execute_table_locked(struct atom_context *ctx, int index,
 	ectx.abort = false;
 	ectx.last_jump = 0;
 	if (ws)
-		ectx.ws = kcalloc(4, ws, GFP_ATOMIC);
+		ectx.ws = kcalloc(4, ws, GFP_KERNEL);
 	else
 		ectx.ws = NULL;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/ci_dpm.c b/drivers/gpu/drm/amd/amdgpu/ci_dpm.c
index c9d45cffca56..d2469453dca2 100644
--- a/drivers/gpu/drm/amd/amdgpu/ci_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/ci_dpm.c
@@ -49,10 +49,10 @@
 #include "gmc/gmc_7_1_d.h"
 #include "gmc/gmc_7_1_sh_mask.h"
 
-MODULE_FIRMWARE("radeon/bonaire_smc.bin");
-MODULE_FIRMWARE("radeon/bonaire_k_smc.bin");
-MODULE_FIRMWARE("radeon/hawaii_smc.bin");
-MODULE_FIRMWARE("radeon/hawaii_k_smc.bin");
+MODULE_FIRMWARE("amdgpu/bonaire_smc.bin");
+MODULE_FIRMWARE("amdgpu/bonaire_k_smc.bin");
+MODULE_FIRMWARE("amdgpu/hawaii_smc.bin");
+MODULE_FIRMWARE("amdgpu/hawaii_k_smc.bin");
 
 #define MC_CG_ARB_FREQ_F0           0x0a
 #define MC_CG_ARB_FREQ_F1           0x0b
@@ -5815,7 +5815,7 @@ static int ci_dpm_init_microcode(struct amdgpu_device *adev)
 	default: BUG();
 	}
 
-	snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
+	snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_smc.bin", chip_name);
 	err = request_firmware(&adev->pm.fw, fw_name, adev->dev);
 	if (err)
 		goto out;
@@ -5846,8 +5846,7 @@ static int ci_dpm_init(struct amdgpu_device *adev)
 	adev->pm.dpm.priv = pi;
 
 	pi->sys_pcie_mask =
-		(adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_MASK) >>
-		CAIL_PCIE_LINK_SPEED_SUPPORT_SHIFT;
+		adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_MASK;
 
 	pi->force_pcie_gen = AMDGPU_PCIE_GEN_INVALID;
 
@@ -6767,6 +6766,19 @@ static int ci_dpm_read_sensor(void *handle, int idx,
 	}
 }
 
+static int ci_set_powergating_by_smu(void *handle,
+				uint32_t block_type, bool gate)
+{
+	switch (block_type) {
+	case AMD_IP_BLOCK_TYPE_UVD:
+		ci_dpm_powergate_uvd(handle, gate);
+		break;
+	default:
+		break;
+	}
+	return 0;
+}
+
 static const struct amd_ip_funcs ci_dpm_ip_funcs = {
 	.name = "ci_dpm",
 	.early_init = ci_dpm_early_init,
@@ -6804,7 +6816,7 @@ static const struct amd_pm_funcs ci_dpm_funcs = {
 	.debugfs_print_current_performance_level = &ci_dpm_debugfs_print_current_performance_level,
 	.force_performance_level = &ci_dpm_force_performance_level,
 	.vblank_too_short = &ci_dpm_vblank_too_short,
-	.powergate_uvd = &ci_dpm_powergate_uvd,
+	.set_powergating_by_smu = &ci_set_powergating_by_smu,
 	.set_fan_control_mode = &ci_dpm_set_fan_control_mode,
 	.get_fan_control_mode = &ci_dpm_get_fan_control_mode,
 	.set_fan_speed_percent = &ci_dpm_set_fan_speed_percent,
diff --git a/drivers/gpu/drm/amd/amdgpu/cik.c b/drivers/gpu/drm/amd/amdgpu/cik.c
index 702e257a483f..78ab939ae5d8 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik.c
+++ b/drivers/gpu/drm/amd/amdgpu/cik.c
@@ -1476,7 +1476,7 @@ static void cik_pcie_gen3_enable(struct amdgpu_device *adev)
 				tmp |= PCIE_LC_CNTL4__LC_REDO_EQ_MASK;
 				WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
 
-				mdelay(100);
+				msleep(100);
 
 				/* linkctl */
 				pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
diff --git a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
index a7576255cc30..d0fa2aac2388 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
+++ b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
@@ -54,16 +54,16 @@ static void cik_sdma_set_buffer_funcs(struct amdgpu_device *adev);
 static void cik_sdma_set_vm_pte_funcs(struct amdgpu_device *adev);
 static int cik_sdma_soft_reset(void *handle);
 
-MODULE_FIRMWARE("radeon/bonaire_sdma.bin");
-MODULE_FIRMWARE("radeon/bonaire_sdma1.bin");
-MODULE_FIRMWARE("radeon/hawaii_sdma.bin");
-MODULE_FIRMWARE("radeon/hawaii_sdma1.bin");
-MODULE_FIRMWARE("radeon/kaveri_sdma.bin");
-MODULE_FIRMWARE("radeon/kaveri_sdma1.bin");
-MODULE_FIRMWARE("radeon/kabini_sdma.bin");
-MODULE_FIRMWARE("radeon/kabini_sdma1.bin");
-MODULE_FIRMWARE("radeon/mullins_sdma.bin");
-MODULE_FIRMWARE("radeon/mullins_sdma1.bin");
+MODULE_FIRMWARE("amdgpu/bonaire_sdma.bin");
+MODULE_FIRMWARE("amdgpu/bonaire_sdma1.bin");
+MODULE_FIRMWARE("amdgpu/hawaii_sdma.bin");
+MODULE_FIRMWARE("amdgpu/hawaii_sdma1.bin");
+MODULE_FIRMWARE("amdgpu/kaveri_sdma.bin");
+MODULE_FIRMWARE("amdgpu/kaveri_sdma1.bin");
+MODULE_FIRMWARE("amdgpu/kabini_sdma.bin");
+MODULE_FIRMWARE("amdgpu/kabini_sdma1.bin");
+MODULE_FIRMWARE("amdgpu/mullins_sdma.bin");
+MODULE_FIRMWARE("amdgpu/mullins_sdma1.bin");
 
 u32 amdgpu_cik_gpu_check_soft_reset(struct amdgpu_device *adev);
 
@@ -132,9 +132,9 @@ static int cik_sdma_init_microcode(struct amdgpu_device *adev)
 
 	for (i = 0; i < adev->sdma.num_instances; i++) {
 		if (i == 0)
-			snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
+			snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_sdma.bin", chip_name);
 		else
-			snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma1.bin", chip_name);
+			snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_sdma1.bin", chip_name);
 		err = request_firmware(&adev->sdma.instance[i].fw, fw_name, adev->dev);
 		if (err)
 			goto out;
@@ -177,9 +177,8 @@ static uint64_t cik_sdma_ring_get_rptr(struct amdgpu_ring *ring)
 static uint64_t cik_sdma_ring_get_wptr(struct amdgpu_ring *ring)
 {
 	struct amdgpu_device *adev = ring->adev;
-	u32 me = (ring == &adev->sdma.instance[0].ring) ? 0 : 1;
 
-	return (RREG32(mmSDMA0_GFX_RB_WPTR + sdma_offsets[me]) & 0x3fffc) >> 2;
+	return (RREG32(mmSDMA0_GFX_RB_WPTR + sdma_offsets[ring->me]) & 0x3fffc) >> 2;
 }
 
 /**
@@ -192,9 +191,8 @@ static uint64_t cik_sdma_ring_get_wptr(struct amdgpu_ring *ring)
 static void cik_sdma_ring_set_wptr(struct amdgpu_ring *ring)
 {
 	struct amdgpu_device *adev = ring->adev;
-	u32 me = (ring == &adev->sdma.instance[0].ring) ? 0 : 1;
 
-	WREG32(mmSDMA0_GFX_RB_WPTR + sdma_offsets[me],
+	WREG32(mmSDMA0_GFX_RB_WPTR + sdma_offsets[ring->me],
 		       	(lower_32_bits(ring->wptr) << 2) & 0x3fffc);
 }
 
@@ -248,7 +246,7 @@ static void cik_sdma_ring_emit_hdp_flush(struct amdgpu_ring *ring)
 			  SDMA_POLL_REG_MEM_EXTRA_FUNC(3)); /* == */
 	u32 ref_and_mask;
 
-	if (ring == &ring->adev->sdma.instance[0].ring)
+	if (ring->me == 0)
 		ref_and_mask = GPU_HDP_FLUSH_DONE__SDMA0_MASK;
 	else
 		ref_and_mask = GPU_HDP_FLUSH_DONE__SDMA1_MASK;
@@ -1290,8 +1288,10 @@ static void cik_sdma_set_ring_funcs(struct amdgpu_device *adev)
 {
 	int i;
 
-	for (i = 0; i < adev->sdma.num_instances; i++)
+	for (i = 0; i < adev->sdma.num_instances; i++) {
 		adev->sdma.instance[i].ring.funcs = &cik_sdma_ring_funcs;
+		adev->sdma.instance[i].ring.me = i;
+	}
 }
 
 static const struct amdgpu_irq_src_funcs cik_sdma_trap_irq_funcs = {
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
index ada241bfeee9..308f9f238bc1 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
@@ -41,6 +41,8 @@
 #include "gmc/gmc_8_1_d.h"
 #include "gmc/gmc_8_1_sh_mask.h"
 
+#include "ivsrcid/ivsrcid_vislands30.h"
+
 static void dce_v10_0_set_display_funcs(struct amdgpu_device *adev);
 static void dce_v10_0_set_irq_funcs(struct amdgpu_device *adev);
 
@@ -1855,15 +1857,14 @@ static int dce_v10_0_crtc_do_set_base(struct drm_crtc *crtc,
 	if (unlikely(r != 0))
 		return r;
 
-	if (atomic) {
-		fb_location = amdgpu_bo_gpu_offset(abo);
-	} else {
-		r = amdgpu_bo_pin(abo, AMDGPU_GEM_DOMAIN_VRAM, &fb_location);
+	if (!atomic) {
+		r = amdgpu_bo_pin(abo, AMDGPU_GEM_DOMAIN_VRAM);
 		if (unlikely(r != 0)) {
 			amdgpu_bo_unreserve(abo);
 			return -EINVAL;
 		}
 	}
+	fb_location = amdgpu_bo_gpu_offset(abo);
 
 	amdgpu_bo_get_tiling_flags(abo, &tiling_flags);
 	amdgpu_bo_unreserve(abo);
@@ -2370,13 +2371,14 @@ static int dce_v10_0_crtc_cursor_set2(struct drm_crtc *crtc,
 		return ret;
 	}
 
-	ret = amdgpu_bo_pin(aobj, AMDGPU_GEM_DOMAIN_VRAM, &amdgpu_crtc->cursor_addr);
+	ret = amdgpu_bo_pin(aobj, AMDGPU_GEM_DOMAIN_VRAM);
 	amdgpu_bo_unreserve(aobj);
 	if (ret) {
 		DRM_ERROR("Failed to pin new cursor BO (%d)\n", ret);
 		drm_gem_object_put_unlocked(obj);
 		return ret;
 	}
+	amdgpu_crtc->cursor_addr = amdgpu_bo_gpu_offset(aobj);
 
 	dce_v10_0_lock_cursor(crtc, true);
 
@@ -2737,14 +2739,14 @@ static int dce_v10_0_sw_init(void *handle)
 			return r;
 	}
 
-	for (i = 8; i < 20; i += 2) {
+	for (i = VISLANDS30_IV_SRCID_D1_GRPH_PFLIP; i < 20; i += 2) {
 		r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, i, &adev->pageflip_irq);
 		if (r)
 			return r;
 	}
 
 	/* HPD hotplug */
-	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, 42, &adev->hpd_irq);
+	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_HOTPLUG_DETECT_A, &adev->hpd_irq);
 	if (r)
 		return r;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c
index a5b96eac3033..76dfb76f7900 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c
@@ -41,6 +41,8 @@
 #include "gmc/gmc_8_1_d.h"
 #include "gmc/gmc_8_1_sh_mask.h"
 
+#include "ivsrcid/ivsrcid_vislands30.h"
+
 static void dce_v11_0_set_display_funcs(struct amdgpu_device *adev);
 static void dce_v11_0_set_irq_funcs(struct amdgpu_device *adev);
 
@@ -1897,15 +1899,14 @@ static int dce_v11_0_crtc_do_set_base(struct drm_crtc *crtc,
 	if (unlikely(r != 0))
 		return r;
 
-	if (atomic) {
-		fb_location = amdgpu_bo_gpu_offset(abo);
-	} else {
-		r = amdgpu_bo_pin(abo, AMDGPU_GEM_DOMAIN_VRAM, &fb_location);
+	if (!atomic) {
+		r = amdgpu_bo_pin(abo, AMDGPU_GEM_DOMAIN_VRAM);
 		if (unlikely(r != 0)) {
 			amdgpu_bo_unreserve(abo);
 			return -EINVAL;
 		}
 	}
+	fb_location = amdgpu_bo_gpu_offset(abo);
 
 	amdgpu_bo_get_tiling_flags(abo, &tiling_flags);
 	amdgpu_bo_unreserve(abo);
@@ -2449,13 +2450,14 @@ static int dce_v11_0_crtc_cursor_set2(struct drm_crtc *crtc,
 		return ret;
 	}
 
-	ret = amdgpu_bo_pin(aobj, AMDGPU_GEM_DOMAIN_VRAM, &amdgpu_crtc->cursor_addr);
+	ret = amdgpu_bo_pin(aobj, AMDGPU_GEM_DOMAIN_VRAM);
 	amdgpu_bo_unreserve(aobj);
 	if (ret) {
 		DRM_ERROR("Failed to pin new cursor BO (%d)\n", ret);
 		drm_gem_object_put_unlocked(obj);
 		return ret;
 	}
+	amdgpu_crtc->cursor_addr = amdgpu_bo_gpu_offset(aobj);
 
 	dce_v11_0_lock_cursor(crtc, true);
 
@@ -2858,14 +2860,14 @@ static int dce_v11_0_sw_init(void *handle)
 			return r;
 	}
 
-	for (i = 8; i < 20; i += 2) {
+	for (i = VISLANDS30_IV_SRCID_D1_GRPH_PFLIP; i < 20; i += 2) {
 		r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, i, &adev->pageflip_irq);
 		if (r)
 			return r;
 	}
 
 	/* HPD hotplug */
-	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, 42, &adev->hpd_irq);
+	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_HOTPLUG_DETECT_A, &adev->hpd_irq);
 	if (r)
 		return r;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c
index 394cc1e8fe20..c9adc627305d 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c
@@ -1811,15 +1811,14 @@ static int dce_v6_0_crtc_do_set_base(struct drm_crtc *crtc,
 	if (unlikely(r != 0))
 		return r;
 
-	if (atomic) {
-		fb_location = amdgpu_bo_gpu_offset(abo);
-	} else {
-		r = amdgpu_bo_pin(abo, AMDGPU_GEM_DOMAIN_VRAM, &fb_location);
+	if (!atomic) {
+		r = amdgpu_bo_pin(abo, AMDGPU_GEM_DOMAIN_VRAM);
 		if (unlikely(r != 0)) {
 			amdgpu_bo_unreserve(abo);
 			return -EINVAL;
 		}
 	}
+	fb_location = amdgpu_bo_gpu_offset(abo);
 
 	amdgpu_bo_get_tiling_flags(abo, &tiling_flags);
 	amdgpu_bo_unreserve(abo);
@@ -2263,13 +2262,14 @@ static int dce_v6_0_crtc_cursor_set2(struct drm_crtc *crtc,
 		return ret;
 	}
 
-	ret = amdgpu_bo_pin(aobj, AMDGPU_GEM_DOMAIN_VRAM, &amdgpu_crtc->cursor_addr);
+	ret = amdgpu_bo_pin(aobj, AMDGPU_GEM_DOMAIN_VRAM);
 	amdgpu_bo_unreserve(aobj);
 	if (ret) {
 		DRM_ERROR("Failed to pin new cursor BO (%d)\n", ret);
 		drm_gem_object_put_unlocked(obj);
 		return ret;
 	}
+	amdgpu_crtc->cursor_addr = amdgpu_bo_gpu_offset(aobj);
 
 	dce_v6_0_lock_cursor(crtc, true);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c
index c9b9ab8f1b05..50cd03beac7d 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c
@@ -1786,15 +1786,14 @@ static int dce_v8_0_crtc_do_set_base(struct drm_crtc *crtc,
 	if (unlikely(r != 0))
 		return r;
 
-	if (atomic) {
-		fb_location = amdgpu_bo_gpu_offset(abo);
-	} else {
-		r = amdgpu_bo_pin(abo, AMDGPU_GEM_DOMAIN_VRAM, &fb_location);
+	if (!atomic) {
+		r = amdgpu_bo_pin(abo, AMDGPU_GEM_DOMAIN_VRAM);
 		if (unlikely(r != 0)) {
 			amdgpu_bo_unreserve(abo);
 			return -EINVAL;
 		}
 	}
+	fb_location = amdgpu_bo_gpu_offset(abo);
 
 	amdgpu_bo_get_tiling_flags(abo, &tiling_flags);
 	amdgpu_bo_unreserve(abo);
@@ -2274,13 +2273,14 @@ static int dce_v8_0_crtc_cursor_set2(struct drm_crtc *crtc,
 		return ret;
 	}
 
-	ret = amdgpu_bo_pin(aobj, AMDGPU_GEM_DOMAIN_VRAM, &amdgpu_crtc->cursor_addr);
+	ret = amdgpu_bo_pin(aobj, AMDGPU_GEM_DOMAIN_VRAM);
 	amdgpu_bo_unreserve(aobj);
 	if (ret) {
 		DRM_ERROR("Failed to pin new cursor BO (%d)\n", ret);
 		drm_gem_object_put_unlocked(obj);
 		return ret;
 	}
+	amdgpu_crtc->cursor_addr = amdgpu_bo_gpu_offset(aobj);
 
 	dce_v8_0_lock_cursor(crtc, true);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_virtual.c b/drivers/gpu/drm/amd/amdgpu/dce_virtual.c
index 677e96a56330..15257634a53a 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_virtual.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_virtual.c
@@ -36,6 +36,7 @@
 #include "dce_v10_0.h"
 #include "dce_v11_0.h"
 #include "dce_virtual.h"
+#include "ivsrcid/ivsrcid_vislands30.h"
 
 #define DCE_VIRTUAL_VBLANK_PERIOD 16666666
 
@@ -371,7 +372,7 @@ static int dce_virtual_sw_init(void *handle)
 	int r, i;
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
-	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, 229, &adev->crtc_irq);
+	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_SMU_DISP_TIMER2_TRIGGER, &adev->crtc_irq);
 	if (r)
 		return r;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c
index cd6bf291a853..de184a886057 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c
@@ -44,30 +44,30 @@ static void gfx_v6_0_set_ring_funcs(struct amdgpu_device *adev);
 static void gfx_v6_0_set_irq_funcs(struct amdgpu_device *adev);
 static void gfx_v6_0_get_cu_info(struct amdgpu_device *adev);
 
-MODULE_FIRMWARE("radeon/tahiti_pfp.bin");
-MODULE_FIRMWARE("radeon/tahiti_me.bin");
-MODULE_FIRMWARE("radeon/tahiti_ce.bin");
-MODULE_FIRMWARE("radeon/tahiti_rlc.bin");
-
-MODULE_FIRMWARE("radeon/pitcairn_pfp.bin");
-MODULE_FIRMWARE("radeon/pitcairn_me.bin");
-MODULE_FIRMWARE("radeon/pitcairn_ce.bin");
-MODULE_FIRMWARE("radeon/pitcairn_rlc.bin");
-
-MODULE_FIRMWARE("radeon/verde_pfp.bin");
-MODULE_FIRMWARE("radeon/verde_me.bin");
-MODULE_FIRMWARE("radeon/verde_ce.bin");
-MODULE_FIRMWARE("radeon/verde_rlc.bin");
-
-MODULE_FIRMWARE("radeon/oland_pfp.bin");
-MODULE_FIRMWARE("radeon/oland_me.bin");
-MODULE_FIRMWARE("radeon/oland_ce.bin");
-MODULE_FIRMWARE("radeon/oland_rlc.bin");
-
-MODULE_FIRMWARE("radeon/hainan_pfp.bin");
-MODULE_FIRMWARE("radeon/hainan_me.bin");
-MODULE_FIRMWARE("radeon/hainan_ce.bin");
-MODULE_FIRMWARE("radeon/hainan_rlc.bin");
+MODULE_FIRMWARE("amdgpu/tahiti_pfp.bin");
+MODULE_FIRMWARE("amdgpu/tahiti_me.bin");
+MODULE_FIRMWARE("amdgpu/tahiti_ce.bin");
+MODULE_FIRMWARE("amdgpu/tahiti_rlc.bin");
+
+MODULE_FIRMWARE("amdgpu/pitcairn_pfp.bin");
+MODULE_FIRMWARE("amdgpu/pitcairn_me.bin");
+MODULE_FIRMWARE("amdgpu/pitcairn_ce.bin");
+MODULE_FIRMWARE("amdgpu/pitcairn_rlc.bin");
+
+MODULE_FIRMWARE("amdgpu/verde_pfp.bin");
+MODULE_FIRMWARE("amdgpu/verde_me.bin");
+MODULE_FIRMWARE("amdgpu/verde_ce.bin");
+MODULE_FIRMWARE("amdgpu/verde_rlc.bin");
+
+MODULE_FIRMWARE("amdgpu/oland_pfp.bin");
+MODULE_FIRMWARE("amdgpu/oland_me.bin");
+MODULE_FIRMWARE("amdgpu/oland_ce.bin");
+MODULE_FIRMWARE("amdgpu/oland_rlc.bin");
+
+MODULE_FIRMWARE("amdgpu/hainan_pfp.bin");
+MODULE_FIRMWARE("amdgpu/hainan_me.bin");
+MODULE_FIRMWARE("amdgpu/hainan_ce.bin");
+MODULE_FIRMWARE("amdgpu/hainan_rlc.bin");
 
 static u32 gfx_v6_0_get_csb_size(struct amdgpu_device *adev);
 static void gfx_v6_0_get_csb_buffer(struct amdgpu_device *adev, volatile u32 *buffer);
@@ -335,7 +335,7 @@ static int gfx_v6_0_init_microcode(struct amdgpu_device *adev)
 	default: BUG();
 	}
 
-	snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
+	snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp.bin", chip_name);
 	err = request_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev);
 	if (err)
 		goto out;
@@ -346,7 +346,7 @@ static int gfx_v6_0_init_microcode(struct amdgpu_device *adev)
 	adev->gfx.pfp_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
 	adev->gfx.pfp_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
 
-	snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
+	snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me.bin", chip_name);
 	err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev);
 	if (err)
 		goto out;
@@ -357,7 +357,7 @@ static int gfx_v6_0_init_microcode(struct amdgpu_device *adev)
 	adev->gfx.me_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
 	adev->gfx.me_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
 
-	snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
+	snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce.bin", chip_name);
 	err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev);
 	if (err)
 		goto out;
@@ -368,7 +368,7 @@ static int gfx_v6_0_init_microcode(struct amdgpu_device *adev)
 	adev->gfx.ce_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
 	adev->gfx.ce_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
 
-	snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
+	snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc.bin", chip_name);
 	err = request_firmware(&adev->gfx.rlc_fw, fw_name, adev->dev);
 	if (err)
 		goto out;
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
index 42b6144c1fd5..95452c5a9df6 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
@@ -57,36 +57,36 @@ static void gfx_v7_0_set_ring_funcs(struct amdgpu_device *adev);
 static void gfx_v7_0_set_irq_funcs(struct amdgpu_device *adev);
 static void gfx_v7_0_set_gds_init(struct amdgpu_device *adev);
 
-MODULE_FIRMWARE("radeon/bonaire_pfp.bin");
-MODULE_FIRMWARE("radeon/bonaire_me.bin");
-MODULE_FIRMWARE("radeon/bonaire_ce.bin");
-MODULE_FIRMWARE("radeon/bonaire_rlc.bin");
-MODULE_FIRMWARE("radeon/bonaire_mec.bin");
-
-MODULE_FIRMWARE("radeon/hawaii_pfp.bin");
-MODULE_FIRMWARE("radeon/hawaii_me.bin");
-MODULE_FIRMWARE("radeon/hawaii_ce.bin");
-MODULE_FIRMWARE("radeon/hawaii_rlc.bin");
-MODULE_FIRMWARE("radeon/hawaii_mec.bin");
-
-MODULE_FIRMWARE("radeon/kaveri_pfp.bin");
-MODULE_FIRMWARE("radeon/kaveri_me.bin");
-MODULE_FIRMWARE("radeon/kaveri_ce.bin");
-MODULE_FIRMWARE("radeon/kaveri_rlc.bin");
-MODULE_FIRMWARE("radeon/kaveri_mec.bin");
-MODULE_FIRMWARE("radeon/kaveri_mec2.bin");
-
-MODULE_FIRMWARE("radeon/kabini_pfp.bin");
-MODULE_FIRMWARE("radeon/kabini_me.bin");
-MODULE_FIRMWARE("radeon/kabini_ce.bin");
-MODULE_FIRMWARE("radeon/kabini_rlc.bin");
-MODULE_FIRMWARE("radeon/kabini_mec.bin");
-
-MODULE_FIRMWARE("radeon/mullins_pfp.bin");
-MODULE_FIRMWARE("radeon/mullins_me.bin");
-MODULE_FIRMWARE("radeon/mullins_ce.bin");
-MODULE_FIRMWARE("radeon/mullins_rlc.bin");
-MODULE_FIRMWARE("radeon/mullins_mec.bin");
+MODULE_FIRMWARE("amdgpu/bonaire_pfp.bin");
+MODULE_FIRMWARE("amdgpu/bonaire_me.bin");
+MODULE_FIRMWARE("amdgpu/bonaire_ce.bin");
+MODULE_FIRMWARE("amdgpu/bonaire_rlc.bin");
+MODULE_FIRMWARE("amdgpu/bonaire_mec.bin");
+
+MODULE_FIRMWARE("amdgpu/hawaii_pfp.bin");
+MODULE_FIRMWARE("amdgpu/hawaii_me.bin");
+MODULE_FIRMWARE("amdgpu/hawaii_ce.bin");
+MODULE_FIRMWARE("amdgpu/hawaii_rlc.bin");
+MODULE_FIRMWARE("amdgpu/hawaii_mec.bin");
+
+MODULE_FIRMWARE("amdgpu/kaveri_pfp.bin");
+MODULE_FIRMWARE("amdgpu/kaveri_me.bin");
+MODULE_FIRMWARE("amdgpu/kaveri_ce.bin");
+MODULE_FIRMWARE("amdgpu/kaveri_rlc.bin");
+MODULE_FIRMWARE("amdgpu/kaveri_mec.bin");
+MODULE_FIRMWARE("amdgpu/kaveri_mec2.bin");
+
+MODULE_FIRMWARE("amdgpu/kabini_pfp.bin");
+MODULE_FIRMWARE("amdgpu/kabini_me.bin");
+MODULE_FIRMWARE("amdgpu/kabini_ce.bin");
+MODULE_FIRMWARE("amdgpu/kabini_rlc.bin");
+MODULE_FIRMWARE("amdgpu/kabini_mec.bin");
+
+MODULE_FIRMWARE("amdgpu/mullins_pfp.bin");
+MODULE_FIRMWARE("amdgpu/mullins_me.bin");
+MODULE_FIRMWARE("amdgpu/mullins_ce.bin");
+MODULE_FIRMWARE("amdgpu/mullins_rlc.bin");
+MODULE_FIRMWARE("amdgpu/mullins_mec.bin");
 
 static const struct amdgpu_gds_reg_offset amdgpu_gds_reg_offset[] =
 {
@@ -925,7 +925,7 @@ static int gfx_v7_0_init_microcode(struct amdgpu_device *adev)
 	default: BUG();
 	}
 
-	snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
+	snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp.bin", chip_name);
 	err = request_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev);
 	if (err)
 		goto out;
@@ -933,7 +933,7 @@ static int gfx_v7_0_init_microcode(struct amdgpu_device *adev)
 	if (err)
 		goto out;
 
-	snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
+	snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me.bin", chip_name);
 	err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev);
 	if (err)
 		goto out;
@@ -941,7 +941,7 @@ static int gfx_v7_0_init_microcode(struct amdgpu_device *adev)
 	if (err)
 		goto out;
 
-	snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
+	snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce.bin", chip_name);
 	err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev);
 	if (err)
 		goto out;
@@ -949,7 +949,7 @@ static int gfx_v7_0_init_microcode(struct amdgpu_device *adev)
 	if (err)
 		goto out;
 
-	snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
+	snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec.bin", chip_name);
 	err = request_firmware(&adev->gfx.mec_fw, fw_name, adev->dev);
 	if (err)
 		goto out;
@@ -958,7 +958,7 @@ static int gfx_v7_0_init_microcode(struct amdgpu_device *adev)
 		goto out;
 
 	if (adev->asic_type == CHIP_KAVERI) {
-		snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec2.bin", chip_name);
+		snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2.bin", chip_name);
 		err = request_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev);
 		if (err)
 			goto out;
@@ -967,7 +967,7 @@ static int gfx_v7_0_init_microcode(struct amdgpu_device *adev)
 			goto out;
 	}
 
-	snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
+	snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc.bin", chip_name);
 	err = request_firmware(&adev->gfx.rlc_fw, fw_name, adev->dev);
 	if (err)
 		goto out;
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
index 807ee0dd623c..5cd45210113f 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
@@ -51,6 +51,8 @@
 
 #include "smu/smu_7_1_3_d.h"
 
+#include "ivsrcid/ivsrcid_vislands30.h"
+
 #define GFX8_NUM_GFX_RINGS     1
 #define GFX8_MEC_HPD_SIZE 2048
 
@@ -704,6 +706,17 @@ static const u32 stoney_mgcg_cgcg_init[] =
 	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
 };
 
+
+static const char * const sq_edc_source_names[] = {
+	"SQ_EDC_INFO_SOURCE_INVALID: No EDC error has occurred",
+	"SQ_EDC_INFO_SOURCE_INST: EDC source is Instruction Fetch",
+	"SQ_EDC_INFO_SOURCE_SGPR: EDC source is SGPR or SQC data return",
+	"SQ_EDC_INFO_SOURCE_VGPR: EDC source is VGPR",
+	"SQ_EDC_INFO_SOURCE_LDS: EDC source is LDS",
+	"SQ_EDC_INFO_SOURCE_GDS: EDC source is GDS",
+	"SQ_EDC_INFO_SOURCE_TA: EDC source is TA",
+};
+
 static void gfx_v8_0_set_ring_funcs(struct amdgpu_device *adev);
 static void gfx_v8_0_set_irq_funcs(struct amdgpu_device *adev);
 static void gfx_v8_0_set_gds_init(struct amdgpu_device *adev);
@@ -2006,6 +2019,8 @@ static int gfx_v8_0_compute_ring_init(struct amdgpu_device *adev, int ring_id,
 	return 0;
 }
 
+static void gfx_v8_0_sq_irq_work_func(struct work_struct *work);
+
 static int gfx_v8_0_sw_init(void *handle)
 {
 	int i, j, k, r, ring_id;
@@ -2034,41 +2049,43 @@ static int gfx_v8_0_sw_init(void *handle)
 	adev->gfx.mec.num_queue_per_pipe = 8;
 
 	/* KIQ event */
-	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, 178, &adev->gfx.kiq.irq);
+	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_CP_INT_IB2, &adev->gfx.kiq.irq);
 	if (r)
 		return r;
 
 	/* EOP Event */
-	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, 181, &adev->gfx.eop_irq);
+	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_CP_END_OF_PIPE, &adev->gfx.eop_irq);
 	if (r)
 		return r;
 
 	/* Privileged reg */
-	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, 184,
+	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_CP_PRIV_REG_FAULT,
 			      &adev->gfx.priv_reg_irq);
 	if (r)
 		return r;
 
 	/* Privileged inst */
-	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, 185,
+	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_CP_PRIV_INSTR_FAULT,
 			      &adev->gfx.priv_inst_irq);
 	if (r)
 		return r;
 
 	/* Add CP EDC/ECC irq  */
-	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, 197,
+	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_CP_ECC_ERROR,
 			      &adev->gfx.cp_ecc_error_irq);
 	if (r)
 		return r;
 
 	/* SQ interrupts. */
-	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, 239,
+	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_SQ_INTERRUPT_MSG,
 			      &adev->gfx.sq_irq);
 	if (r) {
 		DRM_ERROR("amdgpu_irq_add() for SQ failed: %d\n", r);
 		return r;
 	}
 
+	INIT_WORK(&adev->gfx.sq_work.work, gfx_v8_0_sq_irq_work_func);
+
 	adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE;
 
 	gfx_v8_0_scratch_init(adev);
@@ -5581,24 +5598,18 @@ static int gfx_v8_0_late_init(void *handle)
 		return r;
 	}
 
-	amdgpu_device_ip_set_powergating_state(adev,
-					       AMD_IP_BLOCK_TYPE_GFX,
-					       AMD_PG_STATE_GATE);
-
 	return 0;
 }
 
 static void gfx_v8_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *adev,
 						       bool enable)
 {
-	if ((adev->asic_type == CHIP_POLARIS11) ||
+	if (((adev->asic_type == CHIP_POLARIS11) ||
 	    (adev->asic_type == CHIP_POLARIS12) ||
-	    (adev->asic_type == CHIP_VEGAM))
+	    (adev->asic_type == CHIP_VEGAM)) &&
+	    adev->powerplay.pp_funcs->set_powergating_by_smu)
 		/* Send msg to SMU via Powerplay */
-		amdgpu_device_ip_set_powergating_state(adev,
-						       AMD_IP_BLOCK_TYPE_SMC,
-						       enable ?
-						       AMD_PG_STATE_GATE : AMD_PG_STATE_UNGATE);
+		amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, enable);
 
 	WREG32_FIELD(RLC_PG_CNTL, STATIC_PER_CU_PG_ENABLE, enable ? 1 : 0);
 }
@@ -6955,16 +6966,14 @@ static int gfx_v8_0_cp_ecc_error_irq(struct amdgpu_device *adev,
 	return 0;
 }
 
-static int gfx_v8_0_sq_irq(struct amdgpu_device *adev,
-			   struct amdgpu_irq_src *source,
-			   struct amdgpu_iv_entry *entry)
+static void gfx_v8_0_parse_sq_irq(struct amdgpu_device *adev, unsigned ih_data)
 {
-	u8 enc, se_id;
+	u32 enc, se_id, sh_id, cu_id;
 	char type[20];
+	int sq_edc_source = -1;
 
-	/* Parse all fields according to SQ_INTERRUPT* registers */
-	enc = (entry->src_data[0] >> 26) & 0x3;
-	se_id = (entry->src_data[0] >> 24) & 0x3;
+	enc = REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_CMN, ENCODING);
+	se_id = REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_CMN, SE_ID);
 
 	switch (enc) {
 		case 0:
@@ -6974,19 +6983,37 @@ static int gfx_v8_0_sq_irq(struct amdgpu_device *adev,
 					"reg_timestamp %d, thread_trace_buff_full %d,"
 					"wlt %d, thread_trace %d.\n",
 					se_id,
-					(entry->src_data[0] >> 7) & 0x1,
-					(entry->src_data[0] >> 6) & 0x1,
-					(entry->src_data[0] >> 5) & 0x1,
-					(entry->src_data[0] >> 4) & 0x1,
-					(entry->src_data[0] >> 3) & 0x1,
-					(entry->src_data[0] >> 2) & 0x1,
-					(entry->src_data[0] >> 1) & 0x1,
-					entry->src_data[0] & 0x1
+					REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, IMMED_OVERFLOW),
+					REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, HOST_REG_OVERFLOW),
+					REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, HOST_CMD_OVERFLOW),
+					REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, CMD_TIMESTAMP),
+					REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, REG_TIMESTAMP),
+					REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, THREAD_TRACE_BUF_FULL),
+					REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, WLT),
+					REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, THREAD_TRACE)
 					);
 			break;
 		case 1:
 		case 2:
 
+			cu_id = REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, CU_ID);
+			sh_id = REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, SH_ID);
+
+			/*
+			 * This function can be called either directly from ISR
+			 * or from BH in which case we can access SQ_EDC_INFO
+			 * instance
+			 */
+			if (in_task()) {
+				mutex_lock(&adev->grbm_idx_mutex);
+				gfx_v8_0_select_se_sh(adev, se_id, sh_id, cu_id);
+
+				sq_edc_source = REG_GET_FIELD(RREG32(mmSQ_EDC_INFO), SQ_EDC_INFO, SOURCE);
+
+				gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
+				mutex_unlock(&adev->grbm_idx_mutex);
+			}
+
 			if (enc == 1)
 				sprintf(type, "instruction intr");
 			else
@@ -6994,17 +7021,46 @@ static int gfx_v8_0_sq_irq(struct amdgpu_device *adev,
 
 			DRM_INFO(
 				"SQ %s detected: "
-					"se_id %d, cu_id %d, simd_id %d, wave_id %d, vm_id %d\n",
-					type, se_id,
-					(entry->src_data[0] >> 20) & 0xf,
-					(entry->src_data[0] >> 18) & 0x3,
-					(entry->src_data[0] >> 14) & 0xf,
-					(entry->src_data[0] >> 10) & 0xf
-					);
+					"se_id %d, sh_id %d, cu_id %d, simd_id %d, wave_id %d, vm_id %d "
+					"trap %s, sq_ed_info.source %s.\n",
+					type, se_id, sh_id, cu_id,
+					REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, SIMD_ID),
+					REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, WAVE_ID),
+					REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, VM_ID),
+					REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, PRIV) ? "true" : "false",
+					(sq_edc_source != -1) ? sq_edc_source_names[sq_edc_source] : "unavailable"
+				);
 			break;
 		default:
 			DRM_ERROR("SQ invalid encoding type\n.");
-			return -EINVAL;
+	}
+}
+
+static void gfx_v8_0_sq_irq_work_func(struct work_struct *work)
+{
+
+	struct amdgpu_device *adev = container_of(work, struct amdgpu_device, gfx.sq_work.work);
+	struct sq_work *sq_work = container_of(work, struct sq_work, work);
+
+	gfx_v8_0_parse_sq_irq(adev, sq_work->ih_data);
+}
+
+static int gfx_v8_0_sq_irq(struct amdgpu_device *adev,
+			   struct amdgpu_irq_src *source,
+			   struct amdgpu_iv_entry *entry)
+{
+	unsigned ih_data = entry->src_data[0];
+
+	/*
+	 * Try to submit work so SQ_EDC_INFO can be accessed from
+	 * BH. If previous work submission hasn't finished yet
+	 * just print whatever info is possible directly from the ISR.
+	 */
+	if (work_pending(&adev->gfx.sq_work.work)) {
+		gfx_v8_0_parse_sq_irq(adev, ih_data);
+	} else {
+		adev->gfx.sq_work.ih_data = ih_data;
+		schedule_work(&adev->gfx.sq_work.work);
 	}
 
 	return 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
index a69153435ea7..ef00d14f8645 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
@@ -38,6 +38,8 @@
 #include "clearstate_gfx9.h"
 #include "v9_structs.h"
 
+#include "ivsrcid/gfx/irqsrcs_gfx_9_0.h"
+
 #define GFX9_NUM_GFX_RINGS     1
 #define GFX9_MEC_HPD_SIZE 2048
 #define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L
@@ -102,11 +104,22 @@ static const struct soc15_reg_golden golden_settings_gc_9_0_vg10[] =
 {
 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0x0000f000, 0x00012107),
 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080),
 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042),
 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x2a114042),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080),
 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0x00008000, 0x00048000),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080),
 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000),
-	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x00001800, 0x00000800)
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x01000107),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x00001800, 0x00000800),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080)
 };
 
 static const struct soc15_reg_golden golden_settings_gc_9_0_vg20[] =
@@ -648,7 +661,10 @@ static int gfx_v9_0_init_microcode(struct amdgpu_device *adev)
 		adev->firmware.fw_size +=
 			ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
 
-		if (adev->gfx.rlc.is_rlc_v2_1) {
+		if (adev->gfx.rlc.is_rlc_v2_1 &&
+		    adev->gfx.rlc.save_restore_list_cntl_size_bytes &&
+		    adev->gfx.rlc.save_restore_list_gpm_size_bytes &&
+		    adev->gfx.rlc.save_restore_list_srm_size_bytes) {
 			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL];
 			info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL;
 			info->fw = adev->gfx.rlc_fw;
@@ -943,6 +959,7 @@ static int gfx_v9_0_rlc_init(struct amdgpu_device *adev)
 		dst_ptr = adev->gfx.rlc.cs_ptr;
 		gfx_v9_0_get_csb_buffer(adev, dst_ptr);
 		amdgpu_bo_kunmap(adev->gfx.rlc.clear_state_obj);
+		amdgpu_bo_unpin(adev->gfx.rlc.clear_state_obj);
 		amdgpu_bo_unreserve(adev->gfx.rlc.clear_state_obj);
 	}
 
@@ -971,6 +988,39 @@ static int gfx_v9_0_rlc_init(struct amdgpu_device *adev)
 	return 0;
 }
 
+static int gfx_v9_0_csb_vram_pin(struct amdgpu_device *adev)
+{
+	int r;
+
+	r = amdgpu_bo_reserve(adev->gfx.rlc.clear_state_obj, false);
+	if (unlikely(r != 0))
+		return r;
+
+	r = amdgpu_bo_pin(adev->gfx.rlc.clear_state_obj,
+			AMDGPU_GEM_DOMAIN_VRAM);
+	if (!r)
+		adev->gfx.rlc.clear_state_gpu_addr =
+			amdgpu_bo_gpu_offset(adev->gfx.rlc.clear_state_obj);
+
+	amdgpu_bo_unreserve(adev->gfx.rlc.clear_state_obj);
+
+	return r;
+}
+
+static void gfx_v9_0_csb_vram_unpin(struct amdgpu_device *adev)
+{
+	int r;
+
+	if (!adev->gfx.rlc.clear_state_obj)
+		return;
+
+	r = amdgpu_bo_reserve(adev->gfx.rlc.clear_state_obj, true);
+	if (likely(r == 0)) {
+		amdgpu_bo_unpin(adev->gfx.rlc.clear_state_obj);
+		amdgpu_bo_unreserve(adev->gfx.rlc.clear_state_obj);
+	}
+}
+
 static void gfx_v9_0_mec_fini(struct amdgpu_device *adev)
 {
 	amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL);
@@ -1451,23 +1501,23 @@ static int gfx_v9_0_sw_init(void *handle)
 	adev->gfx.mec.num_queue_per_pipe = 8;
 
 	/* KIQ event */
-	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, 178, &adev->gfx.kiq.irq);
+	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_IB2_INTERRUPT_PKT, &adev->gfx.kiq.irq);
 	if (r)
 		return r;
 
 	/* EOP Event */
-	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, 181, &adev->gfx.eop_irq);
+	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_EOP_INTERRUPT, &adev->gfx.eop_irq);
 	if (r)
 		return r;
 
 	/* Privileged reg */
-	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, 184,
+	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_REG_FAULT,
 			      &adev->gfx.priv_reg_irq);
 	if (r)
 		return r;
 
 	/* Privileged inst */
-	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, 185,
+	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_INSTR_FAULT,
 			      &adev->gfx.priv_inst_irq);
 	if (r)
 		return r;
@@ -2148,8 +2198,16 @@ static void gfx_v9_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *ad
 
 static void gfx_v9_0_init_pg(struct amdgpu_device *adev)
 {
-	if (!adev->gfx.rlc.is_rlc_v2_1)
-		return;
+	gfx_v9_0_init_csb(adev);
+
+	/*
+	 * Rlc save restore list is workable since v2_1.
+	 * And it's needed by gfxoff feature.
+	 */
+	if (adev->gfx.rlc.is_rlc_v2_1) {
+		gfx_v9_1_init_rlc_save_restore_list(adev);
+		gfx_v9_0_enable_save_restore_machine(adev);
+	}
 
 	if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
 			      AMD_PG_SUPPORT_GFX_SMG |
@@ -2157,10 +2215,6 @@ static void gfx_v9_0_init_pg(struct amdgpu_device *adev)
 			      AMD_PG_SUPPORT_CP |
 			      AMD_PG_SUPPORT_GDS |
 			      AMD_PG_SUPPORT_RLC_SMU_HS)) {
-		gfx_v9_0_init_csb(adev);
-		gfx_v9_1_init_rlc_save_restore_list(adev);
-		gfx_v9_0_enable_save_restore_machine(adev);
-
 		WREG32(mmRLC_JUMP_TABLE_RESTORE,
 		       adev->gfx.rlc.cp_table_gpu_addr >> 8);
 		gfx_v9_0_init_gfx_power_gating(adev);
@@ -2252,9 +2306,6 @@ static int gfx_v9_0_rlc_resume(struct amdgpu_device *adev)
 	/* disable CG */
 	WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, 0);
 
-	/* disable PG */
-	WREG32_SOC15(GC, 0, mmRLC_PG_CNTL, 0);
-
 	gfx_v9_0_rlc_reset(adev);
 
 	gfx_v9_0_init_pg(adev);
@@ -3116,6 +3167,10 @@ static int gfx_v9_0_hw_init(void *handle)
 
 	gfx_v9_0_gpu_init(adev);
 
+	r = gfx_v9_0_csb_vram_pin(adev);
+	if (r)
+		return r;
+
 	r = gfx_v9_0_rlc_resume(adev);
 	if (r)
 		return r;
@@ -3224,6 +3279,8 @@ static int gfx_v9_0_hw_fini(void *handle)
 	gfx_v9_0_cp_enable(adev, false);
 	gfx_v9_0_rlc_stop(adev);
 
+	gfx_v9_0_csb_vram_unpin(adev);
+
 	return 0;
 }
 
@@ -3433,7 +3490,7 @@ static void gfx_v9_0_enter_rlc_safe_mode(struct amdgpu_device *adev)
 
 		/* wait for RLC_SAFE_MODE */
 		for (i = 0; i < adev->usec_timeout; i++) {
-			if (!REG_GET_FIELD(SOC15_REG_OFFSET(GC, 0, mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD))
+			if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD))
 				break;
 			udelay(1);
 		}
@@ -3510,8 +3567,11 @@ static void gfx_v9_0_update_medium_grain_clock_gating(struct amdgpu_device *adev
 	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) {
 		/* 1 - RLC_CGTT_MGCG_OVERRIDE */
 		def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
-		data &= ~(RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK |
-			  RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
+
+		if (adev->asic_type != CHIP_VEGA12)
+			data &= ~RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK;
+
+		data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
 			  RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK |
 			  RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK);
 
@@ -3541,11 +3601,15 @@ static void gfx_v9_0_update_medium_grain_clock_gating(struct amdgpu_device *adev
 	} else {
 		/* 1 - MGCG_OVERRIDE */
 		def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
-		data |= (RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK |
-			 RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
+
+		if (adev->asic_type != CHIP_VEGA12)
+			data |= RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK;
+
+		data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
 			 RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
 			 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK |
 			 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK);
+
 		if (def != data)
 			WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
 
@@ -3581,9 +3645,11 @@ static void gfx_v9_0_update_3d_clock_gating(struct amdgpu_device *adev,
 		/* update CGCG and CGLS override bits */
 		if (def != data)
 			WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
-		/* enable 3Dcgcg FSM(0x0020003f) */
+
+		/* enable 3Dcgcg FSM(0x0000363f) */
 		def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D);
-		data = (0x2000 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
+
+		data = (0x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
 			RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK;
 		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS)
 			data |= (0x000F << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
@@ -3630,9 +3696,10 @@ static void gfx_v9_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev
 		if (def != data)
 			WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
 
-		/* enable cgcg FSM(0x0020003F) */
+		/* enable cgcg FSM(0x0000363F) */
 		def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL);
-		data = (0x2000 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
+
+		data = (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
 			RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
 		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
 			data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
@@ -3714,6 +3781,15 @@ static int gfx_v9_0_set_powergating_state(void *handle,
 
 		/* update mgcg state */
 		gfx_v9_0_update_gfx_mg_power_gating(adev, enable);
+
+		/* set gfx off through smu */
+		if (enable && adev->powerplay.pp_funcs->set_powergating_by_smu)
+			amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, true);
+		break;
+	case CHIP_VEGA12:
+		/* set gfx off through smu */
+		if (enable && adev->powerplay.pp_funcs->set_powergating_by_smu)
+			amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, true);
 		break;
 	default:
 		break;
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
index 79f9ac29019b..75317f283c69 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
@@ -41,11 +41,11 @@ static void gmc_v6_0_set_gmc_funcs(struct amdgpu_device *adev);
 static void gmc_v6_0_set_irq_funcs(struct amdgpu_device *adev);
 static int gmc_v6_0_wait_for_idle(void *handle);
 
-MODULE_FIRMWARE("radeon/tahiti_mc.bin");
-MODULE_FIRMWARE("radeon/pitcairn_mc.bin");
-MODULE_FIRMWARE("radeon/verde_mc.bin");
-MODULE_FIRMWARE("radeon/oland_mc.bin");
-MODULE_FIRMWARE("radeon/si58_mc.bin");
+MODULE_FIRMWARE("amdgpu/tahiti_mc.bin");
+MODULE_FIRMWARE("amdgpu/pitcairn_mc.bin");
+MODULE_FIRMWARE("amdgpu/verde_mc.bin");
+MODULE_FIRMWARE("amdgpu/oland_mc.bin");
+MODULE_FIRMWARE("amdgpu/si58_mc.bin");
 
 #define MC_SEQ_MISC0__MT__MASK   0xf0000000
 #define MC_SEQ_MISC0__MT__GDDR1  0x10000000
@@ -134,9 +134,9 @@ static int gmc_v6_0_init_microcode(struct amdgpu_device *adev)
 		is_58_fw = true;
 
 	if (is_58_fw)
-		snprintf(fw_name, sizeof(fw_name), "radeon/si58_mc.bin");
+		snprintf(fw_name, sizeof(fw_name), "amdgpu/si58_mc.bin");
 	else
-		snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
+		snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mc.bin", chip_name);
 	err = request_firmware(&adev->gmc.fw, fw_name, adev->dev);
 	if (err)
 		goto out;
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
index 7147bfe25a23..36dc367c4b45 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
@@ -28,6 +28,7 @@
 #include "cik.h"
 #include "gmc_v7_0.h"
 #include "amdgpu_ucode.h"
+#include "amdgpu_amdkfd.h"
 
 #include "bif/bif_4_1_d.h"
 #include "bif/bif_4_1_sh_mask.h"
@@ -43,12 +44,14 @@
 
 #include "amdgpu_atombios.h"
 
+#include "ivsrcid/ivsrcid_vislands30.h"
+
 static void gmc_v7_0_set_gmc_funcs(struct amdgpu_device *adev);
 static void gmc_v7_0_set_irq_funcs(struct amdgpu_device *adev);
 static int gmc_v7_0_wait_for_idle(void *handle);
 
-MODULE_FIRMWARE("radeon/bonaire_mc.bin");
-MODULE_FIRMWARE("radeon/hawaii_mc.bin");
+MODULE_FIRMWARE("amdgpu/bonaire_mc.bin");
+MODULE_FIRMWARE("amdgpu/hawaii_mc.bin");
 MODULE_FIRMWARE("amdgpu/topaz_mc.bin");
 
 static const u32 golden_settings_iceland_a11[] =
@@ -147,10 +150,7 @@ static int gmc_v7_0_init_microcode(struct amdgpu_device *adev)
 	default: BUG();
 	}
 
-	if (adev->asic_type == CHIP_TOPAZ)
-		snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mc.bin", chip_name);
-	else
-		snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
+	snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mc.bin", chip_name);
 
 	err = request_firmware(&adev->gmc.fw, fw_name, adev->dev);
 	if (err)
@@ -999,11 +999,11 @@ static int gmc_v7_0_sw_init(void *handle)
 		adev->gmc.vram_type = gmc_v7_0_convert_vram_type(tmp);
 	}
 
-	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, 146, &adev->gmc.vm_fault);
+	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_GFX_PAGE_INV_FAULT, &adev->gmc.vm_fault);
 	if (r)
 		return r;
 
-	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, 147, &adev->gmc.vm_fault);
+	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_GFX_MEM_PROT_FAULT, &adev->gmc.vm_fault);
 	if (r)
 		return r;
 
@@ -1079,6 +1079,12 @@ static int gmc_v7_0_sw_init(void *handle)
 		adev->vm_manager.vram_base_offset = 0;
 	}
 
+	adev->gmc.vm_fault_info = kmalloc(sizeof(struct kfd_vm_fault_info),
+					GFP_KERNEL);
+	if (!adev->gmc.vm_fault_info)
+		return -ENOMEM;
+	atomic_set(&adev->gmc.vm_fault_info_updated, 0);
+
 	return 0;
 }
 
@@ -1088,6 +1094,7 @@ static int gmc_v7_0_sw_fini(void *handle)
 
 	amdgpu_gem_force_release(adev);
 	amdgpu_vm_manager_fini(adev);
+	kfree(adev->gmc.vm_fault_info);
 	gmc_v7_0_gart_fini(adev);
 	amdgpu_bo_fini(adev);
 	release_firmware(adev->gmc.fw);
@@ -1277,7 +1284,7 @@ static int gmc_v7_0_process_interrupt(struct amdgpu_device *adev,
 				      struct amdgpu_irq_src *source,
 				      struct amdgpu_iv_entry *entry)
 {
-	u32 addr, status, mc_client;
+	u32 addr, status, mc_client, vmid;
 
 	addr = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_ADDR);
 	status = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_STATUS);
@@ -1302,6 +1309,29 @@ static int gmc_v7_0_process_interrupt(struct amdgpu_device *adev,
 					 entry->pasid);
 	}
 
+	vmid = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
+			     VMID);
+	if (amdgpu_amdkfd_is_kfd_vmid(adev, vmid)
+		&& !atomic_read(&adev->gmc.vm_fault_info_updated)) {
+		struct kfd_vm_fault_info *info = adev->gmc.vm_fault_info;
+		u32 protections = REG_GET_FIELD(status,
+					VM_CONTEXT1_PROTECTION_FAULT_STATUS,
+					PROTECTIONS);
+
+		info->vmid = vmid;
+		info->mc_id = REG_GET_FIELD(status,
+					    VM_CONTEXT1_PROTECTION_FAULT_STATUS,
+					    MEMORY_CLIENT_ID);
+		info->status = status;
+		info->page_addr = addr;
+		info->prot_valid = protections & 0x7 ? true : false;
+		info->prot_read = protections & 0x8 ? true : false;
+		info->prot_write = protections & 0x10 ? true : false;
+		info->prot_exec = protections & 0x20 ? true : false;
+		mb();
+		atomic_set(&adev->gmc.vm_fault_info_updated, 1);
+	}
+
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
index 1edbe6b477b5..70fc97b59b4f 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
@@ -26,6 +26,7 @@
 #include "amdgpu.h"
 #include "gmc_v8_0.h"
 #include "amdgpu_ucode.h"
+#include "amdgpu_amdkfd.h"
 
 #include "gmc/gmc_8_1_d.h"
 #include "gmc/gmc_8_1_sh_mask.h"
@@ -44,6 +45,7 @@
 
 #include "amdgpu_atombios.h"
 
+#include "ivsrcid/ivsrcid_vislands30.h"
 
 static void gmc_v8_0_set_gmc_funcs(struct amdgpu_device *adev);
 static void gmc_v8_0_set_irq_funcs(struct amdgpu_device *adev);
@@ -1101,11 +1103,11 @@ static int gmc_v8_0_sw_init(void *handle)
 		adev->gmc.vram_type = gmc_v8_0_convert_vram_type(tmp);
 	}
 
-	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, 146, &adev->gmc.vm_fault);
+	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_GFX_PAGE_INV_FAULT, &adev->gmc.vm_fault);
 	if (r)
 		return r;
 
-	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, 147, &adev->gmc.vm_fault);
+	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_GFX_MEM_PROT_FAULT, &adev->gmc.vm_fault);
 	if (r)
 		return r;
 
@@ -1181,6 +1183,12 @@ static int gmc_v8_0_sw_init(void *handle)
 		adev->vm_manager.vram_base_offset = 0;
 	}
 
+	adev->gmc.vm_fault_info = kmalloc(sizeof(struct kfd_vm_fault_info),
+					GFP_KERNEL);
+	if (!adev->gmc.vm_fault_info)
+		return -ENOMEM;
+	atomic_set(&adev->gmc.vm_fault_info_updated, 0);
+
 	return 0;
 }
 
@@ -1190,6 +1198,7 @@ static int gmc_v8_0_sw_fini(void *handle)
 
 	amdgpu_gem_force_release(adev);
 	amdgpu_vm_manager_fini(adev);
+	kfree(adev->gmc.vm_fault_info);
 	gmc_v8_0_gart_fini(adev);
 	amdgpu_bo_fini(adev);
 	release_firmware(adev->gmc.fw);
@@ -1425,7 +1434,7 @@ static int gmc_v8_0_process_interrupt(struct amdgpu_device *adev,
 				      struct amdgpu_irq_src *source,
 				      struct amdgpu_iv_entry *entry)
 {
-	u32 addr, status, mc_client;
+	u32 addr, status, mc_client, vmid;
 
 	if (amdgpu_sriov_vf(adev)) {
 		dev_err(adev->dev, "GPU fault detected: %d 0x%08x\n",
@@ -1447,8 +1456,13 @@ static int gmc_v8_0_process_interrupt(struct amdgpu_device *adev,
 		gmc_v8_0_set_fault_enable_default(adev, false);
 
 	if (printk_ratelimit()) {
-		dev_err(adev->dev, "GPU fault detected: %d 0x%08x\n",
-			entry->src_id, entry->src_data[0]);
+		struct amdgpu_task_info task_info = { 0 };
+
+		amdgpu_vm_get_task_info(adev, entry->pasid, &task_info);
+
+		dev_err(adev->dev, "GPU fault detected: %d 0x%08x for process %s pid %d thread %s pid %d\n",
+			entry->src_id, entry->src_data[0], task_info.process_name,
+			task_info.tgid, task_info.task_name, task_info.pid);
 		dev_err(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
 			addr);
 		dev_err(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
@@ -1457,6 +1471,29 @@ static int gmc_v8_0_process_interrupt(struct amdgpu_device *adev,
 					 entry->pasid);
 	}
 
+	vmid = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
+			     VMID);
+	if (amdgpu_amdkfd_is_kfd_vmid(adev, vmid)
+		&& !atomic_read(&adev->gmc.vm_fault_info_updated)) {
+		struct kfd_vm_fault_info *info = adev->gmc.vm_fault_info;
+		u32 protections = REG_GET_FIELD(status,
+					VM_CONTEXT1_PROTECTION_FAULT_STATUS,
+					PROTECTIONS);
+
+		info->vmid = vmid;
+		info->mc_id = REG_GET_FIELD(status,
+					    VM_CONTEXT1_PROTECTION_FAULT_STATUS,
+					    MEMORY_CLIENT_ID);
+		info->status = status;
+		info->page_addr = addr;
+		info->prot_valid = protections & 0x7 ? true : false;
+		info->prot_read = protections & 0x8 ? true : false;
+		info->prot_write = protections & 0x10 ? true : false;
+		info->prot_exec = protections & 0x20 ? true : false;
+		mb();
+		atomic_set(&adev->gmc.vm_fault_info_updated, 1);
+	}
+
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
index 3c0a85d4e4ab..399a5db27649 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
@@ -43,6 +43,8 @@
 #include "gfxhub_v1_0.h"
 #include "mmhub_v1_0.h"
 
+#include "ivsrcid/vmc/irqsrcs_vmc_1_0.h"
+
 /* add these here since we already include dce12 headers and these are for DCN */
 #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION                                                          0x055d
 #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_BASE_IDX                                                 2
@@ -257,12 +259,17 @@ static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev,
 	}
 
 	if (printk_ratelimit()) {
+		struct amdgpu_task_info task_info = { 0 };
+
+		amdgpu_vm_get_task_info(adev, entry->pasid, &task_info);
+
 		dev_err(adev->dev,
-			"[%s] VMC page fault (src_id:%u ring:%u vmid:%u pasid:%u)\n",
+			"[%s] VMC page fault (src_id:%u ring:%u vmid:%u pasid:%u, for process %s pid %d thread %s pid %d\n)\n",
 			entry->vmid_src ? "mmhub" : "gfxhub",
 			entry->src_id, entry->ring_id, entry->vmid,
-			entry->pasid);
-		dev_err(adev->dev, "  at page 0x%016llx from %d\n",
+			entry->pasid, task_info.process_name, task_info.tgid,
+			task_info.task_name, task_info.pid);
+		dev_err(adev->dev, "  at address 0x%016llx from %d\n",
 			addr, entry->client_id);
 		if (!amdgpu_sriov_vf(adev))
 			dev_err(adev->dev,
@@ -872,9 +879,9 @@ static int gmc_v9_0_sw_init(void *handle)
 	}
 
 	/* This interrupt is VMC page fault.*/
-	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC, 0,
+	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC, VMC_1_0__SRCID__VM_FAULT,
 				&adev->gmc.vm_fault);
-	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_UTCL2, 0,
+	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_UTCL2, UTCL2_1_0__SRCID__FAULT,
 				&adev->gmc.vm_fault);
 
 	if (r)
diff --git a/drivers/gpu/drm/amd/amdgpu/kv_dpm.c b/drivers/gpu/drm/amd/amdgpu/kv_dpm.c
index 46de1fd18a7b..3f57f6463dc8 100644
--- a/drivers/gpu/drm/amd/amdgpu/kv_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/kv_dpm.c
@@ -3306,6 +3306,19 @@ static int kv_dpm_read_sensor(void *handle, int idx,
 	}
 }
 
+static int kv_set_powergating_by_smu(void *handle,
+				uint32_t block_type, bool gate)
+{
+	switch (block_type) {
+	case AMD_IP_BLOCK_TYPE_UVD:
+		kv_dpm_powergate_uvd(handle, gate);
+		break;
+	default:
+		break;
+	}
+	return 0;
+}
+
 static const struct amd_ip_funcs kv_dpm_ip_funcs = {
 	.name = "kv_dpm",
 	.early_init = kv_dpm_early_init,
@@ -3342,7 +3355,7 @@ static const struct amd_pm_funcs kv_dpm_funcs = {
 	.print_power_state = &kv_dpm_print_power_state,
 	.debugfs_print_current_performance_level = &kv_dpm_debugfs_print_current_performance_level,
 	.force_performance_level = &kv_dpm_force_performance_level,
-	.powergate_uvd = &kv_dpm_powergate_uvd,
+	.set_powergating_by_smu = kv_set_powergating_by_smu,
 	.enable_bapm = &kv_dpm_enable_bapm,
 	.get_vce_clock_state = amdgpu_get_vce_clock_state,
 	.check_state_equal = kv_check_state_equal,
diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c
index 3d53c4413f13..e70a0d4d6db4 100644
--- a/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c
@@ -471,8 +471,8 @@ void mmhub_v1_0_update_power_gating(struct amdgpu_device *adev,
 						RENG_EXECUTE_ON_REG_UPDATE, 1);
 		WREG32_SOC15(MMHUB, 0, mmPCTL1_RENG_EXECUTE, pctl1_reng_execute);
 
-		if (adev->powerplay.pp_funcs->set_mmhub_powergating_by_smu)
-			amdgpu_dpm_set_mmhub_powergating_by_smu(adev);
+		if (adev->powerplay.pp_funcs->set_powergating_by_smu)
+			amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GMC, true);
 
 	} else {
 		pctl0_reng_execute = REG_SET_FIELD(pctl0_reng_execute,
diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v10_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v10_0.c
index 0ff136d02d9b..02be34e72ed9 100644
--- a/drivers/gpu/drm/amd/amdgpu/psp_v10_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/psp_v10_0.c
@@ -88,6 +88,9 @@ psp_v10_0_get_fw_type(struct amdgpu_firmware_info *ucode, enum psp_gfx_fw_type *
 	case AMDGPU_UCODE_ID_VCE:
 		*type = GFX_FW_TYPE_VCE;
 		break;
+	case AMDGPU_UCODE_ID_VCN:
+		*type = GFX_FW_TYPE_VCN;
+		break;
 	case AMDGPU_UCODE_ID_MAXIMUM:
 	default:
 		return -EINVAL;
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
index c7190c39c4f5..15ae4bc9c072 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
@@ -44,6 +44,8 @@
 
 #include "iceland_sdma_pkt_open.h"
 
+#include "ivsrcid/ivsrcid_vislands30.h"
+
 static void sdma_v2_4_set_ring_funcs(struct amdgpu_device *adev);
 static void sdma_v2_4_set_buffer_funcs(struct amdgpu_device *adev);
 static void sdma_v2_4_set_vm_pte_funcs(struct amdgpu_device *adev);
@@ -202,8 +204,7 @@ static uint64_t sdma_v2_4_ring_get_rptr(struct amdgpu_ring *ring)
 static uint64_t sdma_v2_4_ring_get_wptr(struct amdgpu_ring *ring)
 {
 	struct amdgpu_device *adev = ring->adev;
-	int me = (ring == &ring->adev->sdma.instance[0].ring) ? 0 : 1;
-	u32 wptr = RREG32(mmSDMA0_GFX_RB_WPTR + sdma_offsets[me]) >> 2;
+	u32 wptr = RREG32(mmSDMA0_GFX_RB_WPTR + sdma_offsets[ring->me]) >> 2;
 
 	return wptr;
 }
@@ -218,9 +219,8 @@ static uint64_t sdma_v2_4_ring_get_wptr(struct amdgpu_ring *ring)
 static void sdma_v2_4_ring_set_wptr(struct amdgpu_ring *ring)
 {
 	struct amdgpu_device *adev = ring->adev;
-	int me = (ring == &ring->adev->sdma.instance[0].ring) ? 0 : 1;
 
-	WREG32(mmSDMA0_GFX_RB_WPTR + sdma_offsets[me], lower_32_bits(ring->wptr) << 2);
+	WREG32(mmSDMA0_GFX_RB_WPTR + sdma_offsets[ring->me], lower_32_bits(ring->wptr) << 2);
 }
 
 static void sdma_v2_4_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
@@ -273,7 +273,7 @@ static void sdma_v2_4_ring_emit_hdp_flush(struct amdgpu_ring *ring)
 {
 	u32 ref_and_mask = 0;
 
-	if (ring == &ring->adev->sdma.instance[0].ring)
+	if (ring->me == 0)
 		ref_and_mask = REG_SET_FIELD(ref_and_mask, GPU_HDP_FLUSH_DONE, SDMA0, 1);
 	else
 		ref_and_mask = REG_SET_FIELD(ref_and_mask, GPU_HDP_FLUSH_DONE, SDMA1, 1);
@@ -898,7 +898,7 @@ static int sdma_v2_4_sw_init(void *handle)
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
 	/* SDMA trap event */
-	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, 224,
+	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_SDMA_TRAP,
 			      &adev->sdma.trap_irq);
 	if (r)
 		return r;
@@ -910,7 +910,7 @@ static int sdma_v2_4_sw_init(void *handle)
 		return r;
 
 	/* SDMA Privileged inst */
-	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, 247,
+	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_SDMA_SRBM_WRITE,
 			      &adev->sdma.illegal_inst_irq);
 	if (r)
 		return r;
@@ -1213,8 +1213,10 @@ static void sdma_v2_4_set_ring_funcs(struct amdgpu_device *adev)
 {
 	int i;
 
-	for (i = 0; i < adev->sdma.num_instances; i++)
+	for (i = 0; i < adev->sdma.num_instances; i++) {
 		adev->sdma.instance[i].ring.funcs = &sdma_v2_4_ring_funcs;
+		adev->sdma.instance[i].ring.me = i;
+	}
 }
 
 static const struct amdgpu_irq_src_funcs sdma_v2_4_trap_irq_funcs = {
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
index aa9ab299fd32..1e07ff274d73 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
@@ -44,6 +44,8 @@
 
 #include "tonga_sdma_pkt_open.h"
 
+#include "ivsrcid/ivsrcid_vislands30.h"
+
 static void sdma_v3_0_set_ring_funcs(struct amdgpu_device *adev);
 static void sdma_v3_0_set_buffer_funcs(struct amdgpu_device *adev);
 static void sdma_v3_0_set_vm_pte_funcs(struct amdgpu_device *adev);
@@ -365,9 +367,7 @@ static uint64_t sdma_v3_0_ring_get_wptr(struct amdgpu_ring *ring)
 		/* XXX check if swapping is necessary on BE */
 		wptr = ring->adev->wb.wb[ring->wptr_offs] >> 2;
 	} else {
-		int me = (ring == &ring->adev->sdma.instance[0].ring) ? 0 : 1;
-
-		wptr = RREG32(mmSDMA0_GFX_RB_WPTR + sdma_offsets[me]) >> 2;
+		wptr = RREG32(mmSDMA0_GFX_RB_WPTR + sdma_offsets[ring->me]) >> 2;
 	}
 
 	return wptr;
@@ -394,9 +394,7 @@ static void sdma_v3_0_ring_set_wptr(struct amdgpu_ring *ring)
 
 		WRITE_ONCE(*wb, (lower_32_bits(ring->wptr) << 2));
 	} else {
-		int me = (ring == &ring->adev->sdma.instance[0].ring) ? 0 : 1;
-
-		WREG32(mmSDMA0_GFX_RB_WPTR + sdma_offsets[me], lower_32_bits(ring->wptr) << 2);
+		WREG32(mmSDMA0_GFX_RB_WPTR + sdma_offsets[ring->me], lower_32_bits(ring->wptr) << 2);
 	}
 }
 
@@ -450,7 +448,7 @@ static void sdma_v3_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
 {
 	u32 ref_and_mask = 0;
 
-	if (ring == &ring->adev->sdma.instance[0].ring)
+	if (ring->me == 0)
 		ref_and_mask = REG_SET_FIELD(ref_and_mask, GPU_HDP_FLUSH_DONE, SDMA0, 1);
 	else
 		ref_and_mask = REG_SET_FIELD(ref_and_mask, GPU_HDP_FLUSH_DONE, SDMA1, 1);
@@ -1179,7 +1177,7 @@ static int sdma_v3_0_sw_init(void *handle)
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
 	/* SDMA trap event */
-	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, 224,
+	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_SDMA_TRAP,
 			      &adev->sdma.trap_irq);
 	if (r)
 		return r;
@@ -1191,7 +1189,7 @@ static int sdma_v3_0_sw_init(void *handle)
 		return r;
 
 	/* SDMA Privileged inst */
-	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, 247,
+	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_SDMA_SRBM_WRITE,
 			      &adev->sdma.illegal_inst_irq);
 	if (r)
 		return r;
@@ -1655,8 +1653,10 @@ static void sdma_v3_0_set_ring_funcs(struct amdgpu_device *adev)
 {
 	int i;
 
-	for (i = 0; i < adev->sdma.num_instances; i++)
+	for (i = 0; i < adev->sdma.num_instances; i++) {
 		adev->sdma.instance[i].ring.funcs = &sdma_v3_0_ring_funcs;
+		adev->sdma.instance[i].ring.me = i;
+	}
 }
 
 static const struct amdgpu_irq_src_funcs sdma_v3_0_trap_irq_funcs = {
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
index ca53b3fba422..e7ca4623cfb9 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
@@ -38,6 +38,9 @@
 #include "soc15.h"
 #include "vega10_sdma_pkt_open.h"
 
+#include "ivsrcid/sdma0/irqsrcs_sdma0_4_0.h"
+#include "ivsrcid/sdma1/irqsrcs_sdma1_4_0.h"
+
 MODULE_FIRMWARE("amdgpu/vega10_sdma.bin");
 MODULE_FIRMWARE("amdgpu/vega10_sdma1.bin");
 MODULE_FIRMWARE("amdgpu/vega12_sdma.bin");
@@ -296,13 +299,12 @@ static uint64_t sdma_v4_0_ring_get_wptr(struct amdgpu_ring *ring)
 		DRM_DEBUG("wptr/doorbell before shift == 0x%016llx\n", wptr);
 	} else {
 		u32 lowbit, highbit;
-		int me = (ring == &adev->sdma.instance[0].ring) ? 0 : 1;
 
-		lowbit = RREG32(sdma_v4_0_get_reg_offset(adev, me, mmSDMA0_GFX_RB_WPTR)) >> 2;
-		highbit = RREG32(sdma_v4_0_get_reg_offset(adev, me, mmSDMA0_GFX_RB_WPTR_HI)) >> 2;
+		lowbit = RREG32(sdma_v4_0_get_reg_offset(adev, ring->me, mmSDMA0_GFX_RB_WPTR)) >> 2;
+		highbit = RREG32(sdma_v4_0_get_reg_offset(adev, ring->me, mmSDMA0_GFX_RB_WPTR_HI)) >> 2;
 
 		DRM_DEBUG("wptr [%i]high== 0x%08x low==0x%08x\n",
-				me, highbit, lowbit);
+				ring->me, highbit, lowbit);
 		wptr = highbit;
 		wptr = wptr << 32;
 		wptr |= lowbit;
@@ -339,17 +341,15 @@ static void sdma_v4_0_ring_set_wptr(struct amdgpu_ring *ring)
 				ring->doorbell_index, ring->wptr << 2);
 		WDOORBELL64(ring->doorbell_index, ring->wptr << 2);
 	} else {
-		int me = (ring == &ring->adev->sdma.instance[0].ring) ? 0 : 1;
-
 		DRM_DEBUG("Not using doorbell -- "
 				"mmSDMA%i_GFX_RB_WPTR == 0x%08x "
 				"mmSDMA%i_GFX_RB_WPTR_HI == 0x%08x\n",
-				me,
+				ring->me,
 				lower_32_bits(ring->wptr << 2),
-				me,
+				ring->me,
 				upper_32_bits(ring->wptr << 2));
-		WREG32(sdma_v4_0_get_reg_offset(adev, me, mmSDMA0_GFX_RB_WPTR), lower_32_bits(ring->wptr << 2));
-		WREG32(sdma_v4_0_get_reg_offset(adev, me, mmSDMA0_GFX_RB_WPTR_HI), upper_32_bits(ring->wptr << 2));
+		WREG32(sdma_v4_0_get_reg_offset(adev, ring->me, mmSDMA0_GFX_RB_WPTR), lower_32_bits(ring->wptr << 2));
+		WREG32(sdma_v4_0_get_reg_offset(adev, ring->me, mmSDMA0_GFX_RB_WPTR_HI), upper_32_bits(ring->wptr << 2));
 	}
 }
 
@@ -430,7 +430,7 @@ static void sdma_v4_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
 	u32 ref_and_mask = 0;
 	const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio_funcs->hdp_flush_reg;
 
-	if (ring == &ring->adev->sdma.instance[0].ring)
+	if (ring->me == 0)
 		ref_and_mask = nbio_hf_reg->ref_and_mask_sdma0;
 	else
 		ref_and_mask = nbio_hf_reg->ref_and_mask_sdma1;
@@ -1228,13 +1228,13 @@ static int sdma_v4_0_sw_init(void *handle)
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
 	/* SDMA trap event */
-	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_SDMA0, 224,
+	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_SDMA0, SDMA0_4_0__SRCID__SDMA_TRAP,
 			      &adev->sdma.trap_irq);
 	if (r)
 		return r;
 
 	/* SDMA trap event */
-	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_SDMA1, 224,
+	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_SDMA1, SDMA1_4_0__SRCID__SDMA_TRAP,
 			      &adev->sdma.trap_irq);
 	if (r)
 		return r;
@@ -1651,8 +1651,10 @@ static void sdma_v4_0_set_ring_funcs(struct amdgpu_device *adev)
 {
 	int i;
 
-	for (i = 0; i < adev->sdma.num_instances; i++)
+	for (i = 0; i < adev->sdma.num_instances; i++) {
 		adev->sdma.instance[i].ring.funcs = &sdma_v4_0_ring_funcs;
+		adev->sdma.instance[i].ring.me = i;
+	}
 }
 
 static const struct amdgpu_irq_src_funcs sdma_v4_0_trap_irq_funcs = {
diff --git a/drivers/gpu/drm/amd/amdgpu/si_dpm.c b/drivers/gpu/drm/amd/amdgpu/si_dpm.c
index d51318c695e6..db327b412562 100644
--- a/drivers/gpu/drm/amd/amdgpu/si_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/si_dpm.c
@@ -56,16 +56,16 @@
 
 #define BIOS_SCRATCH_4                                    0x5cd
 
-MODULE_FIRMWARE("radeon/tahiti_smc.bin");
-MODULE_FIRMWARE("radeon/pitcairn_smc.bin");
-MODULE_FIRMWARE("radeon/pitcairn_k_smc.bin");
-MODULE_FIRMWARE("radeon/verde_smc.bin");
-MODULE_FIRMWARE("radeon/verde_k_smc.bin");
-MODULE_FIRMWARE("radeon/oland_smc.bin");
-MODULE_FIRMWARE("radeon/oland_k_smc.bin");
-MODULE_FIRMWARE("radeon/hainan_smc.bin");
-MODULE_FIRMWARE("radeon/hainan_k_smc.bin");
-MODULE_FIRMWARE("radeon/banks_k_2_smc.bin");
+MODULE_FIRMWARE("amdgpu/tahiti_smc.bin");
+MODULE_FIRMWARE("amdgpu/pitcairn_smc.bin");
+MODULE_FIRMWARE("amdgpu/pitcairn_k_smc.bin");
+MODULE_FIRMWARE("amdgpu/verde_smc.bin");
+MODULE_FIRMWARE("amdgpu/verde_k_smc.bin");
+MODULE_FIRMWARE("amdgpu/oland_smc.bin");
+MODULE_FIRMWARE("amdgpu/oland_k_smc.bin");
+MODULE_FIRMWARE("amdgpu/hainan_smc.bin");
+MODULE_FIRMWARE("amdgpu/hainan_k_smc.bin");
+MODULE_FIRMWARE("amdgpu/banks_k_2_smc.bin");
 
 static const struct amd_pm_funcs si_dpm_funcs;
 
@@ -7318,8 +7318,7 @@ static int si_dpm_init(struct amdgpu_device *adev)
 	pi = &eg_pi->rv7xx;
 
 	si_pi->sys_pcie_mask =
-		(adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_MASK) >>
-		CAIL_PCIE_LINK_SPEED_SUPPORT_SHIFT;
+		adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_MASK;
 	si_pi->force_pcie_gen = AMDGPU_PCIE_GEN_INVALID;
 	si_pi->boot_pcie_gen = si_get_current_pcie_speed(adev);
 
@@ -7667,7 +7666,7 @@ static int si_dpm_init_microcode(struct amdgpu_device *adev)
 	default: BUG();
 	}
 
-	snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
+	snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_smc.bin", chip_name);
 	err = request_firmware(&adev->pm.fw, fw_name, adev->dev);
 	if (err)
 		goto out;
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c b/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c
index 6fed3d7797a8..8a926d1df939 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c
@@ -123,6 +123,10 @@ static int uvd_v4_2_sw_init(void *handle)
 	ring = &adev->uvd.inst->ring;
 	sprintf(ring->name, "uvd");
 	r = amdgpu_ring_init(adev, ring, 512, &adev->uvd.inst->irq, 0);
+	if (r)
+		return r;
+
+	r = amdgpu_uvd_entity_init(adev);
 
 	return r;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c
index 341ee6d55ce8..50248059412e 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c
@@ -35,6 +35,7 @@
 #include "vi.h"
 #include "smu/smu_7_1_2_d.h"
 #include "smu/smu_7_1_2_sh_mask.h"
+#include "ivsrcid/ivsrcid_vislands30.h"
 
 static void uvd_v5_0_set_ring_funcs(struct amdgpu_device *adev);
 static void uvd_v5_0_set_irq_funcs(struct amdgpu_device *adev);
@@ -104,7 +105,7 @@ static int uvd_v5_0_sw_init(void *handle)
 	int r;
 
 	/* UVD TRAP */
-	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, 124, &adev->uvd.inst->irq);
+	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_UVD_SYSTEM_MESSAGE, &adev->uvd.inst->irq);
 	if (r)
 		return r;
 
@@ -119,6 +120,10 @@ static int uvd_v5_0_sw_init(void *handle)
 	ring = &adev->uvd.inst->ring;
 	sprintf(ring->name, "uvd");
 	r = amdgpu_ring_init(adev, ring, 512, &adev->uvd.inst->irq, 0);
+	if (r)
+		return r;
+
+	r = amdgpu_uvd_entity_init(adev);
 
 	return r;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
index bfddf97dd13e..6ae82cc2e55e 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
@@ -36,6 +36,7 @@
 #include "bif/bif_5_1_d.h"
 #include "gmc/gmc_8_1_d.h"
 #include "vi.h"
+#include "ivsrcid/ivsrcid_vislands30.h"
 
 /* Polaris10/11/12 firmware version */
 #define FW_1_130_16 ((1 << 24) | (130 << 16) | (16 << 8))
@@ -247,12 +248,10 @@ static int uvd_v6_0_enc_get_create_msg(struct amdgpu_ring *ring, uint32_t handle
 	for (i = ib->length_dw; i < ib_size_dw; ++i)
 		ib->ptr[i] = 0x0;
 
-	r = amdgpu_ib_schedule(ring, 1, ib, NULL, &f);
-	job->fence = dma_fence_get(f);
+	r = amdgpu_job_submit_direct(job, ring, &f);
 	if (r)
 		goto err;
 
-	amdgpu_job_free(job);
 	if (fence)
 		*fence = dma_fence_get(f);
 	dma_fence_put(f);
@@ -311,19 +310,13 @@ static int uvd_v6_0_enc_get_destroy_msg(struct amdgpu_ring *ring,
 	for (i = ib->length_dw; i < ib_size_dw; ++i)
 		ib->ptr[i] = 0x0;
 
-	if (direct) {
-		r = amdgpu_ib_schedule(ring, 1, ib, NULL, &f);
-		job->fence = dma_fence_get(f);
-		if (r)
-			goto err;
-
-		amdgpu_job_free(job);
-	} else {
-		r = amdgpu_job_submit(job, ring, &ring->adev->vce.entity,
+	if (direct)
+		r = amdgpu_job_submit_direct(job, ring, &f);
+	else
+		r = amdgpu_job_submit(job, &ring->adev->vce.entity,
 				      AMDGPU_FENCE_OWNER_UNDEFINED, &f);
-		if (r)
-			goto err;
-	}
+	if (r)
+		goto err;
 
 	if (fence)
 		*fence = dma_fence_get(f);
@@ -400,14 +393,14 @@ static int uvd_v6_0_sw_init(void *handle)
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
 	/* UVD TRAP */
-	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, 124, &adev->uvd.inst->irq);
+	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_UVD_SYSTEM_MESSAGE, &adev->uvd.inst->irq);
 	if (r)
 		return r;
 
 	/* UVD ENC TRAP */
 	if (uvd_v6_0_enc_support(adev)) {
 		for (i = 0; i < adev->uvd.num_enc_rings; ++i) {
-			r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, i + 119, &adev->uvd.inst->irq);
+			r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, i + VISLANDS30_IV_SRCID_UVD_ENC_GEN_PURP, &adev->uvd.inst->irq);
 			if (r)
 				return r;
 		}
@@ -425,16 +418,6 @@ static int uvd_v6_0_sw_init(void *handle)
 		adev->uvd.num_enc_rings = 0;
 
 		DRM_INFO("UVD ENC is disabled\n");
-	} else {
-		struct drm_sched_rq *rq;
-		ring = &adev->uvd.inst->ring_enc[0];
-		rq = &ring->sched.sched_rq[DRM_SCHED_PRIORITY_NORMAL];
-		r = drm_sched_entity_init(&ring->sched, &adev->uvd.inst->entity_enc,
-					  rq, NULL);
-		if (r) {
-			DRM_ERROR("Failed setting up UVD ENC run queue.\n");
-			return r;
-		}
 	}
 
 	r = amdgpu_uvd_resume(adev);
@@ -457,6 +440,8 @@ static int uvd_v6_0_sw_init(void *handle)
 		}
 	}
 
+	r = amdgpu_uvd_entity_init(adev);
+
 	return r;
 }
 
@@ -470,8 +455,6 @@ static int uvd_v6_0_sw_fini(void *handle)
 		return r;
 
 	if (uvd_v6_0_enc_support(adev)) {
-		drm_sched_entity_fini(&adev->uvd.inst->ring_enc[0].sched, &adev->uvd.inst->entity_enc);
-
 		for (i = 0; i < adev->uvd.num_enc_rings; ++i)
 			amdgpu_ring_fini(&adev->uvd.inst->ring_enc[i]);
 	}
@@ -1569,7 +1552,6 @@ static const struct amdgpu_ring_funcs uvd_v6_0_ring_phys_funcs = {
 static const struct amdgpu_ring_funcs uvd_v6_0_ring_vm_funcs = {
 	.type = AMDGPU_RING_TYPE_UVD,
 	.align_mask = 0xf,
-	.nop = PACKET0(mmUVD_NO_OP, 0),
 	.support_64bit_ptrs = false,
 	.get_rptr = uvd_v6_0_ring_get_rptr,
 	.get_wptr = uvd_v6_0_ring_get_wptr,
@@ -1587,7 +1569,7 @@ static const struct amdgpu_ring_funcs uvd_v6_0_ring_vm_funcs = {
 	.emit_hdp_flush = uvd_v6_0_ring_emit_hdp_flush,
 	.test_ring = uvd_v6_0_ring_test_ring,
 	.test_ib = amdgpu_uvd_ring_test_ib,
-	.insert_nop = amdgpu_ring_insert_nop,
+	.insert_nop = uvd_v6_0_ring_insert_nop,
 	.pad_ib = amdgpu_ring_generic_pad_ib,
 	.begin_use = amdgpu_uvd_ring_begin_use,
 	.end_use = amdgpu_uvd_ring_end_use,
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c
index 57d32f21b3a6..9b7f8469bc5c 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c
@@ -39,6 +39,13 @@
 #include "hdp/hdp_4_0_offset.h"
 #include "mmhub/mmhub_1_0_offset.h"
 #include "mmhub/mmhub_1_0_sh_mask.h"
+#include "ivsrcid/uvd/irqsrcs_uvd_7_0.h"
+
+#define mmUVD_PG0_CC_UVD_HARVESTING                                                                    0x00c7
+#define mmUVD_PG0_CC_UVD_HARVESTING_BASE_IDX                                                           1
+//UVD_PG0_CC_UVD_HARVESTING
+#define UVD_PG0_CC_UVD_HARVESTING__UVD_DISABLE__SHIFT                                                         0x1
+#define UVD_PG0_CC_UVD_HARVESTING__UVD_DISABLE_MASK                                                           0x00000002L
 
 #define UVD7_MAX_HW_INSTANCES_VEGA20			2
 
@@ -249,12 +256,10 @@ static int uvd_v7_0_enc_get_create_msg(struct amdgpu_ring *ring, uint32_t handle
 	for (i = ib->length_dw; i < ib_size_dw; ++i)
 		ib->ptr[i] = 0x0;
 
-	r = amdgpu_ib_schedule(ring, 1, ib, NULL, &f);
-	job->fence = dma_fence_get(f);
+	r = amdgpu_job_submit_direct(job, ring, &f);
 	if (r)
 		goto err;
 
-	amdgpu_job_free(job);
 	if (fence)
 		*fence = dma_fence_get(f);
 	dma_fence_put(f);
@@ -312,19 +317,13 @@ int uvd_v7_0_enc_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle,
 	for (i = ib->length_dw; i < ib_size_dw; ++i)
 		ib->ptr[i] = 0x0;
 
-	if (direct) {
-		r = amdgpu_ib_schedule(ring, 1, ib, NULL, &f);
-		job->fence = dma_fence_get(f);
-		if (r)
-			goto err;
-
-		amdgpu_job_free(job);
-	} else {
-		r = amdgpu_job_submit(job, ring, &ring->adev->vce.entity,
+	if (direct)
+		r = amdgpu_job_submit_direct(job, ring, &f);
+	else
+		r = amdgpu_job_submit(job, &ring->adev->vce.entity,
 				      AMDGPU_FENCE_OWNER_UNDEFINED, &f);
-		if (r)
-			goto err;
-	}
+	if (r)
+		goto err;
 
 	if (fence)
 		*fence = dma_fence_get(f);
@@ -377,10 +376,25 @@ error:
 static int uvd_v7_0_early_init(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
-	if (adev->asic_type == CHIP_VEGA20)
+
+	if (adev->asic_type == CHIP_VEGA20) {
+		u32 harvest;
+		int i;
+
 		adev->uvd.num_uvd_inst = UVD7_MAX_HW_INSTANCES_VEGA20;
-	else
+		for (i = 0; i < adev->uvd.num_uvd_inst; i++) {
+			harvest = RREG32_SOC15(UVD, i, mmUVD_PG0_CC_UVD_HARVESTING);
+			if (harvest & UVD_PG0_CC_UVD_HARVESTING__UVD_DISABLE_MASK) {
+				adev->uvd.harvest_config |= 1 << i;
+			}
+		}
+		if (adev->uvd.harvest_config == (AMDGPU_UVD_HARVEST_UVD0 |
+						 AMDGPU_UVD_HARVEST_UVD1))
+			/* both instances are harvested, disable the block */
+			return -ENOENT;
+	} else {
 		adev->uvd.num_uvd_inst = 1;
+	}
 
 	if (amdgpu_sriov_vf(adev))
 		adev->uvd.num_enc_rings = 1;
@@ -396,19 +410,21 @@ static int uvd_v7_0_early_init(void *handle)
 static int uvd_v7_0_sw_init(void *handle)
 {
 	struct amdgpu_ring *ring;
-	struct drm_sched_rq *rq;
+
 	int i, j, r;
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
 	for (j = 0; j < adev->uvd.num_uvd_inst; j++) {
+		if (adev->uvd.harvest_config & (1 << j))
+			continue;
 		/* UVD TRAP */
-		r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_uvds[j], 124, &adev->uvd.inst[j].irq);
+		r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_uvds[j], UVD_7_0__SRCID__UVD_SYSTEM_MESSAGE_INTERRUPT, &adev->uvd.inst[j].irq);
 		if (r)
 			return r;
 
 		/* UVD ENC TRAP */
 		for (i = 0; i < adev->uvd.num_enc_rings; ++i) {
-			r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_uvds[j], i + 119, &adev->uvd.inst[j].irq);
+			r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_uvds[j], i + UVD_7_0__SRCID__UVD_ENC_GEN_PURP, &adev->uvd.inst[j].irq);
 			if (r)
 				return r;
 		}
@@ -428,22 +444,13 @@ static int uvd_v7_0_sw_init(void *handle)
 		DRM_INFO("PSP loading UVD firmware\n");
 	}
 
-	for (j = 0; j < adev->uvd.num_uvd_inst; j++) {
-		ring = &adev->uvd.inst[j].ring_enc[0];
-		rq = &ring->sched.sched_rq[DRM_SCHED_PRIORITY_NORMAL];
-		r = drm_sched_entity_init(&ring->sched, &adev->uvd.inst[j].entity_enc,
-					  rq, NULL);
-		if (r) {
-			DRM_ERROR("(%d)Failed setting up UVD ENC run queue.\n", j);
-			return r;
-		}
-	}
-
 	r = amdgpu_uvd_resume(adev);
 	if (r)
 		return r;
 
 	for (j = 0; j < adev->uvd.num_uvd_inst; j++) {
+		if (adev->uvd.harvest_config & (1 << j))
+			continue;
 		if (!amdgpu_sriov_vf(adev)) {
 			ring = &adev->uvd.inst[j].ring;
 			sprintf(ring->name, "uvd<%d>", j);
@@ -472,6 +479,10 @@ static int uvd_v7_0_sw_init(void *handle)
 		}
 	}
 
+	r = amdgpu_uvd_entity_init(adev);
+	if (r)
+		return r;
+
 	r = amdgpu_virt_alloc_mm_table(adev);
 	if (r)
 		return r;
@@ -491,8 +502,8 @@ static int uvd_v7_0_sw_fini(void *handle)
 		return r;
 
 	for (j = 0; j < adev->uvd.num_uvd_inst; ++j) {
-		drm_sched_entity_fini(&adev->uvd.inst[j].ring_enc[0].sched, &adev->uvd.inst[j].entity_enc);
-
+		if (adev->uvd.harvest_config & (1 << j))
+			continue;
 		for (i = 0; i < adev->uvd.num_enc_rings; ++i)
 			amdgpu_ring_fini(&adev->uvd.inst[j].ring_enc[i]);
 	}
@@ -521,6 +532,8 @@ static int uvd_v7_0_hw_init(void *handle)
 		goto done;
 
 	for (j = 0; j < adev->uvd.num_uvd_inst; ++j) {
+		if (adev->uvd.harvest_config & (1 << j))
+			continue;
 		ring = &adev->uvd.inst[j].ring;
 
 		if (!amdgpu_sriov_vf(adev)) {
@@ -600,8 +613,11 @@ static int uvd_v7_0_hw_fini(void *handle)
 		DRM_DEBUG("For SRIOV client, shouldn't do anything.\n");
 	}
 
-	for (i = 0; i < adev->uvd.num_uvd_inst; ++i)
+	for (i = 0; i < adev->uvd.num_uvd_inst; ++i) {
+		if (adev->uvd.harvest_config & (1 << i))
+			continue;
 		adev->uvd.inst[i].ring.ready = false;
+	}
 
 	return 0;
 }
@@ -644,6 +660,8 @@ static void uvd_v7_0_mc_resume(struct amdgpu_device *adev)
 	int i;
 
 	for (i = 0; i < adev->uvd.num_uvd_inst; ++i) {
+		if (adev->uvd.harvest_config & (1 << i))
+			continue;
 		if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
 			WREG32_SOC15(UVD, i, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
 				lower_32_bits(adev->firmware.ucode[AMDGPU_UCODE_ID_UVD].mc_addr));
@@ -716,6 +734,8 @@ static int uvd_v7_0_mmsch_start(struct amdgpu_device *adev,
 	WREG32_SOC15(VCE, 0, mmVCE_MMSCH_VF_MAILBOX_RESP, 0);
 
 	for (i = 0; i < adev->uvd.num_uvd_inst; ++i) {
+		if (adev->uvd.harvest_config & (1 << i))
+			continue;
 		WDOORBELL32(adev->uvd.inst[i].ring_enc[0].doorbell_index, 0);
 		adev->wb.wb[adev->uvd.inst[i].ring_enc[0].wptr_offs] = 0;
 		adev->uvd.inst[i].ring_enc[0].wptr = 0;
@@ -772,6 +792,8 @@ static int uvd_v7_0_sriov_start(struct amdgpu_device *adev)
 		init_table += header->uvd_table_offset;
 
 		for (i = 0; i < adev->uvd.num_uvd_inst; ++i) {
+			if (adev->uvd.harvest_config & (1 << i))
+				continue;
 			ring = &adev->uvd.inst[i].ring;
 			ring->wptr = 0;
 			size = AMDGPU_GPU_PAGE_ALIGN(adev->uvd.fw->size + 4);
@@ -911,6 +933,8 @@ static int uvd_v7_0_start(struct amdgpu_device *adev)
 	int i, j, k, r;
 
 	for (k = 0; k < adev->uvd.num_uvd_inst; ++k) {
+		if (adev->uvd.harvest_config & (1 << k))
+			continue;
 		/* disable DPG */
 		WREG32_P(SOC15_REG_OFFSET(UVD, k, mmUVD_POWER_STATUS), 0,
 				~UVD_POWER_STATUS__UVD_PG_MODE_MASK);
@@ -923,6 +947,8 @@ static int uvd_v7_0_start(struct amdgpu_device *adev)
 	uvd_v7_0_mc_resume(adev);
 
 	for (k = 0; k < adev->uvd.num_uvd_inst; ++k) {
+		if (adev->uvd.harvest_config & (1 << k))
+			continue;
 		ring = &adev->uvd.inst[k].ring;
 		/* disable clock gating */
 		WREG32_P(SOC15_REG_OFFSET(UVD, k, mmUVD_CGC_CTRL), 0,
@@ -1090,6 +1116,8 @@ static void uvd_v7_0_stop(struct amdgpu_device *adev)
 	uint8_t i = 0;
 
 	for (i = 0; i < adev->uvd.num_uvd_inst; ++i) {
+		if (adev->uvd.harvest_config & (1 << i))
+			continue;
 		/* force RBC into idle state */
 		WREG32_SOC15(UVD, i, mmUVD_RBC_RB_CNTL, 0x11010101);
 
@@ -1227,6 +1255,34 @@ static int uvd_v7_0_ring_test_ring(struct amdgpu_ring *ring)
 }
 
 /**
+ * uvd_v7_0_ring_patch_cs_in_place - Patch the IB for command submission.
+ *
+ * @p: the CS parser with the IBs
+ * @ib_idx: which IB to patch
+ *
+ */
+static int uvd_v7_0_ring_patch_cs_in_place(struct amdgpu_cs_parser *p,
+					   uint32_t ib_idx)
+{
+	struct amdgpu_ib *ib = &p->job->ibs[ib_idx];
+	unsigned i;
+
+	/* No patching necessary for the first instance */
+	if (!p->ring->me)
+		return 0;
+
+	for (i = 0; i < ib->length_dw; i += 2) {
+		uint32_t reg = amdgpu_get_ib_value(p, ib_idx, i);
+
+		reg -= p->adev->reg_offset[UVD_HWIP][0][1];
+		reg += p->adev->reg_offset[UVD_HWIP][1][1];
+
+		amdgpu_set_ib_value(p, ib_idx, i, reg);
+	}
+	return 0;
+}
+
+/**
  * uvd_v7_0_ring_emit_ib - execute indirect buffer
  *
  * @ring: amdgpu_ring pointer
@@ -1718,6 +1774,7 @@ static const struct amdgpu_ring_funcs uvd_v7_0_ring_vm_funcs = {
 	.get_rptr = uvd_v7_0_ring_get_rptr,
 	.get_wptr = uvd_v7_0_ring_get_wptr,
 	.set_wptr = uvd_v7_0_ring_set_wptr,
+	.patch_cs_in_place = uvd_v7_0_ring_patch_cs_in_place,
 	.emit_frame_size =
 		6 + /* hdp invalidate */
 		SOC15_FLUSH_GPU_TLB_NUM_WREG * 6 +
@@ -1777,6 +1834,8 @@ static void uvd_v7_0_set_ring_funcs(struct amdgpu_device *adev)
 	int i;
 
 	for (i = 0; i < adev->uvd.num_uvd_inst; i++) {
+		if (adev->uvd.harvest_config & (1 << i))
+			continue;
 		adev->uvd.inst[i].ring.funcs = &uvd_v7_0_ring_vm_funcs;
 		adev->uvd.inst[i].ring.me = i;
 		DRM_INFO("UVD(%d) is enabled in VM mode\n", i);
@@ -1788,6 +1847,8 @@ static void uvd_v7_0_set_enc_ring_funcs(struct amdgpu_device *adev)
 	int i, j;
 
 	for (j = 0; j < adev->uvd.num_uvd_inst; j++) {
+		if (adev->uvd.harvest_config & (1 << j))
+			continue;
 		for (i = 0; i < adev->uvd.num_enc_rings; ++i) {
 			adev->uvd.inst[j].ring_enc[i].funcs = &uvd_v7_0_enc_ring_vm_funcs;
 			adev->uvd.inst[j].ring_enc[i].me = j;
@@ -1807,6 +1868,8 @@ static void uvd_v7_0_set_irq_funcs(struct amdgpu_device *adev)
 	int i;
 
 	for (i = 0; i < adev->uvd.num_uvd_inst; i++) {
+		if (adev->uvd.harvest_config & (1 << i))
+			continue;
 		adev->uvd.inst[i].irq.num_types = adev->uvd.num_enc_rings + 1;
 		adev->uvd.inst[i].irq.funcs = &uvd_v7_0_irq_funcs;
 	}
diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c
index 47f70827195b..7eaa54ba016b 100644
--- a/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c
@@ -56,7 +56,7 @@ static uint64_t vce_v2_0_ring_get_rptr(struct amdgpu_ring *ring)
 {
 	struct amdgpu_device *adev = ring->adev;
 
-	if (ring == &adev->vce.ring[0])
+	if (ring->me == 0)
 		return RREG32(mmVCE_RB_RPTR);
 	else
 		return RREG32(mmVCE_RB_RPTR2);
@@ -73,7 +73,7 @@ static uint64_t vce_v2_0_ring_get_wptr(struct amdgpu_ring *ring)
 {
 	struct amdgpu_device *adev = ring->adev;
 
-	if (ring == &adev->vce.ring[0])
+	if (ring->me == 0)
 		return RREG32(mmVCE_RB_WPTR);
 	else
 		return RREG32(mmVCE_RB_WPTR2);
@@ -90,7 +90,7 @@ static void vce_v2_0_ring_set_wptr(struct amdgpu_ring *ring)
 {
 	struct amdgpu_device *adev = ring->adev;
 
-	if (ring == &adev->vce.ring[0])
+	if (ring->me == 0)
 		WREG32(mmVCE_RB_WPTR, lower_32_bits(ring->wptr));
 	else
 		WREG32(mmVCE_RB_WPTR2, lower_32_bits(ring->wptr));
@@ -439,6 +439,8 @@ static int vce_v2_0_sw_init(void *handle)
 			return r;
 	}
 
+	r = amdgpu_vce_entity_init(adev);
+
 	return r;
 }
 
@@ -627,8 +629,10 @@ static void vce_v2_0_set_ring_funcs(struct amdgpu_device *adev)
 {
 	int i;
 
-	for (i = 0; i < adev->vce.num_rings; i++)
+	for (i = 0; i < adev->vce.num_rings; i++) {
 		adev->vce.ring[i].funcs = &vce_v2_0_ring_funcs;
+		adev->vce.ring[i].me = i;
+	}
 }
 
 static const struct amdgpu_irq_src_funcs vce_v2_0_irq_funcs = {
diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c
index 0999c843f623..c8390f9adfd6 100644
--- a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c
@@ -39,6 +39,7 @@
 #include "smu/smu_7_1_2_sh_mask.h"
 #include "gca/gfx_8_0_d.h"
 #include "gca/gfx_8_0_sh_mask.h"
+#include "ivsrcid/ivsrcid_vislands30.h"
 
 
 #define GRBM_GFX_INDEX__VCE_INSTANCE__SHIFT	0x04
@@ -86,9 +87,9 @@ static uint64_t vce_v3_0_ring_get_rptr(struct amdgpu_ring *ring)
 	else if (adev->vce.harvest_config == AMDGPU_VCE_HARVEST_VCE0)
 		WREG32(mmGRBM_GFX_INDEX, GET_VCE_INSTANCE(1));
 
-	if (ring == &adev->vce.ring[0])
+	if (ring->me == 0)
 		v = RREG32(mmVCE_RB_RPTR);
-	else if (ring == &adev->vce.ring[1])
+	else if (ring->me == 1)
 		v = RREG32(mmVCE_RB_RPTR2);
 	else
 		v = RREG32(mmVCE_RB_RPTR3);
@@ -118,9 +119,9 @@ static uint64_t vce_v3_0_ring_get_wptr(struct amdgpu_ring *ring)
 	else if (adev->vce.harvest_config == AMDGPU_VCE_HARVEST_VCE0)
 		WREG32(mmGRBM_GFX_INDEX, GET_VCE_INSTANCE(1));
 
-	if (ring == &adev->vce.ring[0])
+	if (ring->me == 0)
 		v = RREG32(mmVCE_RB_WPTR);
-	else if (ring == &adev->vce.ring[1])
+	else if (ring->me == 1)
 		v = RREG32(mmVCE_RB_WPTR2);
 	else
 		v = RREG32(mmVCE_RB_WPTR3);
@@ -149,9 +150,9 @@ static void vce_v3_0_ring_set_wptr(struct amdgpu_ring *ring)
 	else if (adev->vce.harvest_config == AMDGPU_VCE_HARVEST_VCE0)
 		WREG32(mmGRBM_GFX_INDEX, GET_VCE_INSTANCE(1));
 
-	if (ring == &adev->vce.ring[0])
+	if (ring->me == 0)
 		WREG32(mmVCE_RB_WPTR, lower_32_bits(ring->wptr));
-	else if (ring == &adev->vce.ring[1])
+	else if (ring->me == 1)
 		WREG32(mmVCE_RB_WPTR2, lower_32_bits(ring->wptr));
 	else
 		WREG32(mmVCE_RB_WPTR3, lower_32_bits(ring->wptr));
@@ -422,7 +423,7 @@ static int vce_v3_0_sw_init(void *handle)
 	int r, i;
 
 	/* VCE */
-	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, 167, &adev->vce.irq);
+	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_VCE_TRAP, &adev->vce.irq);
 	if (r)
 		return r;
 
@@ -447,6 +448,8 @@ static int vce_v3_0_sw_init(void *handle)
 			return r;
 	}
 
+	r = amdgpu_vce_entity_init(adev);
+
 	return r;
 }
 
@@ -900,7 +903,7 @@ static const struct amdgpu_ring_funcs vce_v3_0_ring_phys_funcs = {
 	.emit_frame_size =
 		4 + /* vce_v3_0_emit_pipeline_sync */
 		6, /* amdgpu_vce_ring_emit_fence x1 no user fence */
-	.emit_ib_size = 5, /* vce_v3_0_ring_emit_ib */
+	.emit_ib_size = 4, /* amdgpu_vce_ring_emit_ib */
 	.emit_ib = amdgpu_vce_ring_emit_ib,
 	.emit_fence = amdgpu_vce_ring_emit_fence,
 	.test_ring = amdgpu_vce_ring_test_ring,
@@ -924,7 +927,7 @@ static const struct amdgpu_ring_funcs vce_v3_0_ring_vm_funcs = {
 		6 + /* vce_v3_0_emit_vm_flush */
 		4 + /* vce_v3_0_emit_pipeline_sync */
 		6 + 6, /* amdgpu_vce_ring_emit_fence x2 vm fence */
-	.emit_ib_size = 4, /* amdgpu_vce_ring_emit_ib */
+	.emit_ib_size = 5, /* vce_v3_0_ring_emit_ib */
 	.emit_ib = vce_v3_0_ring_emit_ib,
 	.emit_vm_flush = vce_v3_0_emit_vm_flush,
 	.emit_pipeline_sync = vce_v3_0_emit_pipeline_sync,
@@ -942,12 +945,16 @@ static void vce_v3_0_set_ring_funcs(struct amdgpu_device *adev)
 	int i;
 
 	if (adev->asic_type >= CHIP_STONEY) {
-		for (i = 0; i < adev->vce.num_rings; i++)
+		for (i = 0; i < adev->vce.num_rings; i++) {
 			adev->vce.ring[i].funcs = &vce_v3_0_ring_vm_funcs;
+			adev->vce.ring[i].me = i;
+		}
 		DRM_INFO("VCE enabled in VM mode\n");
 	} else {
-		for (i = 0; i < adev->vce.num_rings; i++)
+		for (i = 0; i < adev->vce.num_rings; i++) {
 			adev->vce.ring[i].funcs = &vce_v3_0_ring_phys_funcs;
+			adev->vce.ring[i].me = i;
+		}
 		DRM_INFO("VCE enabled in physical mode\n");
 	}
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c
index 8fd1b742985a..2e4d1b5f6243 100644
--- a/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c
@@ -39,6 +39,8 @@
 #include "mmhub/mmhub_1_0_offset.h"
 #include "mmhub/mmhub_1_0_sh_mask.h"
 
+#include "ivsrcid/vce/irqsrcs_vce_4_0.h"
+
 #define VCE_STATUS_VCPU_REPORT_FW_LOADED_MASK	0x02
 
 #define VCE_V4_0_FW_SIZE	(384 * 1024)
@@ -60,9 +62,9 @@ static uint64_t vce_v4_0_ring_get_rptr(struct amdgpu_ring *ring)
 {
 	struct amdgpu_device *adev = ring->adev;
 
-	if (ring == &adev->vce.ring[0])
+	if (ring->me == 0)
 		return RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_RPTR));
-	else if (ring == &adev->vce.ring[1])
+	else if (ring->me == 1)
 		return RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_RPTR2));
 	else
 		return RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_RPTR3));
@@ -82,9 +84,9 @@ static uint64_t vce_v4_0_ring_get_wptr(struct amdgpu_ring *ring)
 	if (ring->use_doorbell)
 		return adev->wb.wb[ring->wptr_offs];
 
-	if (ring == &adev->vce.ring[0])
+	if (ring->me == 0)
 		return RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_WPTR));
-	else if (ring == &adev->vce.ring[1])
+	else if (ring->me == 1)
 		return RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_WPTR2));
 	else
 		return RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_WPTR3));
@@ -108,10 +110,10 @@ static void vce_v4_0_ring_set_wptr(struct amdgpu_ring *ring)
 		return;
 	}
 
-	if (ring == &adev->vce.ring[0])
+	if (ring->me == 0)
 		WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_WPTR),
 			lower_32_bits(ring->wptr));
-	else if (ring == &adev->vce.ring[1])
+	else if (ring->me == 1)
 		WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_WPTR2),
 			lower_32_bits(ring->wptr));
 	else
@@ -417,6 +419,7 @@ static int vce_v4_0_sw_init(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 	struct amdgpu_ring *ring;
+
 	unsigned size;
 	int r, i;
 
@@ -436,7 +439,7 @@ static int vce_v4_0_sw_init(void *handle)
 		const struct common_firmware_header *hdr;
 		unsigned size = amdgpu_bo_size(adev->vce.vcpu_bo);
 
-		adev->vce.saved_bo = kmalloc(size, GFP_KERNEL);
+		adev->vce.saved_bo = kvmalloc(size, GFP_KERNEL);
 		if (!adev->vce.saved_bo)
 			return -ENOMEM;
 
@@ -472,6 +475,11 @@ static int vce_v4_0_sw_init(void *handle)
 			return r;
 	}
 
+
+	r = amdgpu_vce_entity_init(adev);
+	if (r)
+		return r;
+
 	r = amdgpu_virt_alloc_mm_table(adev);
 	if (r)
 		return r;
@@ -488,7 +496,7 @@ static int vce_v4_0_sw_fini(void *handle)
 	amdgpu_virt_free_mm_table(adev);
 
 	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
-		kfree(adev->vce.saved_bo);
+		kvfree(adev->vce.saved_bo);
 		adev->vce.saved_bo = NULL;
 	}
 
@@ -1088,8 +1096,10 @@ static void vce_v4_0_set_ring_funcs(struct amdgpu_device *adev)
 {
 	int i;
 
-	for (i = 0; i < adev->vce.num_rings; i++)
+	for (i = 0; i < adev->vce.num_rings; i++) {
 		adev->vce.ring[i].funcs = &vce_v4_0_ring_vm_funcs;
+		adev->vce.ring[i].me = i;
+	}
 	DRM_INFO("VCE enabled in VM mode\n");
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c
index b82c92084b6f..072371ef5975 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c
@@ -35,6 +35,8 @@
 #include "mmhub/mmhub_9_1_offset.h"
 #include "mmhub/mmhub_9_1_sh_mask.h"
 
+#include "ivsrcid/vcn/irqsrcs_vcn_1_0.h"
+
 static int vcn_v1_0_stop(struct amdgpu_device *adev);
 static void vcn_v1_0_set_dec_ring_funcs(struct amdgpu_device *adev);
 static void vcn_v1_0_set_enc_ring_funcs(struct amdgpu_device *adev);
@@ -77,13 +79,13 @@ static int vcn_v1_0_sw_init(void *handle)
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
 	/* VCN DEC TRAP */
-	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN, 124, &adev->vcn.irq);
+	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN, VCN_1_0__SRCID__UVD_SYSTEM_MESSAGE_INTERRUPT, &adev->vcn.irq);
 	if (r)
 		return r;
 
 	/* VCN ENC TRAP */
 	for (i = 0; i < adev->vcn.num_enc_rings; ++i) {
-		r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN, i + 119,
+		r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN, i + VCN_1_0__SRCID__UVD_ENC_GENERAL_PURPOSE,
 					&adev->vcn.irq);
 		if (r)
 			return r;
@@ -98,6 +100,16 @@ static int vcn_v1_0_sw_init(void *handle)
 	if (r)
 		return r;
 
+	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
+		const struct common_firmware_header *hdr;
+		hdr = (const struct common_firmware_header *)adev->vcn.fw->data;
+		adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].ucode_id = AMDGPU_UCODE_ID_VCN;
+		adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].fw = adev->vcn.fw;
+		adev->firmware.fw_size +=
+			ALIGN(le32_to_cpu(hdr->ucode_size_bytes), PAGE_SIZE);
+		DRM_INFO("PSP loading VCN firmware\n");
+	}
+
 	r = amdgpu_vcn_resume(adev);
 	if (r)
 		return r;
@@ -263,26 +275,38 @@ static int vcn_v1_0_resume(void *handle)
 static void vcn_v1_0_mc_resume(struct amdgpu_device *adev)
 {
 	uint32_t size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw->size + 4);
-
-	WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
+	uint32_t offset;
+
+	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
+		WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
+			     (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].tmr_mc_addr_lo));
+		WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
+			     (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].tmr_mc_addr_hi));
+		WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_OFFSET0, 0);
+		offset = 0;
+	} else {
+		WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
 			lower_32_bits(adev->vcn.gpu_addr));
-	WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
+		WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
 			upper_32_bits(adev->vcn.gpu_addr));
-	WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_OFFSET0,
-				AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
+		offset = size;
+		WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_OFFSET0,
+			     AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
+	}
+
 	WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_SIZE0, size);
 
 	WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW,
-			lower_32_bits(adev->vcn.gpu_addr + size));
+		     lower_32_bits(adev->vcn.gpu_addr + offset));
 	WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH,
-			upper_32_bits(adev->vcn.gpu_addr + size));
+		     upper_32_bits(adev->vcn.gpu_addr + offset));
 	WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_OFFSET1, 0);
 	WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_SIZE1, AMDGPU_VCN_HEAP_SIZE);
 
 	WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW,
-			lower_32_bits(adev->vcn.gpu_addr + size + AMDGPU_VCN_HEAP_SIZE));
+		     lower_32_bits(adev->vcn.gpu_addr + offset + AMDGPU_VCN_HEAP_SIZE));
 	WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH,
-			upper_32_bits(adev->vcn.gpu_addr + size + AMDGPU_VCN_HEAP_SIZE));
+		     upper_32_bits(adev->vcn.gpu_addr + offset + AMDGPU_VCN_HEAP_SIZE));
 	WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_OFFSET2, 0);
 	WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_SIZE2,
 			AMDGPU_VCN_STACK_SIZE + (AMDGPU_VCN_SESSION_SIZE * 40));
@@ -600,12 +624,12 @@ static int vcn_v1_0_start(struct amdgpu_device *adev)
 	/* disable byte swapping */
 	lmi_swap_cntl = 0;
 
-	vcn_v1_0_mc_resume(adev);
-
 	vcn_1_0_disable_static_power_gating(adev);
 	/* disable clock gating */
 	vcn_v1_0_disable_clock_gating(adev);
 
+	vcn_v1_0_mc_resume(adev);
+
 	/* disable interupt */
 	WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_MASTINT_EN), 0,
 			~UVD_MASTINT_EN__VCPU_EN_MASK);
diff --git a/drivers/gpu/drm/amd/amdgpu/vega10_reg_init.c b/drivers/gpu/drm/amd/amdgpu/vega10_reg_init.c
index 45aafca7f315..c5c9b2bc190d 100644
--- a/drivers/gpu/drm/amd/amdgpu/vega10_reg_init.c
+++ b/drivers/gpu/drm/amd/amdgpu/vega10_reg_init.c
@@ -51,6 +51,7 @@ int vega10_reg_base_init(struct amdgpu_device *adev)
 		adev->reg_offset[PWR_HWIP][i] = (uint32_t *)(&(PWR_BASE.instance[i]));
 		adev->reg_offset[NBIF_HWIP][i] = (uint32_t *)(&(NBIF_BASE.instance[i]));
 		adev->reg_offset[THM_HWIP][i] = (uint32_t *)(&(THM_BASE.instance[i]));
+		adev->reg_offset[CLK_HWIP][i] = (uint32_t *)(&(CLK_BASE.instance[i]));
 	}
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/vega20_reg_init.c b/drivers/gpu/drm/amd/amdgpu/vega20_reg_init.c
index 52778de93ab0..2d4473557b0d 100644
--- a/drivers/gpu/drm/amd/amdgpu/vega20_reg_init.c
+++ b/drivers/gpu/drm/amd/amdgpu/vega20_reg_init.c
@@ -38,6 +38,7 @@ int vega20_reg_base_init(struct amdgpu_device *adev)
 		adev->reg_offset[ATHUB_HWIP][i] = (uint32_t *)(&(ATHUB_BASE.instance[i]));
 		adev->reg_offset[NBIO_HWIP][i] = (uint32_t *)(&(NBIO_BASE.instance[i]));
 		adev->reg_offset[MP0_HWIP][i] = (uint32_t *)(&(MP0_BASE.instance[i]));
+		adev->reg_offset[MP1_HWIP][i] = (uint32_t *)(&(MP1_BASE.instance[i]));
 		adev->reg_offset[UVD_HWIP][i] = (uint32_t *)(&(UVD_BASE.instance[i]));
 		adev->reg_offset[VCE_HWIP][i] = (uint32_t *)(&(VCE_BASE.instance[i]));
 		adev->reg_offset[DF_HWIP][i] = (uint32_t *)(&(DF_BASE.instance[i]));
@@ -46,6 +47,8 @@ int vega20_reg_base_init(struct amdgpu_device *adev)
 		adev->reg_offset[SDMA0_HWIP][i] = (uint32_t *)(&(SDMA0_BASE.instance[i]));
 		adev->reg_offset[SDMA1_HWIP][i] = (uint32_t *)(&(SDMA1_BASE.instance[i]));
 		adev->reg_offset[SMUIO_HWIP][i] = (uint32_t *)(&(SMUIO_BASE.instance[i]));
+		adev->reg_offset[NBIF_HWIP][i] = (uint32_t *)(&(NBIO_BASE.instance[i]));
+		adev->reg_offset[THM_HWIP][i] = (uint32_t *)(&(THM_BASE.instance[i]));
 	}
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c b/drivers/gpu/drm/amd/amdgpu/vi.c
index 4ac1288ab7df..88b57a5e9489 100644
--- a/drivers/gpu/drm/amd/amdgpu/vi.c
+++ b/drivers/gpu/drm/amd/amdgpu/vi.c
@@ -112,8 +112,8 @@ static u32 vi_smc_rreg(struct amdgpu_device *adev, u32 reg)
 	u32 r;
 
 	spin_lock_irqsave(&adev->smc_idx_lock, flags);
-	WREG32(mmSMC_IND_INDEX_11, (reg));
-	r = RREG32(mmSMC_IND_DATA_11);
+	WREG32_NO_KIQ(mmSMC_IND_INDEX_11, (reg));
+	r = RREG32_NO_KIQ(mmSMC_IND_DATA_11);
 	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 	return r;
 }
@@ -1363,11 +1363,11 @@ static int vi_common_set_clockgating_state_by_smu(void *handle,
 
 	if (adev->cg_flags & (AMD_CG_SUPPORT_MC_LS | AMD_CG_SUPPORT_MC_MGCG)) {
 		if (adev->cg_flags & AMD_CG_SUPPORT_MC_LS) {
-			pp_support_state = AMD_CG_SUPPORT_MC_LS;
+			pp_support_state = PP_STATE_SUPPORT_LS;
 			pp_state = PP_STATE_LS;
 		}
 		if (adev->cg_flags & AMD_CG_SUPPORT_MC_MGCG) {
-			pp_support_state |= AMD_CG_SUPPORT_MC_MGCG;
+			pp_support_state |= PP_STATE_SUPPORT_CG;
 			pp_state |= PP_STATE_CG;
 		}
 		if (state == AMD_CG_STATE_UNGATE)
@@ -1382,11 +1382,11 @@ static int vi_common_set_clockgating_state_by_smu(void *handle,
 
 	if (adev->cg_flags & (AMD_CG_SUPPORT_SDMA_LS | AMD_CG_SUPPORT_SDMA_MGCG)) {
 		if (adev->cg_flags & AMD_CG_SUPPORT_SDMA_LS) {
-			pp_support_state = AMD_CG_SUPPORT_SDMA_LS;
+			pp_support_state = PP_STATE_SUPPORT_LS;
 			pp_state = PP_STATE_LS;
 		}
 		if (adev->cg_flags & AMD_CG_SUPPORT_SDMA_MGCG) {
-			pp_support_state |= AMD_CG_SUPPORT_SDMA_MGCG;
+			pp_support_state |= PP_STATE_SUPPORT_CG;
 			pp_state |= PP_STATE_CG;
 		}
 		if (state == AMD_CG_STATE_UNGATE)
@@ -1401,11 +1401,11 @@ static int vi_common_set_clockgating_state_by_smu(void *handle,
 
 	if (adev->cg_flags & (AMD_CG_SUPPORT_HDP_LS | AMD_CG_SUPPORT_HDP_MGCG)) {
 		if (adev->cg_flags & AMD_CG_SUPPORT_HDP_LS) {
-			pp_support_state = AMD_CG_SUPPORT_HDP_LS;
+			pp_support_state = PP_STATE_SUPPORT_LS;
 			pp_state = PP_STATE_LS;
 		}
 		if (adev->cg_flags & AMD_CG_SUPPORT_HDP_MGCG) {
-			pp_support_state |= AMD_CG_SUPPORT_HDP_MGCG;
+			pp_support_state |= PP_STATE_SUPPORT_CG;
 			pp_state |= PP_STATE_CG;
 		}
 		if (state == AMD_CG_STATE_UNGATE)
diff --git a/drivers/gpu/drm/amd/amdkfd/cik_event_interrupt.c b/drivers/gpu/drm/amd/amdkfd/cik_event_interrupt.c
index 49df6c791cfc..5d2475d5392c 100644
--- a/drivers/gpu/drm/amd/amdkfd/cik_event_interrupt.c
+++ b/drivers/gpu/drm/amd/amdkfd/cik_event_interrupt.c
@@ -25,12 +25,39 @@
 #include "cik_int.h"
 
 static bool cik_event_interrupt_isr(struct kfd_dev *dev,
-					const uint32_t *ih_ring_entry)
+					const uint32_t *ih_ring_entry,
+					uint32_t *patched_ihre,
+					bool *patched_flag)
 {
 	const struct cik_ih_ring_entry *ihre =
 			(const struct cik_ih_ring_entry *)ih_ring_entry;
+	const struct kfd2kgd_calls *f2g = dev->kfd2kgd;
 	unsigned int vmid, pasid;
 
+	/* This workaround is due to HW/FW limitation on Hawaii that
+	 * VMID and PASID are not written into ih_ring_entry
+	 */
+	if ((ihre->source_id == CIK_INTSRC_GFX_PAGE_INV_FAULT ||
+		ihre->source_id == CIK_INTSRC_GFX_MEM_PROT_FAULT) &&
+		dev->device_info->asic_family == CHIP_HAWAII) {
+		struct cik_ih_ring_entry *tmp_ihre =
+			(struct cik_ih_ring_entry *)patched_ihre;
+
+		*patched_flag = true;
+		*tmp_ihre = *ihre;
+
+		vmid = f2g->read_vmid_from_vmfault_reg(dev->kgd);
+		pasid = f2g->get_atc_vmid_pasid_mapping_pasid(dev->kgd, vmid);
+
+		tmp_ihre->ring_id &= 0x000000ff;
+		tmp_ihre->ring_id |= vmid << 8;
+		tmp_ihre->ring_id |= pasid << 16;
+
+		return (pasid != 0) &&
+			vmid >= dev->vm_info.first_vmid_kfd &&
+			vmid <= dev->vm_info.last_vmid_kfd;
+	}
+
 	/* Only handle interrupts from KFD VMIDs */
 	vmid  = (ihre->ring_id & 0x0000ff00) >> 8;
 	if (vmid < dev->vm_info.first_vmid_kfd ||
@@ -48,18 +75,19 @@ static bool cik_event_interrupt_isr(struct kfd_dev *dev,
 	return ihre->source_id == CIK_INTSRC_CP_END_OF_PIPE ||
 		ihre->source_id == CIK_INTSRC_SDMA_TRAP ||
 		ihre->source_id == CIK_INTSRC_SQ_INTERRUPT_MSG ||
-		ihre->source_id == CIK_INTSRC_CP_BAD_OPCODE;
+		ihre->source_id == CIK_INTSRC_CP_BAD_OPCODE ||
+		ihre->source_id == CIK_INTSRC_GFX_PAGE_INV_FAULT ||
+		ihre->source_id == CIK_INTSRC_GFX_MEM_PROT_FAULT;
 }
 
 static void cik_event_interrupt_wq(struct kfd_dev *dev,
 					const uint32_t *ih_ring_entry)
 {
-	unsigned int pasid;
 	const struct cik_ih_ring_entry *ihre =
 			(const struct cik_ih_ring_entry *)ih_ring_entry;
 	uint32_t context_id = ihre->data & 0xfffffff;
-
-	pasid = (ihre->ring_id & 0xffff0000) >> 16;
+	unsigned int vmid  = (ihre->ring_id & 0x0000ff00) >> 8;
+	unsigned int pasid = (ihre->ring_id & 0xffff0000) >> 16;
 
 	if (pasid == 0)
 		return;
@@ -72,6 +100,22 @@ static void cik_event_interrupt_wq(struct kfd_dev *dev,
 		kfd_signal_event_interrupt(pasid, context_id & 0xff, 8);
 	else if (ihre->source_id == CIK_INTSRC_CP_BAD_OPCODE)
 		kfd_signal_hw_exception_event(pasid);
+	else if (ihre->source_id == CIK_INTSRC_GFX_PAGE_INV_FAULT ||
+		ihre->source_id == CIK_INTSRC_GFX_MEM_PROT_FAULT) {
+		struct kfd_vm_fault_info info;
+
+		kfd_process_vm_fault(dev->dqm, pasid);
+
+		memset(&info, 0, sizeof(info));
+		dev->kfd2kgd->get_vm_fault_info(dev->kgd, &info);
+		if (!info.page_addr && !info.status)
+			return;
+
+		if (info.vmid == vmid)
+			kfd_signal_vm_fault_event(dev, pasid, &info);
+		else
+			kfd_signal_vm_fault_event(dev, pasid, NULL);
+	}
 }
 
 const struct kfd_event_interrupt_class event_interrupt_class_cik = {
diff --git a/drivers/gpu/drm/amd/amdkfd/cik_int.h b/drivers/gpu/drm/amd/amdkfd/cik_int.h
index 109298b9d507..76f8677a7926 100644
--- a/drivers/gpu/drm/amd/amdkfd/cik_int.h
+++ b/drivers/gpu/drm/amd/amdkfd/cik_int.h
@@ -20,8 +20,8 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#ifndef HSA_RADEON_CIK_INT_H_INCLUDED
-#define HSA_RADEON_CIK_INT_H_INCLUDED
+#ifndef CIK_INT_H_INCLUDED
+#define CIK_INT_H_INCLUDED
 
 #include <linux/types.h>
 
@@ -34,9 +34,10 @@ struct cik_ih_ring_entry {
 
 #define CIK_INTSRC_CP_END_OF_PIPE	0xB5
 #define CIK_INTSRC_CP_BAD_OPCODE	0xB7
-#define CIK_INTSRC_DEQUEUE_COMPLETE	0xC6
 #define CIK_INTSRC_SDMA_TRAP		0xE0
 #define CIK_INTSRC_SQ_INTERRUPT_MSG	0xEF
+#define CIK_INTSRC_GFX_PAGE_INV_FAULT	0x92
+#define CIK_INTSRC_GFX_MEM_PROT_FAULT	0x93
 
 #endif
 
diff --git a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h
index f68aef02fc1f..3621efbd5759 100644
--- a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h
+++ b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h
@@ -21,18 +21,21 @@
  */
 
 static const uint32_t cwsr_trap_gfx8_hex[] = {
-	0xbf820001, 0xbf820125,
+	0xbf820001, 0xbf82012b,
 	0xb8f4f802, 0x89748674,
 	0xb8f5f803, 0x8675ff75,
-	0x00000400, 0xbf850011,
+	0x00000400, 0xbf850017,
 	0xc00a1e37, 0x00000000,
 	0xbf8c007f, 0x87777978,
-	0xbf840002, 0xb974f802,
-	0xbe801d78, 0xb8f5f803,
-	0x8675ff75, 0x000001ff,
-	0xbf850002, 0x80708470,
-	0x82718071, 0x8671ff71,
-	0x0000ffff, 0xb974f802,
+	0xbf840005, 0x8f728374,
+	0xb972e0c2, 0xbf800002,
+	0xb9740002, 0xbe801d78,
+	0xb8f5f803, 0x8675ff75,
+	0x000001ff, 0xbf850002,
+	0x80708470, 0x82718071,
+	0x8671ff71, 0x0000ffff,
+	0x8f728374, 0xb972e0c2,
+	0xbf800002, 0xb9740002,
 	0xbe801f70, 0xb8f5f803,
 	0x8675ff75, 0x00000100,
 	0xbf840006, 0xbefa0080,
@@ -168,7 +171,7 @@ static const uint32_t cwsr_trap_gfx8_hex[] = {
 	0x807c847c, 0x806eff6e,
 	0x00000400, 0xbf0a757c,
 	0xbf85ffef, 0xbf9c0000,
-	0xbf8200ca, 0xbef8007e,
+	0xbf8200cd, 0xbef8007e,
 	0x8679ff7f, 0x0000ffff,
 	0x8779ff79, 0x00040000,
 	0xbefa0080, 0xbefb00ff,
@@ -268,16 +271,18 @@ static const uint32_t cwsr_trap_gfx8_hex[] = {
 	0x8f739773, 0xb976f807,
 	0x8671ff71, 0x0000ffff,
 	0x86fe7e7e, 0x86ea6a6a,
-	0xb974f802, 0xbf8a0000,
-	0x95807370, 0xbf810000,
+	0x8f768374, 0xb976e0c2,
+	0xbf800002, 0xb9740002,
+	0xbf8a0000, 0x95807370,
+	0xbf810000, 0x00000000,
 };
 
 
 static const uint32_t cwsr_trap_gfx9_hex[] = {
-	0xbf820001, 0xbf82015a,
+	0xbf820001, 0xbf82015d,
 	0xb8f8f802, 0x89788678,
 	0xb8f1f803, 0x866eff71,
-	0x00000400, 0xbf850034,
+	0x00000400, 0xbf850037,
 	0x866eff71, 0x00000800,
 	0xbf850003, 0x866eff71,
 	0x00000100, 0xbf840008,
@@ -303,258 +308,261 @@ static const uint32_t cwsr_trap_gfx9_hex[] = {
 	0x8f6e8b77, 0x866eff6e,
 	0x001f8000, 0xb96ef807,
 	0x86fe7e7e, 0x86ea6a6a,
-	0xb978f802, 0xbe801f6c,
-	0x866dff6d, 0x0000ffff,
-	0xbef00080, 0xb9700283,
-	0xb8f02407, 0x8e709c70,
-	0x876d706d, 0xb8f003c7,
-	0x8e709b70, 0x876d706d,
-	0xb8f0f807, 0x8670ff70,
-	0x00007fff, 0xb970f807,
-	0xbeee007e, 0xbeef007f,
-	0xbefe0180, 0xbf900004,
-	0x87708478, 0xb970f802,
-	0xbf8e0002, 0xbf88fffe,
-	0xb8f02a05, 0x80708170,
-	0x8e708a70, 0xb8f11605,
-	0x80718171, 0x8e718671,
-	0x80707170, 0x80707e70,
-	0x8271807f, 0x8671ff71,
-	0x0000ffff, 0xc0471cb8,
-	0x00000040, 0xbf8cc07f,
-	0xc04b1d38, 0x00000048,
-	0xbf8cc07f, 0xc0431e78,
-	0x00000058, 0xbf8cc07f,
-	0xc0471eb8, 0x0000005c,
-	0xbf8cc07f, 0xbef4007e,
-	0x8675ff7f, 0x0000ffff,
-	0x8775ff75, 0x00040000,
-	0xbef60080, 0xbef700ff,
-	0x00807fac, 0x8670ff7f,
-	0x08000000, 0x8f708370,
-	0x87777077, 0x8670ff7f,
-	0x70000000, 0x8f708170,
-	0x87777077, 0xbefb007c,
-	0xbefa0080, 0xb8fa2a05,
-	0x807a817a, 0x8e7a8a7a,
-	0xb8f01605, 0x80708170,
-	0x8e708670, 0x807a707a,
-	0xbef60084, 0xbef600ff,
-	0x01000000, 0xbefe007c,
-	0xbefc007a, 0xc0611efa,
-	0x0000007c, 0xbf8cc07f,
-	0x807a847a, 0xbefc007e,
+	0x8f6e8378, 0xb96ee0c2,
+	0xbf800002, 0xb9780002,
+	0xbe801f6c, 0x866dff6d,
+	0x0000ffff, 0xbef00080,
+	0xb9700283, 0xb8f02407,
+	0x8e709c70, 0x876d706d,
+	0xb8f003c7, 0x8e709b70,
+	0x876d706d, 0xb8f0f807,
+	0x8670ff70, 0x00007fff,
+	0xb970f807, 0xbeee007e,
+	0xbeef007f, 0xbefe0180,
+	0xbf900004, 0x87708478,
+	0xb970f802, 0xbf8e0002,
+	0xbf88fffe, 0xb8f02a05,
+	0x80708170, 0x8e708a70,
+	0xb8f11605, 0x80718171,
+	0x8e718671, 0x80707170,
+	0x80707e70, 0x8271807f,
+	0x8671ff71, 0x0000ffff,
+	0xc0471cb8, 0x00000040,
+	0xbf8cc07f, 0xc04b1d38,
+	0x00000048, 0xbf8cc07f,
+	0xc0431e78, 0x00000058,
+	0xbf8cc07f, 0xc0471eb8,
+	0x0000005c, 0xbf8cc07f,
+	0xbef4007e, 0x8675ff7f,
+	0x0000ffff, 0x8775ff75,
+	0x00040000, 0xbef60080,
+	0xbef700ff, 0x00807fac,
+	0x8670ff7f, 0x08000000,
+	0x8f708370, 0x87777077,
+	0x8670ff7f, 0x70000000,
+	0x8f708170, 0x87777077,
+	0xbefb007c, 0xbefa0080,
+	0xb8fa2a05, 0x807a817a,
+	0x8e7a8a7a, 0xb8f01605,
+	0x80708170, 0x8e708670,
+	0x807a707a, 0xbef60084,
+	0xbef600ff, 0x01000000,
 	0xbefe007c, 0xbefc007a,
-	0xc0611b3a, 0x0000007c,
+	0xc0611efa, 0x0000007c,
 	0xbf8cc07f, 0x807a847a,
 	0xbefc007e, 0xbefe007c,
-	0xbefc007a, 0xc0611b7a,
+	0xbefc007a, 0xc0611b3a,
 	0x0000007c, 0xbf8cc07f,
 	0x807a847a, 0xbefc007e,
 	0xbefe007c, 0xbefc007a,
-	0xc0611bba, 0x0000007c,
+	0xc0611b7a, 0x0000007c,
 	0xbf8cc07f, 0x807a847a,
 	0xbefc007e, 0xbefe007c,
-	0xbefc007a, 0xc0611bfa,
+	0xbefc007a, 0xc0611bba,
 	0x0000007c, 0xbf8cc07f,
 	0x807a847a, 0xbefc007e,
 	0xbefe007c, 0xbefc007a,
-	0xc0611e3a, 0x0000007c,
-	0xbf8cc07f, 0x807a847a,
-	0xbefc007e, 0xb8f1f803,
-	0xbefe007c, 0xbefc007a,
-	0xc0611c7a, 0x0000007c,
+	0xc0611bfa, 0x0000007c,
 	0xbf8cc07f, 0x807a847a,
 	0xbefc007e, 0xbefe007c,
-	0xbefc007a, 0xc0611a3a,
+	0xbefc007a, 0xc0611e3a,
+	0x0000007c, 0xbf8cc07f,
+	0x807a847a, 0xbefc007e,
+	0xb8f1f803, 0xbefe007c,
+	0xbefc007a, 0xc0611c7a,
 	0x0000007c, 0xbf8cc07f,
 	0x807a847a, 0xbefc007e,
 	0xbefe007c, 0xbefc007a,
-	0xc0611a7a, 0x0000007c,
-	0xbf8cc07f, 0x807a847a,
-	0xbefc007e, 0xb8fbf801,
-	0xbefe007c, 0xbefc007a,
-	0xc0611efa, 0x0000007c,
+	0xc0611a3a, 0x0000007c,
 	0xbf8cc07f, 0x807a847a,
-	0xbefc007e, 0x8670ff7f,
-	0x04000000, 0xbeef0080,
-	0x876f6f70, 0xb8fa2a05,
+	0xbefc007e, 0xbefe007c,
+	0xbefc007a, 0xc0611a7a,
+	0x0000007c, 0xbf8cc07f,
+	0x807a847a, 0xbefc007e,
+	0xb8fbf801, 0xbefe007c,
+	0xbefc007a, 0xc0611efa,
+	0x0000007c, 0xbf8cc07f,
+	0x807a847a, 0xbefc007e,
+	0x8670ff7f, 0x04000000,
+	0xbeef0080, 0x876f6f70,
+	0xb8fa2a05, 0x807a817a,
+	0x8e7a8a7a, 0xb8f11605,
+	0x80718171, 0x8e718471,
+	0x8e768271, 0xbef600ff,
+	0x01000000, 0xbef20174,
+	0x80747a74, 0x82758075,
+	0xbefc0080, 0xbf800000,
+	0xbe802b00, 0xbe822b02,
+	0xbe842b04, 0xbe862b06,
+	0xbe882b08, 0xbe8a2b0a,
+	0xbe8c2b0c, 0xbe8e2b0e,
+	0xc06b003a, 0x00000000,
+	0xbf8cc07f, 0xc06b013a,
+	0x00000010, 0xbf8cc07f,
+	0xc06b023a, 0x00000020,
+	0xbf8cc07f, 0xc06b033a,
+	0x00000030, 0xbf8cc07f,
+	0x8074c074, 0x82758075,
+	0x807c907c, 0xbf0a717c,
+	0xbf85ffe7, 0xbef40172,
+	0xbefa0080, 0xbefe00c1,
+	0xbeff00c1, 0xbee80080,
+	0xbee90080, 0xbef600ff,
+	0x01000000, 0xe0724000,
+	0x7a1d0000, 0xe0724100,
+	0x7a1d0100, 0xe0724200,
+	0x7a1d0200, 0xe0724300,
+	0x7a1d0300, 0xbefe00c1,
+	0xbeff00c1, 0xb8f14306,
+	0x8671c171, 0xbf84002c,
+	0xbf8a0000, 0x8670ff6f,
+	0x04000000, 0xbf840028,
+	0x8e718671, 0x8e718271,
+	0xbef60071, 0xb8fa2a05,
 	0x807a817a, 0x8e7a8a7a,
-	0xb8f11605, 0x80718171,
-	0x8e718471, 0x8e768271,
+	0xb8f01605, 0x80708170,
+	0x8e708670, 0x807a707a,
+	0x807aff7a, 0x00000080,
 	0xbef600ff, 0x01000000,
-	0xbef20174, 0x80747a74,
-	0x82758075, 0xbefc0080,
-	0xbf800000, 0xbe802b00,
-	0xbe822b02, 0xbe842b04,
-	0xbe862b06, 0xbe882b08,
-	0xbe8a2b0a, 0xbe8c2b0c,
-	0xbe8e2b0e, 0xc06b003a,
-	0x00000000, 0xbf8cc07f,
-	0xc06b013a, 0x00000010,
-	0xbf8cc07f, 0xc06b023a,
-	0x00000020, 0xbf8cc07f,
-	0xc06b033a, 0x00000030,
-	0xbf8cc07f, 0x8074c074,
-	0x82758075, 0x807c907c,
-	0xbf0a717c, 0xbf85ffe7,
-	0xbef40172, 0xbefa0080,
+	0xbefc0080, 0xd28c0002,
+	0x000100c1, 0xd28d0003,
+	0x000204c1, 0xd1060002,
+	0x00011103, 0x7e0602ff,
+	0x00000200, 0xbefc00ff,
+	0x00010000, 0xbe800077,
+	0x8677ff77, 0xff7fffff,
+	0x8777ff77, 0x00058000,
+	0xd8ec0000, 0x00000002,
+	0xbf8cc07f, 0xe0765000,
+	0x7a1d0002, 0x68040702,
+	0xd0c9006a, 0x0000e302,
+	0xbf87fff7, 0xbef70000,
+	0xbefa00ff, 0x00000400,
 	0xbefe00c1, 0xbeff00c1,
-	0xbee80080, 0xbee90080,
+	0xb8f12a05, 0x80718171,
+	0x8e718271, 0x8e768871,
 	0xbef600ff, 0x01000000,
+	0xbefc0084, 0xbf0a717c,
+	0xbf840015, 0xbf11017c,
+	0x8071ff71, 0x00001000,
+	0x7e000300, 0x7e020301,
+	0x7e040302, 0x7e060303,
 	0xe0724000, 0x7a1d0000,
 	0xe0724100, 0x7a1d0100,
 	0xe0724200, 0x7a1d0200,
 	0xe0724300, 0x7a1d0300,
+	0x807c847c, 0x807aff7a,
+	0x00000400, 0xbf0a717c,
+	0xbf85ffef, 0xbf9c0000,
+	0xbf8200dc, 0xbef4007e,
+	0x8675ff7f, 0x0000ffff,
+	0x8775ff75, 0x00040000,
+	0xbef60080, 0xbef700ff,
+	0x00807fac, 0x866eff7f,
+	0x08000000, 0x8f6e836e,
+	0x87776e77, 0x866eff7f,
+	0x70000000, 0x8f6e816e,
+	0x87776e77, 0x866eff7f,
+	0x04000000, 0xbf84001e,
 	0xbefe00c1, 0xbeff00c1,
-	0xb8f14306, 0x8671c171,
-	0xbf84002c, 0xbf8a0000,
-	0x8670ff6f, 0x04000000,
-	0xbf840028, 0x8e718671,
-	0x8e718271, 0xbef60071,
-	0xb8fa2a05, 0x807a817a,
-	0x8e7a8a7a, 0xb8f01605,
-	0x80708170, 0x8e708670,
-	0x807a707a, 0x807aff7a,
+	0xb8ef4306, 0x866fc16f,
+	0xbf840019, 0x8e6f866f,
+	0x8e6f826f, 0xbef6006f,
+	0xb8f82a05, 0x80788178,
+	0x8e788a78, 0xb8ee1605,
+	0x806e816e, 0x8e6e866e,
+	0x80786e78, 0x8078ff78,
 	0x00000080, 0xbef600ff,
 	0x01000000, 0xbefc0080,
-	0xd28c0002, 0x000100c1,
-	0xd28d0003, 0x000204c1,
-	0xd1060002, 0x00011103,
-	0x7e0602ff, 0x00000200,
-	0xbefc00ff, 0x00010000,
-	0xbe800077, 0x8677ff77,
-	0xff7fffff, 0x8777ff77,
-	0x00058000, 0xd8ec0000,
-	0x00000002, 0xbf8cc07f,
-	0xe0765000, 0x7a1d0002,
-	0x68040702, 0xd0c9006a,
-	0x0000e302, 0xbf87fff7,
-	0xbef70000, 0xbefa00ff,
-	0x00000400, 0xbefe00c1,
-	0xbeff00c1, 0xb8f12a05,
-	0x80718171, 0x8e718271,
-	0x8e768871, 0xbef600ff,
-	0x01000000, 0xbefc0084,
-	0xbf0a717c, 0xbf840015,
-	0xbf11017c, 0x8071ff71,
-	0x00001000, 0x7e000300,
+	0xe0510000, 0x781d0000,
+	0xe0510100, 0x781d0000,
+	0x807cff7c, 0x00000200,
+	0x8078ff78, 0x00000200,
+	0xbf0a6f7c, 0xbf85fff6,
+	0xbef80080, 0xbefe00c1,
+	0xbeff00c1, 0xb8ef2a05,
+	0x806f816f, 0x8e6f826f,
+	0x8e76886f, 0xbef600ff,
+	0x01000000, 0xbeee0078,
+	0x8078ff78, 0x00000400,
+	0xbefc0084, 0xbf11087c,
+	0x806fff6f, 0x00008000,
+	0xe0524000, 0x781d0000,
+	0xe0524100, 0x781d0100,
+	0xe0524200, 0x781d0200,
+	0xe0524300, 0x781d0300,
+	0xbf8c0f70, 0x7e000300,
 	0x7e020301, 0x7e040302,
-	0x7e060303, 0xe0724000,
-	0x7a1d0000, 0xe0724100,
-	0x7a1d0100, 0xe0724200,
-	0x7a1d0200, 0xe0724300,
-	0x7a1d0300, 0x807c847c,
-	0x807aff7a, 0x00000400,
-	0xbf0a717c, 0xbf85ffef,
-	0xbf9c0000, 0xbf8200d9,
-	0xbef4007e, 0x8675ff7f,
-	0x0000ffff, 0x8775ff75,
-	0x00040000, 0xbef60080,
-	0xbef700ff, 0x00807fac,
-	0x866eff7f, 0x08000000,
-	0x8f6e836e, 0x87776e77,
-	0x866eff7f, 0x70000000,
-	0x8f6e816e, 0x87776e77,
-	0x866eff7f, 0x04000000,
-	0xbf84001e, 0xbefe00c1,
-	0xbeff00c1, 0xb8ef4306,
-	0x866fc16f, 0xbf840019,
-	0x8e6f866f, 0x8e6f826f,
-	0xbef6006f, 0xb8f82a05,
+	0x7e060303, 0x807c847c,
+	0x8078ff78, 0x00000400,
+	0xbf0a6f7c, 0xbf85ffee,
+	0xbf9c0000, 0xe0524000,
+	0x6e1d0000, 0xe0524100,
+	0x6e1d0100, 0xe0524200,
+	0x6e1d0200, 0xe0524300,
+	0x6e1d0300, 0xb8f82a05,
 	0x80788178, 0x8e788a78,
 	0xb8ee1605, 0x806e816e,
 	0x8e6e866e, 0x80786e78,
-	0x8078ff78, 0x00000080,
-	0xbef600ff, 0x01000000,
-	0xbefc0080, 0xe0510000,
-	0x781d0000, 0xe0510100,
-	0x781d0000, 0x807cff7c,
-	0x00000200, 0x8078ff78,
-	0x00000200, 0xbf0a6f7c,
-	0xbf85fff6, 0xbef80080,
-	0xbefe00c1, 0xbeff00c1,
-	0xb8ef2a05, 0x806f816f,
-	0x8e6f826f, 0x8e76886f,
-	0xbef600ff, 0x01000000,
-	0xbeee0078, 0x8078ff78,
-	0x00000400, 0xbefc0084,
-	0xbf11087c, 0x806fff6f,
-	0x00008000, 0xe0524000,
-	0x781d0000, 0xe0524100,
-	0x781d0100, 0xe0524200,
-	0x781d0200, 0xe0524300,
-	0x781d0300, 0xbf8c0f70,
-	0x7e000300, 0x7e020301,
-	0x7e040302, 0x7e060303,
-	0x807c847c, 0x8078ff78,
-	0x00000400, 0xbf0a6f7c,
-	0xbf85ffee, 0xbf9c0000,
-	0xe0524000, 0x6e1d0000,
-	0xe0524100, 0x6e1d0100,
-	0xe0524200, 0x6e1d0200,
-	0xe0524300, 0x6e1d0300,
+	0x80f8c078, 0xb8ef1605,
+	0x806f816f, 0x8e6f846f,
+	0x8e76826f, 0xbef600ff,
+	0x01000000, 0xbefc006f,
+	0xc031003a, 0x00000078,
+	0x80f8c078, 0xbf8cc07f,
+	0x80fc907c, 0xbf800000,
+	0xbe802d00, 0xbe822d02,
+	0xbe842d04, 0xbe862d06,
+	0xbe882d08, 0xbe8a2d0a,
+	0xbe8c2d0c, 0xbe8e2d0e,
+	0xbf06807c, 0xbf84fff0,
 	0xb8f82a05, 0x80788178,
 	0x8e788a78, 0xb8ee1605,
 	0x806e816e, 0x8e6e866e,
-	0x80786e78, 0x80f8c078,
-	0xb8ef1605, 0x806f816f,
-	0x8e6f846f, 0x8e76826f,
+	0x80786e78, 0xbef60084,
 	0xbef600ff, 0x01000000,
-	0xbefc006f, 0xc031003a,
-	0x00000078, 0x80f8c078,
-	0xbf8cc07f, 0x80fc907c,
-	0xbf800000, 0xbe802d00,
-	0xbe822d02, 0xbe842d04,
-	0xbe862d06, 0xbe882d08,
-	0xbe8a2d0a, 0xbe8c2d0c,
-	0xbe8e2d0e, 0xbf06807c,
-	0xbf84fff0, 0xb8f82a05,
-	0x80788178, 0x8e788a78,
-	0xb8ee1605, 0x806e816e,
-	0x8e6e866e, 0x80786e78,
-	0xbef60084, 0xbef600ff,
-	0x01000000, 0xc0211bfa,
+	0xc0211bfa, 0x00000078,
+	0x80788478, 0xc0211b3a,
 	0x00000078, 0x80788478,
-	0xc0211b3a, 0x00000078,
-	0x80788478, 0xc0211b7a,
+	0xc0211b7a, 0x00000078,
+	0x80788478, 0xc0211eba,
 	0x00000078, 0x80788478,
-	0xc0211eba, 0x00000078,
-	0x80788478, 0xc0211efa,
+	0xc0211efa, 0x00000078,
+	0x80788478, 0xc0211c3a,
 	0x00000078, 0x80788478,
-	0xc0211c3a, 0x00000078,
-	0x80788478, 0xc0211c7a,
+	0xc0211c7a, 0x00000078,
+	0x80788478, 0xc0211a3a,
 	0x00000078, 0x80788478,
-	0xc0211a3a, 0x00000078,
-	0x80788478, 0xc0211a7a,
+	0xc0211a7a, 0x00000078,
+	0x80788478, 0xc0211cfa,
 	0x00000078, 0x80788478,
-	0xc0211cfa, 0x00000078,
-	0x80788478, 0xbf8cc07f,
-	0xbefc006f, 0xbefe007a,
-	0xbeff007b, 0x866f71ff,
-	0x000003ff, 0xb96f4803,
-	0x866f71ff, 0xfffff800,
-	0x8f6f8b6f, 0xb96fa2c3,
-	0xb973f801, 0xb8ee2a05,
-	0x806e816e, 0x8e6e8a6e,
-	0xb8ef1605, 0x806f816f,
-	0x8e6f866f, 0x806e6f6e,
-	0x806e746e, 0x826f8075,
-	0x866fff6f, 0x0000ffff,
-	0xc0071cb7, 0x00000040,
-	0xc00b1d37, 0x00000048,
-	0xc0031e77, 0x00000058,
-	0xc0071eb7, 0x0000005c,
-	0xbf8cc07f, 0x866fff6d,
-	0xf0000000, 0x8f6f9c6f,
-	0x8e6f906f, 0xbeee0080,
-	0x876e6f6e, 0x866fff6d,
-	0x08000000, 0x8f6f9b6f,
-	0x8e6f8f6f, 0x876e6f6e,
-	0x866fff70, 0x00800000,
-	0x8f6f976f, 0xb96ef807,
-	0x866dff6d, 0x0000ffff,
-	0x86fe7e7e, 0x86ea6a6a,
-	0xb970f802, 0xbf8a0000,
+	0xbf8cc07f, 0xbefc006f,
+	0xbefe007a, 0xbeff007b,
+	0x866f71ff, 0x000003ff,
+	0xb96f4803, 0x866f71ff,
+	0xfffff800, 0x8f6f8b6f,
+	0xb96fa2c3, 0xb973f801,
+	0xb8ee2a05, 0x806e816e,
+	0x8e6e8a6e, 0xb8ef1605,
+	0x806f816f, 0x8e6f866f,
+	0x806e6f6e, 0x806e746e,
+	0x826f8075, 0x866fff6f,
+	0x0000ffff, 0xc0071cb7,
+	0x00000040, 0xc00b1d37,
+	0x00000048, 0xc0031e77,
+	0x00000058, 0xc0071eb7,
+	0x0000005c, 0xbf8cc07f,
+	0x866fff6d, 0xf0000000,
+	0x8f6f9c6f, 0x8e6f906f,
+	0xbeee0080, 0x876e6f6e,
+	0x866fff6d, 0x08000000,
+	0x8f6f9b6f, 0x8e6f8f6f,
+	0x876e6f6e, 0x866fff70,
+	0x00800000, 0x8f6f976f,
+	0xb96ef807, 0x866dff6d,
+	0x0000ffff, 0x86fe7e7e,
+	0x86ea6a6a, 0x8f6e8370,
+	0xb96ee0c2, 0xbf800002,
+	0xb9700002, 0xbf8a0000,
 	0x95806f6c, 0xbf810000,
 };
diff --git a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx8.asm b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx8.asm
index a2a04bb64096..abe1a5da29fb 100644
--- a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx8.asm
+++ b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx8.asm
@@ -103,6 +103,10 @@ var SQ_WAVE_STATUS_INST_ATC_SHIFT  = 23
 var SQ_WAVE_STATUS_INST_ATC_MASK   = 0x00800000
 var SQ_WAVE_STATUS_SPI_PRIO_SHIFT  = 1
 var SQ_WAVE_STATUS_SPI_PRIO_MASK   = 0x00000006
+var SQ_WAVE_STATUS_PRE_SPI_PRIO_SHIFT   = 0
+var SQ_WAVE_STATUS_PRE_SPI_PRIO_SIZE    = 1
+var SQ_WAVE_STATUS_POST_SPI_PRIO_SHIFT  = 3
+var SQ_WAVE_STATUS_POST_SPI_PRIO_SIZE   = 29
 
 var SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT    = 12
 var SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE     = 9
@@ -251,7 +255,7 @@ if (!EMU_RUN_HACK)
     s_waitcnt lgkmcnt(0)
     s_or_b32        ttmp7, ttmp8, ttmp9
     s_cbranch_scc0  L_NO_NEXT_TRAP //next level trap handler not been set
-    s_setreg_b32    hwreg(HW_REG_STATUS), s_save_status //restore HW status(SCC)
+    set_status_without_spi_prio(s_save_status, ttmp2) //restore HW status(SCC)
     s_setpc_b64     [ttmp8,ttmp9] //jump to next level trap handler
 
 L_NO_NEXT_TRAP:
@@ -262,7 +266,7 @@ L_NO_NEXT_TRAP:
     s_addc_u32  ttmp1, ttmp1, 0
 L_EXCP_CASE:
     s_and_b32   ttmp1, ttmp1, 0xFFFF
-    s_setreg_b32    hwreg(HW_REG_STATUS), s_save_status //restore HW status(SCC)
+    set_status_without_spi_prio(s_save_status, ttmp2) //restore HW status(SCC)
     s_rfe_b64       [ttmp0, ttmp1]
 end
     // *********        End handling of non-CWSR traps   *******************
@@ -1053,7 +1057,7 @@ end
     s_and_b32 s_restore_pc_hi, s_restore_pc_hi, 0x0000ffff      //pc[47:32]        //Do it here in order not to affect STATUS
     s_and_b64    exec, exec, exec  // Restore STATUS.EXECZ, not writable by s_setreg_b32
     s_and_b64    vcc, vcc, vcc  // Restore STATUS.VCCZ, not writable by s_setreg_b32
-    s_setreg_b32    hwreg(HW_REG_STATUS),   s_restore_status     // SCC is included, which is changed by previous salu
+    set_status_without_spi_prio(s_restore_status, s_restore_tmp) // SCC is included, which is changed by previous salu
 
     s_barrier                                                   //barrier to ensure the readiness of LDS before access attempts from any other wave in the same TG //FIXME not performance-optimal at this time
 
@@ -1134,3 +1138,11 @@ end
 function get_hwreg_size_bytes
     return 128 //HWREG size 128 bytes
 end
+
+function set_status_without_spi_prio(status, tmp)
+    // Do not restore STATUS.SPI_PRIO since scheduler may have raised it.
+    s_lshr_b32      tmp, status, SQ_WAVE_STATUS_POST_SPI_PRIO_SHIFT
+    s_setreg_b32    hwreg(HW_REG_STATUS, SQ_WAVE_STATUS_POST_SPI_PRIO_SHIFT, SQ_WAVE_STATUS_POST_SPI_PRIO_SIZE), tmp
+    s_nop           0x2 // avoid S_SETREG => S_SETREG hazard
+    s_setreg_b32    hwreg(HW_REG_STATUS, SQ_WAVE_STATUS_PRE_SPI_PRIO_SHIFT, SQ_WAVE_STATUS_PRE_SPI_PRIO_SIZE), status
+end
diff --git a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx9.asm b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx9.asm
index 998be96be736..0bb9c577b3a2 100644
--- a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx9.asm
+++ b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx9.asm
@@ -103,6 +103,10 @@ var SQ_WAVE_STATUS_INST_ATC_MASK   = 0x00800000
 var SQ_WAVE_STATUS_SPI_PRIO_SHIFT  = 1
 var SQ_WAVE_STATUS_SPI_PRIO_MASK   = 0x00000006
 var SQ_WAVE_STATUS_HALT_MASK       = 0x2000
+var SQ_WAVE_STATUS_PRE_SPI_PRIO_SHIFT   = 0
+var SQ_WAVE_STATUS_PRE_SPI_PRIO_SIZE    = 1
+var SQ_WAVE_STATUS_POST_SPI_PRIO_SHIFT  = 3
+var SQ_WAVE_STATUS_POST_SPI_PRIO_SIZE   = 29
 
 var SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT	= 12
 var SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE	= 9
@@ -317,7 +321,7 @@ L_EXCP_CASE:
     // Restore SQ_WAVE_STATUS.
     s_and_b64       exec, exec, exec // Restore STATUS.EXECZ, not writable by s_setreg_b32
     s_and_b64       vcc, vcc, vcc    // Restore STATUS.VCCZ, not writable by s_setreg_b32
-    s_setreg_b32    hwreg(HW_REG_STATUS), s_save_status
+    set_status_without_spi_prio(s_save_status, ttmp2)
 
     s_rfe_b64       [ttmp0, ttmp1]
 end
@@ -1120,7 +1124,7 @@ end
     s_and_b32 s_restore_pc_hi, s_restore_pc_hi, 0x0000ffff	//pc[47:32]	   //Do it here in order not to affect STATUS
     s_and_b64	 exec, exec, exec  // Restore STATUS.EXECZ, not writable by s_setreg_b32
     s_and_b64	 vcc, vcc, vcc	// Restore STATUS.VCCZ, not writable by s_setreg_b32
-    s_setreg_b32    hwreg(HW_REG_STATUS),   s_restore_status	 // SCC is included, which is changed by previous salu
+    set_status_without_spi_prio(s_restore_status, s_restore_tmp) // SCC is included, which is changed by previous salu
 
     s_barrier							//barrier to ensure the readiness of LDS before access attempts from any other wave in the same TG //FIXME not performance-optimal at this time
 
@@ -1212,3 +1216,11 @@ function ack_sqc_store_workaround
         s_waitcnt lgkmcnt(0)
     end
 end
+
+function set_status_without_spi_prio(status, tmp)
+    // Do not restore STATUS.SPI_PRIO since scheduler may have raised it.
+    s_lshr_b32      tmp, status, SQ_WAVE_STATUS_POST_SPI_PRIO_SHIFT
+    s_setreg_b32    hwreg(HW_REG_STATUS, SQ_WAVE_STATUS_POST_SPI_PRIO_SHIFT, SQ_WAVE_STATUS_POST_SPI_PRIO_SIZE), tmp
+    s_nop           0x2 // avoid S_SETREG => S_SETREG hazard
+    s_setreg_b32    hwreg(HW_REG_STATUS, SQ_WAVE_STATUS_PRE_SPI_PRIO_SHIFT, SQ_WAVE_STATUS_PRE_SPI_PRIO_SIZE), status
+end
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
index f64c5551cdba..297b36c26a05 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
@@ -122,6 +122,9 @@ static int kfd_open(struct inode *inode, struct file *filep)
 	if (IS_ERR(process))
 		return PTR_ERR(process);
 
+	if (kfd_is_locked())
+		return -EAGAIN;
+
 	dev_dbg(kfd_device, "process %d opened, compat mode (32 bit) - %d\n",
 		process->pasid, process->is_32bit_user_mode);
 
@@ -389,6 +392,61 @@ static int kfd_ioctl_update_queue(struct file *filp, struct kfd_process *p,
 	return retval;
 }
 
+static int kfd_ioctl_set_cu_mask(struct file *filp, struct kfd_process *p,
+					void *data)
+{
+	int retval;
+	const int max_num_cus = 1024;
+	struct kfd_ioctl_set_cu_mask_args *args = data;
+	struct queue_properties properties;
+	uint32_t __user *cu_mask_ptr = (uint32_t __user *)args->cu_mask_ptr;
+	size_t cu_mask_size = sizeof(uint32_t) * (args->num_cu_mask / 32);
+
+	if ((args->num_cu_mask % 32) != 0) {
+		pr_debug("num_cu_mask 0x%x must be a multiple of 32",
+				args->num_cu_mask);
+		return -EINVAL;
+	}
+
+	properties.cu_mask_count = args->num_cu_mask;
+	if (properties.cu_mask_count == 0) {
+		pr_debug("CU mask cannot be 0");
+		return -EINVAL;
+	}
+
+	/* To prevent an unreasonably large CU mask size, set an arbitrary
+	 * limit of max_num_cus bits.  We can then just drop any CU mask bits
+	 * past max_num_cus bits and just use the first max_num_cus bits.
+	 */
+	if (properties.cu_mask_count > max_num_cus) {
+		pr_debug("CU mask cannot be greater than 1024 bits");
+		properties.cu_mask_count = max_num_cus;
+		cu_mask_size = sizeof(uint32_t) * (max_num_cus/32);
+	}
+
+	properties.cu_mask = kzalloc(cu_mask_size, GFP_KERNEL);
+	if (!properties.cu_mask)
+		return -ENOMEM;
+
+	retval = copy_from_user(properties.cu_mask, cu_mask_ptr, cu_mask_size);
+	if (retval) {
+		pr_debug("Could not copy CU mask from userspace");
+		kfree(properties.cu_mask);
+		return -EFAULT;
+	}
+
+	mutex_lock(&p->mutex);
+
+	retval = pqm_set_cu_mask(&p->pqm, args->queue_id, &properties);
+
+	mutex_unlock(&p->mutex);
+
+	if (retval)
+		kfree(properties.cu_mask);
+
+	return retval;
+}
+
 static int kfd_ioctl_set_memory_policy(struct file *filep,
 					struct kfd_process *p, void *data)
 {
@@ -754,7 +812,6 @@ static int kfd_ioctl_get_clock_counters(struct file *filep,
 {
 	struct kfd_ioctl_get_clock_counters_args *args = data;
 	struct kfd_dev *dev;
-	struct timespec64 time;
 
 	dev = kfd_device_by_id(args->gpu_id);
 	if (dev)
@@ -766,11 +823,8 @@ static int kfd_ioctl_get_clock_counters(struct file *filep,
 		args->gpu_clock_counter = 0;
 
 	/* No access to rdtsc. Using raw monotonic time */
-	getrawmonotonic64(&time);
-	args->cpu_clock_counter = (uint64_t)timespec64_to_ns(&time);
-
-	get_monotonic_boottime64(&time);
-	args->system_clock_counter = (uint64_t)timespec64_to_ns(&time);
+	args->cpu_clock_counter = ktime_get_raw_ns();
+	args->system_clock_counter = ktime_get_boot_ns();
 
 	/* Since the counter is in nano-seconds we use 1GHz frequency */
 	args->system_clock_freq = 1000000000;
@@ -1558,6 +1612,9 @@ static const struct amdkfd_ioctl_desc amdkfd_ioctls[] = {
 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU,
 			kfd_ioctl_unmap_memory_from_gpu, 0),
 
+	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_CU_MASK,
+			kfd_ioctl_set_cu_mask, 0),
+
 };
 
 #define AMDKFD_CORE_IOCTL_COUNT	ARRAY_SIZE(amdkfd_ioctls)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
index 296b3f230280..ee4996029a86 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
@@ -189,6 +189,21 @@ static int kfd_parse_subtype_cu(struct crat_subtype_computeunit *cu,
 	return 0;
 }
 
+static struct kfd_mem_properties *
+find_subtype_mem(uint32_t heap_type, uint32_t flags, uint32_t width,
+		struct kfd_topology_device *dev)
+{
+	struct kfd_mem_properties *props;
+
+	list_for_each_entry(props, &dev->mem_props, list) {
+		if (props->heap_type == heap_type
+				&& props->flags == flags
+				&& props->width == width)
+			return props;
+	}
+
+	return NULL;
+}
 /* kfd_parse_subtype_mem - parse memory subtypes and attach it to correct
  * topology device present in the device_list
  */
@@ -197,36 +212,56 @@ static int kfd_parse_subtype_mem(struct crat_subtype_memory *mem,
 {
 	struct kfd_mem_properties *props;
 	struct kfd_topology_device *dev;
+	uint32_t heap_type;
+	uint64_t size_in_bytes;
+	uint32_t flags = 0;
+	uint32_t width;
 
 	pr_debug("Found memory entry in CRAT table with proximity_domain=%d\n",
 			mem->proximity_domain);
 	list_for_each_entry(dev, device_list, list) {
 		if (mem->proximity_domain == dev->proximity_domain) {
-			props = kfd_alloc_struct(props);
-			if (!props)
-				return -ENOMEM;
-
 			/* We're on GPU node */
 			if (dev->node_props.cpu_cores_count == 0) {
 				/* APU */
 				if (mem->visibility_type == 0)
-					props->heap_type =
+					heap_type =
 						HSA_MEM_HEAP_TYPE_FB_PRIVATE;
 				/* dGPU */
 				else
-					props->heap_type = mem->visibility_type;
+					heap_type = mem->visibility_type;
 			} else
-				props->heap_type = HSA_MEM_HEAP_TYPE_SYSTEM;
+				heap_type = HSA_MEM_HEAP_TYPE_SYSTEM;
 
 			if (mem->flags & CRAT_MEM_FLAGS_HOT_PLUGGABLE)
-				props->flags |= HSA_MEM_FLAGS_HOT_PLUGGABLE;
+				flags |= HSA_MEM_FLAGS_HOT_PLUGGABLE;
 			if (mem->flags & CRAT_MEM_FLAGS_NON_VOLATILE)
-				props->flags |= HSA_MEM_FLAGS_NON_VOLATILE;
+				flags |= HSA_MEM_FLAGS_NON_VOLATILE;
 
-			props->size_in_bytes =
+			size_in_bytes =
 				((uint64_t)mem->length_high << 32) +
 							mem->length_low;
-			props->width = mem->width;
+			width = mem->width;
+
+			/* Multiple banks of the same type are aggregated into
+			 * one. User mode doesn't care about multiple physical
+			 * memory segments. It's managed as a single virtual
+			 * heap for user mode.
+			 */
+			props = find_subtype_mem(heap_type, flags, width, dev);
+			if (props) {
+				props->size_in_bytes += size_in_bytes;
+				break;
+			}
+
+			props = kfd_alloc_struct(props);
+			if (!props)
+				return -ENOMEM;
+
+			props->heap_type = heap_type;
+			props->flags = flags;
+			props->size_in_bytes = size_in_bytes;
+			props->width = width;
 
 			dev->node_props.mem_banks_count++;
 			list_add_tail(&props->list, &dev->mem_props);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.c b/drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.c
index afb26f205d29..a3441b0e385b 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.c
@@ -38,7 +38,6 @@
 #include "kfd_dbgmgr.h"
 #include "kfd_dbgdev.h"
 #include "kfd_device_queue_manager.h"
-#include "../../radeon/cik_reg.h"
 
 static void dbgdev_address_watch_disable_nodiq(struct kfd_dev *dev)
 {
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.h b/drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.h
index 03424c20920c..0619c777b47e 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.h
@@ -60,6 +60,9 @@ enum {
 	SH_REG_SIZE = SH_REG_END - SH_REG_BASE
 };
 
+/* SQ_CMD definitions */
+#define SQ_CMD						0x8DEC
+
 enum SQ_IND_CMD_CMD {
 	SQ_IND_CMD_CMD_NULL = 0x00000000,
 	SQ_IND_CMD_CMD_HALT = 0x00000001,
@@ -190,4 +193,38 @@ union ULARGE_INTEGER {
 void kfd_dbgdev_init(struct kfd_dbgdev *pdbgdev, struct kfd_dev *pdev,
 			enum DBGDEV_TYPE type);
 
+union TCP_WATCH_CNTL_BITS {
+	struct {
+		uint32_t mask:24;
+		uint32_t vmid:4;
+		uint32_t atc:1;
+		uint32_t mode:2;
+		uint32_t valid:1;
+	} bitfields, bits;
+	uint32_t u32All;
+	signed int i32All;
+	float f32All;
+};
+
+enum {
+	ADDRESS_WATCH_REG_CNTL_ATC_BIT = 0x10000000UL,
+	ADDRESS_WATCH_REG_CNTL_DEFAULT_MASK = 0x00FFFFFF,
+	ADDRESS_WATCH_REG_ADDLOW_MASK_EXTENSION = 0x03000000,
+	/* extend the mask to 26 bits in order to match the low address field */
+	ADDRESS_WATCH_REG_ADDLOW_SHIFT = 6,
+	ADDRESS_WATCH_REG_ADDHIGH_MASK = 0xFFFF
+};
+
+enum {
+	MAX_TRAPID = 8,		/* 3 bits in the bitfield. */
+	MAX_WATCH_ADDRESSES = 4
+};
+
+enum {
+	ADDRESS_WATCH_REG_ADDR_HI = 0,
+	ADDRESS_WATCH_REG_ADDR_LO,
+	ADDRESS_WATCH_REG_CNTL,
+	ADDRESS_WATCH_REG_MAX
+};
+
 #endif	/* KFD_DBGDEV_H_ */
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_debugfs.c b/drivers/gpu/drm/amd/amdkfd/kfd_debugfs.c
index 4bd6ebfaf425..ab37d36d9cd6 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_debugfs.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_debugfs.c
@@ -21,6 +21,8 @@
  */
 
 #include <linux/debugfs.h>
+#include <linux/uaccess.h>
+
 #include "kfd_priv.h"
 
 static struct dentry *debugfs_root;
@@ -32,6 +34,38 @@ static int kfd_debugfs_open(struct inode *inode, struct file *file)
 	return single_open(file, show, NULL);
 }
 
+static ssize_t kfd_debugfs_hang_hws_write(struct file *file,
+	const char __user *user_buf, size_t size, loff_t *ppos)
+{
+	struct kfd_dev *dev;
+	char tmp[16];
+	uint32_t gpu_id;
+	int ret = -EINVAL;
+
+	memset(tmp, 0, 16);
+	if (size >= 16) {
+		pr_err("Invalid input for gpu id.\n");
+		goto out;
+	}
+	if (copy_from_user(tmp, user_buf, size)) {
+		ret = -EFAULT;
+		goto out;
+	}
+	if (kstrtoint(tmp, 10, &gpu_id)) {
+		pr_err("Invalid input for gpu id.\n");
+		goto out;
+	}
+	dev = kfd_device_by_id(gpu_id);
+	if (dev) {
+		kfd_debugfs_hang_hws(dev);
+		ret = size;
+	} else
+		pr_err("Cannot find device %d.\n", gpu_id);
+
+out:
+	return ret;
+}
+
 static const struct file_operations kfd_debugfs_fops = {
 	.owner = THIS_MODULE,
 	.open = kfd_debugfs_open,
@@ -40,6 +74,15 @@ static const struct file_operations kfd_debugfs_fops = {
 	.release = single_release,
 };
 
+static const struct file_operations kfd_debugfs_hang_hws_fops = {
+	.owner = THIS_MODULE,
+	.open = kfd_debugfs_open,
+	.read = seq_read,
+	.write = kfd_debugfs_hang_hws_write,
+	.llseek = seq_lseek,
+	.release = single_release,
+};
+
 void kfd_debugfs_init(void)
 {
 	struct dentry *ent;
@@ -65,6 +108,11 @@ void kfd_debugfs_init(void)
 	ent = debugfs_create_file("rls", S_IFREG | 0444, debugfs_root,
 				  kfd_debugfs_rls_by_device,
 				  &kfd_debugfs_fops);
+
+	ent = debugfs_create_file("hang_hws", S_IFREG | 0644, debugfs_root,
+				  NULL,
+				  &kfd_debugfs_hang_hws_fops);
+
 	if (!ent)
 		pr_warn("Failed to create rls in kfd debugfs\n");
 }
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
index 7ee6cec2c060..1b048715ab8a 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
@@ -30,7 +30,13 @@
 #include "kfd_iommu.h"
 
 #define MQD_SIZE_ALIGNED 768
-static atomic_t kfd_device_suspended = ATOMIC_INIT(0);
+
+/*
+ * kfd_locked is used to lock the kfd driver during suspend or reset
+ * once locked, kfd driver will stop any further GPU execution.
+ * create process (open) will return -EAGAIN.
+ */
+static atomic_t kfd_locked = ATOMIC_INIT(0);
 
 #ifdef KFD_SUPPORT_IOMMU_V2
 static const struct kfd_device_info kaveri_device_info = {
@@ -46,6 +52,7 @@ static const struct kfd_device_info kaveri_device_info = {
 	.supports_cwsr = false,
 	.needs_iommu_device = true,
 	.needs_pci_atomics = false,
+	.num_sdma_engines = 2,
 };
 
 static const struct kfd_device_info carrizo_device_info = {
@@ -61,6 +68,22 @@ static const struct kfd_device_info carrizo_device_info = {
 	.supports_cwsr = true,
 	.needs_iommu_device = true,
 	.needs_pci_atomics = false,
+	.num_sdma_engines = 2,
+};
+
+static const struct kfd_device_info raven_device_info = {
+	.asic_family = CHIP_RAVEN,
+	.max_pasid_bits = 16,
+	.max_no_of_hqd  = 24,
+	.doorbell_size  = 8,
+	.ih_ring_entry_size = 8 * sizeof(uint32_t),
+	.event_interrupt_class = &event_interrupt_class_v9,
+	.num_of_watch_points = 4,
+	.mqd_size_aligned = MQD_SIZE_ALIGNED,
+	.supports_cwsr = true,
+	.needs_iommu_device = true,
+	.needs_pci_atomics = true,
+	.num_sdma_engines = 1,
 };
 #endif
 
@@ -77,6 +100,7 @@ static const struct kfd_device_info hawaii_device_info = {
 	.supports_cwsr = false,
 	.needs_iommu_device = false,
 	.needs_pci_atomics = false,
+	.num_sdma_engines = 2,
 };
 
 static const struct kfd_device_info tonga_device_info = {
@@ -91,6 +115,7 @@ static const struct kfd_device_info tonga_device_info = {
 	.supports_cwsr = false,
 	.needs_iommu_device = false,
 	.needs_pci_atomics = true,
+	.num_sdma_engines = 2,
 };
 
 static const struct kfd_device_info tonga_vf_device_info = {
@@ -105,6 +130,7 @@ static const struct kfd_device_info tonga_vf_device_info = {
 	.supports_cwsr = false,
 	.needs_iommu_device = false,
 	.needs_pci_atomics = false,
+	.num_sdma_engines = 2,
 };
 
 static const struct kfd_device_info fiji_device_info = {
@@ -119,6 +145,7 @@ static const struct kfd_device_info fiji_device_info = {
 	.supports_cwsr = true,
 	.needs_iommu_device = false,
 	.needs_pci_atomics = true,
+	.num_sdma_engines = 2,
 };
 
 static const struct kfd_device_info fiji_vf_device_info = {
@@ -133,6 +160,7 @@ static const struct kfd_device_info fiji_vf_device_info = {
 	.supports_cwsr = true,
 	.needs_iommu_device = false,
 	.needs_pci_atomics = false,
+	.num_sdma_engines = 2,
 };
 
 
@@ -148,6 +176,7 @@ static const struct kfd_device_info polaris10_device_info = {
 	.supports_cwsr = true,
 	.needs_iommu_device = false,
 	.needs_pci_atomics = true,
+	.num_sdma_engines = 2,
 };
 
 static const struct kfd_device_info polaris10_vf_device_info = {
@@ -162,6 +191,7 @@ static const struct kfd_device_info polaris10_vf_device_info = {
 	.supports_cwsr = true,
 	.needs_iommu_device = false,
 	.needs_pci_atomics = false,
+	.num_sdma_engines = 2,
 };
 
 static const struct kfd_device_info polaris11_device_info = {
@@ -176,6 +206,7 @@ static const struct kfd_device_info polaris11_device_info = {
 	.supports_cwsr = true,
 	.needs_iommu_device = false,
 	.needs_pci_atomics = true,
+	.num_sdma_engines = 2,
 };
 
 static const struct kfd_device_info vega10_device_info = {
@@ -190,6 +221,7 @@ static const struct kfd_device_info vega10_device_info = {
 	.supports_cwsr = true,
 	.needs_iommu_device = false,
 	.needs_pci_atomics = false,
+	.num_sdma_engines = 2,
 };
 
 static const struct kfd_device_info vega10_vf_device_info = {
@@ -204,6 +236,7 @@ static const struct kfd_device_info vega10_vf_device_info = {
 	.supports_cwsr = true,
 	.needs_iommu_device = false,
 	.needs_pci_atomics = false,
+	.num_sdma_engines = 2,
 };
 
 
@@ -241,6 +274,7 @@ static const struct kfd_deviceid supported_devices[] = {
 	{ 0x9875, &carrizo_device_info },	/* Carrizo */
 	{ 0x9876, &carrizo_device_info },	/* Carrizo */
 	{ 0x9877, &carrizo_device_info },	/* Carrizo */
+	{ 0x15DD, &raven_device_info },		/* Raven */
 #endif
 	{ 0x67A0, &hawaii_device_info },	/* Hawaii */
 	{ 0x67A1, &hawaii_device_info },	/* Hawaii */
@@ -514,13 +548,54 @@ void kgd2kfd_device_exit(struct kfd_dev *kfd)
 	kfree(kfd);
 }
 
+int kgd2kfd_pre_reset(struct kfd_dev *kfd)
+{
+	if (!kfd->init_complete)
+		return 0;
+	kgd2kfd_suspend(kfd);
+
+	/* hold dqm->lock to prevent further execution*/
+	dqm_lock(kfd->dqm);
+
+	kfd_signal_reset_event(kfd);
+	return 0;
+}
+
+/*
+ * Fix me. KFD won't be able to resume existing process for now.
+ * We will keep all existing process in a evicted state and
+ * wait the process to be terminated.
+ */
+
+int kgd2kfd_post_reset(struct kfd_dev *kfd)
+{
+	int ret, count;
+
+	if (!kfd->init_complete)
+		return 0;
+
+	dqm_unlock(kfd->dqm);
+
+	ret = kfd_resume(kfd);
+	if (ret)
+		return ret;
+	count = atomic_dec_return(&kfd_locked);
+	WARN_ONCE(count != 0, "KFD reset ref. error");
+	return 0;
+}
+
+bool kfd_is_locked(void)
+{
+	return  (atomic_read(&kfd_locked) > 0);
+}
+
 void kgd2kfd_suspend(struct kfd_dev *kfd)
 {
 	if (!kfd->init_complete)
 		return;
 
 	/* For first KFD device suspend all the KFD processes */
-	if (atomic_inc_return(&kfd_device_suspended) == 1)
+	if (atomic_inc_return(&kfd_locked) == 1)
 		kfd_suspend_all_processes();
 
 	kfd->dqm->ops.stop(kfd->dqm);
@@ -539,7 +614,7 @@ int kgd2kfd_resume(struct kfd_dev *kfd)
 	if (ret)
 		return ret;
 
-	count = atomic_dec_return(&kfd_device_suspended);
+	count = atomic_dec_return(&kfd_locked);
 	WARN_ONCE(count < 0, "KFD suspend / resume ref. error");
 	if (count == 0)
 		ret = kfd_resume_all_processes();
@@ -577,14 +652,24 @@ dqm_start_error:
 /* This is called directly from KGD at ISR. */
 void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry)
 {
+	uint32_t patched_ihre[KFD_MAX_RING_ENTRY_SIZE];
+	bool is_patched = false;
+
 	if (!kfd->init_complete)
 		return;
 
+	if (kfd->device_info->ih_ring_entry_size > sizeof(patched_ihre)) {
+		dev_err_once(kfd_device, "Ring entry too small\n");
+		return;
+	}
+
 	spin_lock(&kfd->interrupt_lock);
 
 	if (kfd->interrupts_active
-	    && interrupt_is_wanted(kfd, ih_ring_entry)
-	    && enqueue_ih_ring_entry(kfd, ih_ring_entry))
+	    && interrupt_is_wanted(kfd, ih_ring_entry,
+				   patched_ihre, &is_patched)
+	    && enqueue_ih_ring_entry(kfd,
+				     is_patched ? patched_ihre : ih_ring_entry))
 		queue_work(kfd->ih_wq, &kfd->interrupt_work);
 
 	spin_unlock(&kfd->interrupt_lock);
@@ -739,8 +824,8 @@ int kfd_gtt_sa_allocate(struct kfd_dev *kfd, unsigned int size,
 	if (size > kfd->gtt_sa_num_of_chunks * kfd->gtt_sa_chunk_size)
 		return -ENOMEM;
 
-	*mem_obj = kzalloc(sizeof(struct kfd_mem_obj), GFP_NOIO);
-	if ((*mem_obj) == NULL)
+	*mem_obj = kzalloc(sizeof(struct kfd_mem_obj), GFP_KERNEL);
+	if (!(*mem_obj))
 		return -ENOMEM;
 
 	pr_debug("Allocated mem_obj = %p for size = %d\n", *mem_obj, size);
@@ -857,3 +942,26 @@ int kfd_gtt_sa_free(struct kfd_dev *kfd, struct kfd_mem_obj *mem_obj)
 	kfree(mem_obj);
 	return 0;
 }
+
+#if defined(CONFIG_DEBUG_FS)
+
+/* This function will send a package to HIQ to hang the HWS
+ * which will trigger a GPU reset and bring the HWS back to normal state
+ */
+int kfd_debugfs_hang_hws(struct kfd_dev *dev)
+{
+	int r = 0;
+
+	if (dev->dqm->sched_policy != KFD_SCHED_POLICY_HWS) {
+		pr_err("HWS is not enabled");
+		return -EINVAL;
+	}
+
+	r = pm_debugfs_hang_hws(&dev->dqm->packets);
+	if (!r)
+		r = dqm_debugfs_execute_queues(dev->dqm);
+
+	return r;
+}
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
index 668ad07ebe1f..ec0d62a16e53 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
@@ -61,6 +61,8 @@ static int create_sdma_queue_nocpsch(struct device_queue_manager *dqm,
 static void deallocate_sdma_queue(struct device_queue_manager *dqm,
 				unsigned int sdma_queue_id);
 
+static void kfd_process_hw_exception(struct work_struct *work);
+
 static inline
 enum KFD_MQD_TYPE get_mqd_type_from_queue_type(enum kfd_queue_type type)
 {
@@ -99,6 +101,17 @@ unsigned int get_pipes_per_mec(struct device_queue_manager *dqm)
 	return dqm->dev->shared_resources.num_pipe_per_mec;
 }
 
+static unsigned int get_num_sdma_engines(struct device_queue_manager *dqm)
+{
+	return dqm->dev->device_info->num_sdma_engines;
+}
+
+unsigned int get_num_sdma_queues(struct device_queue_manager *dqm)
+{
+	return dqm->dev->device_info->num_sdma_engines
+			* KFD_SDMA_QUEUES_PER_ENGINE;
+}
+
 void program_sh_mem_settings(struct device_queue_manager *dqm,
 					struct qcm_process_device *qpd)
 {
@@ -240,7 +253,7 @@ static int create_queue_nocpsch(struct device_queue_manager *dqm,
 
 	print_queue(q);
 
-	mutex_lock(&dqm->lock);
+	dqm_lock(dqm);
 
 	if (dqm->total_queue_count >= max_num_of_queues_per_device) {
 		pr_warn("Can't create new usermode queue because %d queues were already created\n",
@@ -297,7 +310,7 @@ static int create_queue_nocpsch(struct device_queue_manager *dqm,
 			dqm->total_queue_count);
 
 out_unlock:
-	mutex_unlock(&dqm->lock);
+	dqm_unlock(dqm);
 	return retval;
 }
 
@@ -346,10 +359,10 @@ static int create_compute_queue_nocpsch(struct device_queue_manager *dqm,
 					struct qcm_process_device *qpd)
 {
 	int retval;
-	struct mqd_manager *mqd;
+	struct mqd_manager *mqd_mgr;
 
-	mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_COMPUTE);
-	if (!mqd)
+	mqd_mgr = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_COMPUTE);
+	if (!mqd_mgr)
 		return -ENOMEM;
 
 	retval = allocate_hqd(dqm, q);
@@ -360,7 +373,7 @@ static int create_compute_queue_nocpsch(struct device_queue_manager *dqm,
 	if (retval)
 		goto out_deallocate_hqd;
 
-	retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,
+	retval = mqd_mgr->init_mqd(mqd_mgr, &q->mqd, &q->mqd_mem_obj,
 				&q->gart_mqd_addr, &q->properties);
 	if (retval)
 		goto out_deallocate_doorbell;
@@ -374,15 +387,15 @@ static int create_compute_queue_nocpsch(struct device_queue_manager *dqm,
 	if (!q->properties.is_active)
 		return 0;
 
-	retval = mqd->load_mqd(mqd, q->mqd, q->pipe, q->queue, &q->properties,
-			       q->process->mm);
+	retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe, q->queue,
+			&q->properties, q->process->mm);
 	if (retval)
 		goto out_uninit_mqd;
 
 	return 0;
 
 out_uninit_mqd:
-	mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
+	mqd_mgr->uninit_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
 out_deallocate_doorbell:
 	deallocate_doorbell(qpd, q);
 out_deallocate_hqd:
@@ -399,11 +412,11 @@ static int destroy_queue_nocpsch_locked(struct device_queue_manager *dqm,
 				struct queue *q)
 {
 	int retval;
-	struct mqd_manager *mqd;
+	struct mqd_manager *mqd_mgr;
 
-	mqd = dqm->ops.get_mqd_manager(dqm,
+	mqd_mgr = dqm->ops.get_mqd_manager(dqm,
 		get_mqd_type_from_queue_type(q->properties.type));
-	if (!mqd)
+	if (!mqd_mgr)
 		return -ENOMEM;
 
 	if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) {
@@ -420,14 +433,14 @@ static int destroy_queue_nocpsch_locked(struct device_queue_manager *dqm,
 
 	deallocate_doorbell(qpd, q);
 
-	retval = mqd->destroy_mqd(mqd, q->mqd,
+	retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
 				KFD_PREEMPT_TYPE_WAVEFRONT_RESET,
 				KFD_UNMAP_LATENCY_MS,
 				q->pipe, q->queue);
 	if (retval == -ETIME)
 		qpd->reset_wavefronts = true;
 
-	mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
+	mqd_mgr->uninit_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
 
 	list_del(&q->list);
 	if (list_empty(&qpd->queues_list)) {
@@ -457,9 +470,9 @@ static int destroy_queue_nocpsch(struct device_queue_manager *dqm,
 {
 	int retval;
 
-	mutex_lock(&dqm->lock);
+	dqm_lock(dqm);
 	retval = destroy_queue_nocpsch_locked(dqm, qpd, q);
-	mutex_unlock(&dqm->lock);
+	dqm_unlock(dqm);
 
 	return retval;
 }
@@ -467,19 +480,19 @@ static int destroy_queue_nocpsch(struct device_queue_manager *dqm,
 static int update_queue(struct device_queue_manager *dqm, struct queue *q)
 {
 	int retval;
-	struct mqd_manager *mqd;
+	struct mqd_manager *mqd_mgr;
 	struct kfd_process_device *pdd;
 	bool prev_active = false;
 
-	mutex_lock(&dqm->lock);
+	dqm_lock(dqm);
 	pdd = kfd_get_process_device_data(q->device, q->process);
 	if (!pdd) {
 		retval = -ENODEV;
 		goto out_unlock;
 	}
-	mqd = dqm->ops.get_mqd_manager(dqm,
+	mqd_mgr = dqm->ops.get_mqd_manager(dqm,
 			get_mqd_type_from_queue_type(q->properties.type));
-	if (!mqd) {
+	if (!mqd_mgr) {
 		retval = -ENOMEM;
 		goto out_unlock;
 	}
@@ -506,7 +519,7 @@ static int update_queue(struct device_queue_manager *dqm, struct queue *q)
 	} else if (prev_active &&
 		   (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
 		    q->properties.type == KFD_QUEUE_TYPE_SDMA)) {
-		retval = mqd->destroy_mqd(mqd, q->mqd,
+		retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
 				KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN,
 				KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);
 		if (retval) {
@@ -515,7 +528,7 @@ static int update_queue(struct device_queue_manager *dqm, struct queue *q)
 		}
 	}
 
-	retval = mqd->update_mqd(mqd, q->mqd, &q->properties);
+	retval = mqd_mgr->update_mqd(mqd_mgr, q->mqd, &q->properties);
 
 	/*
 	 * check active state vs. the previous state and modify
@@ -533,44 +546,44 @@ static int update_queue(struct device_queue_manager *dqm, struct queue *q)
 	else if (q->properties.is_active &&
 		 (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
 		  q->properties.type == KFD_QUEUE_TYPE_SDMA))
-		retval = mqd->load_mqd(mqd, q->mqd, q->pipe, q->queue,
+		retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe, q->queue,
 				       &q->properties, q->process->mm);
 
 out_unlock:
-	mutex_unlock(&dqm->lock);
+	dqm_unlock(dqm);
 	return retval;
 }
 
 static struct mqd_manager *get_mqd_manager(
 		struct device_queue_manager *dqm, enum KFD_MQD_TYPE type)
 {
-	struct mqd_manager *mqd;
+	struct mqd_manager *mqd_mgr;
 
 	if (WARN_ON(type >= KFD_MQD_TYPE_MAX))
 		return NULL;
 
 	pr_debug("mqd type %d\n", type);
 
-	mqd = dqm->mqds[type];
-	if (!mqd) {
-		mqd = mqd_manager_init(type, dqm->dev);
-		if (!mqd)
+	mqd_mgr = dqm->mqd_mgrs[type];
+	if (!mqd_mgr) {
+		mqd_mgr = mqd_manager_init(type, dqm->dev);
+		if (!mqd_mgr)
 			pr_err("mqd manager is NULL");
-		dqm->mqds[type] = mqd;
+		dqm->mqd_mgrs[type] = mqd_mgr;
 	}
 
-	return mqd;
+	return mqd_mgr;
 }
 
 static int evict_process_queues_nocpsch(struct device_queue_manager *dqm,
 					struct qcm_process_device *qpd)
 {
 	struct queue *q;
-	struct mqd_manager *mqd;
+	struct mqd_manager *mqd_mgr;
 	struct kfd_process_device *pdd;
 	int retval = 0;
 
-	mutex_lock(&dqm->lock);
+	dqm_lock(dqm);
 	if (qpd->evicted++ > 0) /* already evicted, do nothing */
 		goto out;
 
@@ -582,16 +595,16 @@ static int evict_process_queues_nocpsch(struct device_queue_manager *dqm,
 	list_for_each_entry(q, &qpd->queues_list, list) {
 		if (!q->properties.is_active)
 			continue;
-		mqd = dqm->ops.get_mqd_manager(dqm,
+		mqd_mgr = dqm->ops.get_mqd_manager(dqm,
 			get_mqd_type_from_queue_type(q->properties.type));
-		if (!mqd) { /* should not be here */
+		if (!mqd_mgr) { /* should not be here */
 			pr_err("Cannot evict queue, mqd mgr is NULL\n");
 			retval = -ENOMEM;
 			goto out;
 		}
 		q->properties.is_evicted = true;
 		q->properties.is_active = false;
-		retval = mqd->destroy_mqd(mqd, q->mqd,
+		retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
 				KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN,
 				KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);
 		if (retval)
@@ -600,7 +613,7 @@ static int evict_process_queues_nocpsch(struct device_queue_manager *dqm,
 	}
 
 out:
-	mutex_unlock(&dqm->lock);
+	dqm_unlock(dqm);
 	return retval;
 }
 
@@ -611,7 +624,7 @@ static int evict_process_queues_cpsch(struct device_queue_manager *dqm,
 	struct kfd_process_device *pdd;
 	int retval = 0;
 
-	mutex_lock(&dqm->lock);
+	dqm_lock(dqm);
 	if (qpd->evicted++ > 0) /* already evicted, do nothing */
 		goto out;
 
@@ -633,7 +646,7 @@ static int evict_process_queues_cpsch(struct device_queue_manager *dqm,
 				KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
 
 out:
-	mutex_unlock(&dqm->lock);
+	dqm_unlock(dqm);
 	return retval;
 }
 
@@ -641,7 +654,7 @@ static int restore_process_queues_nocpsch(struct device_queue_manager *dqm,
 					  struct qcm_process_device *qpd)
 {
 	struct queue *q;
-	struct mqd_manager *mqd;
+	struct mqd_manager *mqd_mgr;
 	struct kfd_process_device *pdd;
 	uint32_t pd_base;
 	int retval = 0;
@@ -650,7 +663,7 @@ static int restore_process_queues_nocpsch(struct device_queue_manager *dqm,
 	/* Retrieve PD base */
 	pd_base = dqm->dev->kfd2kgd->get_process_page_dir(pdd->vm);
 
-	mutex_lock(&dqm->lock);
+	dqm_lock(dqm);
 	if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */
 		goto out;
 	if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */
@@ -677,16 +690,16 @@ static int restore_process_queues_nocpsch(struct device_queue_manager *dqm,
 	list_for_each_entry(q, &qpd->queues_list, list) {
 		if (!q->properties.is_evicted)
 			continue;
-		mqd = dqm->ops.get_mqd_manager(dqm,
+		mqd_mgr = dqm->ops.get_mqd_manager(dqm,
 			get_mqd_type_from_queue_type(q->properties.type));
-		if (!mqd) { /* should not be here */
+		if (!mqd_mgr) { /* should not be here */
 			pr_err("Cannot restore queue, mqd mgr is NULL\n");
 			retval = -ENOMEM;
 			goto out;
 		}
 		q->properties.is_evicted = false;
 		q->properties.is_active = true;
-		retval = mqd->load_mqd(mqd, q->mqd, q->pipe,
+		retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe,
 				       q->queue, &q->properties,
 				       q->process->mm);
 		if (retval)
@@ -695,7 +708,7 @@ static int restore_process_queues_nocpsch(struct device_queue_manager *dqm,
 	}
 	qpd->evicted = 0;
 out:
-	mutex_unlock(&dqm->lock);
+	dqm_unlock(dqm);
 	return retval;
 }
 
@@ -711,7 +724,7 @@ static int restore_process_queues_cpsch(struct device_queue_manager *dqm,
 	/* Retrieve PD base */
 	pd_base = dqm->dev->kfd2kgd->get_process_page_dir(pdd->vm);
 
-	mutex_lock(&dqm->lock);
+	dqm_lock(dqm);
 	if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */
 		goto out;
 	if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */
@@ -739,7 +752,7 @@ static int restore_process_queues_cpsch(struct device_queue_manager *dqm,
 	if (!retval)
 		qpd->evicted = 0;
 out:
-	mutex_unlock(&dqm->lock);
+	dqm_unlock(dqm);
 	return retval;
 }
 
@@ -761,7 +774,7 @@ static int register_process(struct device_queue_manager *dqm,
 	/* Retrieve PD base */
 	pd_base = dqm->dev->kfd2kgd->get_process_page_dir(pdd->vm);
 
-	mutex_lock(&dqm->lock);
+	dqm_lock(dqm);
 	list_add(&n->list, &dqm->queues);
 
 	/* Update PD Base in QPD */
@@ -769,9 +782,10 @@ static int register_process(struct device_queue_manager *dqm,
 
 	retval = dqm->asic_ops.update_qpd(dqm, qpd);
 
-	dqm->processes_count++;
+	if (dqm->processes_count++ == 0)
+		dqm->dev->kfd2kgd->set_compute_idle(dqm->dev->kgd, false);
 
-	mutex_unlock(&dqm->lock);
+	dqm_unlock(dqm);
 
 	return retval;
 }
@@ -786,20 +800,22 @@ static int unregister_process(struct device_queue_manager *dqm,
 			list_empty(&qpd->queues_list) ? "empty" : "not empty");
 
 	retval = 0;
-	mutex_lock(&dqm->lock);
+	dqm_lock(dqm);
 
 	list_for_each_entry_safe(cur, next, &dqm->queues, list) {
 		if (qpd == cur->qpd) {
 			list_del(&cur->list);
 			kfree(cur);
-			dqm->processes_count--;
+			if (--dqm->processes_count == 0)
+				dqm->dev->kfd2kgd->set_compute_idle(
+					dqm->dev->kgd, true);
 			goto out;
 		}
 	}
 	/* qpd not found in dqm list */
 	retval = 1;
 out:
-	mutex_unlock(&dqm->lock);
+	dqm_unlock(dqm);
 	return retval;
 }
 
@@ -838,7 +854,7 @@ static int initialize_nocpsch(struct device_queue_manager *dqm)
 	if (!dqm->allocated_queues)
 		return -ENOMEM;
 
-	mutex_init(&dqm->lock);
+	mutex_init(&dqm->lock_hidden);
 	INIT_LIST_HEAD(&dqm->queues);
 	dqm->queue_count = dqm->next_pipe_to_allocate = 0;
 	dqm->sdma_queue_count = 0;
@@ -853,7 +869,7 @@ static int initialize_nocpsch(struct device_queue_manager *dqm)
 	}
 
 	dqm->vmid_bitmap = (1 << dqm->dev->vm_info.vmid_num_kfd) - 1;
-	dqm->sdma_bitmap = (1 << CIK_SDMA_QUEUES) - 1;
+	dqm->sdma_bitmap = (1 << get_num_sdma_queues(dqm)) - 1;
 
 	return 0;
 }
@@ -866,8 +882,8 @@ static void uninitialize(struct device_queue_manager *dqm)
 
 	kfree(dqm->allocated_queues);
 	for (i = 0 ; i < KFD_MQD_TYPE_MAX ; i++)
-		kfree(dqm->mqds[i]);
-	mutex_destroy(&dqm->lock);
+		kfree(dqm->mqd_mgrs[i]);
+	mutex_destroy(&dqm->lock_hidden);
 	kfd_gtt_sa_free(dqm->dev, dqm->pipeline_mem);
 }
 
@@ -901,7 +917,7 @@ static int allocate_sdma_queue(struct device_queue_manager *dqm,
 static void deallocate_sdma_queue(struct device_queue_manager *dqm,
 				unsigned int sdma_queue_id)
 {
-	if (sdma_queue_id >= CIK_SDMA_QUEUES)
+	if (sdma_queue_id >= get_num_sdma_queues(dqm))
 		return;
 	dqm->sdma_bitmap |= (1 << sdma_queue_id);
 }
@@ -910,19 +926,19 @@ static int create_sdma_queue_nocpsch(struct device_queue_manager *dqm,
 					struct queue *q,
 					struct qcm_process_device *qpd)
 {
-	struct mqd_manager *mqd;
+	struct mqd_manager *mqd_mgr;
 	int retval;
 
-	mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_SDMA);
-	if (!mqd)
+	mqd_mgr = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_SDMA);
+	if (!mqd_mgr)
 		return -ENOMEM;
 
 	retval = allocate_sdma_queue(dqm, &q->sdma_id);
 	if (retval)
 		return retval;
 
-	q->properties.sdma_queue_id = q->sdma_id / CIK_SDMA_QUEUES_PER_ENGINE;
-	q->properties.sdma_engine_id = q->sdma_id % CIK_SDMA_QUEUES_PER_ENGINE;
+	q->properties.sdma_queue_id = q->sdma_id / get_num_sdma_engines(dqm);
+	q->properties.sdma_engine_id = q->sdma_id % get_num_sdma_engines(dqm);
 
 	retval = allocate_doorbell(qpd, q);
 	if (retval)
@@ -933,19 +949,20 @@ static int create_sdma_queue_nocpsch(struct device_queue_manager *dqm,
 	pr_debug("SDMA engine id: %d\n", q->properties.sdma_engine_id);
 
 	dqm->asic_ops.init_sdma_vm(dqm, q, qpd);
-	retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,
+	retval = mqd_mgr->init_mqd(mqd_mgr, &q->mqd, &q->mqd_mem_obj,
 				&q->gart_mqd_addr, &q->properties);
 	if (retval)
 		goto out_deallocate_doorbell;
 
-	retval = mqd->load_mqd(mqd, q->mqd, 0, 0, &q->properties, NULL);
+	retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, 0, 0, &q->properties,
+				NULL);
 	if (retval)
 		goto out_uninit_mqd;
 
 	return 0;
 
 out_uninit_mqd:
-	mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
+	mqd_mgr->uninit_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
 out_deallocate_doorbell:
 	deallocate_doorbell(qpd, q);
 out_deallocate_sdma_queue:
@@ -1003,12 +1020,14 @@ static int initialize_cpsch(struct device_queue_manager *dqm)
 {
 	pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm));
 
-	mutex_init(&dqm->lock);
+	mutex_init(&dqm->lock_hidden);
 	INIT_LIST_HEAD(&dqm->queues);
 	dqm->queue_count = dqm->processes_count = 0;
 	dqm->sdma_queue_count = 0;
 	dqm->active_runlist = false;
-	dqm->sdma_bitmap = (1 << CIK_SDMA_QUEUES) - 1;
+	dqm->sdma_bitmap = (1 << get_num_sdma_queues(dqm)) - 1;
+
+	INIT_WORK(&dqm->hw_exception_work, kfd_process_hw_exception);
 
 	return 0;
 }
@@ -1041,9 +1060,11 @@ static int start_cpsch(struct device_queue_manager *dqm)
 
 	init_interrupts(dqm);
 
-	mutex_lock(&dqm->lock);
+	dqm_lock(dqm);
+	/* clear hang status when driver try to start the hw scheduler */
+	dqm->is_hws_hang = false;
 	execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
-	mutex_unlock(&dqm->lock);
+	dqm_unlock(dqm);
 
 	return 0;
 fail_allocate_vidmem:
@@ -1055,9 +1076,9 @@ fail_packet_manager_init:
 
 static int stop_cpsch(struct device_queue_manager *dqm)
 {
-	mutex_lock(&dqm->lock);
+	dqm_lock(dqm);
 	unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0);
-	mutex_unlock(&dqm->lock);
+	dqm_unlock(dqm);
 
 	kfd_gtt_sa_free(dqm->dev, dqm->fence_mem);
 	pm_uninit(&dqm->packets);
@@ -1069,11 +1090,11 @@ static int create_kernel_queue_cpsch(struct device_queue_manager *dqm,
 					struct kernel_queue *kq,
 					struct qcm_process_device *qpd)
 {
-	mutex_lock(&dqm->lock);
+	dqm_lock(dqm);
 	if (dqm->total_queue_count >= max_num_of_queues_per_device) {
 		pr_warn("Can't create new kernel queue because %d queues were already created\n",
 				dqm->total_queue_count);
-		mutex_unlock(&dqm->lock);
+		dqm_unlock(dqm);
 		return -EPERM;
 	}
 
@@ -1089,7 +1110,7 @@ static int create_kernel_queue_cpsch(struct device_queue_manager *dqm,
 	dqm->queue_count++;
 	qpd->is_debug = true;
 	execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
-	mutex_unlock(&dqm->lock);
+	dqm_unlock(dqm);
 
 	return 0;
 }
@@ -1098,7 +1119,7 @@ static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm,
 					struct kernel_queue *kq,
 					struct qcm_process_device *qpd)
 {
-	mutex_lock(&dqm->lock);
+	dqm_lock(dqm);
 	list_del(&kq->list);
 	dqm->queue_count--;
 	qpd->is_debug = false;
@@ -1110,18 +1131,18 @@ static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm,
 	dqm->total_queue_count--;
 	pr_debug("Total of %d queues are accountable so far\n",
 			dqm->total_queue_count);
-	mutex_unlock(&dqm->lock);
+	dqm_unlock(dqm);
 }
 
 static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
 			struct qcm_process_device *qpd)
 {
 	int retval;
-	struct mqd_manager *mqd;
+	struct mqd_manager *mqd_mgr;
 
 	retval = 0;
 
-	mutex_lock(&dqm->lock);
+	dqm_lock(dqm);
 
 	if (dqm->total_queue_count >= max_num_of_queues_per_device) {
 		pr_warn("Can't create new usermode queue because %d queues were already created\n",
@@ -1135,19 +1156,19 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
 		if (retval)
 			goto out_unlock;
 		q->properties.sdma_queue_id =
-			q->sdma_id / CIK_SDMA_QUEUES_PER_ENGINE;
+			q->sdma_id / get_num_sdma_engines(dqm);
 		q->properties.sdma_engine_id =
-			q->sdma_id % CIK_SDMA_QUEUES_PER_ENGINE;
+			q->sdma_id % get_num_sdma_engines(dqm);
 	}
 
 	retval = allocate_doorbell(qpd, q);
 	if (retval)
 		goto out_deallocate_sdma_queue;
 
-	mqd = dqm->ops.get_mqd_manager(dqm,
+	mqd_mgr = dqm->ops.get_mqd_manager(dqm,
 			get_mqd_type_from_queue_type(q->properties.type));
 
-	if (!mqd) {
+	if (!mqd_mgr) {
 		retval = -ENOMEM;
 		goto out_deallocate_doorbell;
 	}
@@ -1164,7 +1185,7 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
 
 	q->properties.tba_addr = qpd->tba_addr;
 	q->properties.tma_addr = qpd->tma_addr;
-	retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,
+	retval = mqd_mgr->init_mqd(mqd_mgr, &q->mqd, &q->mqd_mem_obj,
 				&q->gart_mqd_addr, &q->properties);
 	if (retval)
 		goto out_deallocate_doorbell;
@@ -1188,7 +1209,7 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
 	pr_debug("Total of %d queues are accountable so far\n",
 			dqm->total_queue_count);
 
-	mutex_unlock(&dqm->lock);
+	dqm_unlock(dqm);
 	return retval;
 
 out_deallocate_doorbell:
@@ -1197,7 +1218,8 @@ out_deallocate_sdma_queue:
 	if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
 		deallocate_sdma_queue(dqm, q->sdma_id);
 out_unlock:
-	mutex_unlock(&dqm->lock);
+	dqm_unlock(dqm);
+
 	return retval;
 }
 
@@ -1210,6 +1232,13 @@ int amdkfd_fence_wait_timeout(unsigned int *fence_addr,
 	while (*fence_addr != fence_value) {
 		if (time_after(jiffies, end_jiffies)) {
 			pr_err("qcm fence wait loop timeout expired\n");
+			/* In HWS case, this is used to halt the driver thread
+			 * in order not to mess up CP states before doing
+			 * scandumps for FW debugging.
+			 */
+			while (halt_if_hws_hang)
+				schedule();
+
 			return -ETIME;
 		}
 		schedule();
@@ -1254,6 +1283,8 @@ static int unmap_queues_cpsch(struct device_queue_manager *dqm,
 {
 	int retval = 0;
 
+	if (dqm->is_hws_hang)
+		return -EIO;
 	if (!dqm->active_runlist)
 		return retval;
 
@@ -1292,9 +1323,13 @@ static int execute_queues_cpsch(struct device_queue_manager *dqm,
 {
 	int retval;
 
+	if (dqm->is_hws_hang)
+		return -EIO;
 	retval = unmap_queues_cpsch(dqm, filter, filter_param);
 	if (retval) {
 		pr_err("The cp might be in an unrecoverable state due to an unsuccessful queues preemption\n");
+		dqm->is_hws_hang = true;
+		schedule_work(&dqm->hw_exception_work);
 		return retval;
 	}
 
@@ -1306,7 +1341,7 @@ static int destroy_queue_cpsch(struct device_queue_manager *dqm,
 				struct queue *q)
 {
 	int retval;
-	struct mqd_manager *mqd;
+	struct mqd_manager *mqd_mgr;
 	bool preempt_all_queues;
 
 	preempt_all_queues = false;
@@ -1314,7 +1349,7 @@ static int destroy_queue_cpsch(struct device_queue_manager *dqm,
 	retval = 0;
 
 	/* remove queue from list to prevent rescheduling after preemption */
-	mutex_lock(&dqm->lock);
+	dqm_lock(dqm);
 
 	if (qpd->is_debug) {
 		/*
@@ -1326,9 +1361,9 @@ static int destroy_queue_cpsch(struct device_queue_manager *dqm,
 
 	}
 
-	mqd = dqm->ops.get_mqd_manager(dqm,
+	mqd_mgr = dqm->ops.get_mqd_manager(dqm,
 			get_mqd_type_from_queue_type(q->properties.type));
-	if (!mqd) {
+	if (!mqd_mgr) {
 		retval = -ENOMEM;
 		goto failed;
 	}
@@ -1350,7 +1385,7 @@ static int destroy_queue_cpsch(struct device_queue_manager *dqm,
 			qpd->reset_wavefronts = true;
 	}
 
-	mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
+	mqd_mgr->uninit_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
 
 	/*
 	 * Unconditionally decrement this counter, regardless of the queue's
@@ -1360,14 +1395,14 @@ static int destroy_queue_cpsch(struct device_queue_manager *dqm,
 	pr_debug("Total of %d queues are accountable so far\n",
 			dqm->total_queue_count);
 
-	mutex_unlock(&dqm->lock);
+	dqm_unlock(dqm);
 
 	return retval;
 
 failed:
 failed_try_destroy_debugged_queue:
 
-	mutex_unlock(&dqm->lock);
+	dqm_unlock(dqm);
 	return retval;
 }
 
@@ -1391,7 +1426,7 @@ static bool set_cache_memory_policy(struct device_queue_manager *dqm,
 	if (!dqm->asic_ops.set_cache_memory_policy)
 		return retval;
 
-	mutex_lock(&dqm->lock);
+	dqm_lock(dqm);
 
 	if (alternate_aperture_size == 0) {
 		/* base > limit disables APE1 */
@@ -1437,7 +1472,7 @@ static bool set_cache_memory_policy(struct device_queue_manager *dqm,
 		qpd->sh_mem_ape1_limit);
 
 out:
-	mutex_unlock(&dqm->lock);
+	dqm_unlock(dqm);
 	return retval;
 }
 
@@ -1468,7 +1503,7 @@ static int process_termination_nocpsch(struct device_queue_manager *dqm,
 	struct device_process_node *cur, *next_dpn;
 	int retval = 0;
 
-	mutex_lock(&dqm->lock);
+	dqm_lock(dqm);
 
 	/* Clear all user mode queues */
 	list_for_each_entry_safe(q, next, &qpd->queues_list, list) {
@@ -1489,7 +1524,7 @@ static int process_termination_nocpsch(struct device_queue_manager *dqm,
 		}
 	}
 
-	mutex_unlock(&dqm->lock);
+	dqm_unlock(dqm);
 	return retval;
 }
 
@@ -1500,14 +1535,14 @@ static int process_termination_cpsch(struct device_queue_manager *dqm,
 	int retval;
 	struct queue *q, *next;
 	struct kernel_queue *kq, *kq_next;
-	struct mqd_manager *mqd;
+	struct mqd_manager *mqd_mgr;
 	struct device_process_node *cur, *next_dpn;
 	enum kfd_unmap_queues_filter filter =
 		KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES;
 
 	retval = 0;
 
-	mutex_lock(&dqm->lock);
+	dqm_lock(dqm);
 
 	/* Clean all kernel queues */
 	list_for_each_entry_safe(kq, kq_next, &qpd->priv_queue_list, list) {
@@ -1542,7 +1577,7 @@ static int process_termination_cpsch(struct device_queue_manager *dqm,
 	}
 
 	retval = execute_queues_cpsch(dqm, filter, 0);
-	if (retval || qpd->reset_wavefronts) {
+	if ((!dqm->is_hws_hang) && (retval || qpd->reset_wavefronts)) {
 		pr_warn("Resetting wave fronts (cpsch) on dev %p\n", dqm->dev);
 		dbgdev_wave_reset_wavefronts(dqm->dev, qpd->pqm->process);
 		qpd->reset_wavefronts = false;
@@ -1550,19 +1585,19 @@ static int process_termination_cpsch(struct device_queue_manager *dqm,
 
 	/* lastly, free mqd resources */
 	list_for_each_entry_safe(q, next, &qpd->queues_list, list) {
-		mqd = dqm->ops.get_mqd_manager(dqm,
+		mqd_mgr = dqm->ops.get_mqd_manager(dqm,
 			get_mqd_type_from_queue_type(q->properties.type));
-		if (!mqd) {
+		if (!mqd_mgr) {
 			retval = -ENOMEM;
 			goto out;
 		}
 		list_del(&q->list);
 		qpd->queue_count--;
-		mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
+		mqd_mgr->uninit_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
 	}
 
 out:
-	mutex_unlock(&dqm->lock);
+	dqm_unlock(dqm);
 	return retval;
 }
 
@@ -1683,6 +1718,30 @@ void device_queue_manager_uninit(struct device_queue_manager *dqm)
 	kfree(dqm);
 }
 
+int kfd_process_vm_fault(struct device_queue_manager *dqm,
+			 unsigned int pasid)
+{
+	struct kfd_process_device *pdd;
+	struct kfd_process *p = kfd_lookup_process_by_pasid(pasid);
+	int ret = 0;
+
+	if (!p)
+		return -EINVAL;
+	pdd = kfd_get_process_device_data(dqm->dev, p);
+	if (pdd)
+		ret = dqm->ops.evict_process_queues(dqm, &pdd->qpd);
+	kfd_unref_process(p);
+
+	return ret;
+}
+
+static void kfd_process_hw_exception(struct work_struct *work)
+{
+	struct device_queue_manager *dqm = container_of(work,
+			struct device_queue_manager, hw_exception_work);
+	dqm->dev->kfd2kgd->gpu_recover(dqm->dev->kgd);
+}
+
 #if defined(CONFIG_DEBUG_FS)
 
 static void seq_reg_dump(struct seq_file *m,
@@ -1746,8 +1805,8 @@ int dqm_debugfs_hqds(struct seq_file *m, void *data)
 		}
 	}
 
-	for (pipe = 0; pipe < CIK_SDMA_ENGINE_NUM; pipe++) {
-		for (queue = 0; queue < CIK_SDMA_QUEUES_PER_ENGINE; queue++) {
+	for (pipe = 0; pipe < get_num_sdma_engines(dqm); pipe++) {
+		for (queue = 0; queue < KFD_SDMA_QUEUES_PER_ENGINE; queue++) {
 			r = dqm->dev->kfd2kgd->hqd_sdma_dump(
 				dqm->dev->kgd, pipe, queue, &dump, &n_regs);
 			if (r)
@@ -1764,4 +1823,16 @@ int dqm_debugfs_hqds(struct seq_file *m, void *data)
 	return r;
 }
 
+int dqm_debugfs_execute_queues(struct device_queue_manager *dqm)
+{
+	int r = 0;
+
+	dqm_lock(dqm);
+	dqm->active_runlist = true;
+	r = execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0);
+	dqm_unlock(dqm);
+
+	return r;
+}
+
 #endif
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
index 59a6b1956932..00da3169a004 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
@@ -26,15 +26,14 @@
 
 #include <linux/rwsem.h>
 #include <linux/list.h>
+#include <linux/mutex.h>
+#include <linux/sched/mm.h>
 #include "kfd_priv.h"
 #include "kfd_mqd_manager.h"
 
 #define KFD_UNMAP_LATENCY_MS			(4000)
 #define QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS (2 * KFD_UNMAP_LATENCY_MS + 1000)
-
-#define CIK_SDMA_QUEUES				(4)
-#define CIK_SDMA_QUEUES_PER_ENGINE		(2)
-#define CIK_SDMA_ENGINE_NUM			(2)
+#define KFD_SDMA_QUEUES_PER_ENGINE		(2)
 
 struct device_process_node {
 	struct qcm_process_device *qpd;
@@ -170,11 +169,12 @@ struct device_queue_manager {
 	struct device_queue_manager_ops ops;
 	struct device_queue_manager_asic_ops asic_ops;
 
-	struct mqd_manager	*mqds[KFD_MQD_TYPE_MAX];
+	struct mqd_manager	*mqd_mgrs[KFD_MQD_TYPE_MAX];
 	struct packet_manager	packets;
 	struct kfd_dev		*dev;
-	struct mutex		lock;
+	struct mutex		lock_hidden; /* use dqm_lock/unlock(dqm) */
 	struct list_head	queues;
+	unsigned int		saved_flags;
 	unsigned int		processes_count;
 	unsigned int		queue_count;
 	unsigned int		sdma_queue_count;
@@ -190,6 +190,10 @@ struct device_queue_manager {
 	struct kfd_mem_obj	*fence_mem;
 	bool			active_runlist;
 	int			sched_policy;
+
+	/* hw exception  */
+	bool			is_hws_hang;
+	struct work_struct	hw_exception_work;
 };
 
 void device_queue_manager_init_cik(
@@ -207,6 +211,7 @@ void program_sh_mem_settings(struct device_queue_manager *dqm,
 unsigned int get_queues_num(struct device_queue_manager *dqm);
 unsigned int get_queues_per_pipe(struct device_queue_manager *dqm);
 unsigned int get_pipes_per_mec(struct device_queue_manager *dqm);
+unsigned int get_num_sdma_queues(struct device_queue_manager *dqm);
 
 static inline unsigned int get_sh_mem_bases_32(struct kfd_process_device *pdd)
 {
@@ -219,4 +224,19 @@ get_sh_mem_bases_nybble_64(struct kfd_process_device *pdd)
 	return (pdd->lds_base >> 60) & 0x0E;
 }
 
+/* The DQM lock can be taken in MMU notifiers. Make sure no reclaim-FS
+ * happens while holding this lock anywhere to prevent deadlocks when
+ * an MMU notifier runs in reclaim-FS context.
+ */
+static inline void dqm_lock(struct device_queue_manager *dqm)
+{
+	mutex_lock(&dqm->lock_hidden);
+	dqm->saved_flags = memalloc_nofs_save();
+}
+static inline void dqm_unlock(struct device_queue_manager *dqm)
+{
+	memalloc_nofs_restore(dqm->saved_flags);
+	mutex_unlock(&dqm->lock_hidden);
+}
+
 #endif /* KFD_DEVICE_QUEUE_MANAGER_H_ */
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v9.c
index 79e5bcf6367c..417515332c35 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v9.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v9.c
@@ -60,7 +60,7 @@ static int update_qpd_v9(struct device_queue_manager *dqm,
 		qpd->sh_mem_config =
 				SH_MEM_ALIGNMENT_MODE_UNALIGNED <<
 					SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT;
-		if (vega10_noretry &&
+		if (noretry &&
 		    !dqm->dev->device_info->needs_iommu_device)
 			qpd->sh_mem_config |=
 				1 << SH_MEM_CONFIG__RETRY_DISABLE__SHIFT;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c b/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c
index c3744d89352c..ebe79bf00145 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c
@@ -188,9 +188,9 @@ void __iomem *kfd_get_kernel_doorbell(struct kfd_dev *kfd,
 	*doorbell_off = kfd->doorbell_id_offset + inx;
 
 	pr_debug("Get kernel queue doorbell\n"
-			 "     doorbell offset   == 0x%08X\n"
-			 "     kernel address    == %p\n",
-		*doorbell_off, (kfd->doorbell_kernel_ptr + inx));
+			"     doorbell offset   == 0x%08X\n"
+			"     doorbell index    == 0x%x\n",
+		*doorbell_off, inx);
 
 	return kfd->doorbell_kernel_ptr + inx;
 }
@@ -199,7 +199,8 @@ void kfd_release_kernel_doorbell(struct kfd_dev *kfd, u32 __iomem *db_addr)
 {
 	unsigned int inx;
 
-	inx = (unsigned int)(db_addr - kfd->doorbell_kernel_ptr);
+	inx = (unsigned int)(db_addr - kfd->doorbell_kernel_ptr)
+		* sizeof(u32) / kfd->device_info->doorbell_size;
 
 	mutex_lock(&kfd->doorbell_mutex);
 	__clear_bit(inx, kfd->doorbell_available_index);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_events.c b/drivers/gpu/drm/amd/amdkfd/kfd_events.c
index 5562e94e786a..e9f0e0a1b41c 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_events.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_events.c
@@ -850,6 +850,13 @@ static void lookup_events_by_type_and_signal(struct kfd_process *p,
 				ev->memory_exception_data = *ev_data;
 		}
 
+	if (type == KFD_EVENT_TYPE_MEMORY) {
+		dev_warn(kfd_device,
+			"Sending SIGSEGV to HSA Process with PID %d ",
+				p->lead_thread->pid);
+		send_sig(SIGSEGV, p->lead_thread, 0);
+	}
+
 	/* Send SIGTERM no event of type "type" has been found*/
 	if (send_signal) {
 		if (send_sigterm) {
@@ -904,34 +911,41 @@ void kfd_signal_iommu_event(struct kfd_dev *dev, unsigned int pasid,
 	memory_exception_data.failure.NotPresent = 1;
 	memory_exception_data.failure.NoExecute = 0;
 	memory_exception_data.failure.ReadOnly = 0;
-	if (vma) {
-		if (vma->vm_start > address) {
-			memory_exception_data.failure.NotPresent = 1;
-			memory_exception_data.failure.NoExecute = 0;
+	if (vma && address >= vma->vm_start) {
+		memory_exception_data.failure.NotPresent = 0;
+
+		if (is_write_requested && !(vma->vm_flags & VM_WRITE))
+			memory_exception_data.failure.ReadOnly = 1;
+		else
 			memory_exception_data.failure.ReadOnly = 0;
-		} else {
-			memory_exception_data.failure.NotPresent = 0;
-			if (is_write_requested && !(vma->vm_flags & VM_WRITE))
-				memory_exception_data.failure.ReadOnly = 1;
-			else
-				memory_exception_data.failure.ReadOnly = 0;
-			if (is_execute_requested && !(vma->vm_flags & VM_EXEC))
-				memory_exception_data.failure.NoExecute = 1;
-			else
-				memory_exception_data.failure.NoExecute = 0;
-		}
+
+		if (is_execute_requested && !(vma->vm_flags & VM_EXEC))
+			memory_exception_data.failure.NoExecute = 1;
+		else
+			memory_exception_data.failure.NoExecute = 0;
 	}
 
 	up_read(&mm->mmap_sem);
 	mmput(mm);
 
-	mutex_lock(&p->event_mutex);
+	pr_debug("notpresent %d, noexecute %d, readonly %d\n",
+			memory_exception_data.failure.NotPresent,
+			memory_exception_data.failure.NoExecute,
+			memory_exception_data.failure.ReadOnly);
 
-	/* Lookup events by type and signal them */
-	lookup_events_by_type_and_signal(p, KFD_EVENT_TYPE_MEMORY,
-			&memory_exception_data);
+	/* Workaround on Raven to not kill the process when memory is freed
+	 * before IOMMU is able to finish processing all the excessive PPRs
+	 */
+	if (dev->device_info->asic_family != CHIP_RAVEN) {
+		mutex_lock(&p->event_mutex);
+
+		/* Lookup events by type and signal them */
+		lookup_events_by_type_and_signal(p, KFD_EVENT_TYPE_MEMORY,
+				&memory_exception_data);
+
+		mutex_unlock(&p->event_mutex);
+	}
 
-	mutex_unlock(&p->event_mutex);
 	kfd_unref_process(p);
 }
 #endif /* KFD_SUPPORT_IOMMU_V2 */
@@ -956,3 +970,67 @@ void kfd_signal_hw_exception_event(unsigned int pasid)
 	mutex_unlock(&p->event_mutex);
 	kfd_unref_process(p);
 }
+
+void kfd_signal_vm_fault_event(struct kfd_dev *dev, unsigned int pasid,
+				struct kfd_vm_fault_info *info)
+{
+	struct kfd_event *ev;
+	uint32_t id;
+	struct kfd_process *p = kfd_lookup_process_by_pasid(pasid);
+	struct kfd_hsa_memory_exception_data memory_exception_data;
+
+	if (!p)
+		return; /* Presumably process exited. */
+	memset(&memory_exception_data, 0, sizeof(memory_exception_data));
+	memory_exception_data.gpu_id = dev->id;
+	memory_exception_data.failure.imprecise = 1;
+	/* Set failure reason */
+	if (info) {
+		memory_exception_data.va = (info->page_addr) << PAGE_SHIFT;
+		memory_exception_data.failure.NotPresent =
+			info->prot_valid ? 1 : 0;
+		memory_exception_data.failure.NoExecute =
+			info->prot_exec ? 1 : 0;
+		memory_exception_data.failure.ReadOnly =
+			info->prot_write ? 1 : 0;
+		memory_exception_data.failure.imprecise = 0;
+	}
+	mutex_lock(&p->event_mutex);
+
+	id = KFD_FIRST_NONSIGNAL_EVENT_ID;
+	idr_for_each_entry_continue(&p->event_idr, ev, id)
+		if (ev->type == KFD_EVENT_TYPE_MEMORY) {
+			ev->memory_exception_data = memory_exception_data;
+			set_event(ev);
+		}
+
+	mutex_unlock(&p->event_mutex);
+	kfd_unref_process(p);
+}
+
+void kfd_signal_reset_event(struct kfd_dev *dev)
+{
+	struct kfd_hsa_hw_exception_data hw_exception_data;
+	struct kfd_process *p;
+	struct kfd_event *ev;
+	unsigned int temp;
+	uint32_t id, idx;
+
+	/* Whole gpu reset caused by GPU hang and memory is lost */
+	memset(&hw_exception_data, 0, sizeof(hw_exception_data));
+	hw_exception_data.gpu_id = dev->id;
+	hw_exception_data.memory_lost = 1;
+
+	idx = srcu_read_lock(&kfd_processes_srcu);
+	hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
+		mutex_lock(&p->event_mutex);
+		id = KFD_FIRST_NONSIGNAL_EVENT_ID;
+		idr_for_each_entry_continue(&p->event_idr, ev, id)
+			if (ev->type == KFD_EVENT_TYPE_HW_EXCEPTION) {
+				ev->hw_exception_data = hw_exception_data;
+				set_event(ev);
+			}
+		mutex_unlock(&p->event_mutex);
+	}
+	srcu_read_unlock(&kfd_processes_srcu, idx);
+}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_events.h b/drivers/gpu/drm/amd/amdkfd/kfd_events.h
index abca5bfebbff..c7ac6c73af86 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_events.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_events.h
@@ -66,6 +66,7 @@ struct kfd_event {
 	/* type specific data */
 	union {
 		struct kfd_hsa_memory_exception_data memory_exception_data;
+		struct kfd_hsa_hw_exception_data hw_exception_data;
 	};
 };
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c
index 37029baa3346..f836897bbf58 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c
@@ -26,7 +26,9 @@
 
 
 static bool event_interrupt_isr_v9(struct kfd_dev *dev,
-					const uint32_t *ih_ring_entry)
+					const uint32_t *ih_ring_entry,
+					uint32_t *patched_ihre,
+					bool *patched_flag)
 {
 	uint16_t source_id, client_id, pasid, vmid;
 	const uint32_t *data = ih_ring_entry;
@@ -57,7 +59,9 @@ static bool event_interrupt_isr_v9(struct kfd_dev *dev,
 	return source_id == SOC15_INTSRC_CP_END_OF_PIPE ||
 		source_id == SOC15_INTSRC_SDMA_TRAP ||
 		source_id == SOC15_INTSRC_SQ_INTERRUPT_MSG ||
-		source_id == SOC15_INTSRC_CP_BAD_OPCODE;
+		source_id == SOC15_INTSRC_CP_BAD_OPCODE ||
+		client_id == SOC15_IH_CLIENTID_VMC ||
+		client_id == SOC15_IH_CLIENTID_UTCL2;
 }
 
 static void event_interrupt_wq_v9(struct kfd_dev *dev,
@@ -82,7 +86,19 @@ static void event_interrupt_wq_v9(struct kfd_dev *dev,
 		kfd_signal_hw_exception_event(pasid);
 	else if (client_id == SOC15_IH_CLIENTID_VMC ||
 		 client_id == SOC15_IH_CLIENTID_UTCL2) {
-		/* TODO */
+		struct kfd_vm_fault_info info = {0};
+		uint16_t ring_id = SOC15_RING_ID_FROM_IH_ENTRY(ih_ring_entry);
+
+		info.vmid = vmid;
+		info.mc_id = client_id;
+		info.page_addr = ih_ring_entry[4] |
+			(uint64_t)(ih_ring_entry[5] & 0xf) << 32;
+		info.prot_valid = ring_id & 0x08;
+		info.prot_read  = ring_id & 0x10;
+		info.prot_write = ring_id & 0x20;
+
+		kfd_process_vm_fault(dev->dqm, pasid);
+		kfd_signal_vm_fault_event(dev, pasid, &info);
 	}
 }
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_interrupt.c b/drivers/gpu/drm/amd/amdkfd/kfd_interrupt.c
index db6d9336b80d..c56ac47cd318 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_interrupt.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_interrupt.c
@@ -151,13 +151,15 @@ static void interrupt_wq(struct work_struct *work)
 								ih_ring_entry);
 }
 
-bool interrupt_is_wanted(struct kfd_dev *dev, const uint32_t *ih_ring_entry)
+bool interrupt_is_wanted(struct kfd_dev *dev,
+			const uint32_t *ih_ring_entry,
+			uint32_t *patched_ihre, bool *flag)
 {
 	/* integer and bitwise OR so there is no boolean short-circuiting */
 	unsigned int wanted = 0;
 
 	wanted |= dev->device_info->event_interrupt_class->interrupt_isr(dev,
-								ih_ring_entry);
+					 ih_ring_entry, patched_ihre, flag);
 
 	return wanted != 0;
 }
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_iommu.c b/drivers/gpu/drm/amd/amdkfd/kfd_iommu.c
index c71817963eea..7a61f38c09e6 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_iommu.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_iommu.c
@@ -190,7 +190,7 @@ static int iommu_invalid_ppr_cb(struct pci_dev *pdev, int pasid,
 {
 	struct kfd_dev *dev;
 
-	dev_warn(kfd_device,
+	dev_warn_ratelimited(kfd_device,
 			"Invalid PPR device %x:%x.%x pasid %d address 0x%lX flags 0x%X",
 			PCI_BUS_NUM(pdev->devfn),
 			PCI_SLOT(pdev->devfn),
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c b/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c
index 476951d8c91c..9f84b4d9fb88 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c
@@ -59,7 +59,7 @@ static bool initialize(struct kernel_queue *kq, struct kfd_dev *dev,
 	switch (type) {
 	case KFD_QUEUE_TYPE_DIQ:
 	case KFD_QUEUE_TYPE_HIQ:
-		kq->mqd = dev->dqm->ops.get_mqd_manager(dev->dqm,
+		kq->mqd_mgr = dev->dqm->ops.get_mqd_manager(dev->dqm,
 						KFD_MQD_TYPE_HIQ);
 		break;
 	default:
@@ -67,7 +67,7 @@ static bool initialize(struct kernel_queue *kq, struct kfd_dev *dev,
 		return false;
 	}
 
-	if (!kq->mqd)
+	if (!kq->mqd_mgr)
 		return false;
 
 	prop.doorbell_ptr = kfd_get_kernel_doorbell(dev, &prop.doorbell_off);
@@ -123,6 +123,7 @@ static bool initialize(struct kernel_queue *kq, struct kfd_dev *dev,
 	prop.write_ptr = (uint32_t *) kq->wptr_gpu_addr;
 	prop.eop_ring_buffer_address = kq->eop_gpu_addr;
 	prop.eop_ring_buffer_size = PAGE_SIZE;
+	prop.cu_mask = NULL;
 
 	if (init_queue(&kq->queue, &prop) != 0)
 		goto err_init_queue;
@@ -130,7 +131,7 @@ static bool initialize(struct kernel_queue *kq, struct kfd_dev *dev,
 	kq->queue->device = dev;
 	kq->queue->process = kfd_get_process(current);
 
-	retval = kq->mqd->init_mqd(kq->mqd, &kq->queue->mqd,
+	retval = kq->mqd_mgr->init_mqd(kq->mqd_mgr, &kq->queue->mqd,
 					&kq->queue->mqd_mem_obj,
 					&kq->queue->gart_mqd_addr,
 					&kq->queue->properties);
@@ -142,9 +143,9 @@ static bool initialize(struct kernel_queue *kq, struct kfd_dev *dev,
 		pr_debug("Assigning hiq to hqd\n");
 		kq->queue->pipe = KFD_CIK_HIQ_PIPE;
 		kq->queue->queue = KFD_CIK_HIQ_QUEUE;
-		kq->mqd->load_mqd(kq->mqd, kq->queue->mqd, kq->queue->pipe,
-				  kq->queue->queue, &kq->queue->properties,
-				  NULL);
+		kq->mqd_mgr->load_mqd(kq->mqd_mgr, kq->queue->mqd,
+				kq->queue->pipe, kq->queue->queue,
+				&kq->queue->properties, NULL);
 	} else {
 		/* allocate fence for DIQ */
 
@@ -182,7 +183,7 @@ err_get_kernel_doorbell:
 static void uninitialize(struct kernel_queue *kq)
 {
 	if (kq->queue->properties.type == KFD_QUEUE_TYPE_HIQ)
-		kq->mqd->destroy_mqd(kq->mqd,
+		kq->mqd_mgr->destroy_mqd(kq->mqd_mgr,
 					kq->queue->mqd,
 					KFD_PREEMPT_TYPE_WAVEFRONT_RESET,
 					KFD_UNMAP_LATENCY_MS,
@@ -191,7 +192,8 @@ static void uninitialize(struct kernel_queue *kq)
 	else if (kq->queue->properties.type == KFD_QUEUE_TYPE_DIQ)
 		kfd_gtt_sa_free(kq->dev, kq->fence_mem_obj);
 
-	kq->mqd->uninit_mqd(kq->mqd, kq->queue->mqd, kq->queue->mqd_mem_obj);
+	kq->mqd_mgr->uninit_mqd(kq->mqd_mgr, kq->queue->mqd,
+				kq->queue->mqd_mem_obj);
 
 	kfd_gtt_sa_free(kq->dev, kq->rptr_mem);
 	kfd_gtt_sa_free(kq->dev, kq->wptr_mem);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.h b/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.h
index 97aff2041a5d..a7116a939029 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.h
@@ -70,7 +70,7 @@ struct kernel_queue {
 
 	/* data */
 	struct kfd_dev		*dev;
-	struct mqd_manager	*mqd;
+	struct mqd_manager	*mqd_mgr;
 	struct queue		*queue;
 	uint64_t		pending_wptr64;
 	uint32_t		pending_wptr;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_module.c b/drivers/gpu/drm/amd/amdkfd/kfd_module.c
index 76bf2dc8aec4..6e1f5c7c2d4b 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_module.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_module.c
@@ -47,6 +47,8 @@ static const struct kgd2kfd_calls kgd2kfd = {
 	.resume_mm	= kgd2kfd_resume_mm,
 	.schedule_evict_and_restore_process =
 			  kgd2kfd_schedule_evict_and_restore_process,
+	.pre_reset	= kgd2kfd_pre_reset,
+	.post_reset	= kgd2kfd_post_reset,
 };
 
 int sched_policy = KFD_SCHED_POLICY_HWS;
@@ -61,7 +63,7 @@ MODULE_PARM_DESC(hws_max_conc_proc,
 
 int cwsr_enable = 1;
 module_param(cwsr_enable, int, 0444);
-MODULE_PARM_DESC(cwsr_enable, "CWSR enable (0 = Off, 1 = On (Default))");
+MODULE_PARM_DESC(cwsr_enable, "CWSR enable (0 = off, 1 = on (default))");
 
 int max_num_of_queues_per_device = KFD_MAX_NUM_OF_QUEUES_PER_DEVICE_DEFAULT;
 module_param(max_num_of_queues_per_device, int, 0444);
@@ -83,13 +85,19 @@ module_param(ignore_crat, int, 0444);
 MODULE_PARM_DESC(ignore_crat,
 	"Ignore CRAT table during KFD initialization (0 = use CRAT (default), 1 = ignore CRAT)");
 
-int vega10_noretry;
-module_param_named(noretry, vega10_noretry, int, 0644);
+int noretry;
+module_param(noretry, int, 0644);
 MODULE_PARM_DESC(noretry,
-	"Set sh_mem_config.retry_disable on Vega10 (0 = retry enabled (default), 1 = retry disabled)");
+	"Set sh_mem_config.retry_disable on GFXv9+ dGPUs (0 = retry enabled (default), 1 = retry disabled)");
+
+int halt_if_hws_hang;
+module_param(halt_if_hws_hang, int, 0644);
+MODULE_PARM_DESC(halt_if_hws_hang, "Halt if HWS hang is detected (0 = off (default), 1 = on)");
+
 
 static int amdkfd_init_completed;
 
+
 int kgd2kfd_init(unsigned int interface_version,
 		const struct kgd2kfd_calls **g2f)
 {
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c
index 4b8eb506642b..3bc25ab84f34 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c
@@ -21,7 +21,7 @@
  *
  */
 
-#include "kfd_priv.h"
+#include "kfd_mqd_manager.h"
 
 struct mqd_manager *mqd_manager_init(enum KFD_MQD_TYPE type,
 					struct kfd_dev *dev)
@@ -48,3 +48,42 @@ struct mqd_manager *mqd_manager_init(enum KFD_MQD_TYPE type,
 
 	return NULL;
 }
+
+void mqd_symmetrically_map_cu_mask(struct mqd_manager *mm,
+		const uint32_t *cu_mask, uint32_t cu_mask_count,
+		uint32_t *se_mask)
+{
+	struct kfd_cu_info cu_info;
+	uint32_t cu_per_sh[4] = {0};
+	int i, se, cu = 0;
+
+	mm->dev->kfd2kgd->get_cu_info(mm->dev->kgd, &cu_info);
+
+	if (cu_mask_count > cu_info.cu_active_number)
+		cu_mask_count = cu_info.cu_active_number;
+
+	for (se = 0; se < cu_info.num_shader_engines; se++)
+		for (i = 0; i < 4; i++)
+			cu_per_sh[se] += hweight32(cu_info.cu_bitmap[se][i]);
+
+	/* Symmetrically map cu_mask to all SEs:
+	 * cu_mask[0] bit0 -> se_mask[0] bit0;
+	 * cu_mask[0] bit1 -> se_mask[1] bit0;
+	 * ... (if # SE is 4)
+	 * cu_mask[0] bit4 -> se_mask[0] bit1;
+	 * ...
+	 */
+	se = 0;
+	for (i = 0; i < cu_mask_count; i++) {
+		if (cu_mask[i / 32] & (1 << (i % 32)))
+			se_mask[se] |= 1 << cu;
+
+		do {
+			se++;
+			if (se == cu_info.num_shader_engines) {
+				se = 0;
+				cu++;
+			}
+		} while (cu >= cu_per_sh[se] && cu < 32);
+	}
+}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h
index 8972bcfbf701..4e84052d4e21 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h
@@ -93,4 +93,8 @@ struct mqd_manager {
 	struct kfd_dev	*dev;
 };
 
+void mqd_symmetrically_map_cu_mask(struct mqd_manager *mm,
+		const uint32_t *cu_mask, uint32_t cu_mask_count,
+		uint32_t *se_mask);
+
 #endif /* KFD_MQD_MANAGER_H_ */
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
index 06eaa218eba6..47243165a082 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
@@ -41,6 +41,31 @@ static inline struct cik_sdma_rlc_registers *get_sdma_mqd(void *mqd)
 	return (struct cik_sdma_rlc_registers *)mqd;
 }
 
+static void update_cu_mask(struct mqd_manager *mm, void *mqd,
+			struct queue_properties *q)
+{
+	struct cik_mqd *m;
+	uint32_t se_mask[4] = {0}; /* 4 is the max # of SEs */
+
+	if (q->cu_mask_count == 0)
+		return;
+
+	mqd_symmetrically_map_cu_mask(mm,
+		q->cu_mask, q->cu_mask_count, se_mask);
+
+	m = get_mqd(mqd);
+	m->compute_static_thread_mgmt_se0 = se_mask[0];
+	m->compute_static_thread_mgmt_se1 = se_mask[1];
+	m->compute_static_thread_mgmt_se2 = se_mask[2];
+	m->compute_static_thread_mgmt_se3 = se_mask[3];
+
+	pr_debug("Update cu mask to %#x %#x %#x %#x\n",
+		m->compute_static_thread_mgmt_se0,
+		m->compute_static_thread_mgmt_se1,
+		m->compute_static_thread_mgmt_se2,
+		m->compute_static_thread_mgmt_se3);
+}
+
 static int init_mqd(struct mqd_manager *mm, void **mqd,
 		struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
 		struct queue_properties *q)
@@ -196,6 +221,8 @@ static int __update_mqd(struct mqd_manager *mm, void *mqd,
 	if (q->format == KFD_QUEUE_FORMAT_AQL)
 		m->cp_hqd_pq_control |= NO_UPDATE_RPTR;
 
+	update_cu_mask(mm, mqd, q);
+
 	q->is_active = (q->queue_size > 0 &&
 			q->queue_address != 0 &&
 			q->queue_percent > 0 &&
@@ -408,7 +435,7 @@ struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type,
 	if (WARN_ON(type >= KFD_MQD_TYPE_MAX))
 		return NULL;
 
-	mqd = kzalloc(sizeof(*mqd), GFP_NOIO);
+	mqd = kzalloc(sizeof(*mqd), GFP_KERNEL);
 	if (!mqd)
 		return NULL;
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
index 684054ff02cd..f5fc3675f21e 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
@@ -41,6 +41,31 @@ static inline struct v9_sdma_mqd *get_sdma_mqd(void *mqd)
 	return (struct v9_sdma_mqd *)mqd;
 }
 
+static void update_cu_mask(struct mqd_manager *mm, void *mqd,
+			struct queue_properties *q)
+{
+	struct v9_mqd *m;
+	uint32_t se_mask[4] = {0}; /* 4 is the max # of SEs */
+
+	if (q->cu_mask_count == 0)
+		return;
+
+	mqd_symmetrically_map_cu_mask(mm,
+		q->cu_mask, q->cu_mask_count, se_mask);
+
+	m = get_mqd(mqd);
+	m->compute_static_thread_mgmt_se0 = se_mask[0];
+	m->compute_static_thread_mgmt_se1 = se_mask[1];
+	m->compute_static_thread_mgmt_se2 = se_mask[2];
+	m->compute_static_thread_mgmt_se3 = se_mask[3];
+
+	pr_debug("update cu mask to %#x %#x %#x %#x\n",
+		m->compute_static_thread_mgmt_se0,
+		m->compute_static_thread_mgmt_se1,
+		m->compute_static_thread_mgmt_se2,
+		m->compute_static_thread_mgmt_se3);
+}
+
 static int init_mqd(struct mqd_manager *mm, void **mqd,
 			struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
 			struct queue_properties *q)
@@ -55,7 +80,7 @@ static int init_mqd(struct mqd_manager *mm, void **mqd,
 	 * instead of sub-allocation function.
 	 */
 	if (kfd->cwsr_enabled && (q->type == KFD_QUEUE_TYPE_COMPUTE)) {
-		*mqd_mem_obj = kzalloc(sizeof(struct kfd_mem_obj), GFP_NOIO);
+		*mqd_mem_obj = kzalloc(sizeof(struct kfd_mem_obj), GFP_KERNEL);
 		if (!*mqd_mem_obj)
 			return -ENOMEM;
 		retval = kfd->kfd2kgd->init_gtt_mem_allocation(kfd->kgd,
@@ -198,6 +223,8 @@ static int update_mqd(struct mqd_manager *mm, void *mqd,
 	if (mm->dev->cwsr_enabled && q->ctx_save_restore_area_address)
 		m->cp_hqd_ctx_save_control = 0;
 
+	update_cu_mask(mm, mqd, q);
+
 	q->is_active = (q->queue_size > 0 &&
 			q->queue_address != 0 &&
 			q->queue_percent > 0 &&
@@ -393,7 +420,7 @@ struct mqd_manager *mqd_manager_init_v9(enum KFD_MQD_TYPE type,
 	if (WARN_ON(type >= KFD_MQD_TYPE_MAX))
 		return NULL;
 
-	mqd = kzalloc(sizeof(*mqd), GFP_NOIO);
+	mqd = kzalloc(sizeof(*mqd), GFP_KERNEL);
 	if (!mqd)
 		return NULL;
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
index 481307b8b4db..b81fda3754da 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
@@ -43,6 +43,31 @@ static inline struct vi_sdma_mqd *get_sdma_mqd(void *mqd)
 	return (struct vi_sdma_mqd *)mqd;
 }
 
+static void update_cu_mask(struct mqd_manager *mm, void *mqd,
+			struct queue_properties *q)
+{
+	struct vi_mqd *m;
+	uint32_t se_mask[4] = {0}; /* 4 is the max # of SEs */
+
+	if (q->cu_mask_count == 0)
+		return;
+
+	mqd_symmetrically_map_cu_mask(mm,
+		q->cu_mask, q->cu_mask_count, se_mask);
+
+	m = get_mqd(mqd);
+	m->compute_static_thread_mgmt_se0 = se_mask[0];
+	m->compute_static_thread_mgmt_se1 = se_mask[1];
+	m->compute_static_thread_mgmt_se2 = se_mask[2];
+	m->compute_static_thread_mgmt_se3 = se_mask[3];
+
+	pr_debug("Update cu mask to %#x %#x %#x %#x\n",
+		m->compute_static_thread_mgmt_se0,
+		m->compute_static_thread_mgmt_se1,
+		m->compute_static_thread_mgmt_se2,
+		m->compute_static_thread_mgmt_se3);
+}
+
 static int init_mqd(struct mqd_manager *mm, void **mqd,
 			struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
 			struct queue_properties *q)
@@ -196,6 +221,8 @@ static int __update_mqd(struct mqd_manager *mm, void *mqd,
 			atc_bit << CP_HQD_CTX_SAVE_CONTROL__ATC__SHIFT |
 			mtype << CP_HQD_CTX_SAVE_CONTROL__MTYPE__SHIFT;
 
+	update_cu_mask(mm, mqd, q);
+
 	q->is_active = (q->queue_size > 0 &&
 			q->queue_address != 0 &&
 			q->queue_percent > 0 &&
@@ -394,7 +421,7 @@ struct mqd_manager *mqd_manager_init_vi(enum KFD_MQD_TYPE type,
 	if (WARN_ON(type >= KFD_MQD_TYPE_MAX))
 		return NULL;
 
-	mqd = kzalloc(sizeof(*mqd), GFP_NOIO);
+	mqd = kzalloc(sizeof(*mqd), GFP_KERNEL);
 	if (!mqd)
 		return NULL;
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c
index c317feb43f69..1092631765cb 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c
@@ -418,4 +418,30 @@ out:
 	return 0;
 }
 
+int pm_debugfs_hang_hws(struct packet_manager *pm)
+{
+	uint32_t *buffer, size;
+	int r = 0;
+
+	size = pm->pmf->query_status_size;
+	mutex_lock(&pm->lock);
+	pm->priv_queue->ops.acquire_packet_buffer(pm->priv_queue,
+			size / sizeof(uint32_t), (unsigned int **)&buffer);
+	if (!buffer) {
+		pr_err("Failed to allocate buffer on kernel queue\n");
+		r = -ENOMEM;
+		goto out;
+	}
+	memset(buffer, 0x55, size);
+	pm->priv_queue->ops.submit_packet(pm->priv_queue);
+
+	pr_info("Submitting %x %x %x %x %x %x %x to HIQ to hang the HWS.",
+		buffer[0], buffer[1], buffer[2], buffer[3],
+		buffer[4], buffer[5], buffer[6]);
+out:
+	mutex_unlock(&pm->lock);
+	return r;
+}
+
+
 #endif
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index 5e3990bb4c4b..f971710f1c91 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -73,7 +73,7 @@
 
 /*
  * When working with cp scheduler we should assign the HIQ manually or via
- * the radeon driver to a fixed hqd slot, here are the fixed HIQ hqd slot
+ * the amdgpu driver to a fixed hqd slot, here are the fixed HIQ hqd slot
  * definitions for Kaveri. In Kaveri only the first ME queues participates
  * in the cp scheduling taking that in mind we set the HIQ slot in the
  * second ME.
@@ -142,7 +142,12 @@ extern int ignore_crat;
 /*
  * Set sh_mem_config.retry_disable on Vega10
  */
-extern int vega10_noretry;
+extern int noretry;
+
+/*
+ * Halt if HWS hang is detected
+ */
+extern int halt_if_hws_hang;
 
 /**
  * enum kfd_sched_policy
@@ -180,9 +185,10 @@ enum cache_policy {
 
 struct kfd_event_interrupt_class {
 	bool (*interrupt_isr)(struct kfd_dev *dev,
-				const uint32_t *ih_ring_entry);
+			const uint32_t *ih_ring_entry, uint32_t *patched_ihre,
+			bool *patched_flag);
 	void (*interrupt_wq)(struct kfd_dev *dev,
-				const uint32_t *ih_ring_entry);
+			const uint32_t *ih_ring_entry);
 };
 
 struct kfd_device_info {
@@ -197,6 +203,7 @@ struct kfd_device_info {
 	bool supports_cwsr;
 	bool needs_iommu_device;
 	bool needs_pci_atomics;
+	unsigned int num_sdma_engines;
 };
 
 struct kfd_mem_obj {
@@ -415,6 +422,9 @@ struct queue_properties {
 	uint32_t ctl_stack_size;
 	uint64_t tba_addr;
 	uint64_t tma_addr;
+	/* Relevant for CU */
+	uint32_t cu_mask_count; /* Must be a multiple of 32 */
+	uint32_t *cu_mask;
 };
 
 /**
@@ -806,12 +816,18 @@ int kfd_interrupt_init(struct kfd_dev *dev);
 void kfd_interrupt_exit(struct kfd_dev *dev);
 void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry);
 bool enqueue_ih_ring_entry(struct kfd_dev *kfd,	const void *ih_ring_entry);
-bool interrupt_is_wanted(struct kfd_dev *dev, const uint32_t *ih_ring_entry);
+bool interrupt_is_wanted(struct kfd_dev *dev,
+				const uint32_t *ih_ring_entry,
+				uint32_t *patched_ihre, bool *flag);
 
 /* Power Management */
 void kgd2kfd_suspend(struct kfd_dev *kfd);
 int kgd2kfd_resume(struct kfd_dev *kfd);
 
+/* GPU reset */
+int kgd2kfd_pre_reset(struct kfd_dev *kfd);
+int kgd2kfd_post_reset(struct kfd_dev *kfd);
+
 /* amdkfd Apertures */
 int kfd_init_apertures(struct kfd_process *process);
 
@@ -838,6 +854,7 @@ void device_queue_manager_uninit(struct device_queue_manager *dqm);
 struct kernel_queue *kernel_queue_init(struct kfd_dev *dev,
 					enum kfd_queue_type type);
 void kernel_queue_uninit(struct kernel_queue *kq);
+int kfd_process_vm_fault(struct device_queue_manager *dqm, unsigned int pasid);
 
 /* Process Queue Manager */
 struct process_queue_node {
@@ -858,6 +875,8 @@ int pqm_create_queue(struct process_queue_manager *pqm,
 int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid);
 int pqm_update_queue(struct process_queue_manager *pqm, unsigned int qid,
 			struct queue_properties *p);
+int pqm_set_cu_mask(struct process_queue_manager *pqm, unsigned int qid,
+			struct queue_properties *p);
 struct kernel_queue *pqm_get_kernel_queue(struct process_queue_manager *pqm,
 						unsigned int qid);
 
@@ -964,10 +983,17 @@ int kfd_event_create(struct file *devkfd, struct kfd_process *p,
 		     uint64_t *event_page_offset, uint32_t *event_slot_index);
 int kfd_event_destroy(struct kfd_process *p, uint32_t event_id);
 
+void kfd_signal_vm_fault_event(struct kfd_dev *dev, unsigned int pasid,
+				struct kfd_vm_fault_info *info);
+
+void kfd_signal_reset_event(struct kfd_dev *dev);
+
 void kfd_flush_tlb(struct kfd_process_device *pdd);
 
 int dbgdev_wave_reset_wavefronts(struct kfd_dev *dev, struct kfd_process *p);
 
+bool kfd_is_locked(void);
+
 /* Debugfs */
 #if defined(CONFIG_DEBUG_FS)
 
@@ -980,6 +1006,10 @@ int dqm_debugfs_hqds(struct seq_file *m, void *data);
 int kfd_debugfs_rls_by_device(struct seq_file *m, void *data);
 int pm_debugfs_runlist(struct seq_file *m, void *data);
 
+int kfd_debugfs_hang_hws(struct kfd_dev *dev);
+int pm_debugfs_hang_hws(struct packet_manager *pm);
+int dqm_debugfs_execute_queues(struct device_queue_manager *dqm);
+
 #else
 
 static inline void kfd_debugfs_init(void) {}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
index 1d80b4f7c681..4694386cc623 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
@@ -244,6 +244,8 @@ struct kfd_process *kfd_get_process(const struct task_struct *thread)
 		return ERR_PTR(-EINVAL);
 
 	process = find_process(thread);
+	if (!process)
+		return ERR_PTR(-EINVAL);
 
 	return process;
 }
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
index d65ce0436b31..c8cad9c078ae 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
@@ -186,8 +186,7 @@ int pqm_create_queue(struct process_queue_manager *pqm,
 
 	switch (type) {
 	case KFD_QUEUE_TYPE_SDMA:
-		if (dev->dqm->queue_count >=
-			CIK_SDMA_QUEUES_PER_ENGINE * CIK_SDMA_ENGINE_NUM) {
+		if (dev->dqm->queue_count >= get_num_sdma_queues(dev->dqm)) {
 			pr_err("Over-subscription is not allowed for SDMA.\n");
 			retval = -EPERM;
 			goto err_create_queue;
@@ -209,7 +208,7 @@ int pqm_create_queue(struct process_queue_manager *pqm,
 		     KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION) &&
 		((dev->dqm->processes_count >= dev->vm_info.vmid_num_kfd) ||
 		(dev->dqm->queue_count >= get_queues_num(dev->dqm)))) {
-			pr_err("Over-subscription is not allowed in radeon_kfd.sched_policy == 1\n");
+			pr_debug("Over-subscription is not allowed when amdkfd.sched_policy == 1\n");
 			retval = -EPERM;
 			goto err_create_queue;
 		}
@@ -326,6 +325,8 @@ int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid)
 			if (retval != -ETIME)
 				goto err_destroy_queue;
 		}
+		kfree(pqn->q->properties.cu_mask);
+		pqn->q->properties.cu_mask = NULL;
 		uninit_queue(pqn->q);
 	}
 
@@ -366,6 +367,34 @@ int pqm_update_queue(struct process_queue_manager *pqm, unsigned int qid,
 	return 0;
 }
 
+int pqm_set_cu_mask(struct process_queue_manager *pqm, unsigned int qid,
+			struct queue_properties *p)
+{
+	int retval;
+	struct process_queue_node *pqn;
+
+	pqn = get_queue_by_qid(pqm, qid);
+	if (!pqn) {
+		pr_debug("No queue %d exists for update operation\n", qid);
+		return -EFAULT;
+	}
+
+	/* Free the old CU mask memory if it is already allocated, then
+	 * allocate memory for the new CU mask.
+	 */
+	kfree(pqn->q->properties.cu_mask);
+
+	pqn->q->properties.cu_mask_count = p->cu_mask_count;
+	pqn->q->properties.cu_mask = p->cu_mask;
+
+	retval = pqn->q->device->dqm->ops.update_queue(pqn->q->device->dqm,
+							pqn->q);
+	if (retval != 0)
+		return retval;
+
+	return 0;
+}
+
 struct kernel_queue *pqm_get_kernel_queue(
 					struct process_queue_manager *pqm,
 					unsigned int qid)
@@ -387,7 +416,7 @@ int pqm_debugfs_mqds(struct seq_file *m, void *data)
 	struct process_queue_node *pqn;
 	struct queue *q;
 	enum KFD_MQD_TYPE mqd_type;
-	struct mqd_manager *mqd_manager;
+	struct mqd_manager *mqd_mgr;
 	int r = 0;
 
 	list_for_each_entry(pqn, &pqm->queues, process_queue_list) {
@@ -410,11 +439,11 @@ int pqm_debugfs_mqds(struct seq_file *m, void *data)
 					   q->properties.type, q->device->id);
 				continue;
 			}
-			mqd_manager = q->device->dqm->ops.get_mqd_manager(
+			mqd_mgr = q->device->dqm->ops.get_mqd_manager(
 				q->device->dqm, mqd_type);
 		} else if (pqn->kq) {
 			q = pqn->kq->queue;
-			mqd_manager = pqn->kq->mqd;
+			mqd_mgr = pqn->kq->mqd_mgr;
 			switch (q->properties.type) {
 			case KFD_QUEUE_TYPE_DIQ:
 				seq_printf(m, "  DIQ on device %x\n",
@@ -434,7 +463,7 @@ int pqm_debugfs_mqds(struct seq_file *m, void *data)
 			continue;
 		}
 
-		r = mqd_manager->debugfs_show_mqd(m, q->mqd);
+		r = mqd_mgr->debugfs_show_mqd(m, q->mqd);
 		if (r != 0)
 			break;
 	}
diff --git a/drivers/gpu/drm/amd/display/Kconfig b/drivers/gpu/drm/amd/display/Kconfig
index 4c35625eb2c7..ed654a76c76a 100644
--- a/drivers/gpu/drm/amd/display/Kconfig
+++ b/drivers/gpu/drm/amd/display/Kconfig
@@ -4,18 +4,16 @@ menu "Display Engine Configuration"
 config DRM_AMD_DC
 	bool "AMD DC - Enable new display engine"
 	default y
+	select DRM_AMD_DC_DCN1_0 if X86 && !(KCOV_INSTRUMENT_ALL && KCOV_ENABLE_COMPARISONS)
 	help
 	  Choose this option if you want to use the new display engine
 	  support for AMDGPU. This adds required support for Vega and
 	  Raven ASICs.
 
 config DRM_AMD_DC_DCN1_0
-	bool "DCN 1.0 Raven family"
-	depends on DRM_AMD_DC && X86
-	default y
+	def_bool n
 	help
-	  Choose this option if you want to have
-	  RV family for display engine
+	  RV family support for display engine
 
 config DEBUG_KERNEL_DC
 	bool "Enable kgdb break in DC"
diff --git a/drivers/gpu/drm/amd/display/TODO b/drivers/gpu/drm/amd/display/TODO
index 357d59648401..a8a6c106e8c7 100644
--- a/drivers/gpu/drm/amd/display/TODO
+++ b/drivers/gpu/drm/amd/display/TODO
@@ -97,10 +97,10 @@ share it with drivers. But that's a very long term goal, and by far not just an
 issue with DC - other drivers, especially around DP sink handling, are equally
 guilty.
 
-19. The DC logger is still a rather sore thing, but I know that the DRM_DEBUG
-stuff just isn't up to the challenges either. We need to figure out something
-that integrates better with DRM and linux debug printing, while not being
-useless with filtering output. dynamic debug printing might be an option.
+19. DONE - The DC logger is still a rather sore thing, but I know that the
+DRM_DEBUG stuff just isn't up to the challenges either. We need to figure out
+something that integrates better with DRM and linux debug printing, while not
+being useless with filtering output. dynamic debug printing might be an option.
 
 20. Use kernel i2c device to program HDMI retimer. Some boards have an HDMI
 retimer that we need to program to pass PHY compliance. Currently that's
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/Makefile b/drivers/gpu/drm/amd/display/amdgpu_dm/Makefile
index af16973f2c41..94911871eb9b 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/Makefile
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/Makefile
@@ -28,11 +28,11 @@
 AMDGPUDM = amdgpu_dm.o amdgpu_dm_irq.o amdgpu_dm_mst_types.o amdgpu_dm_color.o
 
 ifneq ($(CONFIG_DRM_AMD_DC),)
-AMDGPUDM += amdgpu_dm_services.o amdgpu_dm_helpers.o
+AMDGPUDM += amdgpu_dm_services.o amdgpu_dm_helpers.o amdgpu_dm_pp_smu.o
 endif
 
 ifneq ($(CONFIG_DEBUG_FS),)
-AMDGPUDM += amdgpu_dm_crc.o
+AMDGPUDM += amdgpu_dm_crc.o amdgpu_dm_debugfs.o
 endif
 
 subdir-ccflags-y += -I$(FULL_AMD_DISPLAY_PATH)/dc
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
index a5c1617e55c5..11d6dd23eeb8 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
@@ -39,6 +39,9 @@
 #include "dm_helpers.h"
 #include "dm_services_types.h"
 #include "amdgpu_dm_mst_types.h"
+#if defined(CONFIG_DEBUG_FS)
+#include "amdgpu_dm_debugfs.h"
+#endif
 
 #include "ivsrcid/ivsrcid_vislands30.h"
 
@@ -55,8 +58,6 @@
 #include <drm/drm_fb_helper.h>
 #include <drm/drm_edid.h>
 
-#include "modules/inc/mod_freesync.h"
-
 #if defined(CONFIG_DRM_AMD_DC_DCN1_0)
 #include "ivsrcid/irqsrcs_dcn_1_0.h"
 
@@ -1038,7 +1039,7 @@ static void handle_hpd_rx_irq(void *param)
 	if (dc_link->type != dc_connection_mst_branch)
 		mutex_lock(&aconnector->hpd_lock);
 
-	if (dc_link_handle_hpd_rx_irq(dc_link, NULL) &&
+	if (dc_link_handle_hpd_rx_irq(dc_link, NULL, NULL) &&
 			!is_mst_root_connector) {
 		/* Downstream Port status changed. */
 		if (dc_link_detect(dc_link, DETECT_REASON_HPDRX)) {
@@ -1317,7 +1318,12 @@ static int amdgpu_dm_backlight_update_status(struct backlight_device *bd)
 
 static int amdgpu_dm_backlight_get_brightness(struct backlight_device *bd)
 {
-	return bd->props.brightness;
+	struct amdgpu_display_manager *dm = bl_get_data(bd);
+	int ret = dc_link_get_backlight_level(dm->backlight_link);
+
+	if (ret == DC_ERROR_UNEXPECTED)
+		return bd->props.brightness;
+	return ret;
 }
 
 static const struct backlight_ops amdgpu_dm_backlight_ops = {
@@ -1332,6 +1338,7 @@ amdgpu_dm_register_backlight_device(struct amdgpu_display_manager *dm)
 	struct backlight_properties props = { 0 };
 
 	props.max_brightness = AMDGPU_MAX_BL_LEVEL;
+	props.brightness = AMDGPU_MAX_BL_LEVEL;
 	props.type = BACKLIGHT_RAW;
 
 	snprintf(bl_name, sizeof(bl_name), "amdgpu_bl%d",
@@ -1529,10 +1536,6 @@ static int amdgpu_dm_initialize_drm_device(struct amdgpu_device *adev)
 			DRM_ERROR("DM: Failed to initialize IRQ\n");
 			goto fail;
 		}
-		/*
-		 * Temporary disable until pplib/smu interaction is implemented
-		 */
-		dm->dc->debug.disable_stutter = true;
 		break;
 #endif
 	default:
@@ -1540,6 +1543,9 @@ static int amdgpu_dm_initialize_drm_device(struct amdgpu_device *adev)
 		goto fail;
 	}
 
+	if (adev->asic_type != CHIP_CARRIZO && adev->asic_type != CHIP_STONEY)
+		dm->dc->debug.disable_stutter = amdgpu_pp_feature_mask & PP_STUTTER_MODE ? false : true;
+
 	return 0;
 fail:
 	kfree(aencoder);
@@ -1571,18 +1577,6 @@ static void dm_bandwidth_update(struct amdgpu_device *adev)
 	/* TODO: implement later */
 }
 
-static void dm_set_backlight_level(struct amdgpu_encoder *amdgpu_encoder,
-				     u8 level)
-{
-	/* TODO: translate amdgpu_encoder to display_index and call DAL */
-}
-
-static u8 dm_get_backlight_level(struct amdgpu_encoder *amdgpu_encoder)
-{
-	/* TODO: translate amdgpu_encoder to display_index and call DAL */
-	return 0;
-}
-
 static int amdgpu_notify_freesync(struct drm_device *dev, void *data,
 				struct drm_file *filp)
 {
@@ -1611,10 +1605,8 @@ static int amdgpu_notify_freesync(struct drm_device *dev, void *data,
 static const struct amdgpu_display_funcs dm_display_funcs = {
 	.bandwidth_update = dm_bandwidth_update, /* called unconditionally */
 	.vblank_get_counter = dm_vblank_get_counter,/* called unconditionally */
-	.backlight_set_level =
-		dm_set_backlight_level,/* called unconditionally */
-	.backlight_get_level =
-		dm_get_backlight_level,/* called unconditionally */
+	.backlight_set_level = NULL, /* never called for DC */
+	.backlight_get_level = NULL, /* never called for DC */
 	.hpd_sense = NULL,/* called unconditionally */
 	.hpd_set_polarity = NULL, /* called unconditionally */
 	.hpd_get_gpio_reg = NULL, /* VBIOS parsing. DAL does it. */
@@ -2121,13 +2113,8 @@ convert_color_depth_from_display_info(const struct drm_connector *connector)
 static enum dc_aspect_ratio
 get_aspect_ratio(const struct drm_display_mode *mode_in)
 {
-	int32_t width = mode_in->crtc_hdisplay * 9;
-	int32_t height = mode_in->crtc_vdisplay * 16;
-
-	if ((width - height) < 10 && (width - height) > -10)
-		return ASPECT_RATIO_16_9;
-	else
-		return ASPECT_RATIO_4_3;
+	/* 1-1 mapping, since both enums follow the HDMI spec. */
+	return (enum dc_aspect_ratio) mode_in->picture_aspect_ratio;
 }
 
 static enum dc_color_space
@@ -2173,6 +2160,46 @@ get_output_color_space(const struct dc_crtc_timing *dc_crtc_timing)
 	return color_space;
 }
 
+static void reduce_mode_colour_depth(struct dc_crtc_timing *timing_out)
+{
+	if (timing_out->display_color_depth <= COLOR_DEPTH_888)
+		return;
+
+	timing_out->display_color_depth--;
+}
+
+static void adjust_colour_depth_from_display_info(struct dc_crtc_timing *timing_out,
+						const struct drm_display_info *info)
+{
+	int normalized_clk;
+	if (timing_out->display_color_depth <= COLOR_DEPTH_888)
+		return;
+	do {
+		normalized_clk = timing_out->pix_clk_khz;
+		/* YCbCr 4:2:0 requires additional adjustment of 1/2 */
+		if (timing_out->pixel_encoding == PIXEL_ENCODING_YCBCR420)
+			normalized_clk /= 2;
+		/* Adjusting pix clock following on HDMI spec based on colour depth */
+		switch (timing_out->display_color_depth) {
+		case COLOR_DEPTH_101010:
+			normalized_clk = (normalized_clk * 30) / 24;
+			break;
+		case COLOR_DEPTH_121212:
+			normalized_clk = (normalized_clk * 36) / 24;
+			break;
+		case COLOR_DEPTH_161616:
+			normalized_clk = (normalized_clk * 48) / 24;
+			break;
+		default:
+			return;
+		}
+		if (normalized_clk <= info->max_tmds_clock)
+			return;
+		reduce_mode_colour_depth(timing_out);
+
+	} while (timing_out->display_color_depth > COLOR_DEPTH_888);
+
+}
 /*****************************************************************************/
 
 static void
@@ -2181,6 +2208,7 @@ fill_stream_properties_from_drm_display_mode(struct dc_stream_state *stream,
 					     const struct drm_connector *connector)
 {
 	struct dc_crtc_timing *timing_out = &stream->timing;
+	const struct drm_display_info *info = &connector->display_info;
 
 	memset(timing_out, 0, sizeof(struct dc_crtc_timing));
 
@@ -2189,8 +2217,10 @@ fill_stream_properties_from_drm_display_mode(struct dc_stream_state *stream,
 	timing_out->v_border_top = 0;
 	timing_out->v_border_bottom = 0;
 	/* TODO: un-hardcode */
-
-	if ((connector->display_info.color_formats & DRM_COLOR_FORMAT_YCRCB444)
+	if (drm_mode_is_420_only(info, mode_in)
+			&& stream->sink->sink_signal == SIGNAL_TYPE_HDMI_TYPE_A)
+		timing_out->pixel_encoding = PIXEL_ENCODING_YCBCR420;
+	else if ((connector->display_info.color_formats & DRM_COLOR_FORMAT_YCRCB444)
 			&& stream->sink->sink_signal == SIGNAL_TYPE_HDMI_TYPE_A)
 		timing_out->pixel_encoding = PIXEL_ENCODING_YCBCR444;
 	else
@@ -2226,6 +2256,8 @@ fill_stream_properties_from_drm_display_mode(struct dc_stream_state *stream,
 
 	stream->out_transfer_func->type = TF_TYPE_PREDEFINED;
 	stream->out_transfer_func->tf = TRANSFER_FUNCTION_SRGB;
+	if (stream->sink->sink_signal == SIGNAL_TYPE_HDMI_TYPE_A)
+		adjust_colour_depth_from_display_info(timing_out, info);
 }
 
 static void fill_audio_info(struct audio_info *audio_info,
@@ -3047,14 +3079,24 @@ static int dm_plane_helper_prepare_fb(struct drm_plane *plane,
 	else
 		domain = AMDGPU_GEM_DOMAIN_VRAM;
 
-	r = amdgpu_bo_pin(rbo, domain, &afb->address);
-	amdgpu_bo_unreserve(rbo);
-
+	r = amdgpu_bo_pin(rbo, domain);
 	if (unlikely(r != 0)) {
 		if (r != -ERESTARTSYS)
 			DRM_ERROR("Failed to pin framebuffer with error %d\n", r);
+		amdgpu_bo_unreserve(rbo);
+		return r;
+	}
+
+	r = amdgpu_ttm_alloc_gart(&rbo->tbo);
+	if (unlikely(r != 0)) {
+		amdgpu_bo_unpin(rbo);
+		amdgpu_bo_unreserve(rbo);
+		DRM_ERROR("%p bind failed\n", rbo);
 		return r;
 	}
+	amdgpu_bo_unreserve(rbo);
+
+	afb->address = amdgpu_bo_gpu_offset(rbo);
 
 	amdgpu_bo_ref(rbo);
 
@@ -3452,7 +3494,6 @@ void amdgpu_dm_connector_init_helper(struct amdgpu_display_manager *dm,
 	aconnector->base.stereo_allowed = false;
 	aconnector->base.dpms = DRM_MODE_DPMS_OFF;
 	aconnector->hpd.hpd = AMDGPU_HPD_NONE; /* not used */
-
 	mutex_init(&aconnector->hpd_lock);
 
 	/* configure support HPD hot plug connector_>polled default value is 0
@@ -3461,9 +3502,13 @@ void amdgpu_dm_connector_init_helper(struct amdgpu_display_manager *dm,
 	switch (connector_type) {
 	case DRM_MODE_CONNECTOR_HDMIA:
 		aconnector->base.polled = DRM_CONNECTOR_POLL_HPD;
+		aconnector->base.ycbcr_420_allowed =
+			link->link_enc->features.ycbcr420_supported ? true : false;
 		break;
 	case DRM_MODE_CONNECTOR_DisplayPort:
 		aconnector->base.polled = DRM_CONNECTOR_POLL_HPD;
+		aconnector->base.ycbcr_420_allowed =
+			link->link_enc->features.ycbcr420_supported ? true : false;
 		break;
 	case DRM_MODE_CONNECTOR_DVID:
 		aconnector->base.polled = DRM_CONNECTOR_POLL_HPD;
@@ -3620,6 +3665,13 @@ static int amdgpu_dm_connector_init(struct amdgpu_display_manager *dm,
 		&aconnector->base, &aencoder->base);
 
 	drm_connector_register(&aconnector->base);
+#if defined(CONFIG_DEBUG_FS)
+	res = connector_debugfs_init(aconnector);
+	if (res) {
+		DRM_ERROR("Failed to create debugfs for connector");
+		goto out_free;
+	}
+#endif
 
 	if (connector_type == DRM_MODE_CONNECTOR_DisplayPort
 		|| connector_type == DRM_MODE_CONNECTOR_eDP)
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c
index b329393307e5..326f6fb7e0bc 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c
@@ -231,18 +231,21 @@ void amdgpu_dm_set_ctm(struct dm_crtc_state *crtc)
  * preparation for hardware commit. If no lut is specified by user, we default
  * to SRGB degamma.
  *
- * Currently, we only support degamma bypass, or preprogrammed SRGB degamma.
- * Programmable degamma is not supported, and an attempt to do so will return
- * -EINVAL.
+ * We support degamma bypass, predefined SRGB, and custom degamma
  *
  * RETURNS:
- * 0 on success, -EINVAL if custom degamma curve is given.
+ * 0 on success
+ * -EINVAL if crtc_state has a degamma_lut of invalid size
+ * -ENOMEM if gamma allocation fails
  */
 int amdgpu_dm_set_degamma_lut(struct drm_crtc_state *crtc_state,
 			      struct dc_plane_state *dc_plane_state)
 {
 	struct drm_property_blob *blob = crtc_state->degamma_lut;
 	struct drm_color_lut *lut;
+	uint32_t lut_size;
+	struct dc_gamma *gamma;
+	bool ret;
 
 	if (!blob) {
 		/* Default to SRGB */
@@ -258,11 +261,30 @@ int amdgpu_dm_set_degamma_lut(struct drm_crtc_state *crtc_state,
 		return 0;
 	}
 
-	/* Otherwise, assume SRGB, since programmable degamma is not
-	 * supported.
-	 */
-	dc_plane_state->in_transfer_func->type = TF_TYPE_PREDEFINED;
-	dc_plane_state->in_transfer_func->tf = TRANSFER_FUNCTION_SRGB;
-	return -EINVAL;
+	gamma = dc_create_gamma();
+	if (!gamma)
+		return -ENOMEM;
+
+	lut_size = blob->length / sizeof(struct drm_color_lut);
+	gamma->num_entries = lut_size;
+	if (gamma->num_entries == MAX_COLOR_LUT_ENTRIES)
+		gamma->type = GAMMA_CUSTOM;
+	else {
+		dc_gamma_release(&gamma);
+		return -EINVAL;
+	}
+
+	__drm_lut_to_dc_gamma(lut, gamma, false);
+
+	dc_plane_state->in_transfer_func->type = TF_TYPE_DISTRIBUTED_POINTS;
+	ret = mod_color_calculate_degamma_params(dc_plane_state->in_transfer_func, gamma, true);
+	dc_gamma_release(&gamma);
+	if (!ret) {
+		dc_plane_state->in_transfer_func->type = TF_TYPE_BYPASS;
+		DRM_ERROR("Out of memory when calculating degamma params\n");
+		return -ENOMEM;
+	}
+
+	return 0;
 }
 
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.c
index e7ad528f5853..01fc5717b657 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.c
@@ -112,10 +112,16 @@ int amdgpu_dm_crtc_set_crc_source(struct drm_crtc *crtc, const char *src_name)
  */
 void amdgpu_dm_crtc_handle_crc_irq(struct drm_crtc *crtc)
 {
-	struct dm_crtc_state *crtc_state = to_dm_crtc_state(crtc->state);
-	struct dc_stream_state *stream_state = crtc_state->stream;
+	struct dm_crtc_state *crtc_state;
+	struct dc_stream_state *stream_state;
 	uint32_t crcs[3];
 
+	if (crtc == NULL)
+		return;
+
+	crtc_state = to_dm_crtc_state(crtc->state);
+	stream_state = crtc_state->stream;
+
 	/* Early return if CRC capture is not enabled. */
 	if (!crtc_state->crc_enabled)
 		return;
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c
new file mode 100644
index 000000000000..0d9e410ca01e
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c
@@ -0,0 +1,722 @@
+/*
+ * Copyright 2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#include <linux/debugfs.h>
+
+#include "dc.h"
+#include "amdgpu.h"
+#include "amdgpu_dm.h"
+#include "amdgpu_dm_debugfs.h"
+
+/* function description
+ * get/ set DP configuration: lane_count, link_rate, spread_spectrum
+ *
+ * valid lane count value: 1, 2, 4
+ * valid link rate value:
+ * 06h = 1.62Gbps per lane
+ * 0Ah = 2.7Gbps per lane
+ * 0Ch = 3.24Gbps per lane
+ * 14h = 5.4Gbps per lane
+ * 1Eh = 8.1Gbps per lane
+ *
+ * debugfs is located at /sys/kernel/debug/dri/0/DP-x/link_settings
+ *
+ * --- to get dp configuration
+ *
+ * cat link_settings
+ *
+ * It will list current, verified, reported, preferred dp configuration.
+ * current -- for current video mode
+ * verified --- maximum configuration which pass link training
+ * reported --- DP rx report caps (DPCD register offset 0, 1 2)
+ * preferred --- user force settings
+ *
+ * --- set (or force) dp configuration
+ *
+ * echo <lane_count>  <link_rate> > link_settings
+ *
+ * for example, to force to  2 lane, 2.7GHz,
+ * echo 4 0xa > link_settings
+ *
+ * spread_spectrum could not be changed dynamically.
+ *
+ * in case invalid lane count, link rate are force, no hw programming will be
+ * done. please check link settings after force operation to see if HW get
+ * programming.
+ *
+ * cat link_settings
+ *
+ * check current and preferred settings.
+ *
+ */
+static ssize_t dp_link_settings_read(struct file *f, char __user *buf,
+				 size_t size, loff_t *pos)
+{
+	struct amdgpu_dm_connector *connector = file_inode(f)->i_private;
+	struct dc_link *link = connector->dc_link;
+	char *rd_buf = NULL;
+	char *rd_buf_ptr = NULL;
+	const uint32_t rd_buf_size = 100;
+	uint32_t result = 0;
+	uint8_t str_len = 0;
+	int r;
+
+	if (*pos & 3 || size & 3)
+		return -EINVAL;
+
+	rd_buf = kcalloc(rd_buf_size, sizeof(char), GFP_KERNEL);
+	if (!rd_buf)
+		return 0;
+
+	rd_buf_ptr = rd_buf;
+
+	str_len = strlen("Current:  %d  %d  %d  ");
+	snprintf(rd_buf_ptr, str_len, "Current:  %d  %d  %d  ",
+			link->cur_link_settings.lane_count,
+			link->cur_link_settings.link_rate,
+			link->cur_link_settings.link_spread);
+	rd_buf_ptr += str_len;
+
+	str_len = strlen("Verified:  %d  %d  %d  ");
+	snprintf(rd_buf_ptr, str_len, "Verified:  %d  %d  %d  ",
+			link->verified_link_cap.lane_count,
+			link->verified_link_cap.link_rate,
+			link->verified_link_cap.link_spread);
+	rd_buf_ptr += str_len;
+
+	str_len = strlen("Reported:  %d  %d  %d  ");
+	snprintf(rd_buf_ptr, str_len, "Reported:  %d  %d  %d  ",
+			link->reported_link_cap.lane_count,
+			link->reported_link_cap.link_rate,
+			link->reported_link_cap.link_spread);
+	rd_buf_ptr += str_len;
+
+	str_len = strlen("Preferred:  %d  %d  %d  ");
+	snprintf(rd_buf_ptr, str_len, "Preferred:  %d  %d  %d\n",
+			link->preferred_link_setting.lane_count,
+			link->preferred_link_setting.link_rate,
+			link->preferred_link_setting.link_spread);
+
+	while (size) {
+		if (*pos >= rd_buf_size)
+			break;
+
+		r = put_user(*(rd_buf + result), buf);
+		if (r)
+			return r; /* r = -EFAULT */
+
+		buf += 1;
+		size -= 1;
+		*pos += 1;
+		result += 1;
+	}
+
+	kfree(rd_buf);
+	return result;
+}
+
+static ssize_t dp_link_settings_write(struct file *f, const char __user *buf,
+				 size_t size, loff_t *pos)
+{
+	struct amdgpu_dm_connector *connector = file_inode(f)->i_private;
+	struct dc_link *link = connector->dc_link;
+	struct dc *dc = (struct dc *)link->dc;
+	struct dc_link_settings prefer_link_settings;
+	char *wr_buf = NULL;
+	char *wr_buf_ptr = NULL;
+	const uint32_t wr_buf_size = 40;
+	int r;
+	int bytes_from_user;
+	char *sub_str;
+	/* 0: lane_count; 1: link_rate */
+	uint8_t param_index = 0;
+	long param[2];
+	const char delimiter[3] = {' ', '\n', '\0'};
+	bool valid_input = false;
+
+	if (size == 0)
+		return -EINVAL;
+
+	wr_buf = kcalloc(wr_buf_size, sizeof(char), GFP_KERNEL);
+	if (!wr_buf)
+		return -EINVAL;
+	wr_buf_ptr = wr_buf;
+
+	r = copy_from_user(wr_buf_ptr, buf, wr_buf_size);
+
+	/* r is bytes not be copied */
+	if (r >= wr_buf_size) {
+		kfree(wr_buf);
+		DRM_DEBUG_DRIVER("user data not read\n");
+		return -EINVAL;
+	}
+
+	bytes_from_user = wr_buf_size - r;
+
+	while (isspace(*wr_buf_ptr))
+		wr_buf_ptr++;
+
+	while ((*wr_buf_ptr != '\0') && (param_index < 2)) {
+
+		sub_str = strsep(&wr_buf_ptr, delimiter);
+
+		r = kstrtol(sub_str, 16, &param[param_index]);
+
+		if (r)
+			DRM_DEBUG_DRIVER("string to int convert error code: %d\n", r);
+
+		param_index++;
+		while (isspace(*wr_buf_ptr))
+			wr_buf_ptr++;
+	}
+
+	switch (param[0]) {
+	case LANE_COUNT_ONE:
+	case LANE_COUNT_TWO:
+	case LANE_COUNT_FOUR:
+		valid_input = true;
+		break;
+	default:
+		break;
+	}
+
+	switch (param[1]) {
+	case LINK_RATE_LOW:
+	case LINK_RATE_HIGH:
+	case LINK_RATE_RBR2:
+	case LINK_RATE_HIGH2:
+	case LINK_RATE_HIGH3:
+		valid_input = true;
+		break;
+	default:
+		break;
+	}
+
+	if (!valid_input) {
+		kfree(wr_buf);
+		DRM_DEBUG_DRIVER("Invalid Input value No HW will be programmed\n");
+		return bytes_from_user;
+	}
+
+	/* save user force lane_count, link_rate to preferred settings
+	 * spread spectrum will not be changed
+	 */
+	prefer_link_settings.link_spread = link->cur_link_settings.link_spread;
+	prefer_link_settings.lane_count = param[0];
+	prefer_link_settings.link_rate = param[1];
+
+	dc_link_set_preferred_link_settings(dc, &prefer_link_settings, link);
+
+	kfree(wr_buf);
+	return bytes_from_user;
+}
+
+/* function: get current DP PHY settings: voltage swing, pre-emphasis,
+ * post-cursor2 (defined by VESA DP specification)
+ *
+ * valid values
+ * voltage swing: 0,1,2,3
+ * pre-emphasis : 0,1,2,3
+ * post cursor2 : 0,1,2,3
+ *
+ *
+ * how to use this debugfs
+ *
+ * debugfs is located at /sys/kernel/debug/dri/0/DP-x
+ *
+ * there will be directories, like DP-1, DP-2,DP-3, etc. for DP display
+ *
+ * To figure out which DP-x is the display for DP to be check,
+ * cd DP-x
+ * ls -ll
+ * There should be debugfs file, like link_settings, phy_settings.
+ * cat link_settings
+ * from lane_count, link_rate to figure which DP-x is for display to be worked
+ * on
+ *
+ * To get current DP PHY settings,
+ * cat phy_settings
+ *
+ * To change DP PHY settings,
+ * echo <voltage_swing> <pre-emphasis> <post_cursor2> > phy_settings
+ * for examle, to change voltage swing to 2, pre-emphasis to 3, post_cursor2 to
+ * 0,
+ * echo 2 3 0 > phy_settings
+ *
+ * To check if change be applied, get current phy settings by
+ * cat phy_settings
+ *
+ * In case invalid values are set by user, like
+ * echo 1 4 0 > phy_settings
+ *
+ * HW will NOT be programmed by these settings.
+ * cat phy_settings will show the previous valid settings.
+ */
+static ssize_t dp_phy_settings_read(struct file *f, char __user *buf,
+				 size_t size, loff_t *pos)
+{
+	struct amdgpu_dm_connector *connector = file_inode(f)->i_private;
+	struct dc_link *link = connector->dc_link;
+	char *rd_buf = NULL;
+	const uint32_t rd_buf_size = 20;
+	uint32_t result = 0;
+	int r;
+
+	if (*pos & 3 || size & 3)
+		return -EINVAL;
+
+	rd_buf = kcalloc(rd_buf_size, sizeof(char), GFP_KERNEL);
+	if (!rd_buf)
+		return -EINVAL;
+
+	snprintf(rd_buf, rd_buf_size, "  %d  %d  %d  ",
+			link->cur_lane_setting.VOLTAGE_SWING,
+			link->cur_lane_setting.PRE_EMPHASIS,
+			link->cur_lane_setting.POST_CURSOR2);
+
+	while (size) {
+		if (*pos >= rd_buf_size)
+			break;
+
+		r = put_user((*(rd_buf + result)), buf);
+		if (r)
+			return r; /* r = -EFAULT */
+
+		buf += 1;
+		size -= 1;
+		*pos += 1;
+		result += 1;
+	}
+
+	kfree(rd_buf);
+	return result;
+}
+
+static ssize_t dp_phy_settings_write(struct file *f, const char __user *buf,
+				 size_t size, loff_t *pos)
+{
+	struct amdgpu_dm_connector *connector = file_inode(f)->i_private;
+	struct dc_link *link = connector->dc_link;
+	struct dc *dc = (struct dc *)link->dc;
+	char *wr_buf = NULL;
+	char *wr_buf_ptr = NULL;
+	uint32_t wr_buf_size = 40;
+	int r;
+	int bytes_from_user;
+	char *sub_str;
+	uint8_t param_index = 0;
+	long param[3];
+	const char delimiter[3] = {' ', '\n', '\0'};
+	bool use_prefer_link_setting;
+	struct link_training_settings link_lane_settings;
+
+	if (size == 0)
+		return 0;
+
+	wr_buf = kcalloc(wr_buf_size, sizeof(char), GFP_KERNEL);
+	if (!wr_buf)
+		return 0;
+	wr_buf_ptr = wr_buf;
+
+	r = copy_from_user(wr_buf_ptr, buf, wr_buf_size);
+
+	/* r is bytes not be copied */
+	if (r >= wr_buf_size) {
+		kfree(wr_buf);
+		DRM_DEBUG_DRIVER("user data not be read\n");
+		return 0;
+	}
+
+	bytes_from_user = wr_buf_size - r;
+
+	while (isspace(*wr_buf_ptr))
+		wr_buf_ptr++;
+
+	while ((*wr_buf_ptr != '\0') && (param_index < 3)) {
+
+		sub_str = strsep(&wr_buf_ptr, delimiter);
+
+		r = kstrtol(sub_str, 16, &param[param_index]);
+
+		if (r)
+			DRM_DEBUG_DRIVER("string to int convert error code: %d\n", r);
+
+		param_index++;
+		while (isspace(*wr_buf_ptr))
+			wr_buf_ptr++;
+	}
+
+	if ((param[0] > VOLTAGE_SWING_MAX_LEVEL) ||
+			(param[1] > PRE_EMPHASIS_MAX_LEVEL) ||
+			(param[2] > POST_CURSOR2_MAX_LEVEL)) {
+		kfree(wr_buf);
+		DRM_DEBUG_DRIVER("Invalid Input No HW will be programmed\n");
+		return bytes_from_user;
+	}
+
+	/* get link settings: lane count, link rate */
+	use_prefer_link_setting =
+		((link->preferred_link_setting.link_rate != LINK_RATE_UNKNOWN) &&
+		(link->test_pattern_enabled));
+
+	memset(&link_lane_settings, 0, sizeof(link_lane_settings));
+
+	if (use_prefer_link_setting) {
+		link_lane_settings.link_settings.lane_count =
+				link->preferred_link_setting.lane_count;
+		link_lane_settings.link_settings.link_rate =
+				link->preferred_link_setting.link_rate;
+		link_lane_settings.link_settings.link_spread =
+				link->preferred_link_setting.link_spread;
+	} else {
+		link_lane_settings.link_settings.lane_count =
+				link->cur_link_settings.lane_count;
+		link_lane_settings.link_settings.link_rate =
+				link->cur_link_settings.link_rate;
+		link_lane_settings.link_settings.link_spread =
+				link->cur_link_settings.link_spread;
+	}
+
+	/* apply phy settings from user */
+	for (r = 0; r < link_lane_settings.link_settings.lane_count; r++) {
+		link_lane_settings.lane_settings[r].VOLTAGE_SWING =
+				(enum dc_voltage_swing) (param[0]);
+		link_lane_settings.lane_settings[r].PRE_EMPHASIS =
+				(enum dc_pre_emphasis) (param[1]);
+		link_lane_settings.lane_settings[r].POST_CURSOR2 =
+				(enum dc_post_cursor2) (param[2]);
+	}
+
+	/* program ASIC registers and DPCD registers */
+	dc_link_set_drive_settings(dc, &link_lane_settings, link);
+
+	kfree(wr_buf);
+	return bytes_from_user;
+}
+
+/* function description
+ *
+ * set PHY layer or Link layer test pattern
+ * PHY test pattern is used for PHY SI check.
+ * Link layer test will not affect PHY SI.
+ *
+ * Reset Test Pattern:
+ * 0 = DP_TEST_PATTERN_VIDEO_MODE
+ *
+ * PHY test pattern supported:
+ * 1 = DP_TEST_PATTERN_D102
+ * 2 = DP_TEST_PATTERN_SYMBOL_ERROR
+ * 3 = DP_TEST_PATTERN_PRBS7
+ * 4 = DP_TEST_PATTERN_80BIT_CUSTOM
+ * 5 = DP_TEST_PATTERN_CP2520_1
+ * 6 = DP_TEST_PATTERN_CP2520_2 = DP_TEST_PATTERN_HBR2_COMPLIANCE_EYE
+ * 7 = DP_TEST_PATTERN_CP2520_3
+ *
+ * DP PHY Link Training Patterns
+ * 8 = DP_TEST_PATTERN_TRAINING_PATTERN1
+ * 9 = DP_TEST_PATTERN_TRAINING_PATTERN2
+ * a = DP_TEST_PATTERN_TRAINING_PATTERN3
+ * b = DP_TEST_PATTERN_TRAINING_PATTERN4
+ *
+ * DP Link Layer Test pattern
+ * c = DP_TEST_PATTERN_COLOR_SQUARES
+ * d = DP_TEST_PATTERN_COLOR_SQUARES_CEA
+ * e = DP_TEST_PATTERN_VERTICAL_BARS
+ * f = DP_TEST_PATTERN_HORIZONTAL_BARS
+ * 10= DP_TEST_PATTERN_COLOR_RAMP
+ *
+ * debugfs phy_test_pattern is located at /syskernel/debug/dri/0/DP-x
+ *
+ * --- set test pattern
+ * echo <test pattern #> > test_pattern
+ *
+ * If test pattern # is not supported, NO HW programming will be done.
+ * for DP_TEST_PATTERN_80BIT_CUSTOM, it needs extra 10 bytes of data
+ * for the user pattern. input 10 bytes data are separated by space
+ *
+ * echo 0x4 0x11 0x22 0x33 0x44 0x55 0x66 0x77 0x88 0x99 0xaa > test_pattern
+ *
+ * --- reset test pattern
+ * echo 0 > test_pattern
+ *
+ * --- HPD detection is disabled when set PHY test pattern
+ *
+ * when PHY test pattern (pattern # within [1,7]) is set, HPD pin of HW ASIC
+ * is disable. User could unplug DP display from DP connected and plug scope to
+ * check test pattern PHY SI.
+ * If there is need unplug scope and plug DP display back, do steps below:
+ * echo 0 > phy_test_pattern
+ * unplug scope
+ * plug DP display.
+ *
+ * "echo 0 > phy_test_pattern" will re-enable HPD pin again so that video sw
+ * driver could detect "unplug scope" and "plug DP display"
+ */
+static ssize_t dp_phy_test_pattern_debugfs_write(struct file *f, const char __user *buf,
+				 size_t size, loff_t *pos)
+{
+	struct amdgpu_dm_connector *connector = file_inode(f)->i_private;
+	struct dc_link *link = connector->dc_link;
+	char *wr_buf = NULL;
+	char *wr_buf_ptr = NULL;
+	uint32_t wr_buf_size = 100;
+	uint32_t wr_buf_count = 0;
+	int r;
+	int bytes_from_user;
+	char *sub_str = NULL;
+	uint8_t param_index = 0;
+	uint8_t param_nums = 0;
+	long param[11] = {0x0};
+	const char delimiter[3] = {' ', '\n', '\0'};
+	enum dp_test_pattern test_pattern = DP_TEST_PATTERN_UNSUPPORTED;
+	bool disable_hpd = false;
+	bool valid_test_pattern = false;
+	/* init with defalut 80bit custom pattern */
+	uint8_t custom_pattern[10] = {
+			0x1f, 0x7c, 0xf0, 0xc1, 0x07,
+			0x1f, 0x7c, 0xf0, 0xc1, 0x07
+			};
+	struct dc_link_settings prefer_link_settings = {LANE_COUNT_UNKNOWN,
+			LINK_RATE_UNKNOWN, LINK_SPREAD_DISABLED};
+	struct dc_link_settings cur_link_settings = {LANE_COUNT_UNKNOWN,
+			LINK_RATE_UNKNOWN, LINK_SPREAD_DISABLED};
+	struct link_training_settings link_training_settings;
+	int i;
+
+	if (size == 0)
+		return 0;
+
+	wr_buf = kcalloc(wr_buf_size, sizeof(char), GFP_KERNEL);
+	if (!wr_buf)
+		return 0;
+	wr_buf_ptr = wr_buf;
+
+	r = copy_from_user(wr_buf_ptr, buf, wr_buf_size);
+
+	/* r is bytes not be copied */
+	if (r >= wr_buf_size) {
+		kfree(wr_buf);
+		DRM_DEBUG_DRIVER("user data not be read\n");
+		return 0;
+	}
+
+	bytes_from_user = wr_buf_size - r;
+
+	/* check number of parameters. isspace could not differ space and \n */
+	while ((*wr_buf_ptr != 0xa) && (wr_buf_count < wr_buf_size)) {
+		/* skip space*/
+		while (isspace(*wr_buf_ptr) && (wr_buf_count < wr_buf_size)) {
+			wr_buf_ptr++;
+			wr_buf_count++;
+			}
+
+		if (wr_buf_count == wr_buf_size)
+			break;
+
+		/* skip non-space*/
+		while ((!isspace(*wr_buf_ptr)) && (wr_buf_count < wr_buf_size)) {
+			wr_buf_ptr++;
+			wr_buf_count++;
+			}
+
+		param_nums++;
+
+		if (wr_buf_count == wr_buf_size)
+			break;
+	}
+
+	/* max 11 parameters */
+	if (param_nums > 11)
+		param_nums = 11;
+
+	wr_buf_ptr = wr_buf; /* reset buf pinter */
+	wr_buf_count = 0; /* number of char already checked */
+
+	while (isspace(*wr_buf_ptr) && (wr_buf_count < wr_buf_size)) {
+		wr_buf_ptr++;
+		wr_buf_count++;
+	}
+
+	while (param_index < param_nums) {
+		/* after strsep, wr_buf_ptr will be moved to after space */
+		sub_str = strsep(&wr_buf_ptr, delimiter);
+
+		r = kstrtol(sub_str, 16, &param[param_index]);
+
+		if (r)
+			DRM_DEBUG_DRIVER("string to int convert error code: %d\n", r);
+
+		param_index++;
+	}
+
+	test_pattern = param[0];
+
+	switch (test_pattern) {
+	case DP_TEST_PATTERN_VIDEO_MODE:
+	case DP_TEST_PATTERN_COLOR_SQUARES:
+	case DP_TEST_PATTERN_COLOR_SQUARES_CEA:
+	case DP_TEST_PATTERN_VERTICAL_BARS:
+	case DP_TEST_PATTERN_HORIZONTAL_BARS:
+	case DP_TEST_PATTERN_COLOR_RAMP:
+		valid_test_pattern = true;
+		break;
+
+	case DP_TEST_PATTERN_D102:
+	case DP_TEST_PATTERN_SYMBOL_ERROR:
+	case DP_TEST_PATTERN_PRBS7:
+	case DP_TEST_PATTERN_80BIT_CUSTOM:
+	case DP_TEST_PATTERN_HBR2_COMPLIANCE_EYE:
+	case DP_TEST_PATTERN_TRAINING_PATTERN4:
+		disable_hpd = true;
+		valid_test_pattern = true;
+		break;
+
+	default:
+		valid_test_pattern = false;
+		test_pattern = DP_TEST_PATTERN_UNSUPPORTED;
+		break;
+	}
+
+	if (!valid_test_pattern) {
+		kfree(wr_buf);
+		DRM_DEBUG_DRIVER("Invalid Test Pattern Parameters\n");
+		return bytes_from_user;
+	}
+
+	if (test_pattern == DP_TEST_PATTERN_80BIT_CUSTOM) {
+		for (i = 0; i < 10; i++) {
+			if ((uint8_t) param[i + 1] != 0x0)
+				break;
+		}
+
+		if (i < 10) {
+			/* not use default value */
+			for (i = 0; i < 10; i++)
+				custom_pattern[i] = (uint8_t) param[i + 1];
+		}
+	}
+
+	/* Usage: set DP physical test pattern using debugfs with normal DP
+	 * panel. Then plug out DP panel and connect a scope to measure
+	 * For normal video mode and test pattern generated from CRCT,
+	 * they are visibile to user. So do not disable HPD.
+	 * Video Mode is also set to clear the test pattern, so enable HPD
+	 * because it might have been disabled after a test pattern was set.
+	 * AUX depends on HPD * sequence dependent, do not move!
+	 */
+	if (!disable_hpd)
+		dc_link_enable_hpd(link);
+
+	prefer_link_settings.lane_count = link->verified_link_cap.lane_count;
+	prefer_link_settings.link_rate = link->verified_link_cap.link_rate;
+	prefer_link_settings.link_spread = link->verified_link_cap.link_spread;
+
+	cur_link_settings.lane_count = link->cur_link_settings.lane_count;
+	cur_link_settings.link_rate = link->cur_link_settings.link_rate;
+	cur_link_settings.link_spread = link->cur_link_settings.link_spread;
+
+	link_training_settings.link_settings = cur_link_settings;
+
+
+	if (test_pattern != DP_TEST_PATTERN_VIDEO_MODE) {
+		if (prefer_link_settings.lane_count != LANE_COUNT_UNKNOWN &&
+			prefer_link_settings.link_rate !=  LINK_RATE_UNKNOWN &&
+			(prefer_link_settings.lane_count != cur_link_settings.lane_count ||
+			prefer_link_settings.link_rate != cur_link_settings.link_rate))
+			link_training_settings.link_settings = prefer_link_settings;
+	}
+
+	for (i = 0; i < (unsigned int)(link_training_settings.link_settings.lane_count); i++)
+		link_training_settings.lane_settings[i] = link->cur_lane_setting;
+
+	dc_link_set_test_pattern(
+		link,
+		test_pattern,
+		&link_training_settings,
+		custom_pattern,
+		10);
+
+	/* Usage: Set DP physical test pattern using AMDDP with normal DP panel
+	 * Then plug out DP panel and connect a scope to measure DP PHY signal.
+	 * Need disable interrupt to avoid SW driver disable DP output. This is
+	 * done after the test pattern is set.
+	 */
+	if (valid_test_pattern && disable_hpd)
+		dc_link_disable_hpd(link);
+
+	kfree(wr_buf);
+
+	return bytes_from_user;
+}
+
+static const struct file_operations dp_link_settings_debugfs_fops = {
+	.owner = THIS_MODULE,
+	.read = dp_link_settings_read,
+	.write = dp_link_settings_write,
+	.llseek = default_llseek
+};
+
+static const struct file_operations dp_phy_settings_debugfs_fop = {
+	.owner = THIS_MODULE,
+	.read = dp_phy_settings_read,
+	.write = dp_phy_settings_write,
+	.llseek = default_llseek
+};
+
+static const struct file_operations dp_phy_test_pattern_fops = {
+	.owner = THIS_MODULE,
+	.write = dp_phy_test_pattern_debugfs_write,
+	.llseek = default_llseek
+};
+
+static const struct {
+	char *name;
+	const struct file_operations *fops;
+} dp_debugfs_entries[] = {
+		{"link_settings", &dp_link_settings_debugfs_fops},
+		{"phy_settings", &dp_phy_settings_debugfs_fop},
+		{"test_pattern", &dp_phy_test_pattern_fops}
+};
+
+int connector_debugfs_init(struct amdgpu_dm_connector *connector)
+{
+	int i;
+	struct dentry *ent, *dir = connector->base.debugfs_entry;
+
+	if (connector->base.connector_type == DRM_MODE_CONNECTOR_DisplayPort) {
+		for (i = 0; i < ARRAY_SIZE(dp_debugfs_entries); i++) {
+			ent = debugfs_create_file(dp_debugfs_entries[i].name,
+						  0644,
+						  dir,
+						  connector,
+						  dp_debugfs_entries[i].fops);
+			if (IS_ERR(ent))
+				return PTR_ERR(ent);
+		}
+	}
+
+	return 0;
+}
+
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.h
new file mode 100644
index 000000000000..d9ed1b2aa811
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.h
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef __AMDGPU_DM_DEBUGFS_H__
+#define __AMDGPU_DM_DEBUGFS_H__
+
+#include "amdgpu.h"
+#include "amdgpu_dm.h"
+
+int connector_debugfs_init(struct amdgpu_dm_connector *connector);
+
+#endif
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c
index b19dc4cfc030..8403b6a9a77b 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c
@@ -169,6 +169,11 @@ static void get_payload_table(
 	mutex_unlock(&mst_mgr->payload_lock);
 }
 
+void dm_helpers_dp_update_branch_info(
+	struct dc_context *ctx,
+	const struct dc_link *link)
+{}
+
 /*
  * Writes payload allocation table in immediate downstream device.
  */
@@ -454,6 +459,22 @@ bool dm_helpers_submit_i2c(
 	return result;
 }
 
+bool dm_helpers_is_dp_sink_present(struct dc_link *link)
+{
+	bool dp_sink_present;
+	struct amdgpu_dm_connector *aconnector = link->priv;
+
+	if (!aconnector) {
+		BUG_ON("Failed to found connector for link!");
+		return true;
+	}
+
+	mutex_lock(&aconnector->dm_dp_aux.aux.hw_mutex);
+	dp_sink_present = dc_link_is_dp_sink_present(link);
+	mutex_unlock(&aconnector->dm_dp_aux.aux.hw_mutex);
+	return dp_sink_present;
+}
+
 enum dc_edid_status dm_helpers_read_local_edid(
 		struct dc_context *ctx,
 		struct dc_link *link,
@@ -498,8 +519,8 @@ enum dc_edid_status dm_helpers_read_local_edid(
 				edid_status,
 				aconnector->base.name);
 	if (link->aux_mode) {
-		union test_request test_request = {0};
-		union test_response test_response = {0};
+		union test_request test_request = { {0} };
+		union test_response test_response = { {0} };
 
 		dm_helpers_dp_read_dpcd(ctx,
 					link,
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c
index 65f210d3497b..9a300732ba37 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c
@@ -80,55 +80,72 @@ static void log_dpcd(uint8_t type,
 static ssize_t dm_dp_aux_transfer(struct drm_dp_aux *aux,
 				  struct drm_dp_aux_msg *msg)
 {
-	enum i2c_mot_mode mot = (msg->request & DP_AUX_I2C_MOT) ?
-		I2C_MOT_TRUE : I2C_MOT_FALSE;
-	enum ddc_result res;
-	uint32_t read_bytes = msg->size;
+	ssize_t result = 0;
+	enum i2caux_transaction_action action;
+	enum aux_transaction_type type;
 
 	if (WARN_ON(msg->size > 16))
 		return -E2BIG;
 
 	switch (msg->request & ~DP_AUX_I2C_MOT) {
 	case DP_AUX_NATIVE_READ:
-		res = dal_ddc_service_read_dpcd_data(
-				TO_DM_AUX(aux)->ddc_service,
-				false,
-				I2C_MOT_UNDEF,
-				msg->address,
-				msg->buffer,
-				msg->size,
-				&read_bytes);
+		type = AUX_TRANSACTION_TYPE_DP;
+		action = I2CAUX_TRANSACTION_ACTION_DP_READ;
+
+		result = dc_link_aux_transfer(TO_DM_AUX(aux)->ddc_service,
+					      msg->address,
+					      &msg->reply,
+					      msg->buffer,
+					      msg->size,
+					      type,
+					      action);
 		break;
 	case DP_AUX_NATIVE_WRITE:
-		res = dal_ddc_service_write_dpcd_data(
-				TO_DM_AUX(aux)->ddc_service,
-				false,
-				I2C_MOT_UNDEF,
-				msg->address,
-				msg->buffer,
-				msg->size);
+		type = AUX_TRANSACTION_TYPE_DP;
+		action = I2CAUX_TRANSACTION_ACTION_DP_WRITE;
+
+		dc_link_aux_transfer(TO_DM_AUX(aux)->ddc_service,
+				     msg->address,
+				     &msg->reply,
+				     msg->buffer,
+				     msg->size,
+				     type,
+				     action);
+		result = msg->size;
 		break;
 	case DP_AUX_I2C_READ:
-		res = dal_ddc_service_read_dpcd_data(
-				TO_DM_AUX(aux)->ddc_service,
-				true,
-				mot,
-				msg->address,
-				msg->buffer,
-				msg->size,
-				&read_bytes);
+		type = AUX_TRANSACTION_TYPE_I2C;
+		if (msg->request & DP_AUX_I2C_MOT)
+			action = I2CAUX_TRANSACTION_ACTION_I2C_READ_MOT;
+		else
+			action = I2CAUX_TRANSACTION_ACTION_I2C_READ;
+
+		result = dc_link_aux_transfer(TO_DM_AUX(aux)->ddc_service,
+					      msg->address,
+					      &msg->reply,
+					      msg->buffer,
+					      msg->size,
+					      type,
+					      action);
 		break;
 	case DP_AUX_I2C_WRITE:
-		res = dal_ddc_service_write_dpcd_data(
-				TO_DM_AUX(aux)->ddc_service,
-				true,
-				mot,
-				msg->address,
-				msg->buffer,
-				msg->size);
+		type = AUX_TRANSACTION_TYPE_I2C;
+		if (msg->request & DP_AUX_I2C_MOT)
+			action = I2CAUX_TRANSACTION_ACTION_I2C_WRITE_MOT;
+		else
+			action = I2CAUX_TRANSACTION_ACTION_I2C_WRITE;
+
+		dc_link_aux_transfer(TO_DM_AUX(aux)->ddc_service,
+				     msg->address,
+				     &msg->reply,
+				     msg->buffer,
+				     msg->size,
+				     type,
+				     action);
+		result = msg->size;
 		break;
 	default:
-		return 0;
+		return -EINVAL;
 	}
 
 #ifdef TRACE_DPCD
@@ -139,9 +156,10 @@ static ssize_t dm_dp_aux_transfer(struct drm_dp_aux *aux,
 		 r == DDC_RESULT_SUCESSFULL);
 #endif
 
-	if (res != DDC_RESULT_SUCESSFULL)
-		return -EIO;
-	return read_bytes;
+	if (result < 0) /* DC doesn't know about kernel error codes */
+		result = -EIO;
+
+	return result;
 }
 
 static enum drm_connector_status
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_pp_smu.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_pp_smu.c
new file mode 100644
index 000000000000..fbe878ae1e8c
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_pp_smu.c
@@ -0,0 +1,562 @@
+/*
+ * Copyright 2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ */
+#include <linux/string.h>
+#include <linux/acpi.h>
+
+#include <drm/drmP.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/amdgpu_drm.h>
+#include "dm_services.h"
+#include "amdgpu.h"
+#include "amdgpu_dm.h"
+#include "amdgpu_dm_irq.h"
+#include "amdgpu_pm.h"
+#include "dm_pp_smu.h"
+
+
+bool dm_pp_apply_display_requirements(
+		const struct dc_context *ctx,
+		const struct dm_pp_display_configuration *pp_display_cfg)
+{
+	struct amdgpu_device *adev = ctx->driver_context;
+	int i;
+
+	if (adev->pm.dpm_enabled) {
+
+		memset(&adev->pm.pm_display_cfg, 0,
+				sizeof(adev->pm.pm_display_cfg));
+
+		adev->pm.pm_display_cfg.cpu_cc6_disable =
+			pp_display_cfg->cpu_cc6_disable;
+
+		adev->pm.pm_display_cfg.cpu_pstate_disable =
+			pp_display_cfg->cpu_pstate_disable;
+
+		adev->pm.pm_display_cfg.cpu_pstate_separation_time =
+			pp_display_cfg->cpu_pstate_separation_time;
+
+		adev->pm.pm_display_cfg.nb_pstate_switch_disable =
+			pp_display_cfg->nb_pstate_switch_disable;
+
+		adev->pm.pm_display_cfg.num_display =
+				pp_display_cfg->display_count;
+		adev->pm.pm_display_cfg.num_path_including_non_display =
+				pp_display_cfg->display_count;
+
+		adev->pm.pm_display_cfg.min_core_set_clock =
+				pp_display_cfg->min_engine_clock_khz/10;
+		adev->pm.pm_display_cfg.min_core_set_clock_in_sr =
+				pp_display_cfg->min_engine_clock_deep_sleep_khz/10;
+		adev->pm.pm_display_cfg.min_mem_set_clock =
+				pp_display_cfg->min_memory_clock_khz/10;
+
+		adev->pm.pm_display_cfg.min_dcef_deep_sleep_set_clk =
+				pp_display_cfg->min_engine_clock_deep_sleep_khz/10;
+		adev->pm.pm_display_cfg.min_dcef_set_clk =
+				pp_display_cfg->min_dcfclock_khz/10;
+
+		adev->pm.pm_display_cfg.multi_monitor_in_sync =
+				pp_display_cfg->all_displays_in_sync;
+		adev->pm.pm_display_cfg.min_vblank_time =
+				pp_display_cfg->avail_mclk_switch_time_us;
+
+		adev->pm.pm_display_cfg.display_clk =
+				pp_display_cfg->disp_clk_khz/10;
+
+		adev->pm.pm_display_cfg.dce_tolerable_mclk_in_active_latency =
+				pp_display_cfg->avail_mclk_switch_time_in_disp_active_us;
+
+		adev->pm.pm_display_cfg.crtc_index = pp_display_cfg->crtc_index;
+		adev->pm.pm_display_cfg.line_time_in_us =
+				pp_display_cfg->line_time_in_us;
+
+		adev->pm.pm_display_cfg.vrefresh = pp_display_cfg->disp_configs[0].v_refresh;
+		adev->pm.pm_display_cfg.crossfire_display_index = -1;
+		adev->pm.pm_display_cfg.min_bus_bandwidth = 0;
+
+		for (i = 0; i < pp_display_cfg->display_count; i++) {
+			const struct dm_pp_single_disp_config *dc_cfg =
+						&pp_display_cfg->disp_configs[i];
+			adev->pm.pm_display_cfg.displays[i].controller_id = dc_cfg->pipe_idx + 1;
+		}
+
+		/* TODO: complete implementation of
+		 * pp_display_configuration_change().
+		 * Follow example of:
+		 * PHM_StoreDALConfigurationData - powerplay\hwmgr\hardwaremanager.c
+		 * PP_IRI_DisplayConfigurationChange - powerplay\eventmgr\iri.c */
+		if (adev->powerplay.pp_funcs->display_configuration_change)
+			adev->powerplay.pp_funcs->display_configuration_change(
+				adev->powerplay.pp_handle,
+				&adev->pm.pm_display_cfg);
+
+		/* TODO: replace by a separate call to 'apply display cfg'? */
+		amdgpu_pm_compute_clocks(adev);
+	}
+
+	return true;
+}
+
+static void get_default_clock_levels(
+		enum dm_pp_clock_type clk_type,
+		struct dm_pp_clock_levels *clks)
+{
+	uint32_t disp_clks_in_khz[6] = {
+			300000, 400000, 496560, 626090, 685720, 757900 };
+	uint32_t sclks_in_khz[6] = {
+			300000, 360000, 423530, 514290, 626090, 720000 };
+	uint32_t mclks_in_khz[2] = { 333000, 800000 };
+
+	switch (clk_type) {
+	case DM_PP_CLOCK_TYPE_DISPLAY_CLK:
+		clks->num_levels = 6;
+		memmove(clks->clocks_in_khz, disp_clks_in_khz,
+				sizeof(disp_clks_in_khz));
+		break;
+	case DM_PP_CLOCK_TYPE_ENGINE_CLK:
+		clks->num_levels = 6;
+		memmove(clks->clocks_in_khz, sclks_in_khz,
+				sizeof(sclks_in_khz));
+		break;
+	case DM_PP_CLOCK_TYPE_MEMORY_CLK:
+		clks->num_levels = 2;
+		memmove(clks->clocks_in_khz, mclks_in_khz,
+				sizeof(mclks_in_khz));
+		break;
+	default:
+		clks->num_levels = 0;
+		break;
+	}
+}
+
+static enum amd_pp_clock_type dc_to_pp_clock_type(
+		enum dm_pp_clock_type dm_pp_clk_type)
+{
+	enum amd_pp_clock_type amd_pp_clk_type = 0;
+
+	switch (dm_pp_clk_type) {
+	case DM_PP_CLOCK_TYPE_DISPLAY_CLK:
+		amd_pp_clk_type = amd_pp_disp_clock;
+		break;
+	case DM_PP_CLOCK_TYPE_ENGINE_CLK:
+		amd_pp_clk_type = amd_pp_sys_clock;
+		break;
+	case DM_PP_CLOCK_TYPE_MEMORY_CLK:
+		amd_pp_clk_type = amd_pp_mem_clock;
+		break;
+	case DM_PP_CLOCK_TYPE_DCEFCLK:
+		amd_pp_clk_type  = amd_pp_dcef_clock;
+		break;
+	case DM_PP_CLOCK_TYPE_DCFCLK:
+		amd_pp_clk_type = amd_pp_dcf_clock;
+		break;
+	case DM_PP_CLOCK_TYPE_PIXELCLK:
+		amd_pp_clk_type = amd_pp_pixel_clock;
+		break;
+	case DM_PP_CLOCK_TYPE_FCLK:
+		amd_pp_clk_type = amd_pp_f_clock;
+		break;
+	case DM_PP_CLOCK_TYPE_DISPLAYPHYCLK:
+		amd_pp_clk_type = amd_pp_phy_clock;
+		break;
+	case DM_PP_CLOCK_TYPE_DPPCLK:
+		amd_pp_clk_type = amd_pp_dpp_clock;
+		break;
+	default:
+		DRM_ERROR("DM_PPLIB: invalid clock type: %d!\n",
+				dm_pp_clk_type);
+		break;
+	}
+
+	return amd_pp_clk_type;
+}
+
+static enum dm_pp_clocks_state pp_to_dc_powerlevel_state(
+			enum PP_DAL_POWERLEVEL max_clocks_state)
+{
+	switch (max_clocks_state) {
+	case PP_DAL_POWERLEVEL_0:
+		return DM_PP_CLOCKS_DPM_STATE_LEVEL_0;
+	case PP_DAL_POWERLEVEL_1:
+		return DM_PP_CLOCKS_DPM_STATE_LEVEL_1;
+	case PP_DAL_POWERLEVEL_2:
+		return DM_PP_CLOCKS_DPM_STATE_LEVEL_2;
+	case PP_DAL_POWERLEVEL_3:
+		return DM_PP_CLOCKS_DPM_STATE_LEVEL_3;
+	case PP_DAL_POWERLEVEL_4:
+		return DM_PP_CLOCKS_DPM_STATE_LEVEL_4;
+	case PP_DAL_POWERLEVEL_5:
+		return DM_PP_CLOCKS_DPM_STATE_LEVEL_5;
+	case PP_DAL_POWERLEVEL_6:
+		return DM_PP_CLOCKS_DPM_STATE_LEVEL_6;
+	case PP_DAL_POWERLEVEL_7:
+		return DM_PP_CLOCKS_DPM_STATE_LEVEL_7;
+	default:
+		DRM_ERROR("DM_PPLIB: invalid powerlevel state: %d!\n",
+				max_clocks_state);
+		return DM_PP_CLOCKS_STATE_INVALID;
+	}
+}
+
+static void pp_to_dc_clock_levels(
+		const struct amd_pp_clocks *pp_clks,
+		struct dm_pp_clock_levels *dc_clks,
+		enum dm_pp_clock_type dc_clk_type)
+{
+	uint32_t i;
+
+	if (pp_clks->count > DM_PP_MAX_CLOCK_LEVELS) {
+		DRM_INFO("DM_PPLIB: Warning: %s clock: number of levels %d exceeds maximum of %d!\n",
+				DC_DECODE_PP_CLOCK_TYPE(dc_clk_type),
+				pp_clks->count,
+				DM_PP_MAX_CLOCK_LEVELS);
+
+		dc_clks->num_levels = DM_PP_MAX_CLOCK_LEVELS;
+	} else
+		dc_clks->num_levels = pp_clks->count;
+
+	DRM_INFO("DM_PPLIB: values for %s clock\n",
+			DC_DECODE_PP_CLOCK_TYPE(dc_clk_type));
+
+	for (i = 0; i < dc_clks->num_levels; i++) {
+		DRM_INFO("DM_PPLIB:\t %d\n", pp_clks->clock[i]);
+		dc_clks->clocks_in_khz[i] = pp_clks->clock[i];
+	}
+}
+
+static void pp_to_dc_clock_levels_with_latency(
+		const struct pp_clock_levels_with_latency *pp_clks,
+		struct dm_pp_clock_levels_with_latency *clk_level_info,
+		enum dm_pp_clock_type dc_clk_type)
+{
+	uint32_t i;
+
+	if (pp_clks->num_levels > DM_PP_MAX_CLOCK_LEVELS) {
+		DRM_INFO("DM_PPLIB: Warning: %s clock: number of levels %d exceeds maximum of %d!\n",
+				DC_DECODE_PP_CLOCK_TYPE(dc_clk_type),
+				pp_clks->num_levels,
+				DM_PP_MAX_CLOCK_LEVELS);
+
+		clk_level_info->num_levels = DM_PP_MAX_CLOCK_LEVELS;
+	} else
+		clk_level_info->num_levels = pp_clks->num_levels;
+
+	DRM_DEBUG("DM_PPLIB: values for %s clock\n",
+			DC_DECODE_PP_CLOCK_TYPE(dc_clk_type));
+
+	for (i = 0; i < clk_level_info->num_levels; i++) {
+		DRM_DEBUG("DM_PPLIB:\t %d in kHz\n", pp_clks->data[i].clocks_in_khz);
+		clk_level_info->data[i].clocks_in_khz = pp_clks->data[i].clocks_in_khz;
+		clk_level_info->data[i].latency_in_us = pp_clks->data[i].latency_in_us;
+	}
+}
+
+static void pp_to_dc_clock_levels_with_voltage(
+		const struct pp_clock_levels_with_voltage *pp_clks,
+		struct dm_pp_clock_levels_with_voltage *clk_level_info,
+		enum dm_pp_clock_type dc_clk_type)
+{
+	uint32_t i;
+
+	if (pp_clks->num_levels > DM_PP_MAX_CLOCK_LEVELS) {
+		DRM_INFO("DM_PPLIB: Warning: %s clock: number of levels %d exceeds maximum of %d!\n",
+				DC_DECODE_PP_CLOCK_TYPE(dc_clk_type),
+				pp_clks->num_levels,
+				DM_PP_MAX_CLOCK_LEVELS);
+
+		clk_level_info->num_levels = DM_PP_MAX_CLOCK_LEVELS;
+	} else
+		clk_level_info->num_levels = pp_clks->num_levels;
+
+	DRM_INFO("DM_PPLIB: values for %s clock\n",
+			DC_DECODE_PP_CLOCK_TYPE(dc_clk_type));
+
+	for (i = 0; i < clk_level_info->num_levels; i++) {
+		DRM_INFO("DM_PPLIB:\t %d in kHz\n", pp_clks->data[i].clocks_in_khz);
+		clk_level_info->data[i].clocks_in_khz = pp_clks->data[i].clocks_in_khz;
+		clk_level_info->data[i].voltage_in_mv = pp_clks->data[i].voltage_in_mv;
+	}
+}
+
+bool dm_pp_get_clock_levels_by_type(
+		const struct dc_context *ctx,
+		enum dm_pp_clock_type clk_type,
+		struct dm_pp_clock_levels *dc_clks)
+{
+	struct amdgpu_device *adev = ctx->driver_context;
+	void *pp_handle = adev->powerplay.pp_handle;
+	struct amd_pp_clocks pp_clks = { 0 };
+	struct amd_pp_simple_clock_info validation_clks = { 0 };
+	uint32_t i;
+
+	if (adev->powerplay.pp_funcs->get_clock_by_type) {
+		if (adev->powerplay.pp_funcs->get_clock_by_type(pp_handle,
+			dc_to_pp_clock_type(clk_type), &pp_clks)) {
+		/* Error in pplib. Provide default values. */
+			get_default_clock_levels(clk_type, dc_clks);
+			return true;
+		}
+	}
+
+	pp_to_dc_clock_levels(&pp_clks, dc_clks, clk_type);
+
+	if (adev->powerplay.pp_funcs->get_display_mode_validation_clocks) {
+		if (adev->powerplay.pp_funcs->get_display_mode_validation_clocks(
+						pp_handle, &validation_clks)) {
+			/* Error in pplib. Provide default values. */
+			DRM_INFO("DM_PPLIB: Warning: using default validation clocks!\n");
+			validation_clks.engine_max_clock = 72000;
+			validation_clks.memory_max_clock = 80000;
+			validation_clks.level = 0;
+		}
+	}
+
+	DRM_INFO("DM_PPLIB: Validation clocks:\n");
+	DRM_INFO("DM_PPLIB:    engine_max_clock: %d\n",
+			validation_clks.engine_max_clock);
+	DRM_INFO("DM_PPLIB:    memory_max_clock: %d\n",
+			validation_clks.memory_max_clock);
+	DRM_INFO("DM_PPLIB:    level           : %d\n",
+			validation_clks.level);
+
+	/* Translate 10 kHz to kHz. */
+	validation_clks.engine_max_clock *= 10;
+	validation_clks.memory_max_clock *= 10;
+
+	/* Determine the highest non-boosted level from the Validation Clocks */
+	if (clk_type == DM_PP_CLOCK_TYPE_ENGINE_CLK) {
+		for (i = 0; i < dc_clks->num_levels; i++) {
+			if (dc_clks->clocks_in_khz[i] > validation_clks.engine_max_clock) {
+				/* This clock is higher the validation clock.
+				 * Than means the previous one is the highest
+				 * non-boosted one. */
+				DRM_INFO("DM_PPLIB: reducing engine clock level from %d to %d\n",
+						dc_clks->num_levels, i);
+				dc_clks->num_levels = i > 0 ? i : 1;
+				break;
+			}
+		}
+	} else if (clk_type == DM_PP_CLOCK_TYPE_MEMORY_CLK) {
+		for (i = 0; i < dc_clks->num_levels; i++) {
+			if (dc_clks->clocks_in_khz[i] > validation_clks.memory_max_clock) {
+				DRM_INFO("DM_PPLIB: reducing memory clock level from %d to %d\n",
+						dc_clks->num_levels, i);
+				dc_clks->num_levels = i > 0 ? i : 1;
+				break;
+			}
+		}
+	}
+
+	return true;
+}
+
+bool dm_pp_get_clock_levels_by_type_with_latency(
+	const struct dc_context *ctx,
+	enum dm_pp_clock_type clk_type,
+	struct dm_pp_clock_levels_with_latency *clk_level_info)
+{
+	struct amdgpu_device *adev = ctx->driver_context;
+	void *pp_handle = adev->powerplay.pp_handle;
+	struct pp_clock_levels_with_latency pp_clks = { 0 };
+	const struct amd_pm_funcs *pp_funcs = adev->powerplay.pp_funcs;
+
+	if (!pp_funcs || !pp_funcs->get_clock_by_type_with_latency)
+		return false;
+
+	if (pp_funcs->get_clock_by_type_with_latency(pp_handle,
+						     dc_to_pp_clock_type(clk_type),
+						     &pp_clks))
+		return false;
+
+	pp_to_dc_clock_levels_with_latency(&pp_clks, clk_level_info, clk_type);
+
+	return true;
+}
+
+bool dm_pp_get_clock_levels_by_type_with_voltage(
+	const struct dc_context *ctx,
+	enum dm_pp_clock_type clk_type,
+	struct dm_pp_clock_levels_with_voltage *clk_level_info)
+{
+	struct amdgpu_device *adev = ctx->driver_context;
+	void *pp_handle = adev->powerplay.pp_handle;
+	struct pp_clock_levels_with_voltage pp_clk_info = {0};
+	const struct amd_pm_funcs *pp_funcs = adev->powerplay.pp_funcs;
+
+	if (pp_funcs->get_clock_by_type_with_voltage(pp_handle,
+						     dc_to_pp_clock_type(clk_type),
+						     &pp_clk_info))
+		return false;
+
+	pp_to_dc_clock_levels_with_voltage(&pp_clk_info, clk_level_info, clk_type);
+
+	return true;
+}
+
+bool dm_pp_notify_wm_clock_changes(
+	const struct dc_context *ctx,
+	struct dm_pp_wm_sets_with_clock_ranges *wm_with_clock_ranges)
+{
+	/* TODO: to be implemented */
+	return false;
+}
+
+bool dm_pp_apply_power_level_change_request(
+	const struct dc_context *ctx,
+	struct dm_pp_power_level_change_request *level_change_req)
+{
+	/* TODO: to be implemented */
+	return false;
+}
+
+bool dm_pp_apply_clock_for_voltage_request(
+	const struct dc_context *ctx,
+	struct dm_pp_clock_for_voltage_req *clock_for_voltage_req)
+{
+	struct amdgpu_device *adev = ctx->driver_context;
+	struct pp_display_clock_request pp_clock_request = {0};
+	int ret = 0;
+
+	pp_clock_request.clock_type = dc_to_pp_clock_type(clock_for_voltage_req->clk_type);
+	pp_clock_request.clock_freq_in_khz = clock_for_voltage_req->clocks_in_khz;
+
+	if (!pp_clock_request.clock_type)
+		return false;
+
+	if (adev->powerplay.pp_funcs->display_clock_voltage_request)
+		ret = adev->powerplay.pp_funcs->display_clock_voltage_request(
+			adev->powerplay.pp_handle,
+			&pp_clock_request);
+	if (ret)
+		return false;
+	return true;
+}
+
+bool dm_pp_get_static_clocks(
+	const struct dc_context *ctx,
+	struct dm_pp_static_clock_info *static_clk_info)
+{
+	struct amdgpu_device *adev = ctx->driver_context;
+	struct amd_pp_clock_info pp_clk_info = {0};
+	int ret = 0;
+
+	if (adev->powerplay.pp_funcs->get_current_clocks)
+		ret = adev->powerplay.pp_funcs->get_current_clocks(
+			adev->powerplay.pp_handle,
+			&pp_clk_info);
+	if (ret)
+		return false;
+
+	static_clk_info->max_clocks_state = pp_to_dc_powerlevel_state(pp_clk_info.max_clocks_state);
+	static_clk_info->max_mclk_khz = pp_clk_info.max_memory_clock * 10;
+	static_clk_info->max_sclk_khz = pp_clk_info.max_engine_clock * 10;
+
+	return true;
+}
+
+void pp_rv_set_display_requirement(struct pp_smu *pp,
+		struct pp_smu_display_requirement_rv *req)
+{
+	struct dc_context *ctx = pp->ctx;
+	struct amdgpu_device *adev = ctx->driver_context;
+	const struct amd_pm_funcs *pp_funcs = adev->powerplay.pp_funcs;
+
+	if (!pp_funcs || !pp_funcs->display_configuration_changed)
+		return;
+
+	amdgpu_dpm_display_configuration_changed(adev);
+}
+
+void pp_rv_set_wm_ranges(struct pp_smu *pp,
+		struct pp_smu_wm_range_sets *ranges)
+{
+	struct dc_context *ctx = pp->ctx;
+	struct amdgpu_device *adev = ctx->driver_context;
+	void *pp_handle = adev->powerplay.pp_handle;
+	const struct amd_pm_funcs *pp_funcs = adev->powerplay.pp_funcs;
+	struct dm_pp_wm_sets_with_clock_ranges_soc15 wm_with_clock_ranges;
+	struct dm_pp_clock_range_for_dmif_wm_set_soc15 *wm_dce_clocks = wm_with_clock_ranges.wm_dmif_clocks_ranges;
+	struct dm_pp_clock_range_for_mcif_wm_set_soc15 *wm_soc_clocks = wm_with_clock_ranges.wm_mcif_clocks_ranges;
+	int32_t i;
+
+	wm_with_clock_ranges.num_wm_dmif_sets = ranges->num_reader_wm_sets;
+	wm_with_clock_ranges.num_wm_mcif_sets = ranges->num_writer_wm_sets;
+
+	for (i = 0; i < wm_with_clock_ranges.num_wm_dmif_sets; i++) {
+		if (ranges->reader_wm_sets[i].wm_inst > 3)
+			wm_dce_clocks[i].wm_set_id = WM_SET_A;
+		else
+			wm_dce_clocks[i].wm_set_id =
+					ranges->reader_wm_sets[i].wm_inst;
+		wm_dce_clocks[i].wm_max_dcfclk_clk_in_khz =
+				ranges->reader_wm_sets[i].max_drain_clk_khz;
+		wm_dce_clocks[i].wm_min_dcfclk_clk_in_khz =
+				ranges->reader_wm_sets[i].min_drain_clk_khz;
+		wm_dce_clocks[i].wm_max_mem_clk_in_khz =
+				ranges->reader_wm_sets[i].max_fill_clk_khz;
+		wm_dce_clocks[i].wm_min_mem_clk_in_khz =
+				ranges->reader_wm_sets[i].min_fill_clk_khz;
+	}
+
+	for (i = 0; i < wm_with_clock_ranges.num_wm_mcif_sets; i++) {
+		if (ranges->writer_wm_sets[i].wm_inst > 3)
+			wm_soc_clocks[i].wm_set_id = WM_SET_A;
+		else
+			wm_soc_clocks[i].wm_set_id =
+					ranges->writer_wm_sets[i].wm_inst;
+		wm_soc_clocks[i].wm_max_socclk_clk_in_khz =
+				ranges->writer_wm_sets[i].max_fill_clk_khz;
+		wm_soc_clocks[i].wm_min_socclk_clk_in_khz =
+				ranges->writer_wm_sets[i].min_fill_clk_khz;
+		wm_soc_clocks[i].wm_max_mem_clk_in_khz =
+				ranges->writer_wm_sets[i].max_drain_clk_khz;
+		wm_soc_clocks[i].wm_min_mem_clk_in_khz =
+				ranges->writer_wm_sets[i].min_drain_clk_khz;
+	}
+
+	pp_funcs->set_watermarks_for_clocks_ranges(pp_handle, &wm_with_clock_ranges);
+}
+
+void pp_rv_set_pme_wa_enable(struct pp_smu *pp)
+{
+	struct dc_context *ctx = pp->ctx;
+	struct amdgpu_device *adev = ctx->driver_context;
+	void *pp_handle = adev->powerplay.pp_handle;
+	const struct amd_pm_funcs *pp_funcs = adev->powerplay.pp_funcs;
+
+	if (!pp_funcs || !pp_funcs->notify_smu_enable_pwe)
+		return;
+
+	pp_funcs->notify_smu_enable_pwe(pp_handle);
+}
+
+void dm_pp_get_funcs_rv(
+		struct dc_context *ctx,
+		struct pp_smu_funcs_rv *funcs)
+{
+	funcs->pp_smu.ctx = ctx;
+	funcs->set_display_requirement = pp_rv_set_display_requirement;
+	funcs->set_wm_ranges = pp_rv_set_wm_ranges;
+	funcs->set_pme_wa_enable = pp_rv_set_pme_wa_enable;
+}
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_services.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_services.c
index e861929dd981..516795342dd2 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_services.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_services.c
@@ -35,6 +35,8 @@
 #include "amdgpu_dm_irq.h"
 #include "amdgpu_pm.h"
 
+
+
 unsigned long long dm_get_elapse_time_in_ns(struct dc_context *ctx,
 		unsigned long long current_time_stamp,
 		unsigned long long last_time_stamp)
@@ -72,326 +74,3 @@ bool dm_read_persistent_data(struct dc_context *ctx,
 
 /**** power component interfaces ****/
 
-bool dm_pp_apply_display_requirements(
-		const struct dc_context *ctx,
-		const struct dm_pp_display_configuration *pp_display_cfg)
-{
-	struct amdgpu_device *adev = ctx->driver_context;
-
-	if (adev->pm.dpm_enabled) {
-
-		memset(&adev->pm.pm_display_cfg, 0,
-				sizeof(adev->pm.pm_display_cfg));
-
-		adev->pm.pm_display_cfg.cpu_cc6_disable =
-			pp_display_cfg->cpu_cc6_disable;
-
-		adev->pm.pm_display_cfg.cpu_pstate_disable =
-			pp_display_cfg->cpu_pstate_disable;
-
-		adev->pm.pm_display_cfg.cpu_pstate_separation_time =
-			pp_display_cfg->cpu_pstate_separation_time;
-
-		adev->pm.pm_display_cfg.nb_pstate_switch_disable =
-			pp_display_cfg->nb_pstate_switch_disable;
-
-		adev->pm.pm_display_cfg.num_display =
-				pp_display_cfg->display_count;
-		adev->pm.pm_display_cfg.num_path_including_non_display =
-				pp_display_cfg->display_count;
-
-		adev->pm.pm_display_cfg.min_core_set_clock =
-				pp_display_cfg->min_engine_clock_khz/10;
-		adev->pm.pm_display_cfg.min_core_set_clock_in_sr =
-				pp_display_cfg->min_engine_clock_deep_sleep_khz/10;
-		adev->pm.pm_display_cfg.min_mem_set_clock =
-				pp_display_cfg->min_memory_clock_khz/10;
-
-		adev->pm.pm_display_cfg.multi_monitor_in_sync =
-				pp_display_cfg->all_displays_in_sync;
-		adev->pm.pm_display_cfg.min_vblank_time =
-				pp_display_cfg->avail_mclk_switch_time_us;
-
-		adev->pm.pm_display_cfg.display_clk =
-				pp_display_cfg->disp_clk_khz/10;
-
-		adev->pm.pm_display_cfg.dce_tolerable_mclk_in_active_latency =
-				pp_display_cfg->avail_mclk_switch_time_in_disp_active_us;
-
-		adev->pm.pm_display_cfg.crtc_index = pp_display_cfg->crtc_index;
-		adev->pm.pm_display_cfg.line_time_in_us =
-				pp_display_cfg->line_time_in_us;
-
-		adev->pm.pm_display_cfg.vrefresh = pp_display_cfg->disp_configs[0].v_refresh;
-		adev->pm.pm_display_cfg.crossfire_display_index = -1;
-		adev->pm.pm_display_cfg.min_bus_bandwidth = 0;
-
-		/* TODO: complete implementation of
-		 * pp_display_configuration_change().
-		 * Follow example of:
-		 * PHM_StoreDALConfigurationData - powerplay\hwmgr\hardwaremanager.c
-		 * PP_IRI_DisplayConfigurationChange - powerplay\eventmgr\iri.c */
-		if (adev->powerplay.pp_funcs->display_configuration_change)
-			adev->powerplay.pp_funcs->display_configuration_change(
-				adev->powerplay.pp_handle,
-				&adev->pm.pm_display_cfg);
-
-		/* TODO: replace by a separate call to 'apply display cfg'? */
-		amdgpu_pm_compute_clocks(adev);
-	}
-
-	return true;
-}
-
-static void get_default_clock_levels(
-		enum dm_pp_clock_type clk_type,
-		struct dm_pp_clock_levels *clks)
-{
-	uint32_t disp_clks_in_khz[6] = {
-			300000, 400000, 496560, 626090, 685720, 757900 };
-	uint32_t sclks_in_khz[6] = {
-			300000, 360000, 423530, 514290, 626090, 720000 };
-	uint32_t mclks_in_khz[2] = { 333000, 800000 };
-
-	switch (clk_type) {
-	case DM_PP_CLOCK_TYPE_DISPLAY_CLK:
-		clks->num_levels = 6;
-		memmove(clks->clocks_in_khz, disp_clks_in_khz,
-				sizeof(disp_clks_in_khz));
-		break;
-	case DM_PP_CLOCK_TYPE_ENGINE_CLK:
-		clks->num_levels = 6;
-		memmove(clks->clocks_in_khz, sclks_in_khz,
-				sizeof(sclks_in_khz));
-		break;
-	case DM_PP_CLOCK_TYPE_MEMORY_CLK:
-		clks->num_levels = 2;
-		memmove(clks->clocks_in_khz, mclks_in_khz,
-				sizeof(mclks_in_khz));
-		break;
-	default:
-		clks->num_levels = 0;
-		break;
-	}
-}
-
-static enum amd_pp_clock_type dc_to_pp_clock_type(
-		enum dm_pp_clock_type dm_pp_clk_type)
-{
-	enum amd_pp_clock_type amd_pp_clk_type = 0;
-
-	switch (dm_pp_clk_type) {
-	case DM_PP_CLOCK_TYPE_DISPLAY_CLK:
-		amd_pp_clk_type = amd_pp_disp_clock;
-		break;
-	case DM_PP_CLOCK_TYPE_ENGINE_CLK:
-		amd_pp_clk_type = amd_pp_sys_clock;
-		break;
-	case DM_PP_CLOCK_TYPE_MEMORY_CLK:
-		amd_pp_clk_type = amd_pp_mem_clock;
-		break;
-	default:
-		DRM_ERROR("DM_PPLIB: invalid clock type: %d!\n",
-				dm_pp_clk_type);
-		break;
-	}
-
-	return amd_pp_clk_type;
-}
-
-static void pp_to_dc_clock_levels(
-		const struct amd_pp_clocks *pp_clks,
-		struct dm_pp_clock_levels *dc_clks,
-		enum dm_pp_clock_type dc_clk_type)
-{
-	uint32_t i;
-
-	if (pp_clks->count > DM_PP_MAX_CLOCK_LEVELS) {
-		DRM_INFO("DM_PPLIB: Warning: %s clock: number of levels %d exceeds maximum of %d!\n",
-				DC_DECODE_PP_CLOCK_TYPE(dc_clk_type),
-				pp_clks->count,
-				DM_PP_MAX_CLOCK_LEVELS);
-
-		dc_clks->num_levels = DM_PP_MAX_CLOCK_LEVELS;
-	} else
-		dc_clks->num_levels = pp_clks->count;
-
-	DRM_INFO("DM_PPLIB: values for %s clock\n",
-			DC_DECODE_PP_CLOCK_TYPE(dc_clk_type));
-
-	for (i = 0; i < dc_clks->num_levels; i++) {
-		DRM_INFO("DM_PPLIB:\t %d\n", pp_clks->clock[i]);
-		/* translate 10kHz to kHz */
-		dc_clks->clocks_in_khz[i] = pp_clks->clock[i] * 10;
-	}
-}
-
-static void pp_to_dc_clock_levels_with_latency(
-		const struct pp_clock_levels_with_latency *pp_clks,
-		struct dm_pp_clock_levels_with_latency *clk_level_info,
-		enum dm_pp_clock_type dc_clk_type)
-{
-	uint32_t i;
-
-	if (pp_clks->num_levels > DM_PP_MAX_CLOCK_LEVELS) {
-		DRM_INFO("DM_PPLIB: Warning: %s clock: number of levels %d exceeds maximum of %d!\n",
-				DC_DECODE_PP_CLOCK_TYPE(dc_clk_type),
-				pp_clks->num_levels,
-				DM_PP_MAX_CLOCK_LEVELS);
-
-		clk_level_info->num_levels = DM_PP_MAX_CLOCK_LEVELS;
-	} else
-		clk_level_info->num_levels = pp_clks->num_levels;
-
-	DRM_DEBUG("DM_PPLIB: values for %s clock\n",
-			DC_DECODE_PP_CLOCK_TYPE(dc_clk_type));
-
-	for (i = 0; i < clk_level_info->num_levels; i++) {
-		DRM_DEBUG("DM_PPLIB:\t %d\n", pp_clks->data[i].clocks_in_khz);
-		clk_level_info->data[i].clocks_in_khz = pp_clks->data[i].clocks_in_khz;
-		clk_level_info->data[i].latency_in_us = pp_clks->data[i].latency_in_us;
-	}
-}
-
-bool dm_pp_get_clock_levels_by_type(
-		const struct dc_context *ctx,
-		enum dm_pp_clock_type clk_type,
-		struct dm_pp_clock_levels *dc_clks)
-{
-	struct amdgpu_device *adev = ctx->driver_context;
-	void *pp_handle = adev->powerplay.pp_handle;
-	struct amd_pp_clocks pp_clks = { 0 };
-	struct amd_pp_simple_clock_info validation_clks = { 0 };
-	uint32_t i;
-
-	if (adev->powerplay.pp_funcs->get_clock_by_type) {
-		if (adev->powerplay.pp_funcs->get_clock_by_type(pp_handle,
-			dc_to_pp_clock_type(clk_type), &pp_clks)) {
-		/* Error in pplib. Provide default values. */
-			get_default_clock_levels(clk_type, dc_clks);
-			return true;
-		}
-	}
-
-	pp_to_dc_clock_levels(&pp_clks, dc_clks, clk_type);
-
-	if (adev->powerplay.pp_funcs->get_display_mode_validation_clocks) {
-		if (adev->powerplay.pp_funcs->get_display_mode_validation_clocks(
-						pp_handle, &validation_clks)) {
-			/* Error in pplib. Provide default values. */
-			DRM_INFO("DM_PPLIB: Warning: using default validation clocks!\n");
-			validation_clks.engine_max_clock = 72000;
-			validation_clks.memory_max_clock = 80000;
-			validation_clks.level = 0;
-		}
-	}
-
-	DRM_INFO("DM_PPLIB: Validation clocks:\n");
-	DRM_INFO("DM_PPLIB:    engine_max_clock: %d\n",
-			validation_clks.engine_max_clock);
-	DRM_INFO("DM_PPLIB:    memory_max_clock: %d\n",
-			validation_clks.memory_max_clock);
-	DRM_INFO("DM_PPLIB:    level           : %d\n",
-			validation_clks.level);
-
-	/* Translate 10 kHz to kHz. */
-	validation_clks.engine_max_clock *= 10;
-	validation_clks.memory_max_clock *= 10;
-
-	/* Determine the highest non-boosted level from the Validation Clocks */
-	if (clk_type == DM_PP_CLOCK_TYPE_ENGINE_CLK) {
-		for (i = 0; i < dc_clks->num_levels; i++) {
-			if (dc_clks->clocks_in_khz[i] > validation_clks.engine_max_clock) {
-				/* This clock is higher the validation clock.
-				 * Than means the previous one is the highest
-				 * non-boosted one. */
-				DRM_INFO("DM_PPLIB: reducing engine clock level from %d to %d\n",
-						dc_clks->num_levels, i);
-				dc_clks->num_levels = i > 0 ? i : 1;
-				break;
-			}
-		}
-	} else if (clk_type == DM_PP_CLOCK_TYPE_MEMORY_CLK) {
-		for (i = 0; i < dc_clks->num_levels; i++) {
-			if (dc_clks->clocks_in_khz[i] > validation_clks.memory_max_clock) {
-				DRM_INFO("DM_PPLIB: reducing memory clock level from %d to %d\n",
-						dc_clks->num_levels, i);
-				dc_clks->num_levels = i > 0 ? i : 1;
-				break;
-			}
-		}
-	}
-
-	return true;
-}
-
-bool dm_pp_get_clock_levels_by_type_with_latency(
-	const struct dc_context *ctx,
-	enum dm_pp_clock_type clk_type,
-	struct dm_pp_clock_levels_with_latency *clk_level_info)
-{
-	struct amdgpu_device *adev = ctx->driver_context;
-	void *pp_handle = adev->powerplay.pp_handle;
-	struct pp_clock_levels_with_latency pp_clks = { 0 };
-	const struct amd_pm_funcs *pp_funcs = adev->powerplay.pp_funcs;
-
-	if (!pp_funcs || !pp_funcs->get_clock_by_type_with_latency)
-		return false;
-
-	if (pp_funcs->get_clock_by_type_with_latency(pp_handle,
-						     dc_to_pp_clock_type(clk_type),
-						     &pp_clks))
-		return false;
-
-	pp_to_dc_clock_levels_with_latency(&pp_clks, clk_level_info, clk_type);
-
-	return true;
-}
-
-bool dm_pp_get_clock_levels_by_type_with_voltage(
-	const struct dc_context *ctx,
-	enum dm_pp_clock_type clk_type,
-	struct dm_pp_clock_levels_with_voltage *clk_level_info)
-{
-	/* TODO: to be implemented */
-	return false;
-}
-
-bool dm_pp_notify_wm_clock_changes(
-	const struct dc_context *ctx,
-	struct dm_pp_wm_sets_with_clock_ranges *wm_with_clock_ranges)
-{
-	/* TODO: to be implemented */
-	return false;
-}
-
-bool dm_pp_apply_power_level_change_request(
-	const struct dc_context *ctx,
-	struct dm_pp_power_level_change_request *level_change_req)
-{
-	/* TODO: to be implemented */
-	return false;
-}
-
-bool dm_pp_apply_clock_for_voltage_request(
-	const struct dc_context *ctx,
-	struct dm_pp_clock_for_voltage_req *clock_for_voltage_req)
-{
-	/* TODO: to be implemented */
-	return false;
-}
-
-bool dm_pp_get_static_clocks(
-	const struct dc_context *ctx,
-	struct dm_pp_static_clock_info *static_clk_info)
-{
-	/* TODO: to be implemented */
-	return false;
-}
-
-void dm_pp_get_funcs_rv(
-		struct dc_context *ctx,
-		struct pp_smu_funcs_rv *funcs)
-{}
-
-/**** end of power component interfaces ****/
diff --git a/drivers/gpu/drm/amd/display/dc/basics/Makefile b/drivers/gpu/drm/amd/display/dc/basics/Makefile
index b49ea96b5dae..a50a76471107 100644
--- a/drivers/gpu/drm/amd/display/dc/basics/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/basics/Makefile
@@ -25,7 +25,7 @@
 # subcomponents.
 
 BASICS = conversion.o fixpt31_32.o \
-	logger.o log_helpers.o vector.o
+	log_helpers.o vector.o
 
 AMD_DAL_BASICS = $(addprefix $(AMDDALPATH)/dc/basics/,$(BASICS))
 
diff --git a/drivers/gpu/drm/amd/display/dc/basics/log_helpers.c b/drivers/gpu/drm/amd/display/dc/basics/log_helpers.c
index f6c00a51d51a..26583f346c39 100644
--- a/drivers/gpu/drm/amd/display/dc/basics/log_helpers.c
+++ b/drivers/gpu/drm/amd/display/dc/basics/log_helpers.c
@@ -28,77 +28,12 @@
 #include "include/logger_interface.h"
 #include "dm_helpers.h"
 
-#define NUM_ELEMENTS(a) (sizeof(a) / sizeof((a)[0]))
-
-struct dc_signal_type_info {
-	enum signal_type type;
-	char name[MAX_NAME_LEN];
-};
-
-static const struct dc_signal_type_info signal_type_info_tbl[] = {
-		{SIGNAL_TYPE_NONE,             "NC"},
-		{SIGNAL_TYPE_DVI_SINGLE_LINK,  "DVI"},
-		{SIGNAL_TYPE_DVI_DUAL_LINK,    "DDVI"},
-		{SIGNAL_TYPE_HDMI_TYPE_A,      "HDMIA"},
-		{SIGNAL_TYPE_LVDS,             "LVDS"},
-		{SIGNAL_TYPE_RGB,              "VGA"},
-		{SIGNAL_TYPE_DISPLAY_PORT,     "DP"},
-		{SIGNAL_TYPE_DISPLAY_PORT_MST, "MST"},
-		{SIGNAL_TYPE_EDP,              "eDP"},
-		{SIGNAL_TYPE_VIRTUAL,          "Virtual"}
-};
-
-void dc_conn_log(struct dc_context *ctx,
-		const struct dc_link *link,
-		uint8_t *hex_data,
-		int hex_data_count,
-		enum dc_log_type event,
-		const char *msg,
-		...)
+void dc_conn_log_hex_linux(const uint8_t *hex_data, int hex_data_count)
 {
 	int i;
-	va_list args;
-	struct log_entry entry = { 0 };
-	enum signal_type signal;
-
-	if (link->local_sink)
-		signal = link->local_sink->sink_signal;
-	else
-		signal = link->connector_signal;
-
-	if (link->type == dc_connection_mst_branch)
-		signal = SIGNAL_TYPE_DISPLAY_PORT_MST;
-
-	dm_logger_open(ctx->logger, &entry, event);
-
-	for (i = 0; i < NUM_ELEMENTS(signal_type_info_tbl); i++)
-		if (signal == signal_type_info_tbl[i].type)
-			break;
-
-	if (i == NUM_ELEMENTS(signal_type_info_tbl))
-		goto fail;
-
-	dm_logger_append_heading(&entry);
-
-	dm_logger_append(&entry, "[%s][ConnIdx:%d] ",
-			signal_type_info_tbl[i].name,
-			link->link_index);
-
-	va_start(args, msg);
-	dm_logger_append_va(&entry, msg, args);
-
-	if (entry.buf_offset > 0 &&
-	    entry.buf[entry.buf_offset - 1] == '\n')
-		entry.buf_offset--;
 
 	if (hex_data)
 		for (i = 0; i < hex_data_count; i++)
-			dm_logger_append(&entry, "%2.2X ", hex_data[i]);
-
-	dm_logger_append(&entry, "^\n");
-
-fail:
-	dm_logger_close(&entry);
-
-	va_end(args);
+			DC_LOG_DEBUG("%2.2X ", hex_data[i]);
 }
+
diff --git a/drivers/gpu/drm/amd/display/dc/basics/logger.c b/drivers/gpu/drm/amd/display/dc/basics/logger.c
deleted file mode 100644
index a3c56cd8b396..000000000000
--- a/drivers/gpu/drm/amd/display/dc/basics/logger.c
+++ /dev/null
@@ -1,406 +0,0 @@
-/*
- * Copyright 2012-15 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: AMD
- *
- */
-#include "dm_services.h"
-#include "include/logger_interface.h"
-#include "logger.h"
-
-
-#define NUM_ELEMENTS(a) (sizeof(a) / sizeof((a)[0]))
-
-static const struct dc_log_type_info log_type_info_tbl[] = {
-		{LOG_ERROR,                 "Error"},
-		{LOG_WARNING,               "Warning"},
-		{LOG_DEBUG,                 "Debug"},
-		{LOG_DC,                    "DC_Interface"},
-		{LOG_DTN,                   "DTN"},
-		{LOG_SURFACE,               "Surface"},
-		{LOG_HW_HOTPLUG,            "HW_Hotplug"},
-		{LOG_HW_LINK_TRAINING,      "HW_LKTN"},
-		{LOG_HW_SET_MODE,           "HW_Mode"},
-		{LOG_HW_RESUME_S3,          "HW_Resume"},
-		{LOG_HW_AUDIO,              "HW_Audio"},
-		{LOG_HW_HPD_IRQ,            "HW_HPDIRQ"},
-		{LOG_MST,                   "MST"},
-		{LOG_SCALER,                "Scaler"},
-		{LOG_BIOS,                  "BIOS"},
-		{LOG_BANDWIDTH_CALCS,       "BWCalcs"},
-		{LOG_BANDWIDTH_VALIDATION,  "BWValidation"},
-		{LOG_I2C_AUX,               "I2C_AUX"},
-		{LOG_SYNC,                  "Sync"},
-		{LOG_BACKLIGHT,             "Backlight"},
-		{LOG_FEATURE_OVERRIDE,      "Override"},
-		{LOG_DETECTION_EDID_PARSER, "Edid"},
-		{LOG_DETECTION_DP_CAPS,     "DP_Caps"},
-		{LOG_RESOURCE,              "Resource"},
-		{LOG_DML,                   "DML"},
-		{LOG_EVENT_MODE_SET,        "Mode"},
-		{LOG_EVENT_DETECTION,       "Detect"},
-		{LOG_EVENT_LINK_TRAINING,   "LKTN"},
-		{LOG_EVENT_LINK_LOSS,       "LinkLoss"},
-		{LOG_EVENT_UNDERFLOW,       "Underflow"},
-		{LOG_IF_TRACE,              "InterfaceTrace"},
-		{LOG_PERF_TRACE,            "PerfTrace"},
-		{LOG_DISPLAYSTATS,          "DisplayStats"}
-};
-
-
-/* ----------- Object init and destruction ----------- */
-static bool construct(struct dc_context *ctx, struct dal_logger *logger,
-		      uint32_t log_mask)
-{
-	/* malloc buffer and init offsets */
-	logger->log_buffer_size = DAL_LOGGER_BUFFER_MAX_SIZE;
-	logger->log_buffer = kcalloc(logger->log_buffer_size, sizeof(char),
-				     GFP_KERNEL);
-	if (!logger->log_buffer)
-		return false;
-
-	/* Initialize both offsets to start of buffer (empty) */
-	logger->buffer_read_offset = 0;
-	logger->buffer_write_offset = 0;
-
-	logger->open_count = 0;
-
-	logger->flags.bits.ENABLE_CONSOLE = 1;
-	logger->flags.bits.ENABLE_BUFFER = 0;
-
-	logger->ctx = ctx;
-
-	logger->mask = log_mask;
-
-	return true;
-}
-
-static void destruct(struct dal_logger *logger)
-{
-	if (logger->log_buffer) {
-		kfree(logger->log_buffer);
-		logger->log_buffer = NULL;
-	}
-}
-
-struct dal_logger *dal_logger_create(struct dc_context *ctx, uint32_t log_mask)
-{
-	/* malloc struct */
-	struct dal_logger *logger = kzalloc(sizeof(struct dal_logger),
-					    GFP_KERNEL);
-
-	if (!logger)
-		return NULL;
-	if (!construct(ctx, logger, log_mask)) {
-		kfree(logger);
-		return NULL;
-	}
-
-	return logger;
-}
-
-uint32_t dal_logger_destroy(struct dal_logger **logger)
-{
-	if (logger == NULL || *logger == NULL)
-		return 1;
-	destruct(*logger);
-	kfree(*logger);
-	*logger = NULL;
-
-	return 0;
-}
-
-/* ------------------------------------------------------------------------ */
-void dm_logger_append_heading(struct log_entry *entry)
-{
-	int j;
-
-	for (j = 0; j < NUM_ELEMENTS(log_type_info_tbl); j++) {
-
-		const struct dc_log_type_info *info = &log_type_info_tbl[j];
-
-		if (info->type == entry->type)
-			dm_logger_append(entry, "[%s]\t", info->name);
-	}
-}
-
-
-/* Print everything unread existing in log_buffer to debug console*/
-void dm_logger_flush_buffer(struct dal_logger *logger, bool should_warn)
-{
-	char *string_start = &logger->log_buffer[logger->buffer_read_offset];
-
-	if (should_warn)
-		dm_output_to_console(
-			"---------------- FLUSHING LOG BUFFER ----------------\n");
-	while (logger->buffer_read_offset < logger->buffer_write_offset) {
-
-		if (logger->log_buffer[logger->buffer_read_offset] == '\0') {
-			dm_output_to_console("%s", string_start);
-			string_start = logger->log_buffer + logger->buffer_read_offset + 1;
-		}
-		logger->buffer_read_offset++;
-	}
-	if (should_warn)
-		dm_output_to_console(
-			"-------------- END FLUSHING LOG BUFFER --------------\n\n");
-}
-/* ------------------------------------------------------------------------ */
-
-/* Warning: Be careful that 'msg' is null terminated and the total size is
- * less than DAL_LOGGER_BUFFER_MAX_LOG_LINE_SIZE (256) including '\0'
- */
-static bool dal_logger_should_log(
-	struct dal_logger *logger,
-	enum dc_log_type log_type)
-{
-	if (logger->mask & (1 << log_type))
-		return true;
-
-	return false;
-}
-
-static void log_to_debug_console(struct log_entry *entry)
-{
-	struct dal_logger *logger = entry->logger;
-
-	if (logger->flags.bits.ENABLE_CONSOLE == 0)
-		return;
-
-	if (entry->buf_offset) {
-		switch (entry->type) {
-		case LOG_ERROR:
-			dm_error("%s", entry->buf);
-			break;
-		default:
-			dm_output_to_console("%s", entry->buf);
-			break;
-		}
-	}
-}
-
-
-static void log_to_internal_buffer(struct log_entry *entry)
-{
-
-	uint32_t size = entry->buf_offset;
-	struct dal_logger *logger = entry->logger;
-
-	if (logger->flags.bits.ENABLE_BUFFER == 0)
-		return;
-
-	if (logger->log_buffer == NULL)
-		return;
-
-	if (size > 0 && size < logger->log_buffer_size) {
-
-		int buffer_space = logger->log_buffer_size -
-				logger->buffer_write_offset;
-
-		if (logger->buffer_write_offset == logger->buffer_read_offset) {
-			/* Buffer is empty, start writing at beginning */
-			buffer_space = logger->log_buffer_size;
-			logger->buffer_write_offset = 0;
-			logger->buffer_read_offset = 0;
-		}
-
-		if (buffer_space > size) {
-			/* No wrap around, copy 'size' bytes
-			 * from 'entry->buf' to 'log_buffer'
-			 */
-			memmove(logger->log_buffer +
-					logger->buffer_write_offset,
-					entry->buf, size);
-			logger->buffer_write_offset += size;
-
-		} else {
-			/* Not enough room remaining, we should flush
-			 * existing logs */
-
-			/* Flush existing unread logs to console */
-			dm_logger_flush_buffer(logger, true);
-
-			/* Start writing to beginning of buffer */
-			memmove(logger->log_buffer, entry->buf, size);
-			logger->buffer_write_offset = size;
-			logger->buffer_read_offset = 0;
-		}
-
-	}
-}
-
-static void append_entry(
-		struct log_entry *entry,
-		char *buffer,
-		uint32_t buf_size)
-{
-	if (!entry->buf ||
-		entry->buf_offset + buf_size > entry->max_buf_bytes
-	) {
-		BREAK_TO_DEBUGGER();
-		return;
-	}
-
-	/* Todo: check if off by 1 byte due to \0 anywhere */
-	memmove(entry->buf + entry->buf_offset, buffer, buf_size);
-	entry->buf_offset += buf_size;
-}
-
-
-void dm_logger_write(
-	struct dal_logger *logger,
-	enum dc_log_type log_type,
-	const char *msg,
-	...)
-{
-	if (logger && dal_logger_should_log(logger, log_type)) {
-		uint32_t size;
-		va_list args;
-		char buffer[LOG_MAX_LINE_SIZE];
-		struct log_entry entry;
-
-		va_start(args, msg);
-
-		entry.logger = logger;
-
-		entry.buf = buffer;
-
-		entry.buf_offset = 0;
-		entry.max_buf_bytes = DAL_LOGGER_BUFFER_MAX_SIZE * sizeof(char);
-
-		entry.type = log_type;
-
-		dm_logger_append_heading(&entry);
-
-		size = dm_log_to_buffer(
-			buffer, LOG_MAX_LINE_SIZE - 1, msg, args);
-
-		buffer[entry.buf_offset + size] = '\0';
-		entry.buf_offset += size + 1;
-
-		/* --Flush log_entry buffer-- */
-		/* print to kernel console */
-		log_to_debug_console(&entry);
-		/* log internally for dsat */
-		log_to_internal_buffer(&entry);
-
-		va_end(args);
-	}
-}
-
-/* Same as dm_logger_write, except without open() and close(), which must
- * be done separately.
- */
-void dm_logger_append(
-	struct log_entry *entry,
-	const char *msg,
-	...)
-{
-	va_list args;
-
-	va_start(args, msg);
-	dm_logger_append_va(entry, msg, args);
-	va_end(args);
-}
-
-void dm_logger_append_va(
-	struct log_entry *entry,
-	const char *msg,
-	va_list args)
-{
-	struct dal_logger *logger;
-
-	if (!entry) {
-		BREAK_TO_DEBUGGER();
-		return;
-	}
-
-	logger = entry->logger;
-
-	if (logger && logger->open_count > 0 &&
-		dal_logger_should_log(logger, entry->type)) {
-
-		uint32_t size;
-		char buffer[LOG_MAX_LINE_SIZE];
-
-		size = dm_log_to_buffer(
-			buffer, LOG_MAX_LINE_SIZE, msg, args);
-
-		if (size < LOG_MAX_LINE_SIZE - 1) {
-			append_entry(entry, buffer, size);
-		} else {
-			append_entry(entry, "LOG_ERROR, line too long\n", 27);
-		}
-	}
-}
-
-void dm_logger_open(
-		struct dal_logger *logger,
-		struct log_entry *entry, /* out */
-		enum dc_log_type log_type)
-{
-	if (!entry) {
-		BREAK_TO_DEBUGGER();
-		return;
-	}
-
-	entry->type = log_type;
-	entry->logger = logger;
-
-	entry->buf = kzalloc(DAL_LOGGER_BUFFER_MAX_SIZE,
-			     GFP_KERNEL);
-
-	entry->buf_offset = 0;
-	entry->max_buf_bytes = DAL_LOGGER_BUFFER_MAX_SIZE * sizeof(char);
-
-	logger->open_count++;
-
-	dm_logger_append_heading(entry);
-}
-
-void dm_logger_close(struct log_entry *entry)
-{
-	struct dal_logger *logger = entry->logger;
-
-	if (logger && logger->open_count > 0) {
-		logger->open_count--;
-	} else {
-		BREAK_TO_DEBUGGER();
-		goto cleanup;
-	}
-
-	/* --Flush log_entry buffer-- */
-	/* print to kernel console */
-	log_to_debug_console(entry);
-	/* log internally for dsat */
-	log_to_internal_buffer(entry);
-
-	/* TODO: Write end heading */
-
-cleanup:
-	if (entry->buf) {
-		kfree(entry->buf);
-		entry->buf = NULL;
-		entry->buf_offset = 0;
-		entry->max_buf_bytes = 0;
-	}
-}
-
diff --git a/drivers/gpu/drm/amd/display/dc/bios/bios_parser.c b/drivers/gpu/drm/amd/display/dc/bios/bios_parser.c
index c7f0b27e457e..be8a2494355a 100644
--- a/drivers/gpu/drm/amd/display/dc/bios/bios_parser.c
+++ b/drivers/gpu/drm/amd/display/dc/bios/bios_parser.c
@@ -3762,6 +3762,200 @@ static struct integrated_info *bios_parser_create_integrated_info(
 	return NULL;
 }
 
+enum bp_result update_slot_layout_info(
+	struct dc_bios *dcb,
+	unsigned int i,
+	struct slot_layout_info *slot_layout_info,
+	unsigned int record_offset)
+{
+	unsigned int j;
+	struct bios_parser *bp;
+	ATOM_BRACKET_LAYOUT_RECORD *record;
+	ATOM_COMMON_RECORD_HEADER *record_header;
+	enum bp_result result = BP_RESULT_NORECORD;
+
+	bp = BP_FROM_DCB(dcb);
+	record = NULL;
+	record_header = NULL;
+
+	for (;;) {
+
+		record_header = (ATOM_COMMON_RECORD_HEADER *)
+			GET_IMAGE(ATOM_COMMON_RECORD_HEADER, record_offset);
+		if (record_header == NULL) {
+			result = BP_RESULT_BADBIOSTABLE;
+			break;
+		}
+
+		/* the end of the list */
+		if (record_header->ucRecordType == 0xff ||
+			record_header->ucRecordSize == 0)	{
+			break;
+		}
+
+		if (record_header->ucRecordType ==
+			ATOM_BRACKET_LAYOUT_RECORD_TYPE &&
+			sizeof(ATOM_BRACKET_LAYOUT_RECORD)
+			<= record_header->ucRecordSize) {
+			record = (ATOM_BRACKET_LAYOUT_RECORD *)
+				(record_header);
+			result = BP_RESULT_OK;
+			break;
+		}
+
+		record_offset += record_header->ucRecordSize;
+	}
+
+	/* return if the record not found */
+	if (result != BP_RESULT_OK)
+		return result;
+
+	/* get slot sizes */
+	slot_layout_info->length = record->ucLength;
+	slot_layout_info->width = record->ucWidth;
+
+	/* get info for each connector in the slot */
+	slot_layout_info->num_of_connectors = record->ucConnNum;
+	for (j = 0; j < slot_layout_info->num_of_connectors; ++j) {
+		slot_layout_info->connectors[j].connector_type =
+			(enum connector_layout_type)
+			(record->asConnInfo[j].ucConnectorType);
+		switch (record->asConnInfo[j].ucConnectorType) {
+		case CONNECTOR_TYPE_DVI_D:
+			slot_layout_info->connectors[j].connector_type =
+				CONNECTOR_LAYOUT_TYPE_DVI_D;
+			slot_layout_info->connectors[j].length =
+				CONNECTOR_SIZE_DVI;
+			break;
+
+		case CONNECTOR_TYPE_HDMI:
+			slot_layout_info->connectors[j].connector_type =
+				CONNECTOR_LAYOUT_TYPE_HDMI;
+			slot_layout_info->connectors[j].length =
+				CONNECTOR_SIZE_HDMI;
+			break;
+
+		case CONNECTOR_TYPE_DISPLAY_PORT:
+			slot_layout_info->connectors[j].connector_type =
+				CONNECTOR_LAYOUT_TYPE_DP;
+			slot_layout_info->connectors[j].length =
+				CONNECTOR_SIZE_DP;
+			break;
+
+		case CONNECTOR_TYPE_MINI_DISPLAY_PORT:
+			slot_layout_info->connectors[j].connector_type =
+				CONNECTOR_LAYOUT_TYPE_MINI_DP;
+			slot_layout_info->connectors[j].length =
+				CONNECTOR_SIZE_MINI_DP;
+			break;
+
+		default:
+			slot_layout_info->connectors[j].connector_type =
+				CONNECTOR_LAYOUT_TYPE_UNKNOWN;
+			slot_layout_info->connectors[j].length =
+				CONNECTOR_SIZE_UNKNOWN;
+		}
+
+		slot_layout_info->connectors[j].position =
+			record->asConnInfo[j].ucPosition;
+		slot_layout_info->connectors[j].connector_id =
+			object_id_from_bios_object_id(
+				record->asConnInfo[j].usConnectorObjectId);
+	}
+	return result;
+}
+
+
+enum bp_result get_bracket_layout_record(
+	struct dc_bios *dcb,
+	unsigned int bracket_layout_id,
+	struct slot_layout_info *slot_layout_info)
+{
+	unsigned int i;
+	unsigned int record_offset;
+	struct bios_parser *bp;
+	enum bp_result result;
+	ATOM_OBJECT *object;
+	ATOM_OBJECT_TABLE *object_table;
+	unsigned int genericTableOffset;
+
+	bp = BP_FROM_DCB(dcb);
+	object = NULL;
+	if (slot_layout_info == NULL) {
+		DC_LOG_DETECTION_EDID_PARSER("Invalid slot_layout_info\n");
+		return BP_RESULT_BADINPUT;
+	}
+
+
+	genericTableOffset = bp->object_info_tbl_offset +
+		bp->object_info_tbl.v1_3->usMiscObjectTableOffset;
+	object_table = (ATOM_OBJECT_TABLE *)
+		GET_IMAGE(ATOM_OBJECT_TABLE, genericTableOffset);
+	if (!object_table)
+		return BP_RESULT_FAILURE;
+
+	result = BP_RESULT_NORECORD;
+	for (i = 0; i < object_table->ucNumberOfObjects; ++i) {
+
+		if (bracket_layout_id ==
+			object_table->asObjects[i].usObjectID) {
+
+			object = &object_table->asObjects[i];
+			record_offset = object->usRecordOffset +
+				bp->object_info_tbl_offset;
+
+			result = update_slot_layout_info(dcb, i,
+				slot_layout_info, record_offset);
+			break;
+		}
+	}
+	return result;
+}
+
+static enum bp_result bios_get_board_layout_info(
+	struct dc_bios *dcb,
+	struct board_layout_info *board_layout_info)
+{
+	unsigned int i;
+	struct bios_parser *bp;
+	enum bp_result record_result;
+
+	const unsigned int slot_index_to_vbios_id[MAX_BOARD_SLOTS] = {
+		GENERICOBJECT_BRACKET_LAYOUT_ENUM_ID1,
+		GENERICOBJECT_BRACKET_LAYOUT_ENUM_ID2,
+		0, 0
+	};
+
+	bp = BP_FROM_DCB(dcb);
+	if (board_layout_info == NULL) {
+		DC_LOG_DETECTION_EDID_PARSER("Invalid board_layout_info\n");
+		return BP_RESULT_BADINPUT;
+	}
+
+	board_layout_info->num_of_slots = 0;
+
+	for (i = 0; i < MAX_BOARD_SLOTS; ++i) {
+		record_result = get_bracket_layout_record(dcb,
+			slot_index_to_vbios_id[i],
+			&board_layout_info->slots[i]);
+
+		if (record_result == BP_RESULT_NORECORD && i > 0)
+			break; /* no more slots present in bios */
+		else if (record_result != BP_RESULT_OK)
+			return record_result;  /* fail */
+
+		++board_layout_info->num_of_slots;
+	}
+
+	/* all data is valid */
+	board_layout_info->is_number_of_slots_valid = 1;
+	board_layout_info->is_slots_size_valid = 1;
+	board_layout_info->is_connector_offsets_valid = 1;
+	board_layout_info->is_connector_lengths_valid = 1;
+
+	return BP_RESULT_OK;
+}
+
 /******************************************************************************/
 
 static const struct dc_vbios_funcs vbios_funcs = {
@@ -3836,6 +4030,8 @@ static const struct dc_vbios_funcs vbios_funcs = {
 	.post_init = bios_parser_post_init,  /* patch vbios table for mxm module by reading i2c */
 
 	.bios_parser_destroy = bios_parser_destroy,
+
+	.get_board_layout_info = bios_get_board_layout_info,
 };
 
 static bool bios_parser_construct(
diff --git a/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c b/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c
index b8cef7af3c4a..eab007e1793c 100644
--- a/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c
+++ b/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c
@@ -43,6 +43,29 @@
 #include "bios_parser_interface.h"
 
 #include "bios_parser_common.h"
+
+/* Temporarily add in defines until ObjectID.h patch is updated in a few days */
+#ifndef GENERIC_OBJECT_ID_BRACKET_LAYOUT
+#define GENERIC_OBJECT_ID_BRACKET_LAYOUT          0x05
+#endif /* GENERIC_OBJECT_ID_BRACKET_LAYOUT */
+
+#ifndef GENERICOBJECT_BRACKET_LAYOUT_ENUM_ID1
+#define GENERICOBJECT_BRACKET_LAYOUT_ENUM_ID1	\
+	(GRAPH_OBJECT_TYPE_GENERIC << OBJECT_TYPE_SHIFT |\
+	GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\
+	GENERIC_OBJECT_ID_BRACKET_LAYOUT << OBJECT_ID_SHIFT)
+#endif /* GENERICOBJECT_BRACKET_LAYOUT_ENUM_ID1 */
+
+#ifndef GENERICOBJECT_BRACKET_LAYOUT_ENUM_ID2
+#define GENERICOBJECT_BRACKET_LAYOUT_ENUM_ID2	\
+	(GRAPH_OBJECT_TYPE_GENERIC << OBJECT_TYPE_SHIFT |\
+	GRAPH_OBJECT_ENUM_ID2 << ENUM_ID_SHIFT |\
+	GENERIC_OBJECT_ID_BRACKET_LAYOUT << OBJECT_ID_SHIFT)
+#endif /* GENERICOBJECT_BRACKET_LAYOUT_ENUM_ID2 */
+
+#define DC_LOGGER \
+	bp->base.ctx->logger
+
 #define LAST_RECORD_TYPE 0xff
 #define SMU9_SYSPLL0_ID  0
 
@@ -86,7 +109,6 @@ static struct atom_encoder_caps_record *get_encoder_cap_record(
 
 #define DATA_TABLES(table) (bp->master_data_tbl->listOfdatatables.table)
 
-
 static void destruct(struct bios_parser *bp)
 {
 	kfree(bp->base.bios_local_image);
@@ -656,7 +678,7 @@ static enum bp_result bios_parser_get_gpio_pin_info(
 		return BP_RESULT_BADBIOSTABLE;
 
 	if (sizeof(struct atom_common_table_header) +
-			sizeof(struct atom_gpio_pin_lut_v2_1)
+			sizeof(struct atom_gpio_pin_assignment)
 			> le16_to_cpu(header->table_header.structuresize))
 		return BP_RESULT_BADBIOSTABLE;
 
@@ -1854,6 +1876,198 @@ static struct integrated_info *bios_parser_create_integrated_info(
 	return NULL;
 }
 
+static enum bp_result update_slot_layout_info(
+	struct dc_bios *dcb,
+	unsigned int i,
+	struct slot_layout_info *slot_layout_info)
+{
+	unsigned int record_offset;
+	unsigned int j;
+	struct atom_display_object_path_v2 *object;
+	struct atom_bracket_layout_record *record;
+	struct atom_common_record_header *record_header;
+	enum bp_result result;
+	struct bios_parser *bp;
+	struct object_info_table *tbl;
+	struct display_object_info_table_v1_4 *v1_4;
+
+	record = NULL;
+	record_header = NULL;
+	result = BP_RESULT_NORECORD;
+
+	bp = BP_FROM_DCB(dcb);
+	tbl = &bp->object_info_tbl;
+	v1_4 = tbl->v1_4;
+
+	object = &v1_4->display_path[i];
+	record_offset = (unsigned int)
+		(object->disp_recordoffset) +
+		(unsigned int)(bp->object_info_tbl_offset);
+
+	for (;;) {
+
+		record_header = (struct atom_common_record_header *)
+			GET_IMAGE(struct atom_common_record_header,
+			record_offset);
+		if (record_header == NULL) {
+			result = BP_RESULT_BADBIOSTABLE;
+			break;
+		}
+
+		/* the end of the list */
+		if (record_header->record_type == 0xff ||
+			record_header->record_size == 0)	{
+			break;
+		}
+
+		if (record_header->record_type ==
+			ATOM_BRACKET_LAYOUT_RECORD_TYPE &&
+			sizeof(struct atom_bracket_layout_record)
+			<= record_header->record_size) {
+			record = (struct atom_bracket_layout_record *)
+				(record_header);
+			result = BP_RESULT_OK;
+			break;
+		}
+
+		record_offset += record_header->record_size;
+	}
+
+	/* return if the record not found */
+	if (result != BP_RESULT_OK)
+		return result;
+
+	/* get slot sizes */
+	slot_layout_info->length = record->bracketlen;
+	slot_layout_info->width = record->bracketwidth;
+
+	/* get info for each connector in the slot */
+	slot_layout_info->num_of_connectors = record->conn_num;
+	for (j = 0; j < slot_layout_info->num_of_connectors; ++j) {
+		slot_layout_info->connectors[j].connector_type =
+			(enum connector_layout_type)
+			(record->conn_info[j].connector_type);
+		switch (record->conn_info[j].connector_type) {
+		case CONNECTOR_TYPE_DVI_D:
+			slot_layout_info->connectors[j].connector_type =
+				CONNECTOR_LAYOUT_TYPE_DVI_D;
+			slot_layout_info->connectors[j].length =
+				CONNECTOR_SIZE_DVI;
+			break;
+
+		case CONNECTOR_TYPE_HDMI:
+			slot_layout_info->connectors[j].connector_type =
+				CONNECTOR_LAYOUT_TYPE_HDMI;
+			slot_layout_info->connectors[j].length =
+				CONNECTOR_SIZE_HDMI;
+			break;
+
+		case CONNECTOR_TYPE_DISPLAY_PORT:
+			slot_layout_info->connectors[j].connector_type =
+				CONNECTOR_LAYOUT_TYPE_DP;
+			slot_layout_info->connectors[j].length =
+				CONNECTOR_SIZE_DP;
+			break;
+
+		case CONNECTOR_TYPE_MINI_DISPLAY_PORT:
+			slot_layout_info->connectors[j].connector_type =
+				CONNECTOR_LAYOUT_TYPE_MINI_DP;
+			slot_layout_info->connectors[j].length =
+				CONNECTOR_SIZE_MINI_DP;
+			break;
+
+		default:
+			slot_layout_info->connectors[j].connector_type =
+				CONNECTOR_LAYOUT_TYPE_UNKNOWN;
+			slot_layout_info->connectors[j].length =
+				CONNECTOR_SIZE_UNKNOWN;
+		}
+
+		slot_layout_info->connectors[j].position =
+			record->conn_info[j].position;
+		slot_layout_info->connectors[j].connector_id =
+			object_id_from_bios_object_id(
+				record->conn_info[j].connectorobjid);
+	}
+	return result;
+}
+
+
+static enum bp_result get_bracket_layout_record(
+	struct dc_bios *dcb,
+	unsigned int bracket_layout_id,
+	struct slot_layout_info *slot_layout_info)
+{
+	unsigned int i;
+	struct bios_parser *bp = BP_FROM_DCB(dcb);
+	enum bp_result result;
+	struct object_info_table *tbl;
+	struct display_object_info_table_v1_4 *v1_4;
+
+	if (slot_layout_info == NULL) {
+		DC_LOG_DETECTION_EDID_PARSER("Invalid slot_layout_info\n");
+		return BP_RESULT_BADINPUT;
+	}
+	tbl = &bp->object_info_tbl;
+	v1_4 = tbl->v1_4;
+
+	result = BP_RESULT_NORECORD;
+	for (i = 0; i < v1_4->number_of_path; ++i)	{
+
+		if (bracket_layout_id ==
+			v1_4->display_path[i].display_objid) {
+			result = update_slot_layout_info(dcb, i,
+				slot_layout_info);
+			break;
+		}
+	}
+	return result;
+}
+
+static enum bp_result bios_get_board_layout_info(
+	struct dc_bios *dcb,
+	struct board_layout_info *board_layout_info)
+{
+	unsigned int i;
+	struct bios_parser *bp;
+	enum bp_result record_result;
+
+	const unsigned int slot_index_to_vbios_id[MAX_BOARD_SLOTS] = {
+		GENERICOBJECT_BRACKET_LAYOUT_ENUM_ID1,
+		GENERICOBJECT_BRACKET_LAYOUT_ENUM_ID2,
+		0, 0
+	};
+
+	bp = BP_FROM_DCB(dcb);
+	if (board_layout_info == NULL) {
+		DC_LOG_DETECTION_EDID_PARSER("Invalid board_layout_info\n");
+		return BP_RESULT_BADINPUT;
+	}
+
+	board_layout_info->num_of_slots = 0;
+
+	for (i = 0; i < MAX_BOARD_SLOTS; ++i) {
+		record_result = get_bracket_layout_record(dcb,
+			slot_index_to_vbios_id[i],
+			&board_layout_info->slots[i]);
+
+		if (record_result == BP_RESULT_NORECORD && i > 0)
+			break; /* no more slots present in bios */
+		else if (record_result != BP_RESULT_OK)
+			return record_result;  /* fail */
+
+		++board_layout_info->num_of_slots;
+	}
+
+	/* all data is valid */
+	board_layout_info->is_number_of_slots_valid = 1;
+	board_layout_info->is_slots_size_valid = 1;
+	board_layout_info->is_connector_offsets_valid = 1;
+	board_layout_info->is_connector_lengths_valid = 1;
+
+	return BP_RESULT_OK;
+}
+
 static const struct dc_vbios_funcs vbios_funcs = {
 	.get_connectors_number = bios_parser_get_connectors_number,
 
@@ -1925,6 +2139,8 @@ static const struct dc_vbios_funcs vbios_funcs = {
 	.bios_parser_destroy = firmware_parser_destroy,
 
 	.get_smu_clock_info = bios_parser_get_smu_clock_info,
+
+	.get_board_layout_info = bios_get_board_layout_info,
 };
 
 static bool bios_parser_construct(
diff --git a/drivers/gpu/drm/amd/display/dc/bios/command_table.c b/drivers/gpu/drm/amd/display/dc/bios/command_table.c
index 651e1fd4622f..a558bfaa0c46 100644
--- a/drivers/gpu/drm/amd/display/dc/bios/command_table.c
+++ b/drivers/gpu/drm/amd/display/dc/bios/command_table.c
@@ -808,6 +808,24 @@ static enum bp_result transmitter_control_v1_5(
 	 * (=1: 8bpp, =1.25: 10bpp, =1.5:12bpp, =2: 16bpp)
 	 * LVDS mode: usPixelClock = pixel clock
 	 */
+	if  (cntl->signal == SIGNAL_TYPE_HDMI_TYPE_A) {
+		switch (cntl->color_depth) {
+		case COLOR_DEPTH_101010:
+			params.usSymClock =
+				cpu_to_le16((le16_to_cpu(params.usSymClock) * 30) / 24);
+			break;
+		case COLOR_DEPTH_121212:
+			params.usSymClock =
+				cpu_to_le16((le16_to_cpu(params.usSymClock) * 36) / 24);
+			break;
+		case COLOR_DEPTH_161616:
+			params.usSymClock =
+				cpu_to_le16((le16_to_cpu(params.usSymClock) * 48) / 24);
+			break;
+		default:
+			break;
+		}
+	}
 
 	if (EXEC_BIOS_CMD_TABLE(UNIPHYTransmitterControl, params))
 		result = BP_RESULT_OK;
diff --git a/drivers/gpu/drm/amd/display/dc/bios/command_table2.c b/drivers/gpu/drm/amd/display/dc/bios/command_table2.c
index 752b08a42d3e..2b5dc499a35e 100644
--- a/drivers/gpu/drm/amd/display/dc/bios/command_table2.c
+++ b/drivers/gpu/drm/amd/display/dc/bios/command_table2.c
@@ -59,36 +59,7 @@
 	bios_cmd_table_para_revision(bp->base.ctx->driver_context, \
 			GET_INDEX_INTO_MASTER_TABLE(command, fname))
 
-static void init_dig_encoder_control(struct bios_parser *bp);
-static void init_transmitter_control(struct bios_parser *bp);
-static void init_set_pixel_clock(struct bios_parser *bp);
 
-static void init_set_crtc_timing(struct bios_parser *bp);
-
-static void init_select_crtc_source(struct bios_parser *bp);
-static void init_enable_crtc(struct bios_parser *bp);
-
-static void init_external_encoder_control(struct bios_parser *bp);
-static void init_enable_disp_power_gating(struct bios_parser *bp);
-static void init_set_dce_clock(struct bios_parser *bp);
-static void init_get_smu_clock_info(struct bios_parser *bp);
-
-void dal_firmware_parser_init_cmd_tbl(struct bios_parser *bp)
-{
-	init_dig_encoder_control(bp);
-	init_transmitter_control(bp);
-	init_set_pixel_clock(bp);
-
-	init_set_crtc_timing(bp);
-
-	init_select_crtc_source(bp);
-	init_enable_crtc(bp);
-
-	init_external_encoder_control(bp);
-	init_enable_disp_power_gating(bp);
-	init_set_dce_clock(bp);
-	init_get_smu_clock_info(bp);
-}
 
 static uint32_t bios_cmd_table_para_revision(void *dev,
 					     uint32_t index)
@@ -829,3 +800,20 @@ static unsigned int get_smu_clock_info_v3_1(struct bios_parser *bp, uint8_t id)
 	return 0;
 }
 
+void dal_firmware_parser_init_cmd_tbl(struct bios_parser *bp)
+{
+	init_dig_encoder_control(bp);
+	init_transmitter_control(bp);
+	init_set_pixel_clock(bp);
+
+	init_set_crtc_timing(bp);
+
+	init_select_crtc_source(bp);
+	init_enable_crtc(bp);
+
+	init_external_encoder_control(bp);
+	init_enable_disp_power_gating(bp);
+	init_set_dce_clock(bp);
+	init_get_smu_clock_info(bp);
+
+}
diff --git a/drivers/gpu/drm/amd/display/dc/calcs/calcs_logger.h b/drivers/gpu/drm/amd/display/dc/calcs/calcs_logger.h
index fc3f98fb09ea..62435bfc274d 100644
--- a/drivers/gpu/drm/amd/display/dc/calcs/calcs_logger.h
+++ b/drivers/gpu/drm/amd/display/dc/calcs/calcs_logger.h
@@ -25,10 +25,9 @@
 
 #ifndef _CALCS_CALCS_LOGGER_H_
 #define _CALCS_CALCS_LOGGER_H_
-#define DC_LOGGER \
-	logger
+#define DC_LOGGER ctx->logger
 
-static void print_bw_calcs_dceip(struct dal_logger *logger, const struct bw_calcs_dceip *dceip)
+static void print_bw_calcs_dceip(struct dc_context *ctx, const struct bw_calcs_dceip *dceip)
 {
 
 	DC_LOG_BANDWIDTH_CALCS("#####################################################################");
@@ -122,7 +121,7 @@ static void print_bw_calcs_dceip(struct dal_logger *logger, const struct bw_calc
 
 }
 
-static void print_bw_calcs_vbios(struct dal_logger *logger, const struct bw_calcs_vbios *vbios)
+static void print_bw_calcs_vbios(struct dc_context *ctx, const struct bw_calcs_vbios *vbios)
 {
 
 	DC_LOG_BANDWIDTH_CALCS("#####################################################################");
@@ -181,7 +180,7 @@ static void print_bw_calcs_vbios(struct dal_logger *logger, const struct bw_calc
 
 }
 
-static void print_bw_calcs_data(struct dal_logger *logger, struct bw_calcs_data *data)
+static void print_bw_calcs_data(struct dc_context *ctx, struct bw_calcs_data *data)
 {
 
 	int i, j, k;
diff --git a/drivers/gpu/drm/amd/display/dc/calcs/dce_calcs.c b/drivers/gpu/drm/amd/display/dc/calcs/dce_calcs.c
index 2c4e8f0cb2dc..160d11a15eac 100644
--- a/drivers/gpu/drm/amd/display/dc/calcs/dce_calcs.c
+++ b/drivers/gpu/drm/amd/display/dc/calcs/dce_calcs.c
@@ -3010,9 +3010,9 @@ bool bw_calcs(struct dc_context *ctx,
 		struct bw_fixed low_yclk = vbios->low_yclk;
 
 		if (ctx->dc->debug.bandwidth_calcs_trace) {
-			print_bw_calcs_dceip(ctx->logger, dceip);
-			print_bw_calcs_vbios(ctx->logger, vbios);
-			print_bw_calcs_data(ctx->logger, data);
+			print_bw_calcs_dceip(ctx, dceip);
+			print_bw_calcs_vbios(ctx, vbios);
+			print_bw_calcs_data(ctx, data);
 		}
 		calculate_bandwidth(dceip, vbios, data);
 
diff --git a/drivers/gpu/drm/amd/display/dc/calcs/dcn_calcs.c b/drivers/gpu/drm/amd/display/dc/calcs/dcn_calcs.c
index 49a4ea45466d..bd039322f697 100644
--- a/drivers/gpu/drm/amd/display/dc/calcs/dcn_calcs.c
+++ b/drivers/gpu/drm/amd/display/dc/calcs/dcn_calcs.c
@@ -31,6 +31,8 @@
 
 #include "resource.h"
 #include "dcn10/dcn10_resource.h"
+#include "dcn10/dcn10_hubbub.h"
+
 #include "dcn_calc_math.h"
 
 #define DC_LOGGER \
@@ -248,7 +250,24 @@ static void pipe_ctx_to_e2e_pipe_params (
 	else if (pipe->bottom_pipe != NULL && pipe->bottom_pipe->plane_state == pipe->plane_state)
 		input->src.is_hsplit = true;
 
-	input->src.dcc                 = pipe->plane_state->dcc.enable;
+	if (pipe->plane_res.dpp->ctx->dc->debug.optimized_watermark) {
+		/*
+		 * this method requires us to always re-calculate watermark when dcc change
+		 * between flip.
+		 */
+		input->src.dcc = pipe->plane_state->dcc.enable ? 1 : 0;
+	} else {
+		/*
+		 * allow us to disable dcc on the fly without re-calculating WM
+		 *
+		 * extra overhead for DCC is quite small.  for 1080p WM without
+		 * DCC is only 0.417us lower (urgent goes from 6.979us to 6.562us)
+		 */
+		unsigned int bpe;
+
+		input->src.dcc = pipe->plane_res.dpp->ctx->dc->res_pool->hubbub->funcs->
+			dcc_support_pixel_format(pipe->plane_state->format, &bpe) ? 1 : 0;
+	}
 	input->src.dcc_rate            = 1;
 	input->src.meta_pitch          = pipe->plane_state->dcc.grph.meta_pitch;
 	input->src.source_scan         = dm_horz;
@@ -423,6 +442,10 @@ static void dcn_bw_calc_rq_dlg_ttu(
 	int total_flip_bytes = 0;
 	int i;
 
+	memset(dlg_regs, 0, sizeof(*dlg_regs));
+	memset(ttu_regs, 0, sizeof(*ttu_regs));
+	memset(rq_regs, 0, sizeof(*rq_regs));
+
 	for (i = 0; i < number_of_planes; i++) {
 		total_active_bw += v->read_bandwidth[i];
 		total_prefetch_bw += v->prefetch_bandwidth[i];
@@ -501,6 +524,7 @@ static void split_stream_across_pipes(
 	resource_build_scaling_params(secondary_pipe);
 }
 
+#if 0
 static void calc_wm_sets_and_perf_params(
 		struct dc_state *context,
 		struct dcn_bw_internal_vars *v)
@@ -582,6 +606,7 @@ static void calc_wm_sets_and_perf_params(
 	if (v->voltage_level >= 3)
 		context->bw.dcn.watermarks.d = context->bw.dcn.watermarks.a;
 }
+#endif
 
 static bool dcn_bw_apply_registry_override(struct dc *dc)
 {
@@ -651,7 +676,7 @@ static void hack_force_pipe_split(struct dcn_bw_internal_vars *v,
 }
 
 static void hack_bounding_box(struct dcn_bw_internal_vars *v,
-		struct dc_debug *dbg,
+		struct dc_debug_options *dbg,
 		struct dc_state *context)
 {
 	if (dbg->pipe_split_policy == MPC_SPLIT_AVOID)
@@ -883,7 +908,26 @@ bool dcn_validate_bandwidth(
 				ASSERT(pipe->plane_res.scl_data.ratios.vert.value != dc_fixpt_one.value
 					|| v->scaler_rec_out_width[input_idx] == v->viewport_height[input_idx]);
 			}
-			v->dcc_enable[input_idx] = pipe->plane_state->dcc.enable ? dcn_bw_yes : dcn_bw_no;
+
+			if (dc->debug.optimized_watermark) {
+				/*
+				 * this method requires us to always re-calculate watermark when dcc change
+				 * between flip.
+				 */
+				v->dcc_enable[input_idx] = pipe->plane_state->dcc.enable ? dcn_bw_yes : dcn_bw_no;
+			} else {
+				/*
+				 * allow us to disable dcc on the fly without re-calculating WM
+				 *
+				 * extra overhead for DCC is quite small.  for 1080p WM without
+				 * DCC is only 0.417us lower (urgent goes from 6.979us to 6.562us)
+				 */
+				unsigned int bpe;
+
+				v->dcc_enable[input_idx] = dc->res_pool->hubbub->funcs->dcc_support_pixel_format(
+						pipe->plane_state->format, &bpe) ? dcn_bw_yes : dcn_bw_no;
+			}
+
 			v->source_pixel_format[input_idx] = tl_pixel_format_to_bw_defs(
 					pipe->plane_state->format);
 			v->source_surface_mode[input_idx] = tl_sw_mode_to_bw_defs(
@@ -976,43 +1020,60 @@ bool dcn_validate_bandwidth(
 				bw_consumed = v->fabric_and_dram_bandwidth;
 
 		display_pipe_configuration(v);
-		calc_wm_sets_and_perf_params(context, v);
-		context->bw.dcn.calc_clk.fclk_khz = (int)(bw_consumed * 1000000 /
+		/*calc_wm_sets_and_perf_params(context, v);*/
+		/* Only 1 set is used by dcn since no noticeable
+		 * performance improvement was measured and due to hw bug DEGVIDCN10-254
+		 */
+		dispclkdppclkdcfclk_deep_sleep_prefetch_parameters_watermarks_and_performance_calculation(v);
+
+		context->bw.dcn.watermarks.a.cstate_pstate.cstate_exit_ns =
+			v->stutter_exit_watermark * 1000;
+		context->bw.dcn.watermarks.a.cstate_pstate.cstate_enter_plus_exit_ns =
+				v->stutter_enter_plus_exit_watermark * 1000;
+		context->bw.dcn.watermarks.a.cstate_pstate.pstate_change_ns =
+				v->dram_clock_change_watermark * 1000;
+		context->bw.dcn.watermarks.a.pte_meta_urgent_ns = v->ptemeta_urgent_watermark * 1000;
+		context->bw.dcn.watermarks.a.urgent_ns = v->urgent_watermark * 1000;
+		context->bw.dcn.watermarks.b = context->bw.dcn.watermarks.a;
+		context->bw.dcn.watermarks.c = context->bw.dcn.watermarks.a;
+		context->bw.dcn.watermarks.d = context->bw.dcn.watermarks.a;
+
+		context->bw.dcn.clk.fclk_khz = (int)(bw_consumed * 1000000 /
 				(ddr4_dram_factor_single_Channel * v->number_of_channels));
 		if (bw_consumed == v->fabric_and_dram_bandwidth_vmin0p65) {
-			context->bw.dcn.calc_clk.fclk_khz = (int)(bw_consumed * 1000000 / 32);
+			context->bw.dcn.clk.fclk_khz = (int)(bw_consumed * 1000000 / 32);
 		}
 
-		context->bw.dcn.calc_clk.dcfclk_deep_sleep_khz = (int)(v->dcf_clk_deep_sleep * 1000);
-		context->bw.dcn.calc_clk.dcfclk_khz = (int)(v->dcfclk * 1000);
+		context->bw.dcn.clk.dcfclk_deep_sleep_khz = (int)(v->dcf_clk_deep_sleep * 1000);
+		context->bw.dcn.clk.dcfclk_khz = (int)(v->dcfclk * 1000);
 
-		context->bw.dcn.calc_clk.dispclk_khz = (int)(v->dispclk * 1000);
+		context->bw.dcn.clk.dispclk_khz = (int)(v->dispclk * 1000);
 		if (dc->debug.max_disp_clk == true)
-			context->bw.dcn.calc_clk.dispclk_khz = (int)(dc->dcn_soc->max_dispclk_vmax0p9 * 1000);
+			context->bw.dcn.clk.dispclk_khz = (int)(dc->dcn_soc->max_dispclk_vmax0p9 * 1000);
 
-		if (context->bw.dcn.calc_clk.dispclk_khz <
+		if (context->bw.dcn.clk.dispclk_khz <
 				dc->debug.min_disp_clk_khz) {
-			context->bw.dcn.calc_clk.dispclk_khz =
+			context->bw.dcn.clk.dispclk_khz =
 					dc->debug.min_disp_clk_khz;
 		}
 
-		context->bw.dcn.calc_clk.dppclk_khz = context->bw.dcn.calc_clk.dispclk_khz / v->dispclk_dppclk_ratio;
-
+		context->bw.dcn.clk.dppclk_khz = context->bw.dcn.clk.dispclk_khz / v->dispclk_dppclk_ratio;
+		context->bw.dcn.clk.phyclk_khz = v->phyclk_per_state[v->voltage_level];
 		switch (v->voltage_level) {
 		case 0:
-			context->bw.dcn.calc_clk.max_supported_dppclk_khz =
+			context->bw.dcn.clk.max_supported_dppclk_khz =
 					(int)(dc->dcn_soc->max_dppclk_vmin0p65 * 1000);
 			break;
 		case 1:
-			context->bw.dcn.calc_clk.max_supported_dppclk_khz =
+			context->bw.dcn.clk.max_supported_dppclk_khz =
 					(int)(dc->dcn_soc->max_dppclk_vmid0p72 * 1000);
 			break;
 		case 2:
-			context->bw.dcn.calc_clk.max_supported_dppclk_khz =
+			context->bw.dcn.clk.max_supported_dppclk_khz =
 					(int)(dc->dcn_soc->max_dppclk_vnom0p8 * 1000);
 			break;
 		default:
-			context->bw.dcn.calc_clk.max_supported_dppclk_khz =
+			context->bw.dcn.clk.max_supported_dppclk_khz =
 					(int)(dc->dcn_soc->max_dppclk_vmax0p9 * 1000);
 			break;
 		}
@@ -1225,27 +1286,27 @@ static unsigned int dcn_find_normalized_clock_vdd_Level(
 
 unsigned int dcn_find_dcfclk_suits_all(
 	const struct dc *dc,
-	struct clocks_value *clocks)
+	struct dc_clocks *clocks)
 {
 	unsigned vdd_level, vdd_level_temp;
 	unsigned dcf_clk;
 
 	/*find a common supported voltage level*/
 	vdd_level = dcn_find_normalized_clock_vdd_Level(
-		dc, DM_PP_CLOCK_TYPE_DISPLAY_CLK, clocks->dispclk_in_khz);
+		dc, DM_PP_CLOCK_TYPE_DISPLAY_CLK, clocks->dispclk_khz);
 	vdd_level_temp = dcn_find_normalized_clock_vdd_Level(
-		dc, DM_PP_CLOCK_TYPE_DISPLAYPHYCLK, clocks->phyclk_in_khz);
+		dc, DM_PP_CLOCK_TYPE_DISPLAYPHYCLK, clocks->phyclk_khz);
 
 	vdd_level = dcn_bw_max(vdd_level, vdd_level_temp);
 	vdd_level_temp = dcn_find_normalized_clock_vdd_Level(
-		dc, DM_PP_CLOCK_TYPE_DPPCLK, clocks->dppclk_in_khz);
+		dc, DM_PP_CLOCK_TYPE_DPPCLK, clocks->dppclk_khz);
 	vdd_level = dcn_bw_max(vdd_level, vdd_level_temp);
 
 	vdd_level_temp = dcn_find_normalized_clock_vdd_Level(
-		dc, DM_PP_CLOCK_TYPE_MEMORY_CLK, clocks->dcfclock_in_khz);
+		dc, DM_PP_CLOCK_TYPE_MEMORY_CLK, clocks->fclk_khz);
 	vdd_level = dcn_bw_max(vdd_level, vdd_level_temp);
 	vdd_level_temp = dcn_find_normalized_clock_vdd_Level(
-		dc, DM_PP_CLOCK_TYPE_DCFCLK, clocks->dcfclock_in_khz);
+		dc, DM_PP_CLOCK_TYPE_DCFCLK, clocks->dcfclk_khz);
 
 	/*find that level conresponding dcfclk*/
 	vdd_level = dcn_bw_max(vdd_level, vdd_level_temp);
@@ -1331,21 +1392,14 @@ void dcn_bw_notify_pplib_of_wm_ranges(struct dc *dc)
 {
 	struct pp_smu_funcs_rv *pp = dc->res_pool->pp_smu;
 	struct pp_smu_wm_range_sets ranges = {0};
-	int max_fclk_khz, nom_fclk_khz, mid_fclk_khz, min_fclk_khz;
-	int max_dcfclk_khz, min_dcfclk_khz;
-	int socclk_khz;
+	int min_fclk_khz, min_dcfclk_khz, socclk_khz;
 	const int overdrive = 5000000; /* 5 GHz to cover Overdrive */
-	unsigned factor = (ddr4_dram_factor_single_Channel * dc->dcn_soc->number_of_channels);
 
 	if (!pp->set_wm_ranges)
 		return;
 
 	kernel_fpu_begin();
-	max_fclk_khz = dc->dcn_soc->fabric_and_dram_bandwidth_vmax0p9 * 1000000 / factor;
-	nom_fclk_khz = dc->dcn_soc->fabric_and_dram_bandwidth_vnom0p8 * 1000000 / factor;
-	mid_fclk_khz = dc->dcn_soc->fabric_and_dram_bandwidth_vmid0p72 * 1000000 / factor;
 	min_fclk_khz = dc->dcn_soc->fabric_and_dram_bandwidth_vmin0p65 * 1000000 / 32;
-	max_dcfclk_khz = dc->dcn_soc->dcfclkv_max0p9 * 1000;
 	min_dcfclk_khz = dc->dcn_soc->dcfclkv_min0p65 * 1000;
 	socclk_khz = dc->dcn_soc->socclk * 1000;
 	kernel_fpu_end();
@@ -1353,105 +1407,46 @@ void dcn_bw_notify_pplib_of_wm_ranges(struct dc *dc)
 	/* Now notify PPLib/SMU about which Watermarks sets they should select
 	 * depending on DPM state they are in. And update BW MGR GFX Engine and
 	 * Memory clock member variables for Watermarks calculations for each
-	 * Watermark Set
+	 * Watermark Set. Only one watermark set for dcn1 due to hw bug DEGVIDCN10-254.
 	 */
 	/* SOCCLK does not affect anytihng but writeback for DCN so for now we dont
 	 * care what the value is, hence min to overdrive level
 	 */
-	ranges.num_reader_wm_sets = WM_COUNT;
-	ranges.num_writer_wm_sets = WM_COUNT;
+	ranges.num_reader_wm_sets = WM_SET_COUNT;
+	ranges.num_writer_wm_sets = WM_SET_COUNT;
 	ranges.reader_wm_sets[0].wm_inst = WM_A;
 	ranges.reader_wm_sets[0].min_drain_clk_khz = min_dcfclk_khz;
-	ranges.reader_wm_sets[0].max_drain_clk_khz = max_dcfclk_khz;
+	ranges.reader_wm_sets[0].max_drain_clk_khz = overdrive;
 	ranges.reader_wm_sets[0].min_fill_clk_khz = min_fclk_khz;
-	ranges.reader_wm_sets[0].max_fill_clk_khz = min_fclk_khz;
+	ranges.reader_wm_sets[0].max_fill_clk_khz = overdrive;
 	ranges.writer_wm_sets[0].wm_inst = WM_A;
 	ranges.writer_wm_sets[0].min_fill_clk_khz = socclk_khz;
 	ranges.writer_wm_sets[0].max_fill_clk_khz = overdrive;
 	ranges.writer_wm_sets[0].min_drain_clk_khz = min_fclk_khz;
-	ranges.writer_wm_sets[0].max_drain_clk_khz = min_fclk_khz;
-
-	ranges.reader_wm_sets[1].wm_inst = WM_B;
-	ranges.reader_wm_sets[1].min_drain_clk_khz = min_fclk_khz;
-	ranges.reader_wm_sets[1].max_drain_clk_khz = max_dcfclk_khz;
-	ranges.reader_wm_sets[1].min_fill_clk_khz = mid_fclk_khz;
-	ranges.reader_wm_sets[1].max_fill_clk_khz = mid_fclk_khz;
-	ranges.writer_wm_sets[1].wm_inst = WM_B;
-	ranges.writer_wm_sets[1].min_fill_clk_khz = socclk_khz;
-	ranges.writer_wm_sets[1].max_fill_clk_khz = overdrive;
-	ranges.writer_wm_sets[1].min_drain_clk_khz = mid_fclk_khz;
-	ranges.writer_wm_sets[1].max_drain_clk_khz = mid_fclk_khz;
-
-
-	ranges.reader_wm_sets[2].wm_inst = WM_C;
-	ranges.reader_wm_sets[2].min_drain_clk_khz = min_fclk_khz;
-	ranges.reader_wm_sets[2].max_drain_clk_khz = max_dcfclk_khz;
-	ranges.reader_wm_sets[2].min_fill_clk_khz = nom_fclk_khz;
-	ranges.reader_wm_sets[2].max_fill_clk_khz = nom_fclk_khz;
-	ranges.writer_wm_sets[2].wm_inst = WM_C;
-	ranges.writer_wm_sets[2].min_fill_clk_khz = socclk_khz;
-	ranges.writer_wm_sets[2].max_fill_clk_khz = overdrive;
-	ranges.writer_wm_sets[2].min_drain_clk_khz = nom_fclk_khz;
-	ranges.writer_wm_sets[2].max_drain_clk_khz = nom_fclk_khz;
-
-	ranges.reader_wm_sets[3].wm_inst = WM_D;
-	ranges.reader_wm_sets[3].min_drain_clk_khz = min_fclk_khz;
-	ranges.reader_wm_sets[3].max_drain_clk_khz = max_dcfclk_khz;
-	ranges.reader_wm_sets[3].min_fill_clk_khz = max_fclk_khz;
-	ranges.reader_wm_sets[3].max_fill_clk_khz = max_fclk_khz;
-	ranges.writer_wm_sets[3].wm_inst = WM_D;
-	ranges.writer_wm_sets[3].min_fill_clk_khz = socclk_khz;
-	ranges.writer_wm_sets[3].max_fill_clk_khz = overdrive;
-	ranges.writer_wm_sets[3].min_drain_clk_khz = max_fclk_khz;
-	ranges.writer_wm_sets[3].max_drain_clk_khz = max_fclk_khz;
+	ranges.writer_wm_sets[0].max_drain_clk_khz = overdrive;
 
 	if (dc->debug.pplib_wm_report_mode == WM_REPORT_OVERRIDE) {
 		ranges.reader_wm_sets[0].wm_inst = WM_A;
 		ranges.reader_wm_sets[0].min_drain_clk_khz = 300000;
-		ranges.reader_wm_sets[0].max_drain_clk_khz = 654000;
+		ranges.reader_wm_sets[0].max_drain_clk_khz = 5000000;
 		ranges.reader_wm_sets[0].min_fill_clk_khz = 800000;
-		ranges.reader_wm_sets[0].max_fill_clk_khz = 800000;
+		ranges.reader_wm_sets[0].max_fill_clk_khz = 5000000;
 		ranges.writer_wm_sets[0].wm_inst = WM_A;
 		ranges.writer_wm_sets[0].min_fill_clk_khz = 200000;
-		ranges.writer_wm_sets[0].max_fill_clk_khz = 757000;
+		ranges.writer_wm_sets[0].max_fill_clk_khz = 5000000;
 		ranges.writer_wm_sets[0].min_drain_clk_khz = 800000;
-		ranges.writer_wm_sets[0].max_drain_clk_khz = 800000;
-
-		ranges.reader_wm_sets[1].wm_inst = WM_B;
-		ranges.reader_wm_sets[1].min_drain_clk_khz = 300000;
-		ranges.reader_wm_sets[1].max_drain_clk_khz = 654000;
-		ranges.reader_wm_sets[1].min_fill_clk_khz = 933000;
-		ranges.reader_wm_sets[1].max_fill_clk_khz = 933000;
-		ranges.writer_wm_sets[1].wm_inst = WM_B;
-		ranges.writer_wm_sets[1].min_fill_clk_khz = 200000;
-		ranges.writer_wm_sets[1].max_fill_clk_khz = 757000;
-		ranges.writer_wm_sets[1].min_drain_clk_khz = 933000;
-		ranges.writer_wm_sets[1].max_drain_clk_khz = 933000;
-
-
-		ranges.reader_wm_sets[2].wm_inst = WM_C;
-		ranges.reader_wm_sets[2].min_drain_clk_khz = 300000;
-		ranges.reader_wm_sets[2].max_drain_clk_khz = 654000;
-		ranges.reader_wm_sets[2].min_fill_clk_khz = 1067000;
-		ranges.reader_wm_sets[2].max_fill_clk_khz = 1067000;
-		ranges.writer_wm_sets[2].wm_inst = WM_C;
-		ranges.writer_wm_sets[2].min_fill_clk_khz = 200000;
-		ranges.writer_wm_sets[2].max_fill_clk_khz = 757000;
-		ranges.writer_wm_sets[2].min_drain_clk_khz = 1067000;
-		ranges.writer_wm_sets[2].max_drain_clk_khz = 1067000;
-
-		ranges.reader_wm_sets[3].wm_inst = WM_D;
-		ranges.reader_wm_sets[3].min_drain_clk_khz = 300000;
-		ranges.reader_wm_sets[3].max_drain_clk_khz = 654000;
-		ranges.reader_wm_sets[3].min_fill_clk_khz = 1200000;
-		ranges.reader_wm_sets[3].max_fill_clk_khz = 1200000;
-		ranges.writer_wm_sets[3].wm_inst = WM_D;
-		ranges.writer_wm_sets[3].min_fill_clk_khz = 200000;
-		ranges.writer_wm_sets[3].max_fill_clk_khz = 757000;
-		ranges.writer_wm_sets[3].min_drain_clk_khz = 1200000;
-		ranges.writer_wm_sets[3].max_drain_clk_khz = 1200000;
+		ranges.writer_wm_sets[0].max_drain_clk_khz = 5000000;
 	}
 
+	ranges.reader_wm_sets[1] = ranges.writer_wm_sets[0];
+	ranges.reader_wm_sets[1].wm_inst = WM_B;
+
+	ranges.reader_wm_sets[2] = ranges.writer_wm_sets[0];
+	ranges.reader_wm_sets[2].wm_inst = WM_C;
+
+	ranges.reader_wm_sets[3] = ranges.writer_wm_sets[0];
+	ranges.reader_wm_sets[3].wm_inst = WM_D;
+
 	/* Notify PP Lib/SMU which Watermarks to use for which clock ranges */
 	pp->set_wm_ranges(&pp->pp_smu, &ranges);
 }
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c b/drivers/gpu/drm/amd/display/dc/core/dc.c
index 53ce7fa864b4..6ae050dc3220 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc.c
@@ -52,6 +52,8 @@
 #include "dm_helpers.h"
 #include "mem_input.h"
 #include "hubp.h"
+
+#include "dc_link_dp.h"
 #define DC_LOGGER \
 	dc->ctx->logger
 
@@ -384,6 +386,80 @@ void dc_stream_set_static_screen_events(struct dc *dc,
 	dc->hwss.set_static_screen_control(pipes_affected, num_pipes_affected, events);
 }
 
+void dc_link_set_drive_settings(struct dc *dc,
+				struct link_training_settings *lt_settings,
+				const struct dc_link *link)
+{
+
+	int i;
+
+	for (i = 0; i < dc->link_count; i++) {
+		if (dc->links[i] == link)
+			break;
+	}
+
+	if (i >= dc->link_count)
+		ASSERT_CRITICAL(false);
+
+	dc_link_dp_set_drive_settings(dc->links[i], lt_settings);
+}
+
+void dc_link_perform_link_training(struct dc *dc,
+				   struct dc_link_settings *link_setting,
+				   bool skip_video_pattern)
+{
+	int i;
+
+	for (i = 0; i < dc->link_count; i++)
+		dc_link_dp_perform_link_training(
+			dc->links[i],
+			link_setting,
+			skip_video_pattern);
+}
+
+void dc_link_set_preferred_link_settings(struct dc *dc,
+					 struct dc_link_settings *link_setting,
+					 struct dc_link *link)
+{
+	struct dc_link_settings store_settings = *link_setting;
+	struct dc_stream_state *link_stream =
+		link->dc->current_state->res_ctx.pipe_ctx[0].stream;
+
+	link->preferred_link_setting = store_settings;
+	if (link_stream)
+		decide_link_settings(link_stream, &store_settings);
+
+	if ((store_settings.lane_count != LANE_COUNT_UNKNOWN) &&
+		(store_settings.link_rate != LINK_RATE_UNKNOWN))
+		dp_retrain_link_dp_test(link, &store_settings, false);
+}
+
+void dc_link_enable_hpd(const struct dc_link *link)
+{
+	dc_link_dp_enable_hpd(link);
+}
+
+void dc_link_disable_hpd(const struct dc_link *link)
+{
+	dc_link_dp_disable_hpd(link);
+}
+
+
+void dc_link_set_test_pattern(struct dc_link *link,
+			      enum dp_test_pattern test_pattern,
+			      const struct link_training_settings *p_link_settings,
+			      const unsigned char *p_custom_pattern,
+			      unsigned int cust_pattern_size)
+{
+	if (link != NULL)
+		dc_link_dp_set_test_pattern(
+			link,
+			test_pattern,
+			p_link_settings,
+			p_custom_pattern,
+			cust_pattern_size);
+}
+
 static void destruct(struct dc *dc)
 {
 	dc_release_state(dc->current_state);
@@ -402,9 +478,6 @@ static void destruct(struct dc *dc)
 	if (dc->ctx->created_bios)
 		dal_bios_parser_destroy(&dc->ctx->dc_bios);
 
-	if (dc->ctx->logger)
-		dal_logger_destroy(&dc->ctx->logger);
-
 	kfree(dc->ctx);
 	dc->ctx = NULL;
 
@@ -427,7 +500,6 @@ static void destruct(struct dc *dc)
 static bool construct(struct dc *dc,
 		const struct dc_init_data *init_params)
 {
-	struct dal_logger *logger;
 	struct dc_context *dc_ctx;
 	struct bw_calcs_dceip *dc_dceip;
 	struct bw_calcs_vbios *dc_vbios;
@@ -492,14 +564,7 @@ static bool construct(struct dc *dc,
 	}
 
 	/* Create logger */
-	logger = dal_logger_create(dc_ctx, init_params->log_mask);
 
-	if (!logger) {
-		/* can *not* call logger. call base driver 'print error' */
-		dm_error("%s: failed to create Logger!\n", __func__);
-		goto fail;
-	}
-	dc_ctx->logger = logger;
 	dc_ctx->dce_environment = init_params->dce_environment;
 
 	dc_version = resource_parse_asic_id(init_params->asic_id);
@@ -918,9 +983,7 @@ bool dc_commit_state(struct dc *dc, struct dc_state *context)
 	for (i = 0; i < context->stream_count; i++) {
 		struct dc_stream_state *stream = context->streams[i];
 
-		dc_stream_log(stream,
-				dc->ctx->logger,
-				LOG_DC);
+		dc_stream_log(dc, stream);
 	}
 
 	result = dc_commit_state_no_check(dc, context);
@@ -944,12 +1007,7 @@ bool dc_post_update_surfaces_to_stream(struct dc *dc)
 
 	dc->optimized_required = false;
 
-	/* 3rd param should be true, temp w/a for RV*/
-#if defined(CONFIG_DRM_AMD_DC_DCN1_0)
-	dc->hwss.set_bandwidth(dc, context, dc->ctx->dce_version < DCN_VERSION_1_0);
-#else
 	dc->hwss.set_bandwidth(dc, context, true);
-#endif
 	return true;
 }
 
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_debug.c b/drivers/gpu/drm/amd/display/dc/core/dc_debug.c
index 267c76766dea..e1ebdf7b5eaf 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_debug.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_debug.c
@@ -352,19 +352,19 @@ void context_clock_trace(
 	DC_LOGGER_INIT(dc->ctx->logger);
 	CLOCK_TRACE("Current: dispclk_khz:%d  max_dppclk_khz:%d  dcfclk_khz:%d\n"
 			"dcfclk_deep_sleep_khz:%d  fclk_khz:%d  socclk_khz:%d\n",
-			context->bw.dcn.calc_clk.dispclk_khz,
-			context->bw.dcn.calc_clk.dppclk_khz,
-			context->bw.dcn.calc_clk.dcfclk_khz,
-			context->bw.dcn.calc_clk.dcfclk_deep_sleep_khz,
-			context->bw.dcn.calc_clk.fclk_khz,
-			context->bw.dcn.calc_clk.socclk_khz);
+			context->bw.dcn.clk.dispclk_khz,
+			context->bw.dcn.clk.dppclk_khz,
+			context->bw.dcn.clk.dcfclk_khz,
+			context->bw.dcn.clk.dcfclk_deep_sleep_khz,
+			context->bw.dcn.clk.fclk_khz,
+			context->bw.dcn.clk.socclk_khz);
 	CLOCK_TRACE("Calculated: dispclk_khz:%d  max_dppclk_khz:%d  dcfclk_khz:%d\n"
 			"dcfclk_deep_sleep_khz:%d  fclk_khz:%d  socclk_khz:%d\n",
-			context->bw.dcn.calc_clk.dispclk_khz,
-			context->bw.dcn.calc_clk.dppclk_khz,
-			context->bw.dcn.calc_clk.dcfclk_khz,
-			context->bw.dcn.calc_clk.dcfclk_deep_sleep_khz,
-			context->bw.dcn.calc_clk.fclk_khz,
-			context->bw.dcn.calc_clk.socclk_khz);
+			context->bw.dcn.clk.dispclk_khz,
+			context->bw.dcn.clk.dppclk_khz,
+			context->bw.dcn.clk.dcfclk_khz,
+			context->bw.dcn.clk.dcfclk_deep_sleep_khz,
+			context->bw.dcn.clk.fclk_khz,
+			context->bw.dcn.clk.socclk_khz);
 #endif
 }
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link.c b/drivers/gpu/drm/amd/display/dc/core/dc_link.c
index 08b7ee526f0f..567867915d32 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_link.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_link.c
@@ -33,6 +33,7 @@
 #include "dc_link_dp.h"
 #include "dc_link_ddc.h"
 #include "link_hwss.h"
+#include "opp.h"
 
 #include "link_encoder.h"
 #include "hw_sequencer.h"
@@ -59,7 +60,14 @@
 
 enum {
 	LINK_RATE_REF_FREQ_IN_MHZ = 27,
-	PEAK_FACTOR_X1000 = 1006
+	PEAK_FACTOR_X1000 = 1006,
+	/*
+	* Some receivers fail to train on first try and are good
+	* on subsequent tries. 2 retries should be plenty. If we
+	* don't have a successful training then we don't expect to
+	* ever get one.
+	*/
+	LINK_TRAINING_MAX_VERIFY_RETRY = 2
 };
 
 /*******************************************************************************
@@ -312,7 +320,7 @@ static enum signal_type get_basic_signal_type(
  * @brief
  * Check whether there is a dongle on DP connector
  */
-static bool is_dp_sink_present(struct dc_link *link)
+bool dc_link_is_dp_sink_present(struct dc_link *link)
 {
 	enum gpio_result gpio_result;
 	uint32_t clock_pin = 0;
@@ -405,7 +413,7 @@ static enum signal_type link_detect_sink(
 			 * we assume signal is DVI; it could be corrected
 			 * to HDMI after dongle detection
 			 */
-			if (!is_dp_sink_present(link))
+			if (!dm_helpers_is_dp_sink_present(link))
 				result = SIGNAL_TYPE_DVI_SINGLE_LINK;
 		}
 	}
@@ -497,6 +505,10 @@ static bool detect_dp(
 			sink_caps->signal = SIGNAL_TYPE_DISPLAY_PORT_MST;
 			link->type = dc_connection_mst_branch;
 
+			dal_ddc_service_set_transaction_type(
+							link->ddc,
+							sink_caps->transaction_type);
+
 			/*
 			 * This call will initiate MST topology discovery. Which
 			 * will detect MST ports and add new DRM connector DRM
@@ -524,6 +536,10 @@ static bool detect_dp(
 			if (reason == DETECT_REASON_BOOT)
 				boot = true;
 
+			dm_helpers_dp_update_branch_info(
+				link->ctx,
+				link);
+
 			if (!dm_helpers_dp_mst_start_top_mgr(
 				link->ctx,
 				link, boot)) {
@@ -728,6 +744,18 @@ bool dc_link_detect(struct dc_link *link, enum dc_detect_reason reason)
 			break;
 		case EDID_NO_RESPONSE:
 			DC_LOG_ERROR("No EDID read.\n");
+
+			/*
+			 * Abort detection for non-DP connectors if we have
+			 * no EDID
+			 *
+			 * DP needs to report as connected if HDP is high
+			 * even if we have no EDID in order to go to
+			 * fail-safe mode
+			 */
+			if (dc_is_hdmi_signal(link->connector_signal) ||
+			    dc_is_dvi_signal(link->connector_signal))
+				return false;
 		default:
 			break;
 		}
@@ -736,30 +764,41 @@ bool dc_link_detect(struct dc_link *link, enum dc_detect_reason reason)
 		if ((prev_sink != NULL) && ((edid_status == EDID_THE_SAME) || (edid_status == EDID_OK)))
 			same_edid = is_same_edid(&prev_sink->dc_edid, &sink->dc_edid);
 
-		// If both edid and dpcd are the same, then discard new sink and revert back to original sink
-		if ((same_edid) && (same_dpcd)) {
-			link_disconnect_remap(prev_sink, link);
-			sink = prev_sink;
-			prev_sink = NULL;
-		} else {
-			if (link->connector_signal == SIGNAL_TYPE_DISPLAY_PORT &&
-					sink_caps.transaction_type ==
-						DDC_TRANSACTION_TYPE_I2C_OVER_AUX) {
-				/*
-				 * TODO debug why Dell 2413 doesn't like
-				 *  two link trainings
-				 */
+		if (link->connector_signal == SIGNAL_TYPE_DISPLAY_PORT &&
+			sink_caps.transaction_type == DDC_TRANSACTION_TYPE_I2C_OVER_AUX &&
+			reason != DETECT_REASON_HPDRX) {
+			/*
+			 * TODO debug why Dell 2413 doesn't like
+			 *  two link trainings
+			 */
+
+			/* deal with non-mst cases */
+			for (i = 0; i < LINK_TRAINING_MAX_VERIFY_RETRY; i++) {
+				int fail_count = 0;
 
-				/* deal with non-mst cases */
-				dp_hbr_verify_link_cap(link, &link->reported_link_cap);
+				dp_verify_link_cap(link,
+						  &link->reported_link_cap,
+						  &fail_count);
+
+				if (fail_count == 0)
+					break;
 			}
 
-			/* HDMI-DVI Dongle */
-			if (sink->sink_signal == SIGNAL_TYPE_HDMI_TYPE_A &&
-					!sink->edid_caps.edid_hdmi)
-				sink->sink_signal = SIGNAL_TYPE_DVI_SINGLE_LINK;
+		} else {
+			// If edid is the same, then discard new sink and revert back to original sink
+			if (same_edid) {
+				link_disconnect_remap(prev_sink, link);
+				sink = prev_sink;
+				prev_sink = NULL;
+
+			}
 		}
 
+		/* HDMI-DVI Dongle */
+		if (sink->sink_signal == SIGNAL_TYPE_HDMI_TYPE_A &&
+				!sink->edid_caps.edid_hdmi)
+			sink->sink_signal = SIGNAL_TYPE_DVI_SINGLE_LINK;
+
 		/* Connectivity log: detection */
 		for (i = 0; i < sink->dc_edid.length / EDID_BLOCK_SIZE; i++) {
 			CONN_DATA_DETECT(link,
@@ -1007,6 +1046,9 @@ static bool construct(
 		goto create_fail;
 	}
 
+	if (link->dc->res_pool->funcs->link_init)
+		link->dc->res_pool->funcs->link_init(link);
+
 	hpd_gpio = get_hpd_gpio(link->ctx->dc_bios, link->link_id, link->ctx->gpio_service);
 
 	if (hpd_gpio != NULL)
@@ -1284,29 +1326,15 @@ static enum dc_status enable_link_dp(
 		max_link_rate = LINK_RATE_HIGH3;
 
 	if (link_settings.link_rate == max_link_rate) {
-		if (state->dis_clk->funcs->set_min_clocks_state) {
-			if (state->dis_clk->cur_min_clks_state < DM_PP_CLOCKS_STATE_NOMINAL)
-				state->dis_clk->funcs->set_min_clocks_state(
-					state->dis_clk, DM_PP_CLOCKS_STATE_NOMINAL);
-		} else {
-			uint32_t dp_phyclk_in_khz;
-			const struct clocks_value clocks_value =
-					state->dis_clk->cur_clocks_value;
-
-			/* 27mhz = 27000000hz= 27000khz */
-			dp_phyclk_in_khz = link_settings.link_rate * 27000;
-
-			if (((clocks_value.max_non_dp_phyclk_in_khz != 0) &&
-				(dp_phyclk_in_khz > clocks_value.max_non_dp_phyclk_in_khz)) ||
-				(dp_phyclk_in_khz > clocks_value.max_dp_phyclk_in_khz)) {
-				state->dis_clk->funcs->apply_clock_voltage_request(
-						state->dis_clk,
-						DM_PP_CLOCK_TYPE_DISPLAYPHYCLK,
-						dp_phyclk_in_khz,
-						false,
-						true);
-			}
-		}
+		struct dc_clocks clocks = state->bw.dcn.clk;
+
+		/* dce/dcn compat, do not update dispclk */
+		clocks.dispclk_khz = 0;
+		/* 27mhz = 27000000hz= 27000khz */
+		clocks.phyclk_khz = link_settings.link_rate * 27000;
+
+		state->dis_clk->funcs->update_clocks(
+				state->dis_clk, &clocks, false);
 	}
 
 	dp_enable_link_phy(
@@ -1784,6 +1812,8 @@ static void enable_link_hdmi(struct pipe_ctx *pipe_ctx)
 	bool is_vga_mode = (stream->timing.h_addressable == 640)
 			&& (stream->timing.v_addressable == 480);
 
+	if (stream->phy_pix_clk == 0)
+		stream->phy_pix_clk = stream->timing.pix_clk_khz;
 	if (stream->phy_pix_clk > 340000)
 		is_over_340mhz = true;
 
@@ -2001,6 +2031,15 @@ enum dc_status dc_link_validate_mode_timing(
 	return DC_OK;
 }
 
+int dc_link_get_backlight_level(const struct dc_link *link)
+{
+	struct abm *abm = link->ctx->dc->res_pool->abm;
+
+	if (abm == NULL || abm->funcs->get_current_backlight_8_bit == NULL)
+		return DC_ERROR_UNEXPECTED;
+
+	return (int) abm->funcs->get_current_backlight_8_bit(abm);
+}
 
 bool dc_link_set_backlight_level(const struct dc_link *link, uint32_t level,
 		uint32_t frame_ramp, const struct dc_stream_state *stream)
@@ -2396,9 +2435,10 @@ void core_link_enable_stream(
 	core_dc->hwss.enable_audio_stream(pipe_ctx);
 
 	/* turn off otg test pattern if enable */
-	pipe_ctx->stream_res.tg->funcs->set_test_pattern(pipe_ctx->stream_res.tg,
-			CONTROLLER_DP_TEST_PATTERN_VIDEOMODE,
-			COLOR_DEPTH_UNDEFINED);
+	if (pipe_ctx->stream_res.tg->funcs->set_test_pattern)
+		pipe_ctx->stream_res.tg->funcs->set_test_pattern(pipe_ctx->stream_res.tg,
+				CONTROLLER_DP_TEST_PATTERN_VIDEOMODE,
+				COLOR_DEPTH_UNDEFINED);
 
 	core_dc->hwss.enable_stream(pipe_ctx);
 
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link_ddc.c b/drivers/gpu/drm/amd/display/dc/core/dc_link_ddc.c
index ae48d603ebd6..8def0d9fa0ff 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_link_ddc.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_link_ddc.c
@@ -33,6 +33,7 @@
 #include "include/vector.h"
 #include "core_types.h"
 #include "dc_link_ddc.h"
+#include "aux_engine.h"
 
 #define AUX_POWER_UP_WA_DELAY 500
 #define I2C_OVER_AUX_DEFER_WA_DELAY 70
@@ -629,83 +630,61 @@ bool dal_ddc_service_query_ddc_data(
 	return ret;
 }
 
-enum ddc_result dal_ddc_service_read_dpcd_data(
-	struct ddc_service *ddc,
-	bool i2c,
-	enum i2c_mot_mode mot,
-	uint32_t address,
-	uint8_t *data,
-	uint32_t len,
-	uint32_t *read)
+int dc_link_aux_transfer(struct ddc_service *ddc,
+			     unsigned int address,
+			     uint8_t *reply,
+			     void *buffer,
+			     unsigned int size,
+			     enum aux_transaction_type type,
+			     enum i2caux_transaction_action action)
 {
-	struct aux_payload read_payload = {
-		.i2c_over_aux = i2c,
-		.write = false,
-		.address = address,
-		.length = len,
-		.data = data,
-	};
-	struct aux_command command = {
-		.payloads = &read_payload,
-		.number_of_payloads = 1,
-		.defer_delay = 0,
-		.max_defer_write_retry = 0,
-		.mot = mot
-	};
-
-	*read = 0;
-
-	if (len > DEFAULT_AUX_MAX_DATA_SIZE) {
-		BREAK_TO_DEBUGGER();
-		return DDC_RESULT_FAILED_INVALID_OPERATION;
-	}
+	struct ddc *ddc_pin = ddc->ddc_pin;
+	struct aux_engine *aux_engine;
+	enum aux_channel_operation_result operation_result;
+	struct aux_request_transaction_data aux_req;
+	struct aux_reply_transaction_data aux_rep;
+	uint8_t returned_bytes = 0;
+	int res = -1;
+	uint32_t status;
 
-	if (dal_i2caux_submit_aux_command(
-		ddc->ctx->i2caux,
-		ddc->ddc_pin,
-		&command)) {
-		*read = command.payloads->length;
-		return DDC_RESULT_SUCESSFULL;
-	}
+	memset(&aux_req, 0, sizeof(aux_req));
+	memset(&aux_rep, 0, sizeof(aux_rep));
 
-	return DDC_RESULT_FAILED_OPERATION;
-}
+	aux_engine = ddc->ctx->dc->res_pool->engines[ddc_pin->pin_data->en];
+	aux_engine->funcs->acquire(aux_engine, ddc_pin);
 
-enum ddc_result dal_ddc_service_write_dpcd_data(
-	struct ddc_service *ddc,
-	bool i2c,
-	enum i2c_mot_mode mot,
-	uint32_t address,
-	const uint8_t *data,
-	uint32_t len)
-{
-	struct aux_payload write_payload = {
-		.i2c_over_aux = i2c,
-		.write = true,
-		.address = address,
-		.length = len,
-		.data = (uint8_t *)data,
-	};
-	struct aux_command command = {
-		.payloads = &write_payload,
-		.number_of_payloads = 1,
-		.defer_delay = 0,
-		.max_defer_write_retry = 0,
-		.mot = mot
-	};
-
-	if (len > DEFAULT_AUX_MAX_DATA_SIZE) {
-		BREAK_TO_DEBUGGER();
-		return DDC_RESULT_FAILED_INVALID_OPERATION;
-	}
+	aux_req.type = type;
+	aux_req.action = action;
+
+	aux_req.address = address;
+	aux_req.delay = 0;
+	aux_req.length = size;
+	aux_req.data = buffer;
 
-	if (dal_i2caux_submit_aux_command(
-		ddc->ctx->i2caux,
-		ddc->ddc_pin,
-		&command))
-		return DDC_RESULT_SUCESSFULL;
+	aux_engine->funcs->submit_channel_request(aux_engine, &aux_req);
+	operation_result = aux_engine->funcs->get_channel_status(aux_engine, &returned_bytes);
 
-	return DDC_RESULT_FAILED_OPERATION;
+	switch (operation_result) {
+	case AUX_CHANNEL_OPERATION_SUCCEEDED:
+		res = returned_bytes;
+
+		if (res <= size && res >= 0)
+			res = aux_engine->funcs->read_channel_reply(aux_engine, size,
+								buffer, reply,
+								&status);
+
+		break;
+	case AUX_CHANNEL_OPERATION_FAILED_HPD_DISCON:
+		res = 0;
+		break;
+	case AUX_CHANNEL_OPERATION_FAILED_REASON_UNKNOWN:
+	case AUX_CHANNEL_OPERATION_FAILED_INVALID_REPLY:
+	case AUX_CHANNEL_OPERATION_FAILED_TIMEOUT:
+		res = -1;
+		break;
+	}
+	aux_engine->funcs->release_engine(aux_engine);
+	return res;
 }
 
 /*test only function*/
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c b/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c
index 509f265663d2..a7553b6d59c2 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c
@@ -3,6 +3,7 @@
 #include "dc.h"
 #include "dc_link_dp.h"
 #include "dm_helpers.h"
+#include "opp.h"
 
 #include "inc/core_types.h"
 #include "link_hwss.h"
@@ -38,7 +39,7 @@ static bool decide_fallback_link_setting(
 		struct dc_link_settings initial_link_settings,
 		struct dc_link_settings *current_link_setting,
 		enum link_training_result training_result);
-static struct dc_link_settings get_common_supported_link_settings (
+static struct dc_link_settings get_common_supported_link_settings(
 		struct dc_link_settings link_setting_a,
 		struct dc_link_settings link_setting_b);
 
@@ -93,8 +94,8 @@ static void dpcd_set_link_settings(
 	uint8_t rate = (uint8_t)
 	(lt_settings->link_settings.link_rate);
 
-	union down_spread_ctrl downspread = {{0}};
-	union lane_count_set lane_count_set = {{0}};
+	union down_spread_ctrl downspread = { {0} };
+	union lane_count_set lane_count_set = { {0} };
 	uint8_t link_set_buffer[2];
 
 	downspread.raw = (uint8_t)
@@ -164,11 +165,11 @@ static void dpcd_set_lt_pattern_and_lane_settings(
 	const struct link_training_settings *lt_settings,
 	enum hw_dp_training_pattern pattern)
 {
-	union dpcd_training_lane dpcd_lane[LANE_COUNT_DP_MAX] = {{{0}}};
+	union dpcd_training_lane dpcd_lane[LANE_COUNT_DP_MAX] = { { {0} } };
 	const uint32_t dpcd_base_lt_offset =
 	DP_TRAINING_PATTERN_SET;
 	uint8_t dpcd_lt_buffer[5] = {0};
-	union dpcd_training_pattern dpcd_pattern = {{0}};
+	union dpcd_training_pattern dpcd_pattern = { {0} };
 	uint32_t lane;
 	uint32_t size_in_bytes;
 	bool edp_workaround = false; /* TODO link_prop.INTERNAL */
@@ -232,7 +233,7 @@ static void dpcd_set_lt_pattern_and_lane_settings(
 			link,
 			DP_TRAINING_PATTERN_SET,
 			&dpcd_pattern.raw,
-			sizeof(dpcd_pattern.raw) );
+			sizeof(dpcd_pattern.raw));
 
 		core_link_write_dpcd(
 			link,
@@ -246,7 +247,7 @@ static void dpcd_set_lt_pattern_and_lane_settings(
 				link,
 				dpcd_base_lt_offset,
 				dpcd_lt_buffer,
-				size_in_bytes + sizeof(dpcd_pattern.raw) );
+				size_in_bytes + sizeof(dpcd_pattern.raw));
 
 	link->cur_lane_setting = lt_settings->lane_settings[0];
 }
@@ -428,8 +429,8 @@ static void get_lane_status_and_drive_settings(
 	struct link_training_settings *req_settings)
 {
 	uint8_t dpcd_buf[6] = {0};
-	union lane_adjust dpcd_lane_adjust[LANE_COUNT_DP_MAX] = {{{0}}};
-	struct link_training_settings request_settings = {{0}};
+	union lane_adjust dpcd_lane_adjust[LANE_COUNT_DP_MAX] = { { {0} } };
+	struct link_training_settings request_settings = { {0} };
 	uint32_t lane;
 
 	memset(req_settings, '\0', sizeof(struct link_training_settings));
@@ -651,7 +652,7 @@ static bool perform_post_lt_adj_req_sequence(
 
 			if (req_drv_setting_changed) {
 				update_drive_settings(
-					lt_settings,req_settings);
+					lt_settings, req_settings);
 
 				dc_link_dp_set_drive_settings(link,
 						lt_settings);
@@ -724,8 +725,8 @@ static enum link_training_result perform_channel_equalization_sequence(
 	enum hw_dp_training_pattern hw_tr_pattern;
 	uint32_t retries_ch_eq;
 	enum dc_lane_count lane_count = lt_settings->link_settings.lane_count;
-	union lane_align_status_updated dpcd_lane_status_updated = {{0}};
-	union lane_status dpcd_lane_status[LANE_COUNT_DP_MAX] = {{{0}}};
+	union lane_align_status_updated dpcd_lane_status_updated = { {0} };
+	union lane_status dpcd_lane_status[LANE_COUNT_DP_MAX] = { { {0} } };
 
 	hw_tr_pattern = get_supported_tp(link);
 
@@ -952,7 +953,10 @@ enum link_training_result dc_link_dp_perform_link_training(
 	 * LINK_SPREAD_05_DOWNSPREAD_30KHZ :
 	 * LINK_SPREAD_DISABLED;
 	 */
-	lt_settings.link_settings.link_spread = LINK_SPREAD_05_DOWNSPREAD_30KHZ;
+	if (link->dp_ss_off)
+		lt_settings.link_settings.link_spread = LINK_SPREAD_DISABLED;
+	else
+		lt_settings.link_settings.link_spread = LINK_SPREAD_05_DOWNSPREAD_30KHZ;
 
 	/* 1. set link rate, lane count and spread*/
 	dpcd_set_link_settings(link, &lt_settings);
@@ -1027,6 +1031,9 @@ enum link_training_result dc_link_dp_perform_link_training(
 			lt_settings.lane_settings[0].VOLTAGE_SWING,
 			lt_settings.lane_settings[0].PRE_EMPHASIS);
 
+	if (status != LINK_TRAINING_SUCCESS)
+		link->ctx->dc->debug_data.ltFailCount++;
+
 	return status;
 }
 
@@ -1082,9 +1089,10 @@ static struct dc_link_settings get_max_link_cap(struct dc_link *link)
 	return max_link_cap;
 }
 
-bool dp_hbr_verify_link_cap(
+bool dp_verify_link_cap(
 	struct dc_link *link,
-	struct dc_link_settings *known_limit_link_setting)
+	struct dc_link_settings *known_limit_link_setting,
+	int *fail_count)
 {
 	struct dc_link_settings max_link_cap = {0};
 	struct dc_link_settings cur_link_setting = {0};
@@ -1097,6 +1105,11 @@ bool dp_hbr_verify_link_cap(
 	enum clock_source_id dp_cs_id = CLOCK_SOURCE_ID_EXTERNAL;
 	enum link_training_result status;
 
+	if (link->dc->debug.skip_detection_link_training) {
+		link->verified_link_cap = *known_limit_link_setting;
+		return true;
+	}
+
 	success = false;
 	skip_link_training = false;
 
@@ -1151,6 +1164,8 @@ bool dp_hbr_verify_link_cap(
 							skip_video_pattern);
 			if (status == LINK_TRAINING_SUCCESS)
 				success = true;
+			else
+				(*fail_count)++;
 		}
 
 		if (success)
@@ -1182,7 +1197,7 @@ bool dp_hbr_verify_link_cap(
 	return success;
 }
 
-static struct dc_link_settings get_common_supported_link_settings (
+static struct dc_link_settings get_common_supported_link_settings(
 		struct dc_link_settings link_setting_a,
 		struct dc_link_settings link_setting_b)
 {
@@ -1428,6 +1443,7 @@ static uint32_t bandwidth_in_kbps_from_link_settings(
 
 	uint32_t lane_count  = link_setting->lane_count;
 	uint32_t kbps = link_rate_in_kbps;
+
 	kbps *= lane_count;
 	kbps *= 8;   /* 8 bits per byte*/
 
@@ -1445,9 +1461,9 @@ bool dp_validate_mode_timing(
 	const struct dc_link_settings *link_setting;
 
 	/*always DP fail safe mode*/
-	if (timing->pix_clk_khz == (uint32_t)25175 &&
-		timing->h_addressable == (uint32_t)640 &&
-		timing->v_addressable == (uint32_t)480)
+	if (timing->pix_clk_khz == (uint32_t) 25175 &&
+		timing->h_addressable == (uint32_t) 640 &&
+		timing->v_addressable == (uint32_t) 480)
 		return true;
 
 	/* We always use verified link settings */
@@ -1771,12 +1787,10 @@ static void dp_test_send_link_training(struct dc_link *link)
 	dp_retrain_link_dp_test(link, &link_settings, false);
 }
 
-/* TODO hbr2 compliance eye output is unstable
+/* TODO Raven hbr2 compliance eye output is unstable
  * (toggling on and off) with debugger break
  * This caueses intermittent PHY automation failure
  * Need to look into the root cause */
-static uint8_t force_tps4_for_cp2520 = 1;
-
 static void dp_test_send_phy_test_pattern(struct dc_link *link)
 {
 	union phy_test_pattern dpcd_test_pattern;
@@ -1836,13 +1850,13 @@ static void dp_test_send_phy_test_pattern(struct dc_link *link)
 		break;
 	case PHY_TEST_PATTERN_CP2520_1:
 		/* CP2520 pattern is unstable, temporarily use TPS4 instead */
-		test_pattern = (force_tps4_for_cp2520 == 1) ?
+		test_pattern = (link->dc->caps.force_dp_tps4_for_cp2520 == 1) ?
 				DP_TEST_PATTERN_TRAINING_PATTERN4 :
 				DP_TEST_PATTERN_HBR2_COMPLIANCE_EYE;
 		break;
 	case PHY_TEST_PATTERN_CP2520_2:
 		/* CP2520 pattern is unstable, temporarily use TPS4 instead */
-		test_pattern = (force_tps4_for_cp2520 == 1) ?
+		test_pattern = (link->dc->caps.force_dp_tps4_for_cp2520 == 1) ?
 				DP_TEST_PATTERN_TRAINING_PATTERN4 :
 				DP_TEST_PATTERN_HBR2_COMPLIANCE_EYE;
 		break;
@@ -1995,12 +2009,16 @@ static void handle_automated_test(struct dc_link *link)
 			sizeof(test_response));
 }
 
-bool dc_link_handle_hpd_rx_irq(struct dc_link *link, union hpd_irq_data *out_hpd_irq_dpcd_data)
+bool dc_link_handle_hpd_rx_irq(struct dc_link *link, union hpd_irq_data *out_hpd_irq_dpcd_data, bool *out_link_loss)
 {
-	union hpd_irq_data hpd_irq_dpcd_data = {{{{0}}}};
+	union hpd_irq_data hpd_irq_dpcd_data = { { { {0} } } };
 	union device_service_irq device_service_clear = { { 0 } };
-	enum dc_status result = DDC_RESULT_UNKNOWN;
+	enum dc_status result;
+
 	bool status = false;
+
+	if (out_link_loss)
+		*out_link_loss = false;
 	/* For use cases related to down stream connection status change,
 	 * PSR and device auto test, refer to function handle_sst_hpd_irq
 	 * in DAL2.1*/
@@ -2075,6 +2093,8 @@ bool dc_link_handle_hpd_rx_irq(struct dc_link *link, union hpd_irq_data *out_hpd
 			true, LINK_TRAINING_ATTEMPTS);
 
 		status = false;
+		if (out_link_loss)
+			*out_link_loss = true;
 	}
 
 	if (link->type == dc_connection_active_dongle &&
@@ -2261,6 +2281,11 @@ static void get_active_converter_info(
 
 		link->dpcd_caps.branch_hw_revision =
 			dp_hw_fw_revision.ieee_hw_rev;
+
+		memmove(
+			link->dpcd_caps.branch_fw_revision,
+			dp_hw_fw_revision.ieee_fw_rev,
+			sizeof(dp_hw_fw_revision.ieee_fw_rev));
 	}
 }
 
@@ -2316,6 +2341,7 @@ static bool retrieve_link_cap(struct dc_link *link)
 	enum dc_status status = DC_ERROR_UNEXPECTED;
 	uint32_t read_dpcd_retry_cnt = 3;
 	int i;
+	struct dp_sink_hw_fw_revision dp_hw_fw_revision;
 
 	memset(dpcd_data, '\0', sizeof(dpcd_data));
 	memset(&down_strm_port_count,
@@ -2407,6 +2433,25 @@ static bool retrieve_link_cap(struct dc_link *link)
 			(sink_id.ieee_oui[1] << 8) +
 			(sink_id.ieee_oui[2]);
 
+	memmove(
+		link->dpcd_caps.sink_dev_id_str,
+		sink_id.ieee_device_id,
+		sizeof(sink_id.ieee_device_id));
+
+	core_link_read_dpcd(
+		link,
+		DP_SINK_HW_REVISION_START,
+		(uint8_t *)&dp_hw_fw_revision,
+		sizeof(dp_hw_fw_revision));
+
+	link->dpcd_caps.sink_hw_revision =
+		dp_hw_fw_revision.ieee_hw_rev;
+
+	memmove(
+		link->dpcd_caps.sink_fw_revision,
+		dp_hw_fw_revision.ieee_fw_rev,
+		sizeof(dp_hw_fw_revision.ieee_fw_rev));
+
 	/* Connectivity log: detection */
 	CONN_DATA_DETECT(link, dpcd_data, sizeof(dpcd_data), "Rx Caps: ");
 
@@ -2511,8 +2556,8 @@ static void set_crtc_test_pattern(struct dc_link *link,
 		pipe_ctx->stream->bit_depth_params = params;
 		pipe_ctx->stream_res.opp->funcs->
 			opp_program_bit_depth_reduction(pipe_ctx->stream_res.opp, &params);
-
-		pipe_ctx->stream_res.tg->funcs->set_test_pattern(pipe_ctx->stream_res.tg,
+		if (pipe_ctx->stream_res.tg->funcs->set_test_pattern)
+			pipe_ctx->stream_res.tg->funcs->set_test_pattern(pipe_ctx->stream_res.tg,
 				controller_test_pattern, color_depth);
 	}
 	break;
@@ -2524,8 +2569,8 @@ static void set_crtc_test_pattern(struct dc_link *link,
 		pipe_ctx->stream->bit_depth_params = params;
 		pipe_ctx->stream_res.opp->funcs->
 			opp_program_bit_depth_reduction(pipe_ctx->stream_res.opp, &params);
-
-		pipe_ctx->stream_res.tg->funcs->set_test_pattern(pipe_ctx->stream_res.tg,
+		if (pipe_ctx->stream_res.tg->funcs->set_test_pattern)
+			pipe_ctx->stream_res.tg->funcs->set_test_pattern(pipe_ctx->stream_res.tg,
 				CONTROLLER_DP_TEST_PATTERN_VIDEOMODE,
 				color_depth);
 	}
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c
index fca22550417a..ea6beccfd89d 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c
@@ -268,24 +268,30 @@ bool resource_construct(
 
 	return true;
 }
+static int find_matching_clock_source(
+		const struct resource_pool *pool,
+		struct clock_source *clock_source)
+{
 
+	int i;
+
+	for (i = 0; i < pool->clk_src_count; i++) {
+		if (pool->clock_sources[i] == clock_source)
+			return i;
+	}
+	return -1;
+}
 
 void resource_unreference_clock_source(
 		struct resource_context *res_ctx,
 		const struct resource_pool *pool,
 		struct clock_source *clock_source)
 {
-	int i;
-
-	for (i = 0; i < pool->clk_src_count; i++) {
-		if (pool->clock_sources[i] != clock_source)
-			continue;
+	int i = find_matching_clock_source(pool, clock_source);
 
+	if (i > -1)
 		res_ctx->clock_source_ref_count[i]--;
 
-		break;
-	}
-
 	if (pool->dp_clock_source == clock_source)
 		res_ctx->dp_clock_source_ref_count--;
 }
@@ -295,19 +301,31 @@ void resource_reference_clock_source(
 		const struct resource_pool *pool,
 		struct clock_source *clock_source)
 {
-	int i;
-	for (i = 0; i < pool->clk_src_count; i++) {
-		if (pool->clock_sources[i] != clock_source)
-			continue;
+	int i = find_matching_clock_source(pool, clock_source);
 
+	if (i > -1)
 		res_ctx->clock_source_ref_count[i]++;
-		break;
-	}
 
 	if (pool->dp_clock_source == clock_source)
 		res_ctx->dp_clock_source_ref_count++;
 }
 
+int resource_get_clock_source_reference(
+		struct resource_context *res_ctx,
+		const struct resource_pool *pool,
+		struct clock_source *clock_source)
+{
+	int i = find_matching_clock_source(pool, clock_source);
+
+	if (i > -1)
+		return res_ctx->clock_source_ref_count[i];
+
+	if (pool->dp_clock_source == clock_source)
+		return res_ctx->dp_clock_source_ref_count;
+
+	return -1;
+}
+
 bool resource_are_streams_timing_synchronizable(
 	struct dc_stream_state *stream1,
 	struct dc_stream_state *stream2)
@@ -330,6 +348,9 @@ bool resource_are_streams_timing_synchronizable(
 				!= stream2->timing.pix_clk_khz)
 		return false;
 
+	if (stream1->clamping.c_depth != stream2->clamping.c_depth)
+		return false;
+
 	if (stream1->phy_pix_clk != stream2->phy_pix_clk
 			&& (!dc_is_dp_signal(stream1->signal)
 			|| !dc_is_dp_signal(stream2->signal)))
@@ -337,6 +358,20 @@ bool resource_are_streams_timing_synchronizable(
 
 	return true;
 }
+static bool is_dp_and_hdmi_sharable(
+		struct dc_stream_state *stream1,
+		struct dc_stream_state *stream2)
+{
+	if (stream1->ctx->dc->caps.disable_dp_clk_share)
+		return false;
+
+	if (stream1->clamping.c_depth != COLOR_DEPTH_888 ||
+	    stream2->clamping.c_depth != COLOR_DEPTH_888)
+	return false;
+
+	return true;
+
+}
 
 static bool is_sharable_clk_src(
 	const struct pipe_ctx *pipe_with_clk_src,
@@ -348,15 +383,18 @@ static bool is_sharable_clk_src(
 	if (pipe_with_clk_src->stream->signal == SIGNAL_TYPE_VIRTUAL)
 		return false;
 
-	if (dc_is_dp_signal(pipe_with_clk_src->stream->signal))
+	if (dc_is_dp_signal(pipe_with_clk_src->stream->signal) ||
+		(dc_is_dp_signal(pipe->stream->signal) &&
+		!is_dp_and_hdmi_sharable(pipe_with_clk_src->stream,
+				     pipe->stream)))
 		return false;
 
 	if (dc_is_hdmi_signal(pipe_with_clk_src->stream->signal)
-			&& dc_is_dvi_signal(pipe->stream->signal))
+			&& dc_is_dual_link_signal(pipe->stream->signal))
 		return false;
 
 	if (dc_is_hdmi_signal(pipe->stream->signal)
-			&& dc_is_dvi_signal(pipe_with_clk_src->stream->signal))
+			&& dc_is_dual_link_signal(pipe_with_clk_src->stream->signal))
 		return false;
 
 	if (!resource_are_streams_timing_synchronizable(
@@ -1705,8 +1743,8 @@ enum dc_status dc_add_stream_to_ctx(
 	struct dc_context *dc_ctx = dc->ctx;
 	enum dc_status res;
 
-	if (new_ctx->stream_count >= dc->res_pool->pipe_count) {
-		DC_ERROR("Max streams reached, can add stream %p !\n", stream);
+	if (new_ctx->stream_count >= dc->res_pool->timing_generator_count) {
+		DC_ERROR("Max streams reached, can't add stream %p !\n", stream);
 		return DC_ERROR_UNEXPECTED;
 	}
 
@@ -1948,7 +1986,7 @@ void dc_resource_state_construct(
 		const struct dc *dc,
 		struct dc_state *dst_ctx)
 {
-	dst_ctx->dis_clk = dc->res_pool->display_clock;
+	dst_ctx->dis_clk = dc->res_pool->dccg;
 }
 
 enum dc_status dc_validate_global_state(
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_stream.c b/drivers/gpu/drm/amd/display/dc/core/dc_stream.c
index 3732a1de9d6c..fdcc8ab19bf3 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_stream.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_stream.c
@@ -30,6 +30,8 @@
 #include "ipp.h"
 #include "timing_generator.h"
 
+#define DC_LOGGER dc->ctx->logger
+
 /*******************************************************************************
  * Private functions
  ******************************************************************************/
@@ -212,6 +214,8 @@ bool dc_stream_set_cursor_attributes(
 		}
 
 		core_dc->hwss.set_cursor_attribute(pipe_ctx);
+		if (core_dc->hwss.set_cursor_sdr_white_level)
+			core_dc->hwss.set_cursor_sdr_white_level(pipe_ctx);
 	}
 
 	if (pipe_to_program)
@@ -317,16 +321,10 @@ bool dc_stream_get_scanoutpos(const struct dc_stream_state *stream,
 	return ret;
 }
 
-
-void dc_stream_log(
-	const struct dc_stream_state *stream,
-	struct dal_logger *dm_logger,
-	enum dc_log_type log_type)
+void dc_stream_log(const struct dc *dc, const struct dc_stream_state *stream)
 {
-
-	dm_logger_write(dm_logger,
-			log_type,
-			"core_stream 0x%x: src: %d, %d, %d, %d; dst: %d, %d, %d, %d, colorSpace:%d\n",
+	DC_LOG_DC(
+			"core_stream 0x%p: src: %d, %d, %d, %d; dst: %d, %d, %d, %d, colorSpace:%d\n",
 			stream,
 			stream->src.x,
 			stream->src.y,
@@ -337,21 +335,18 @@ void dc_stream_log(
 			stream->dst.width,
 			stream->dst.height,
 			stream->output_color_space);
-	dm_logger_write(dm_logger,
-			log_type,
+	DC_LOG_DC(
 			"\tpix_clk_khz: %d, h_total: %d, v_total: %d, pixelencoder:%d, displaycolorDepth:%d\n",
 			stream->timing.pix_clk_khz,
 			stream->timing.h_total,
 			stream->timing.v_total,
 			stream->timing.pixel_encoding,
 			stream->timing.display_color_depth);
-	dm_logger_write(dm_logger,
-			log_type,
+	DC_LOG_DC(
 			"\tsink name: %s, serial: %d\n",
 			stream->sink->edid_caps.display_name,
 			stream->sink->edid_caps.serial_number);
-	dm_logger_write(dm_logger,
-			log_type,
+	DC_LOG_DC(
 			"\tlink: %d\n",
 			stream->sink->link->link_index);
 }
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_surface.c b/drivers/gpu/drm/amd/display/dc/core/dc_surface.c
index 815dfb50089b..8fb3aefd195c 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_surface.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_surface.c
@@ -192,7 +192,7 @@ void dc_transfer_func_release(struct dc_transfer_func *tf)
 	kref_put(&tf->refcount, dc_transfer_func_free);
 }
 
-struct dc_transfer_func *dc_create_transfer_func()
+struct dc_transfer_func *dc_create_transfer_func(void)
 {
 	struct dc_transfer_func *tf = kvzalloc(sizeof(*tf), GFP_KERNEL);
 
diff --git a/drivers/gpu/drm/amd/display/dc/dc.h b/drivers/gpu/drm/amd/display/dc/dc.h
index 7ebce7669eea..6c9990bef267 100644
--- a/drivers/gpu/drm/amd/display/dc/dc.h
+++ b/drivers/gpu/drm/amd/display/dc/dc.h
@@ -38,7 +38,7 @@
 #include "inc/compressor.h"
 #include "dml/display_mode_lib.h"
 
-#define DC_VER "3.1.47"
+#define DC_VER "3.1.59"
 
 #define MAX_SURFACES 3
 #define MAX_STREAMS 6
@@ -77,6 +77,9 @@ struct dc_caps {
 	bool is_apu;
 	bool dual_link_dvi;
 	bool post_blend_color_processing;
+	bool force_dp_tps4_for_cp2520;
+	bool disable_dp_clk_share;
+	bool psp_setup_panel_mode;
 };
 
 struct dc_dcc_surface_param {
@@ -169,6 +172,12 @@ struct dc_config {
 	bool disable_disp_pll_sharing;
 };
 
+enum visual_confirm {
+	VISUAL_CONFIRM_DISABLE = 0,
+	VISUAL_CONFIRM_SURFACE = 1,
+	VISUAL_CONFIRM_HDR = 2,
+};
+
 enum dcc_option {
 	DCC_ENABLE = 0,
 	DCC_DISABLE = 1,
@@ -186,6 +195,10 @@ enum wm_report_mode {
 	WM_REPORT_OVERRIDE = 1,
 };
 
+/*
+ * For any clocks that may differ per pipe
+ * only the max is stored in this structure
+ */
 struct dc_clocks {
 	int dispclk_khz;
 	int max_supported_dppclk_khz;
@@ -194,10 +207,11 @@ struct dc_clocks {
 	int socclk_khz;
 	int dcfclk_deep_sleep_khz;
 	int fclk_khz;
+	int phyclk_khz;
 };
 
-struct dc_debug {
-	bool surface_visual_confirm;
+struct dc_debug_options {
+	enum visual_confirm visual_confirm;
 	bool sanity_checks;
 	bool max_disp_clk;
 	bool surface_trace;
@@ -228,6 +242,7 @@ struct dc_debug {
 	int urgent_latency_ns;
 	int percent_of_ideal_drambw;
 	int dram_clock_change_latency_ns;
+	bool optimized_watermark;
 	int always_scale;
 	bool disable_pplib_clock_request;
 	bool disable_clock_gate;
@@ -243,8 +258,19 @@ struct dc_debug {
 	bool always_use_regamma;
 	bool p010_mpo_support;
 	bool recovery_enabled;
+	bool avoid_vbios_exec_table;
+	bool scl_reset_length10;
+	bool hdmi20_disable;
+	bool skip_detection_link_training;
+};
 
+struct dc_debug_data {
+	uint32_t ltFailCount;
+	uint32_t i2cErrorCount;
+	uint32_t auxErrorCount;
 };
+
+
 struct dc_state;
 struct resource_pool;
 struct dce_hwseq;
@@ -253,8 +279,7 @@ struct dc {
 	struct dc_caps caps;
 	struct dc_cap_funcs cap_funcs;
 	struct dc_config config;
-	struct dc_debug debug;
-
+	struct dc_debug_options debug;
 	struct dc_context *ctx;
 
 	uint8_t link_count;
@@ -290,6 +315,8 @@ struct dc {
 
 	/* FBC compressor */
 	struct compressor *fbc_compressor;
+
+	struct dc_debug_data debug_data;
 };
 
 enum frame_buffer_mode {
@@ -378,7 +405,8 @@ enum dc_transfer_func_predefined {
 	TRANSFER_FUNCTION_LINEAR,
 	TRANSFER_FUNCTION_UNITY,
 	TRANSFER_FUNCTION_HLG,
-	TRANSFER_FUNCTION_HLG12
+	TRANSFER_FUNCTION_HLG12,
+	TRANSFER_FUNCTION_GAMMA22
 };
 
 struct dc_transfer_func {
@@ -621,9 +649,14 @@ struct dpcd_caps {
 	struct dc_dongle_caps dongle_caps;
 
 	uint32_t sink_dev_id;
+	int8_t sink_dev_id_str[6];
+	int8_t sink_hw_revision;
+	int8_t sink_fw_revision[2];
+
 	uint32_t branch_dev_id;
 	int8_t branch_dev_name[6];
 	int8_t branch_hw_revision;
+	int8_t branch_fw_revision[2];
 
 	bool allow_invalid_MSA_timing_param;
 	bool panel_mode_edp;
diff --git a/drivers/gpu/drm/amd/display/dc/dc_bios_types.h b/drivers/gpu/drm/amd/display/dc/dc_bios_types.h
index d9b84ec7954c..90082bab71f0 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_bios_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_bios_types.h
@@ -198,6 +198,10 @@ struct dc_vbios_funcs {
 	void (*post_init)(struct dc_bios *bios);
 
 	void (*bios_parser_destroy)(struct dc_bios **dcb);
+
+	enum bp_result (*get_board_layout_info)(
+		struct dc_bios *dcb,
+		struct board_layout_info *board_layout_info);
 };
 
 struct bios_registers {
diff --git a/drivers/gpu/drm/amd/display/dc/dc_helper.c b/drivers/gpu/drm/amd/display/dc/dc_helper.c
index bd0fda0ceb91..e68077e65565 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_helper.c
+++ b/drivers/gpu/drm/amd/display/dc/dc_helper.c
@@ -255,3 +255,54 @@ uint32_t generic_reg_wait(const struct dc_context *ctx,
 
 	return reg_val;
 }
+
+void generic_write_indirect_reg(const struct dc_context *ctx,
+		uint32_t addr_index, uint32_t addr_data,
+		uint32_t index, uint32_t data)
+{
+	dm_write_reg(ctx, addr_index, index);
+	dm_write_reg(ctx, addr_data, data);
+}
+
+uint32_t generic_read_indirect_reg(const struct dc_context *ctx,
+		uint32_t addr_index, uint32_t addr_data,
+		uint32_t index)
+{
+	uint32_t value = 0;
+
+	dm_write_reg(ctx, addr_index, index);
+	value = dm_read_reg(ctx, addr_data);
+
+	return value;
+}
+
+
+uint32_t generic_indirect_reg_update_ex(const struct dc_context *ctx,
+		uint32_t addr_index, uint32_t addr_data,
+		uint32_t index, uint32_t reg_val, int n,
+		uint8_t shift1, uint32_t mask1, uint32_t field_value1,
+		...)
+{
+	uint32_t shift, mask, field_value;
+	int i = 1;
+
+	va_list ap;
+
+	va_start(ap, field_value1);
+
+	reg_val = set_reg_field_value_ex(reg_val, field_value1, mask1, shift1);
+
+	while (i < n) {
+		shift = va_arg(ap, uint32_t);
+		mask = va_arg(ap, uint32_t);
+		field_value = va_arg(ap, uint32_t);
+
+		reg_val = set_reg_field_value_ex(reg_val, field_value, mask, shift);
+		i++;
+	}
+
+	generic_write_indirect_reg(ctx, addr_index, addr_data, index, reg_val);
+	va_end(ap);
+
+	return reg_val;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/dc_hw_types.h b/drivers/gpu/drm/amd/display/dc/dc_hw_types.h
index d31023d57b58..b789cb2b354b 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_hw_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_hw_types.h
@@ -192,13 +192,14 @@ enum surface_pixel_format {
 	/*swaped & float*/
 	SURFACE_PIXEL_FORMAT_GRPH_ABGR16161616F,
 	/*grow graphics here if necessary */
-
+	SURFACE_PIXEL_FORMAT_VIDEO_AYCrCb8888,
 	SURFACE_PIXEL_FORMAT_VIDEO_BEGIN,
 	SURFACE_PIXEL_FORMAT_VIDEO_420_YCbCr =
 		SURFACE_PIXEL_FORMAT_VIDEO_BEGIN,
 	SURFACE_PIXEL_FORMAT_VIDEO_420_YCrCb,
 	SURFACE_PIXEL_FORMAT_VIDEO_420_10bpc_YCbCr,
 	SURFACE_PIXEL_FORMAT_VIDEO_420_10bpc_YCrCb,
+		SURFACE_PIXEL_FORMAT_SUBSAMPLE_END,
 	SURFACE_PIXEL_FORMAT_INVALID
 
 	/*grow 444 video here if necessary */
@@ -403,9 +404,11 @@ struct dc_cursor_position {
 struct dc_cursor_mi_param {
 	unsigned int pixel_clk_khz;
 	unsigned int ref_clk_khz;
-	unsigned int viewport_x_start;
-	unsigned int viewport_width;
+	struct rect viewport;
 	struct fixed31_32 h_scale_ratio;
+	struct fixed31_32 v_scale_ratio;
+	enum dc_rotation_angle rotation;
+	bool mirror;
 };
 
 /* IPP related types */
@@ -414,6 +417,7 @@ enum {
 	GAMMA_RGB_256_ENTRIES = 256,
 	GAMMA_RGB_FLOAT_1024_ENTRIES = 1024,
 	GAMMA_CS_TFM_1D_ENTRIES = 4096,
+	GAMMA_CUSTOM_ENTRIES = 4096,
 	GAMMA_MAX_ENTRIES = 4096
 };
 
@@ -421,6 +425,7 @@ enum dc_gamma_type {
 	GAMMA_RGB_256 = 1,
 	GAMMA_RGB_FLOAT_1024 = 2,
 	GAMMA_CS_TFM_1D = 3,
+	GAMMA_CUSTOM = 4,
 };
 
 struct dc_csc_transform {
@@ -489,6 +494,7 @@ struct dc_cursor_attributes {
 	uint32_t height;
 
 	enum dc_cursor_color_format color_format;
+	uint32_t sdr_white_level; // for boosting (SDR) cursor in HDR mode
 
 	/* In case we support HW Cursor rotation in the future */
 	enum dc_rotation_angle rotation_angle;
@@ -496,6 +502,11 @@ struct dc_cursor_attributes {
 	union dc_cursor_attribute_flags attribute_flags;
 };
 
+struct dpp_cursor_attributes {
+	int bias;
+	int scale;
+};
+
 /* OPP */
 
 enum dc_color_space {
diff --git a/drivers/gpu/drm/amd/display/dc/dc_link.h b/drivers/gpu/drm/amd/display/dc/dc_link.h
index 8a716baa1203..d43cefbc43d3 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_link.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_link.h
@@ -73,6 +73,7 @@ struct dc_link {
 	enum dc_irq_source irq_source_hpd;
 	enum dc_irq_source irq_source_hpd_rx;/* aka DP Short Pulse  */
 	bool is_hpd_filter_disabled;
+	bool dp_ss_off;
 
 	/* caps is the same as reported_link_cap. link_traing use
 	 * reported_link_cap. Will clean up.  TODO
@@ -141,6 +142,8 @@ static inline struct dc_link *dc_get_link_at_index(struct dc *dc, uint32_t link_
 bool dc_link_set_backlight_level(const struct dc_link *dc_link, uint32_t level,
 		uint32_t frame_ramp, const struct dc_stream_state *stream);
 
+int dc_link_get_backlight_level(const struct dc_link *dc_link);
+
 bool dc_link_set_abm_disable(const struct dc_link *dc_link);
 
 bool dc_link_set_psr_enable(const struct dc_link *dc_link, bool enable, bool wait);
@@ -172,7 +175,7 @@ bool dc_link_detect(struct dc_link *dc_link, enum dc_detect_reason reason);
  * false - no change in Downstream port status. No further action required
  * from DM. */
 bool dc_link_handle_hpd_rx_irq(struct dc_link *dc_link,
-		union hpd_irq_data *hpd_irq_dpcd_data);
+		union hpd_irq_data *hpd_irq_dpcd_data, bool *out_link_loss);
 
 struct dc_sink_init_data;
 
@@ -210,10 +213,29 @@ bool dc_link_dp_set_test_pattern(
 
 void dc_link_enable_hpd_filter(struct dc_link *link, bool enable);
 
+bool dc_link_is_dp_sink_present(struct dc_link *link);
+
 /*
  * DPCD access interfaces
  */
 
+void dc_link_set_drive_settings(struct dc *dc,
+				struct link_training_settings *lt_settings,
+				const struct dc_link *link);
+void dc_link_perform_link_training(struct dc *dc,
+				   struct dc_link_settings *link_setting,
+				   bool skip_video_pattern);
+void dc_link_set_preferred_link_settings(struct dc *dc,
+					 struct dc_link_settings *link_setting,
+					 struct dc_link *link);
+void dc_link_enable_hpd(const struct dc_link *link);
+void dc_link_disable_hpd(const struct dc_link *link);
+void dc_link_set_test_pattern(struct dc_link *link,
+			enum dp_test_pattern test_pattern,
+			const struct link_training_settings *p_link_settings,
+			const unsigned char *p_custom_pattern,
+			unsigned int cust_pattern_size);
+
 bool dc_submit_i2c(
 		struct dc *dc,
 		uint32_t link_index,
diff --git a/drivers/gpu/drm/amd/display/dc/dc_stream.h b/drivers/gpu/drm/amd/display/dc/dc_stream.h
index af503e0286a7..cbfe418006cb 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_stream.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_stream.h
@@ -100,6 +100,7 @@ struct dc_stream_state {
 
 	struct dc_cursor_attributes cursor_attributes;
 	struct dc_cursor_position cursor_position;
+	uint32_t sdr_white_level; // for boosting (SDR) cursor in HDR mode
 
 	/* from stream struct */
 	struct kref refcount;
@@ -147,10 +148,7 @@ void dc_commit_updates_for_stream(struct dc *dc,
 /*
  * Log the current stream state.
  */
-void dc_stream_log(
-	const struct dc_stream_state *stream,
-	struct dal_logger *dc_logger,
-	enum dc_log_type log_type);
+void dc_stream_log(const struct dc *dc, const struct dc_stream_state *stream);
 
 uint8_t dc_get_current_stream_count(struct dc *dc);
 struct dc_stream_state *dc_get_stream_at_index(struct dc *dc, uint8_t i);
@@ -258,6 +256,7 @@ bool dc_stream_set_cursor_position(
 	struct dc_stream_state *stream,
 	const struct dc_cursor_position *position);
 
+
 bool dc_stream_adjust_vmin_vmax(struct dc *dc,
 				struct dc_stream_state **stream,
 				int num_streams,
diff --git a/drivers/gpu/drm/amd/display/dc/dc_types.h b/drivers/gpu/drm/amd/display/dc/dc_types.h
index c96e526d07bb..8c6eb78b0c3b 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_types.h
@@ -77,8 +77,6 @@ struct dc_context {
 	struct dc *dc;
 
 	void *driver_context; /* e.g. amdgpu_device */
-
-	struct dal_logger *logger;
 	void *cgs_device;
 
 	enum dce_environment dce_environment;
@@ -194,6 +192,7 @@ union display_content_support {
 
 struct dc_panel_patch {
 	unsigned int dppowerup_delay;
+	unsigned int extra_t12_ms;
 };
 
 struct dc_edid_caps {
diff --git a/drivers/gpu/drm/amd/display/dc/dce/Makefile b/drivers/gpu/drm/amd/display/dc/dce/Makefile
index 11401fd8e535..825537bd4545 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/dce/Makefile
@@ -28,7 +28,7 @@
 
 DCE = dce_audio.o dce_stream_encoder.o dce_link_encoder.o dce_hwseq.o \
 dce_mem_input.o dce_clock_source.o dce_scl_filters.o dce_transform.o \
-dce_clocks.o dce_opp.o dce_dmcu.o dce_abm.o dce_ipp.o
+dce_clocks.o dce_opp.o dce_dmcu.o dce_abm.o dce_ipp.o dce_aux.o
 
 
 AMD_DAL_DCE = $(addprefix $(AMDDALPATH)/dc/dce/,$(DCE))
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_aux.c b/drivers/gpu/drm/amd/display/dc/dce/dce_aux.c
new file mode 100644
index 000000000000..3f5b2e6f7553
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_aux.c
@@ -0,0 +1,937 @@
+/*
+ * Copyright 2012-15 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#include "dm_services.h"
+#include "dce_aux.h"
+#include "dce/dce_11_0_sh_mask.h"
+
+#define CTX \
+	aux110->base.ctx
+#define REG(reg_name)\
+	(aux110->regs->reg_name)
+
+#define DC_LOGGER \
+	engine->ctx->logger
+
+#include "reg_helper.h"
+
+#define FROM_AUX_ENGINE(ptr) \
+	container_of((ptr), struct aux_engine_dce110, base)
+
+#define FROM_ENGINE(ptr) \
+	FROM_AUX_ENGINE(container_of((ptr), struct aux_engine, base))
+
+#define FROM_AUX_ENGINE_ENGINE(ptr) \
+	container_of((ptr), struct aux_engine, base)
+enum {
+	AUX_INVALID_REPLY_RETRY_COUNTER = 1,
+	AUX_TIMED_OUT_RETRY_COUNTER = 2,
+	AUX_DEFER_RETRY_COUNTER = 6
+};
+static void release_engine(
+	struct aux_engine *engine)
+{
+	struct aux_engine_dce110 *aux110 = FROM_AUX_ENGINE(engine);
+
+	dal_ddc_close(engine->ddc);
+
+	engine->ddc = NULL;
+
+	REG_UPDATE(AUX_ARB_CONTROL, AUX_SW_DONE_USING_AUX_REG, 1);
+}
+
+#define SW_CAN_ACCESS_AUX 1
+#define DMCU_CAN_ACCESS_AUX 2
+
+static bool is_engine_available(
+	struct aux_engine *engine)
+{
+	struct aux_engine_dce110 *aux110 = FROM_AUX_ENGINE(engine);
+
+	uint32_t value = REG_READ(AUX_ARB_CONTROL);
+	uint32_t field = get_reg_field_value(
+			value,
+			AUX_ARB_CONTROL,
+			AUX_REG_RW_CNTL_STATUS);
+
+	return (field != DMCU_CAN_ACCESS_AUX);
+}
+static bool acquire_engine(
+	struct aux_engine *engine)
+{
+	struct aux_engine_dce110 *aux110 = FROM_AUX_ENGINE(engine);
+
+	uint32_t value = REG_READ(AUX_ARB_CONTROL);
+	uint32_t field = get_reg_field_value(
+			value,
+			AUX_ARB_CONTROL,
+			AUX_REG_RW_CNTL_STATUS);
+	if (field == DMCU_CAN_ACCESS_AUX)
+		return false;
+	/* enable AUX before request SW to access AUX */
+	value = REG_READ(AUX_CONTROL);
+	field = get_reg_field_value(value,
+				AUX_CONTROL,
+				AUX_EN);
+
+	if (field == 0) {
+		set_reg_field_value(
+				value,
+				1,
+				AUX_CONTROL,
+				AUX_EN);
+
+		if (REG(AUX_RESET_MASK)) {
+			/*DP_AUX block as part of the enable sequence*/
+			set_reg_field_value(
+				value,
+				1,
+				AUX_CONTROL,
+				AUX_RESET);
+		}
+
+		REG_WRITE(AUX_CONTROL, value);
+
+		if (REG(AUX_RESET_MASK)) {
+			/*poll HW to make sure reset it done*/
+
+			REG_WAIT(AUX_CONTROL, AUX_RESET_DONE, 1,
+					1, 11);
+
+			set_reg_field_value(
+				value,
+				0,
+				AUX_CONTROL,
+				AUX_RESET);
+
+			REG_WRITE(AUX_CONTROL, value);
+
+			REG_WAIT(AUX_CONTROL, AUX_RESET_DONE, 0,
+					1, 11);
+		}
+	} /*if (field)*/
+
+	/* request SW to access AUX */
+	REG_UPDATE(AUX_ARB_CONTROL, AUX_SW_USE_AUX_REG_REQ, 1);
+
+	value = REG_READ(AUX_ARB_CONTROL);
+	field = get_reg_field_value(
+			value,
+			AUX_ARB_CONTROL,
+			AUX_REG_RW_CNTL_STATUS);
+
+	return (field == SW_CAN_ACCESS_AUX);
+}
+
+#define COMPOSE_AUX_SW_DATA_16_20(command, address) \
+	((command) | ((0xF0000 & (address)) >> 16))
+
+#define COMPOSE_AUX_SW_DATA_8_15(address) \
+	((0xFF00 & (address)) >> 8)
+
+#define COMPOSE_AUX_SW_DATA_0_7(address) \
+	(0xFF & (address))
+
+static void submit_channel_request(
+	struct aux_engine *engine,
+	struct aux_request_transaction_data *request)
+{
+	struct aux_engine_dce110 *aux110 = FROM_AUX_ENGINE(engine);
+	uint32_t value;
+	uint32_t length;
+
+	bool is_write =
+		((request->type == AUX_TRANSACTION_TYPE_DP) &&
+		 (request->action == I2CAUX_TRANSACTION_ACTION_DP_WRITE)) ||
+		((request->type == AUX_TRANSACTION_TYPE_I2C) &&
+		((request->action == I2CAUX_TRANSACTION_ACTION_I2C_WRITE) ||
+		 (request->action == I2CAUX_TRANSACTION_ACTION_I2C_WRITE_MOT)));
+	if (REG(AUXN_IMPCAL)) {
+		/* clear_aux_error */
+		REG_UPDATE_SEQ(AUXN_IMPCAL, AUXN_CALOUT_ERROR_AK,
+				1,
+				0);
+
+		REG_UPDATE_SEQ(AUXP_IMPCAL, AUXP_CALOUT_ERROR_AK,
+				1,
+				0);
+
+		/* force_default_calibrate */
+		REG_UPDATE_1BY1_2(AUXN_IMPCAL,
+				AUXN_IMPCAL_ENABLE, 1,
+				AUXN_IMPCAL_OVERRIDE_ENABLE, 0);
+
+		/* bug? why AUXN update EN and OVERRIDE_EN 1 by 1 while AUX P toggles OVERRIDE? */
+
+		REG_UPDATE_SEQ(AUXP_IMPCAL, AUXP_IMPCAL_OVERRIDE_ENABLE,
+				1,
+				0);
+	}
+	/* set the delay and the number of bytes to write */
+
+	/* The length include
+	 * the 4 bit header and the 20 bit address
+	 * (that is 3 byte).
+	 * If the requested length is non zero this means
+	 * an addition byte specifying the length is required.
+	 */
+
+	length = request->length ? 4 : 3;
+	if (is_write)
+		length += request->length;
+
+	REG_UPDATE_2(AUX_SW_CONTROL,
+			AUX_SW_START_DELAY, request->delay,
+			AUX_SW_WR_BYTES, length);
+
+	/* program action and address and payload data (if 'is_write') */
+	value = REG_UPDATE_4(AUX_SW_DATA,
+			AUX_SW_INDEX, 0,
+			AUX_SW_DATA_RW, 0,
+			AUX_SW_AUTOINCREMENT_DISABLE, 1,
+			AUX_SW_DATA, COMPOSE_AUX_SW_DATA_16_20(request->action, request->address));
+
+	value = REG_SET_2(AUX_SW_DATA, value,
+			AUX_SW_AUTOINCREMENT_DISABLE, 0,
+			AUX_SW_DATA, COMPOSE_AUX_SW_DATA_8_15(request->address));
+
+	value = REG_SET(AUX_SW_DATA, value,
+			AUX_SW_DATA, COMPOSE_AUX_SW_DATA_0_7(request->address));
+
+	if (request->length) {
+		value = REG_SET(AUX_SW_DATA, value,
+				AUX_SW_DATA, request->length - 1);
+	}
+
+	if (is_write) {
+		/* Load the HW buffer with the Data to be sent.
+		 * This is relevant for write operation.
+		 * For read, the data recived data will be
+		 * processed in process_channel_reply().
+		 */
+		uint32_t i = 0;
+
+		while (i < request->length) {
+			value = REG_SET(AUX_SW_DATA, value,
+					AUX_SW_DATA, request->data[i]);
+
+			++i;
+		}
+	}
+
+	REG_UPDATE(AUX_INTERRUPT_CONTROL, AUX_SW_DONE_ACK, 1);
+	REG_WAIT(AUX_SW_STATUS, AUX_SW_DONE, 0,
+				10, aux110->timeout_period/10);
+	REG_UPDATE(AUX_SW_CONTROL, AUX_SW_GO, 1);
+}
+
+static int read_channel_reply(struct aux_engine *engine, uint32_t size,
+			      uint8_t *buffer, uint8_t *reply_result,
+			      uint32_t *sw_status)
+{
+	struct aux_engine_dce110 *aux110 = FROM_AUX_ENGINE(engine);
+	uint32_t bytes_replied;
+	uint32_t reply_result_32;
+
+	*sw_status = REG_GET(AUX_SW_STATUS, AUX_SW_REPLY_BYTE_COUNT,
+			     &bytes_replied);
+
+	/* In case HPD is LOW, exit AUX transaction */
+	if ((*sw_status & AUX_SW_STATUS__AUX_SW_HPD_DISCON_MASK))
+		return -1;
+
+	/* Need at least the status byte */
+	if (!bytes_replied)
+		return -1;
+
+	REG_UPDATE_1BY1_3(AUX_SW_DATA,
+			  AUX_SW_INDEX, 0,
+			  AUX_SW_AUTOINCREMENT_DISABLE, 1,
+			  AUX_SW_DATA_RW, 1);
+
+	REG_GET(AUX_SW_DATA, AUX_SW_DATA, &reply_result_32);
+	reply_result_32 = reply_result_32 >> 4;
+	*reply_result = (uint8_t)reply_result_32;
+
+	if (reply_result_32 == 0) { /* ACK */
+		uint32_t i = 0;
+
+		/* First byte was already used to get the command status */
+		--bytes_replied;
+
+		/* Do not overflow buffer */
+		if (bytes_replied > size)
+			return -1;
+
+		while (i < bytes_replied) {
+			uint32_t aux_sw_data_val;
+
+			REG_GET(AUX_SW_DATA, AUX_SW_DATA, &aux_sw_data_val);
+			buffer[i] = aux_sw_data_val;
+			++i;
+		}
+
+		return i;
+	}
+
+	return 0;
+}
+
+static void process_channel_reply(
+	struct aux_engine *engine,
+	struct aux_reply_transaction_data *reply)
+{
+	int bytes_replied;
+	uint8_t reply_result;
+	uint32_t sw_status;
+
+	bytes_replied = read_channel_reply(engine, reply->length, reply->data,
+					   &reply_result, &sw_status);
+
+	/* in case HPD is LOW, exit AUX transaction */
+	if ((sw_status & AUX_SW_STATUS__AUX_SW_HPD_DISCON_MASK)) {
+		reply->status = AUX_CHANNEL_OPERATION_FAILED_HPD_DISCON;
+		return;
+	}
+
+	if (bytes_replied < 0) {
+		/* Need to handle an error case...
+		 * Hopefully, upper layer function won't call this function if
+		 * the number of bytes in the reply was 0, because there was
+		 * surely an error that was asserted that should have been
+		 * handled for hot plug case, this could happens
+		 */
+		if (!(sw_status & AUX_SW_STATUS__AUX_SW_HPD_DISCON_MASK)) {
+			reply->status = AUX_TRANSACTION_REPLY_INVALID;
+			ASSERT_CRITICAL(false);
+			return;
+		}
+	} else {
+
+		switch (reply_result) {
+		case 0: /* ACK */
+			reply->status = AUX_TRANSACTION_REPLY_AUX_ACK;
+		break;
+		case 1: /* NACK */
+			reply->status = AUX_TRANSACTION_REPLY_AUX_NACK;
+		break;
+		case 2: /* DEFER */
+			reply->status = AUX_TRANSACTION_REPLY_AUX_DEFER;
+		break;
+		case 4: /* AUX ACK / I2C NACK */
+			reply->status = AUX_TRANSACTION_REPLY_I2C_NACK;
+		break;
+		case 8: /* AUX ACK / I2C DEFER */
+			reply->status = AUX_TRANSACTION_REPLY_I2C_DEFER;
+		break;
+		default:
+			reply->status = AUX_TRANSACTION_REPLY_INVALID;
+		}
+	}
+}
+
+static enum aux_channel_operation_result get_channel_status(
+	struct aux_engine *engine,
+	uint8_t *returned_bytes)
+{
+	struct aux_engine_dce110 *aux110 = FROM_AUX_ENGINE(engine);
+
+	uint32_t value;
+
+	if (returned_bytes == NULL) {
+		/*caller pass NULL pointer*/
+		ASSERT_CRITICAL(false);
+		return AUX_CHANNEL_OPERATION_FAILED_REASON_UNKNOWN;
+	}
+	*returned_bytes = 0;
+
+	/* poll to make sure that SW_DONE is asserted */
+	value = REG_WAIT(AUX_SW_STATUS, AUX_SW_DONE, 1,
+				10, aux110->timeout_period/10);
+
+	/* in case HPD is LOW, exit AUX transaction */
+	if ((value & AUX_SW_STATUS__AUX_SW_HPD_DISCON_MASK))
+		return AUX_CHANNEL_OPERATION_FAILED_HPD_DISCON;
+
+	/* Note that the following bits are set in 'status.bits'
+	 * during CTS 4.2.1.2 (FW 3.3.1):
+	 * AUX_SW_RX_MIN_COUNT_VIOL, AUX_SW_RX_INVALID_STOP,
+	 * AUX_SW_RX_RECV_NO_DET, AUX_SW_RX_RECV_INVALID_H.
+	 *
+	 * AUX_SW_RX_MIN_COUNT_VIOL is an internal,
+	 * HW debugging bit and should be ignored.
+	 */
+	if (value & AUX_SW_STATUS__AUX_SW_DONE_MASK) {
+		if ((value & AUX_SW_STATUS__AUX_SW_RX_TIMEOUT_STATE_MASK) ||
+			(value & AUX_SW_STATUS__AUX_SW_RX_TIMEOUT_MASK))
+			return AUX_CHANNEL_OPERATION_FAILED_TIMEOUT;
+
+		else if ((value & AUX_SW_STATUS__AUX_SW_RX_INVALID_STOP_MASK) ||
+			(value & AUX_SW_STATUS__AUX_SW_RX_RECV_NO_DET_MASK) ||
+			(value &
+				AUX_SW_STATUS__AUX_SW_RX_RECV_INVALID_H_MASK) ||
+			(value & AUX_SW_STATUS__AUX_SW_RX_RECV_INVALID_L_MASK))
+			return AUX_CHANNEL_OPERATION_FAILED_INVALID_REPLY;
+
+		*returned_bytes = get_reg_field_value(value,
+				AUX_SW_STATUS,
+				AUX_SW_REPLY_BYTE_COUNT);
+
+		if (*returned_bytes == 0)
+			return
+			AUX_CHANNEL_OPERATION_FAILED_INVALID_REPLY;
+		else {
+			*returned_bytes -= 1;
+			return AUX_CHANNEL_OPERATION_SUCCEEDED;
+		}
+	} else {
+		/*time_elapsed >= aux_engine->timeout_period
+		 *  AUX_SW_STATUS__AUX_SW_HPD_DISCON = at this point
+		 */
+		ASSERT_CRITICAL(false);
+		return AUX_CHANNEL_OPERATION_FAILED_TIMEOUT;
+	}
+}
+static void process_read_reply(
+	struct aux_engine *engine,
+	struct read_command_context *ctx)
+{
+	engine->funcs->process_channel_reply(engine, &ctx->reply);
+
+	switch (ctx->reply.status) {
+	case AUX_TRANSACTION_REPLY_AUX_ACK:
+		ctx->defer_retry_aux = 0;
+		if (ctx->returned_byte > ctx->current_read_length) {
+			ctx->status =
+				I2CAUX_TRANSACTION_STATUS_FAILED_PROTOCOL_ERROR;
+			ctx->operation_succeeded = false;
+		} else if (ctx->returned_byte < ctx->current_read_length) {
+			ctx->current_read_length -= ctx->returned_byte;
+
+			ctx->offset += ctx->returned_byte;
+
+			++ctx->invalid_reply_retry_aux_on_ack;
+
+			if (ctx->invalid_reply_retry_aux_on_ack >
+				AUX_INVALID_REPLY_RETRY_COUNTER) {
+				ctx->status =
+				I2CAUX_TRANSACTION_STATUS_FAILED_PROTOCOL_ERROR;
+				ctx->operation_succeeded = false;
+			}
+		} else {
+			ctx->status = I2CAUX_TRANSACTION_STATUS_SUCCEEDED;
+			ctx->transaction_complete = true;
+			ctx->operation_succeeded = true;
+		}
+	break;
+	case AUX_TRANSACTION_REPLY_AUX_NACK:
+		ctx->status = I2CAUX_TRANSACTION_STATUS_FAILED_NACK;
+		ctx->operation_succeeded = false;
+	break;
+	case AUX_TRANSACTION_REPLY_AUX_DEFER:
+		++ctx->defer_retry_aux;
+
+		if (ctx->defer_retry_aux > AUX_DEFER_RETRY_COUNTER) {
+			ctx->status = I2CAUX_TRANSACTION_STATUS_FAILED_TIMEOUT;
+			ctx->operation_succeeded = false;
+		}
+	break;
+	case AUX_TRANSACTION_REPLY_I2C_DEFER:
+		ctx->defer_retry_aux = 0;
+
+		++ctx->defer_retry_i2c;
+
+		if (ctx->defer_retry_i2c > AUX_DEFER_RETRY_COUNTER) {
+			ctx->status = I2CAUX_TRANSACTION_STATUS_FAILED_TIMEOUT;
+			ctx->operation_succeeded = false;
+		}
+	break;
+	case AUX_TRANSACTION_REPLY_HPD_DISCON:
+		ctx->status = I2CAUX_TRANSACTION_STATUS_FAILED_HPD_DISCON;
+		ctx->operation_succeeded = false;
+	break;
+	default:
+		ctx->status = I2CAUX_TRANSACTION_STATUS_UNKNOWN;
+		ctx->operation_succeeded = false;
+	}
+}
+static void process_read_request(
+	struct aux_engine *engine,
+	struct read_command_context *ctx)
+{
+	enum aux_channel_operation_result operation_result;
+
+	engine->funcs->submit_channel_request(engine, &ctx->request);
+
+	operation_result = engine->funcs->get_channel_status(
+		engine, &ctx->returned_byte);
+
+	switch (operation_result) {
+	case AUX_CHANNEL_OPERATION_SUCCEEDED:
+		if (ctx->returned_byte > ctx->current_read_length) {
+			ctx->status =
+				I2CAUX_TRANSACTION_STATUS_FAILED_PROTOCOL_ERROR;
+			ctx->operation_succeeded = false;
+		} else {
+			ctx->timed_out_retry_aux = 0;
+			ctx->invalid_reply_retry_aux = 0;
+
+			ctx->reply.length = ctx->returned_byte;
+			ctx->reply.data = ctx->buffer;
+
+			process_read_reply(engine, ctx);
+		}
+	break;
+	case AUX_CHANNEL_OPERATION_FAILED_INVALID_REPLY:
+		++ctx->invalid_reply_retry_aux;
+
+		if (ctx->invalid_reply_retry_aux >
+			AUX_INVALID_REPLY_RETRY_COUNTER) {
+			ctx->status =
+				I2CAUX_TRANSACTION_STATUS_FAILED_PROTOCOL_ERROR;
+			ctx->operation_succeeded = false;
+		} else
+			udelay(400);
+	break;
+	case AUX_CHANNEL_OPERATION_FAILED_TIMEOUT:
+		++ctx->timed_out_retry_aux;
+
+		if (ctx->timed_out_retry_aux > AUX_TIMED_OUT_RETRY_COUNTER) {
+			ctx->status = I2CAUX_TRANSACTION_STATUS_FAILED_TIMEOUT;
+			ctx->operation_succeeded = false;
+		} else {
+			/* DP 1.2a, table 2-58:
+			 * "S3: AUX Request CMD PENDING:
+			 * retry 3 times, with 400usec wait on each"
+			 * The HW timeout is set to 550usec,
+			 * so we should not wait here
+			 */
+		}
+	break;
+	case AUX_CHANNEL_OPERATION_FAILED_HPD_DISCON:
+		ctx->status = I2CAUX_TRANSACTION_STATUS_FAILED_HPD_DISCON;
+		ctx->operation_succeeded = false;
+	break;
+	default:
+		ctx->status = I2CAUX_TRANSACTION_STATUS_UNKNOWN;
+		ctx->operation_succeeded = false;
+	}
+}
+static bool read_command(
+	struct aux_engine *engine,
+	struct i2caux_transaction_request *request,
+	bool middle_of_transaction)
+{
+	struct read_command_context ctx;
+
+	ctx.buffer = request->payload.data;
+	ctx.current_read_length = request->payload.length;
+	ctx.offset = 0;
+	ctx.timed_out_retry_aux = 0;
+	ctx.invalid_reply_retry_aux = 0;
+	ctx.defer_retry_aux = 0;
+	ctx.defer_retry_i2c = 0;
+	ctx.invalid_reply_retry_aux_on_ack = 0;
+	ctx.transaction_complete = false;
+	ctx.operation_succeeded = true;
+
+	if (request->payload.address_space ==
+		I2CAUX_TRANSACTION_ADDRESS_SPACE_DPCD) {
+		ctx.request.type = AUX_TRANSACTION_TYPE_DP;
+		ctx.request.action = I2CAUX_TRANSACTION_ACTION_DP_READ;
+		ctx.request.address = request->payload.address;
+	} else if (request->payload.address_space ==
+		I2CAUX_TRANSACTION_ADDRESS_SPACE_I2C) {
+		ctx.request.type = AUX_TRANSACTION_TYPE_I2C;
+		ctx.request.action = middle_of_transaction ?
+			I2CAUX_TRANSACTION_ACTION_I2C_READ_MOT :
+			I2CAUX_TRANSACTION_ACTION_I2C_READ;
+		ctx.request.address = request->payload.address >> 1;
+	} else {
+		/* in DAL2, there was no return in such case */
+		BREAK_TO_DEBUGGER();
+		return false;
+	}
+
+	ctx.request.delay = 0;
+
+	do {
+		memset(ctx.buffer + ctx.offset, 0, ctx.current_read_length);
+
+		ctx.request.data = ctx.buffer + ctx.offset;
+		ctx.request.length = ctx.current_read_length;
+
+		process_read_request(engine, &ctx);
+
+		request->status = ctx.status;
+
+		if (ctx.operation_succeeded && !ctx.transaction_complete)
+			if (ctx.request.type == AUX_TRANSACTION_TYPE_I2C)
+				msleep(engine->delay);
+	} while (ctx.operation_succeeded && !ctx.transaction_complete);
+
+	if (request->payload.address_space ==
+		I2CAUX_TRANSACTION_ADDRESS_SPACE_DPCD) {
+		DC_LOG_I2C_AUX("READ: addr:0x%x  value:0x%x Result:%d",
+				request->payload.address,
+				request->payload.data[0],
+				ctx.operation_succeeded);
+	}
+
+	return ctx.operation_succeeded;
+}
+
+static void process_write_reply(
+	struct aux_engine *engine,
+	struct write_command_context *ctx)
+{
+	engine->funcs->process_channel_reply(engine, &ctx->reply);
+
+	switch (ctx->reply.status) {
+	case AUX_TRANSACTION_REPLY_AUX_ACK:
+		ctx->operation_succeeded = true;
+
+		if (ctx->returned_byte) {
+			ctx->request.action = ctx->mot ?
+			I2CAUX_TRANSACTION_ACTION_I2C_STATUS_REQUEST_MOT :
+			I2CAUX_TRANSACTION_ACTION_I2C_STATUS_REQUEST;
+
+			ctx->current_write_length = 0;
+
+			++ctx->ack_m_retry;
+
+			if (ctx->ack_m_retry > AUX_DEFER_RETRY_COUNTER) {
+				ctx->status =
+				I2CAUX_TRANSACTION_STATUS_FAILED_TIMEOUT;
+				ctx->operation_succeeded = false;
+			} else
+				udelay(300);
+		} else {
+			ctx->status = I2CAUX_TRANSACTION_STATUS_SUCCEEDED;
+			ctx->defer_retry_aux = 0;
+			ctx->ack_m_retry = 0;
+			ctx->transaction_complete = true;
+		}
+	break;
+	case AUX_TRANSACTION_REPLY_AUX_NACK:
+		ctx->status = I2CAUX_TRANSACTION_STATUS_FAILED_NACK;
+		ctx->operation_succeeded = false;
+	break;
+	case AUX_TRANSACTION_REPLY_AUX_DEFER:
+		++ctx->defer_retry_aux;
+
+		if (ctx->defer_retry_aux > ctx->max_defer_retry) {
+			ctx->status = I2CAUX_TRANSACTION_STATUS_FAILED_TIMEOUT;
+			ctx->operation_succeeded = false;
+		}
+	break;
+	case AUX_TRANSACTION_REPLY_I2C_DEFER:
+		ctx->defer_retry_aux = 0;
+		ctx->current_write_length = 0;
+
+		ctx->request.action = ctx->mot ?
+			I2CAUX_TRANSACTION_ACTION_I2C_STATUS_REQUEST_MOT :
+			I2CAUX_TRANSACTION_ACTION_I2C_STATUS_REQUEST;
+
+		++ctx->defer_retry_i2c;
+
+		if (ctx->defer_retry_i2c > ctx->max_defer_retry) {
+			ctx->status = I2CAUX_TRANSACTION_STATUS_FAILED_TIMEOUT;
+			ctx->operation_succeeded = false;
+		}
+	break;
+	case AUX_TRANSACTION_REPLY_HPD_DISCON:
+		ctx->status = I2CAUX_TRANSACTION_STATUS_FAILED_HPD_DISCON;
+		ctx->operation_succeeded = false;
+	break;
+	default:
+		ctx->status = I2CAUX_TRANSACTION_STATUS_UNKNOWN;
+		ctx->operation_succeeded = false;
+	}
+}
+static void process_write_request(
+	struct aux_engine *engine,
+	struct write_command_context *ctx)
+{
+	enum aux_channel_operation_result operation_result;
+
+	engine->funcs->submit_channel_request(engine, &ctx->request);
+
+	operation_result = engine->funcs->get_channel_status(
+		engine, &ctx->returned_byte);
+
+	switch (operation_result) {
+	case AUX_CHANNEL_OPERATION_SUCCEEDED:
+		ctx->timed_out_retry_aux = 0;
+		ctx->invalid_reply_retry_aux = 0;
+
+		ctx->reply.length = ctx->returned_byte;
+		ctx->reply.data = ctx->reply_data;
+
+		process_write_reply(engine, ctx);
+	break;
+	case AUX_CHANNEL_OPERATION_FAILED_INVALID_REPLY:
+		++ctx->invalid_reply_retry_aux;
+
+		if (ctx->invalid_reply_retry_aux >
+			AUX_INVALID_REPLY_RETRY_COUNTER) {
+			ctx->status =
+				I2CAUX_TRANSACTION_STATUS_FAILED_PROTOCOL_ERROR;
+			ctx->operation_succeeded = false;
+		} else
+			udelay(400);
+	break;
+	case AUX_CHANNEL_OPERATION_FAILED_TIMEOUT:
+		++ctx->timed_out_retry_aux;
+
+		if (ctx->timed_out_retry_aux > AUX_TIMED_OUT_RETRY_COUNTER) {
+			ctx->status = I2CAUX_TRANSACTION_STATUS_FAILED_TIMEOUT;
+			ctx->operation_succeeded = false;
+		} else {
+			/* DP 1.2a, table 2-58:
+			 * "S3: AUX Request CMD PENDING:
+			 * retry 3 times, with 400usec wait on each"
+			 * The HW timeout is set to 550usec,
+			 * so we should not wait here
+			 */
+		}
+	break;
+	case AUX_CHANNEL_OPERATION_FAILED_HPD_DISCON:
+		ctx->status = I2CAUX_TRANSACTION_STATUS_FAILED_HPD_DISCON;
+		ctx->operation_succeeded = false;
+	break;
+	default:
+		ctx->status = I2CAUX_TRANSACTION_STATUS_UNKNOWN;
+		ctx->operation_succeeded = false;
+	}
+}
+static bool write_command(
+	struct aux_engine *engine,
+	struct i2caux_transaction_request *request,
+	bool middle_of_transaction)
+{
+	struct write_command_context ctx;
+
+	ctx.mot = middle_of_transaction;
+	ctx.buffer = request->payload.data;
+	ctx.current_write_length = request->payload.length;
+	ctx.timed_out_retry_aux = 0;
+	ctx.invalid_reply_retry_aux = 0;
+	ctx.defer_retry_aux = 0;
+	ctx.defer_retry_i2c = 0;
+	ctx.ack_m_retry = 0;
+	ctx.transaction_complete = false;
+	ctx.operation_succeeded = true;
+
+	if (request->payload.address_space ==
+		I2CAUX_TRANSACTION_ADDRESS_SPACE_DPCD) {
+		ctx.request.type = AUX_TRANSACTION_TYPE_DP;
+		ctx.request.action = I2CAUX_TRANSACTION_ACTION_DP_WRITE;
+		ctx.request.address = request->payload.address;
+	} else if (request->payload.address_space ==
+		I2CAUX_TRANSACTION_ADDRESS_SPACE_I2C) {
+		ctx.request.type = AUX_TRANSACTION_TYPE_I2C;
+		ctx.request.action = middle_of_transaction ?
+			I2CAUX_TRANSACTION_ACTION_I2C_WRITE_MOT :
+			I2CAUX_TRANSACTION_ACTION_I2C_WRITE;
+		ctx.request.address = request->payload.address >> 1;
+	} else {
+		/* in DAL2, there was no return in such case */
+		BREAK_TO_DEBUGGER();
+		return false;
+	}
+
+	ctx.request.delay = 0;
+
+	ctx.max_defer_retry =
+		(engine->max_defer_write_retry > AUX_DEFER_RETRY_COUNTER) ?
+			engine->max_defer_write_retry : AUX_DEFER_RETRY_COUNTER;
+
+	do {
+		ctx.request.data = ctx.buffer;
+		ctx.request.length = ctx.current_write_length;
+
+		process_write_request(engine, &ctx);
+
+		request->status = ctx.status;
+
+		if (ctx.operation_succeeded && !ctx.transaction_complete)
+			if (ctx.request.type == AUX_TRANSACTION_TYPE_I2C)
+				msleep(engine->delay);
+	} while (ctx.operation_succeeded && !ctx.transaction_complete);
+
+	if (request->payload.address_space ==
+		I2CAUX_TRANSACTION_ADDRESS_SPACE_DPCD) {
+		DC_LOG_I2C_AUX("WRITE: addr:0x%x  value:0x%x Result:%d",
+				request->payload.address,
+				request->payload.data[0],
+				ctx.operation_succeeded);
+	}
+
+	return ctx.operation_succeeded;
+}
+static bool end_of_transaction_command(
+	struct aux_engine *engine,
+	struct i2caux_transaction_request *request)
+{
+	struct i2caux_transaction_request dummy_request;
+	uint8_t dummy_data;
+
+	/* [tcheng] We only need to send the stop (read with MOT = 0)
+	 * for I2C-over-Aux, not native AUX
+	 */
+
+	if (request->payload.address_space !=
+		I2CAUX_TRANSACTION_ADDRESS_SPACE_I2C)
+		return false;
+
+	dummy_request.operation = request->operation;
+	dummy_request.payload.address_space = request->payload.address_space;
+	dummy_request.payload.address = request->payload.address;
+
+	/*
+	 * Add a dummy byte due to some receiver quirk
+	 * where one byte is sent along with MOT = 0.
+	 * Ideally this should be 0.
+	 */
+
+	dummy_request.payload.length = 0;
+	dummy_request.payload.data = &dummy_data;
+
+	if (request->operation == I2CAUX_TRANSACTION_READ)
+		return read_command(engine, &dummy_request, false);
+	else
+		return write_command(engine, &dummy_request, false);
+
+	/* according Syed, it does not need now DoDummyMOT */
+}
+static bool submit_request(
+	struct aux_engine *engine,
+	struct i2caux_transaction_request *request,
+	bool middle_of_transaction)
+{
+
+	bool result;
+	bool mot_used = true;
+
+	switch (request->operation) {
+	case I2CAUX_TRANSACTION_READ:
+		result = read_command(engine, request, mot_used);
+	break;
+	case I2CAUX_TRANSACTION_WRITE:
+		result = write_command(engine, request, mot_used);
+	break;
+	default:
+		result = false;
+	}
+
+	/* [tcheng]
+	 * need to send stop for the last transaction to free up the AUX
+	 * if the above command fails, this would be the last transaction
+	 */
+
+	if (!middle_of_transaction || !result)
+		end_of_transaction_command(engine, request);
+
+	/* mask AUX interrupt */
+
+	return result;
+}
+enum i2caux_engine_type get_engine_type(
+		const struct aux_engine *engine)
+{
+	return I2CAUX_ENGINE_TYPE_AUX;
+}
+
+static bool acquire(
+	struct aux_engine *engine,
+	struct ddc *ddc)
+{
+
+	enum gpio_result result;
+
+	if (engine->funcs->is_engine_available) {
+		/*check whether SW could use the engine*/
+		if (!engine->funcs->is_engine_available(engine))
+			return false;
+	}
+
+	result = dal_ddc_open(ddc, GPIO_MODE_HARDWARE,
+		GPIO_DDC_CONFIG_TYPE_MODE_AUX);
+
+	if (result != GPIO_RESULT_OK)
+		return false;
+
+	if (!engine->funcs->acquire_engine(engine)) {
+		dal_ddc_close(ddc);
+		return false;
+	}
+
+	engine->ddc = ddc;
+
+	return true;
+}
+
+static const struct aux_engine_funcs aux_engine_funcs = {
+	.acquire_engine = acquire_engine,
+	.submit_channel_request = submit_channel_request,
+	.process_channel_reply = process_channel_reply,
+	.read_channel_reply = read_channel_reply,
+	.get_channel_status = get_channel_status,
+	.is_engine_available = is_engine_available,
+	.release_engine = release_engine,
+	.destroy_engine = dce110_engine_destroy,
+	.submit_request = submit_request,
+	.get_engine_type = get_engine_type,
+	.acquire = acquire,
+};
+
+void dce110_engine_destroy(struct aux_engine **engine)
+{
+
+	struct aux_engine_dce110 *engine110 = FROM_AUX_ENGINE(*engine);
+
+	kfree(engine110);
+	*engine = NULL;
+
+}
+struct aux_engine *dce110_aux_engine_construct(struct aux_engine_dce110 *aux_engine110,
+		struct dc_context *ctx,
+		uint32_t inst,
+		uint32_t timeout_period,
+		const struct dce110_aux_registers *regs)
+{
+	aux_engine110->base.ddc = NULL;
+	aux_engine110->base.ctx = ctx;
+	aux_engine110->base.delay = 0;
+	aux_engine110->base.max_defer_write_retry = 0;
+	aux_engine110->base.funcs = &aux_engine_funcs;
+	aux_engine110->base.inst = inst;
+	aux_engine110->timeout_period = timeout_period;
+	aux_engine110->regs = regs;
+
+	return &aux_engine110->base;
+}
+
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_aux.h b/drivers/gpu/drm/amd/display/dc/dce/dce_aux.h
new file mode 100644
index 000000000000..f7caab85dc80
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_aux.h
@@ -0,0 +1,111 @@
+/*
+ * Copyright 2012-15 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef __DAL_AUX_ENGINE_DCE110_H__
+#define __DAL_AUX_ENGINE_DCE110_H__
+#include "aux_engine.h"
+
+#define AUX_COMMON_REG_LIST(id)\
+	SRI(AUX_CONTROL, DP_AUX, id), \
+	SRI(AUX_ARB_CONTROL, DP_AUX, id), \
+	SRI(AUX_SW_DATA, DP_AUX, id), \
+	SRI(AUX_SW_CONTROL, DP_AUX, id), \
+	SRI(AUX_INTERRUPT_CONTROL, DP_AUX, id), \
+	SRI(AUX_SW_STATUS, DP_AUX, id), \
+	SR(AUXN_IMPCAL), \
+	SR(AUXP_IMPCAL)
+
+struct dce110_aux_registers {
+	uint32_t AUX_CONTROL;
+	uint32_t AUX_ARB_CONTROL;
+	uint32_t AUX_SW_DATA;
+	uint32_t AUX_SW_CONTROL;
+	uint32_t AUX_INTERRUPT_CONTROL;
+	uint32_t AUX_SW_STATUS;
+	uint32_t AUXN_IMPCAL;
+	uint32_t AUXP_IMPCAL;
+
+	uint32_t AUX_RESET_MASK;
+};
+
+enum {	/* This is the timeout as defined in DP 1.2a,
+	 * 2.3.4 "Detailed uPacket TX AUX CH State Description".
+	 */
+	AUX_TIMEOUT_PERIOD = 400,
+
+	/* Ideally, the SW timeout should be just above 550usec
+	 * which is programmed in HW.
+	 * But the SW timeout of 600usec is not reliable,
+	 * because on some systems, delay_in_microseconds()
+	 * returns faster than it should.
+	 * EPR #379763: by trial-and-error on different systems,
+	 * 700usec is the minimum reliable SW timeout for polling
+	 * the AUX_SW_STATUS.AUX_SW_DONE bit.
+	 * This timeout expires *only* when there is
+	 * AUX Error or AUX Timeout conditions - not during normal operation.
+	 * During normal operation, AUX_SW_STATUS.AUX_SW_DONE bit is set
+	 * at most within ~240usec. That means,
+	 * increasing this timeout will not affect normal operation,
+	 * and we'll timeout after
+	 * SW_AUX_TIMEOUT_PERIOD_MULTIPLIER * AUX_TIMEOUT_PERIOD = 1600usec.
+	 * This timeout is especially important for
+	 * resume from S3 and CTS.
+	 */
+	SW_AUX_TIMEOUT_PERIOD_MULTIPLIER = 4
+};
+struct aux_engine_dce110 {
+	struct aux_engine base;
+	const struct dce110_aux_registers *regs;
+	struct {
+		uint32_t aux_control;
+		uint32_t aux_arb_control;
+		uint32_t aux_sw_data;
+		uint32_t aux_sw_control;
+		uint32_t aux_interrupt_control;
+		uint32_t aux_sw_status;
+	} addr;
+	uint32_t timeout_period;
+};
+
+struct aux_engine_dce110_init_data {
+	uint32_t engine_id;
+	uint32_t timeout_period;
+	struct dc_context *ctx;
+	const struct dce110_aux_registers *regs;
+};
+
+struct aux_engine *dce110_aux_engine_construct(
+		struct aux_engine_dce110 *aux_engine110,
+		struct dc_context *ctx,
+		uint32_t inst,
+		uint32_t timeout_period,
+		const struct dce110_aux_registers *regs);
+
+void dce110_engine_destroy(struct aux_engine **engine);
+
+bool dce110_aux_engine_acquire(
+	struct aux_engine *aux_engine,
+	struct ddc *ddc);
+#endif
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.c b/drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.c
index 88b09dd758ba..ca137757a69e 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.c
@@ -133,7 +133,7 @@ static bool calculate_fb_and_fractional_fb_divider(
 	uint64_t feedback_divider;
 
 	feedback_divider =
-		(uint64_t)(target_pix_clk_khz * ref_divider * post_divider);
+		(uint64_t)target_pix_clk_khz * ref_divider * post_divider;
 	feedback_divider *= 10;
 	/* additional factor, since we divide by 10 afterwards */
 	feedback_divider *= (uint64_t)(calc_pll_cs->fract_fb_divider_factor);
@@ -145,8 +145,8 @@ static bool calculate_fb_and_fractional_fb_divider(
  * of fractional feedback decimal point and the fractional FB Divider precision
  * is 2 then the equation becomes (ullfeedbackDivider + 5*100) / (10*100))*/
 
-	feedback_divider += (uint64_t)
-			(5 * calc_pll_cs->fract_fb_divider_precision_factor);
+	feedback_divider += 5ULL *
+			    calc_pll_cs->fract_fb_divider_precision_factor;
 	feedback_divider =
 		div_u64(feedback_divider,
 			calc_pll_cs->fract_fb_divider_precision_factor * 10);
@@ -203,8 +203,8 @@ static bool calc_fb_divider_checking_tolerance(
 			&fract_feedback_divider);
 
 	/*Actual calculated value*/
-	actual_calc_clk_khz = (uint64_t)(feedback_divider *
-					calc_pll_cs->fract_fb_divider_factor) +
+	actual_calc_clk_khz = (uint64_t)feedback_divider *
+					calc_pll_cs->fract_fb_divider_factor +
 							fract_feedback_divider;
 	actual_calc_clk_khz *= calc_pll_cs->ref_freq_khz;
 	actual_calc_clk_khz =
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_clocks.c b/drivers/gpu/drm/amd/display/dc/dce/dce_clocks.c
index 8a581c67bf2d..fb1f373d08a1 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_clocks.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_clocks.c
@@ -38,7 +38,7 @@
 #include "dal_asic_id.h"
 
 #define TO_DCE_CLOCKS(clocks)\
-	container_of(clocks, struct dce_disp_clk, base)
+	container_of(clocks, struct dce_dccg, base)
 
 #define REG(reg) \
 	(clk_dce->regs->reg)
@@ -101,99 +101,84 @@ static const struct state_dependent_clocks dce120_max_clks_by_state[] = {
 /*ClocksStatePerformance*/
 { .display_clk_khz = 1133000, .pixel_clk_khz = 600000 } };
 
-/* Starting point for each divider range.*/
-enum dce_divider_range_start {
-	DIVIDER_RANGE_01_START = 200, /* 2.00*/
-	DIVIDER_RANGE_02_START = 1600, /* 16.00*/
-	DIVIDER_RANGE_03_START = 3200, /* 32.00*/
-	DIVIDER_RANGE_SCALE_FACTOR = 100 /* Results are scaled up by 100.*/
+/* Starting DID for each range */
+enum dentist_base_divider_id {
+	DENTIST_BASE_DID_1 = 0x08,
+	DENTIST_BASE_DID_2 = 0x40,
+	DENTIST_BASE_DID_3 = 0x60,
+	DENTIST_BASE_DID_4 = 0x7e,
+	DENTIST_MAX_DID = 0x7f
 };
 
-/* Ranges for divider identifiers (Divider ID or DID)
- mmDENTIST_DISPCLK_CNTL.DENTIST_DISPCLK_WDIVIDER*/
-enum dce_divider_id_register_setting {
-	DIVIDER_RANGE_01_BASE_DIVIDER_ID = 0X08,
-	DIVIDER_RANGE_02_BASE_DIVIDER_ID = 0X40,
-	DIVIDER_RANGE_03_BASE_DIVIDER_ID = 0X60,
-	DIVIDER_RANGE_MAX_DIVIDER_ID = 0X80
+/* Starting point and step size for each divider range.*/
+enum dentist_divider_range {
+	DENTIST_DIVIDER_RANGE_1_START = 8,   /* 2.00  */
+	DENTIST_DIVIDER_RANGE_1_STEP  = 1,   /* 0.25  */
+	DENTIST_DIVIDER_RANGE_2_START = 64,  /* 16.00 */
+	DENTIST_DIVIDER_RANGE_2_STEP  = 2,   /* 0.50  */
+	DENTIST_DIVIDER_RANGE_3_START = 128, /* 32.00 */
+	DENTIST_DIVIDER_RANGE_3_STEP  = 4,   /* 1.00  */
+	DENTIST_DIVIDER_RANGE_4_START = 248, /* 62.00 */
+	DENTIST_DIVIDER_RANGE_4_STEP  = 264, /* 66.00 */
+	DENTIST_DIVIDER_RANGE_SCALE_FACTOR = 4
 };
 
-/* Step size between each divider within a range.
- Incrementing the DENTIST_DISPCLK_WDIVIDER by one
- will increment the divider by this much.*/
-enum dce_divider_range_step_size {
-	DIVIDER_RANGE_01_STEP_SIZE = 25, /* 0.25*/
-	DIVIDER_RANGE_02_STEP_SIZE = 50, /* 0.50*/
-	DIVIDER_RANGE_03_STEP_SIZE = 100 /* 1.00 */
-};
-
-static bool dce_divider_range_construct(
-	struct dce_divider_range *div_range,
-	int range_start,
-	int range_step,
-	int did_min,
-	int did_max)
+static int dentist_get_divider_from_did(int did)
 {
-	div_range->div_range_start = range_start;
-	div_range->div_range_step = range_step;
-	div_range->did_min = did_min;
-	div_range->did_max = did_max;
-
-	if (div_range->div_range_step == 0) {
-		div_range->div_range_step = 1;
-		/*div_range_step cannot be zero*/
-		BREAK_TO_DEBUGGER();
+	if (did < DENTIST_BASE_DID_1)
+		did = DENTIST_BASE_DID_1;
+	if (did > DENTIST_MAX_DID)
+		did = DENTIST_MAX_DID;
+
+	if (did < DENTIST_BASE_DID_2) {
+		return DENTIST_DIVIDER_RANGE_1_START + DENTIST_DIVIDER_RANGE_1_STEP
+							* (did - DENTIST_BASE_DID_1);
+	} else if (did < DENTIST_BASE_DID_3) {
+		return DENTIST_DIVIDER_RANGE_2_START + DENTIST_DIVIDER_RANGE_2_STEP
+							* (did - DENTIST_BASE_DID_2);
+	} else if (did < DENTIST_BASE_DID_4) {
+		return DENTIST_DIVIDER_RANGE_3_START + DENTIST_DIVIDER_RANGE_3_STEP
+							* (did - DENTIST_BASE_DID_3);
+	} else {
+		return DENTIST_DIVIDER_RANGE_4_START + DENTIST_DIVIDER_RANGE_4_STEP
+							* (did - DENTIST_BASE_DID_4);
 	}
-	/* Calculate this based on the other inputs.*/
-	/* See DividerRange.h for explanation of */
-	/* the relationship between divider id (DID) and a divider.*/
-	/* Number of Divider IDs = (Maximum Divider ID - Minimum Divider ID)*/
-	/* Maximum divider identified in this range =
-	 * (Number of Divider IDs)*Step size between dividers
-	 *  + The start of this range.*/
-	div_range->div_range_end = (did_max - did_min) * range_step
-		+ range_start;
-	return true;
-}
-
-static int dce_divider_range_calc_divider(
-	struct dce_divider_range *div_range,
-	int did)
-{
-	/* Is this DID within our range?*/
-	if ((did < div_range->did_min) || (did >= div_range->did_max))
-		return INVALID_DIVIDER;
-
-	return ((did - div_range->did_min) * div_range->div_range_step)
-			+ div_range->div_range_start;
-
 }
 
-static int dce_divider_range_get_divider(
-	struct dce_divider_range *div_range,
-	int ranges_num,
-	int did)
+/* SW will adjust DP REF Clock average value for all purposes
+ * (DP DTO / DP Audio DTO and DP GTC)
+ if clock is spread for all cases:
+ -if SS enabled on DP Ref clock and HW de-spreading enabled with SW
+ calculations for DS_INCR/DS_MODULO (this is planned to be default case)
+ -if SS enabled on DP Ref clock and HW de-spreading enabled with HW
+ calculations (not planned to be used, but average clock should still
+ be valid)
+ -if SS enabled on DP Ref clock and HW de-spreading disabled
+ (should not be case with CIK) then SW should program all rates
+ generated according to average value (case as with previous ASICs)
+  */
+static int dccg_adjust_dp_ref_freq_for_ss(struct dce_dccg *clk_dce, int dp_ref_clk_khz)
 {
-	int div = INVALID_DIVIDER;
-	int i;
+	if (clk_dce->ss_on_dprefclk && clk_dce->dprefclk_ss_divider != 0) {
+		struct fixed31_32 ss_percentage = dc_fixpt_div_int(
+				dc_fixpt_from_fraction(clk_dce->dprefclk_ss_percentage,
+							clk_dce->dprefclk_ss_divider), 200);
+		struct fixed31_32 adj_dp_ref_clk_khz;
 
-	for (i = 0; i < ranges_num; i++) {
-		/* Calculate divider with given divider ID*/
-		div = dce_divider_range_calc_divider(&div_range[i], did);
-		/* Found a valid return divider*/
-		if (div != INVALID_DIVIDER)
-			break;
+		ss_percentage = dc_fixpt_sub(dc_fixpt_one, ss_percentage);
+		adj_dp_ref_clk_khz = dc_fixpt_mul_int(ss_percentage, dp_ref_clk_khz);
+		dp_ref_clk_khz = dc_fixpt_floor(adj_dp_ref_clk_khz);
 	}
-	return div;
+	return dp_ref_clk_khz;
 }
 
-static int dce_clocks_get_dp_ref_freq(struct display_clock *clk)
+static int dce_get_dp_ref_freq_khz(struct dccg *clk)
 {
-	struct dce_disp_clk *clk_dce = TO_DCE_CLOCKS(clk);
+	struct dce_dccg *clk_dce = TO_DCE_CLOCKS(clk);
 	int dprefclk_wdivider;
 	int dprefclk_src_sel;
 	int dp_ref_clk_khz = 600000;
-	int target_div = INVALID_DIVIDER;
+	int target_div;
 
 	/* ASSERT DP Reference Clock source is from DFS*/
 	REG_GET(DPREFCLK_CNTL, DPREFCLK_SRC_SEL, &dprefclk_src_sel);
@@ -204,80 +189,27 @@ static int dce_clocks_get_dp_ref_freq(struct display_clock *clk)
 	REG_GET(DENTIST_DISPCLK_CNTL, DENTIST_DPREFCLK_WDIVIDER, &dprefclk_wdivider);
 
 	/* Convert DENTIST_DPREFCLK_WDIVIDERto actual divider*/
-	target_div = dce_divider_range_get_divider(
-			clk_dce->divider_ranges,
-			DIVIDER_RANGE_MAX,
-			dprefclk_wdivider);
-
-	if (target_div != INVALID_DIVIDER) {
-		/* Calculate the current DFS clock, in kHz.*/
-		dp_ref_clk_khz = (DIVIDER_RANGE_SCALE_FACTOR
-			* clk_dce->dentist_vco_freq_khz) / target_div;
-	}
+	target_div = dentist_get_divider_from_did(dprefclk_wdivider);
 
-	/* SW will adjust DP REF Clock average value for all purposes
-	 * (DP DTO / DP Audio DTO and DP GTC)
-	 if clock is spread for all cases:
-	 -if SS enabled on DP Ref clock and HW de-spreading enabled with SW
-	 calculations for DS_INCR/DS_MODULO (this is planned to be default case)
-	 -if SS enabled on DP Ref clock and HW de-spreading enabled with HW
-	 calculations (not planned to be used, but average clock should still
-	 be valid)
-	 -if SS enabled on DP Ref clock and HW de-spreading disabled
-	 (should not be case with CIK) then SW should program all rates
-	 generated according to average value (case as with previous ASICs)
-	  */
-	if (clk_dce->ss_on_dprefclk && clk_dce->dprefclk_ss_divider != 0) {
-		struct fixed31_32 ss_percentage = dc_fixpt_div_int(
-				dc_fixpt_from_fraction(
-						clk_dce->dprefclk_ss_percentage,
-						clk_dce->dprefclk_ss_divider), 200);
-		struct fixed31_32 adj_dp_ref_clk_khz;
+	/* Calculate the current DFS clock, in kHz.*/
+	dp_ref_clk_khz = (DENTIST_DIVIDER_RANGE_SCALE_FACTOR
+		* clk_dce->dentist_vco_freq_khz) / target_div;
 
-		ss_percentage = dc_fixpt_sub(dc_fixpt_one,
-								ss_percentage);
-		adj_dp_ref_clk_khz =
-			dc_fixpt_mul_int(
-				ss_percentage,
-				dp_ref_clk_khz);
-		dp_ref_clk_khz = dc_fixpt_floor(adj_dp_ref_clk_khz);
-	}
-
-	return dp_ref_clk_khz;
+	return dccg_adjust_dp_ref_freq_for_ss(clk_dce, dp_ref_clk_khz);
 }
 
-/* TODO: This is DCN DPREFCLK: it could be program by DENTIST by VBIOS
- * or CLK0_CLK11 by SMU. For DCE120, it is wlays 600Mhz. Will re-visit
- * clock implementation
- */
-static int dce_clocks_get_dp_ref_freq_wrkaround(struct display_clock *clk)
+static int dce12_get_dp_ref_freq_khz(struct dccg *clk)
 {
-	struct dce_disp_clk *clk_dce = TO_DCE_CLOCKS(clk);
-	int dp_ref_clk_khz = 600000;
-
-	if (clk_dce->ss_on_dprefclk && clk_dce->dprefclk_ss_divider != 0) {
-		struct fixed31_32 ss_percentage = dc_fixpt_div_int(
-				dc_fixpt_from_fraction(
-						clk_dce->dprefclk_ss_percentage,
-						clk_dce->dprefclk_ss_divider), 200);
-		struct fixed31_32 adj_dp_ref_clk_khz;
+	struct dce_dccg *clk_dce = TO_DCE_CLOCKS(clk);
 
-		ss_percentage = dc_fixpt_sub(dc_fixpt_one,
-								ss_percentage);
-		adj_dp_ref_clk_khz =
-			dc_fixpt_mul_int(
-				ss_percentage,
-				dp_ref_clk_khz);
-		dp_ref_clk_khz = dc_fixpt_floor(adj_dp_ref_clk_khz);
-	}
-
-	return dp_ref_clk_khz;
+	return dccg_adjust_dp_ref_freq_for_ss(clk_dce, 600000);
 }
+
 static enum dm_pp_clocks_state dce_get_required_clocks_state(
-	struct display_clock *clk,
-	struct state_dependent_clocks *req_clocks)
+	struct dccg *clk,
+	struct dc_clocks *req_clocks)
 {
-	struct dce_disp_clk *clk_dce = TO_DCE_CLOCKS(clk);
+	struct dce_dccg *clk_dce = TO_DCE_CLOCKS(clk);
 	int i;
 	enum dm_pp_clocks_state low_req_clk;
 
@@ -286,53 +218,30 @@ static enum dm_pp_clocks_state dce_get_required_clocks_state(
 	 * all required clocks
 	 */
 	for (i = clk->max_clks_state; i >= DM_PP_CLOCKS_STATE_ULTRA_LOW; i--)
-		if (req_clocks->display_clk_khz >
+		if (req_clocks->dispclk_khz >
 				clk_dce->max_clks_by_state[i].display_clk_khz
-			|| req_clocks->pixel_clk_khz >
+			|| req_clocks->phyclk_khz >
 				clk_dce->max_clks_by_state[i].pixel_clk_khz)
 			break;
 
 	low_req_clk = i + 1;
 	if (low_req_clk > clk->max_clks_state) {
-		DC_LOG_WARNING("%s: clocks unsupported disp_clk %d pix_clk %d",
-				__func__,
-				req_clocks->display_clk_khz,
-				req_clocks->pixel_clk_khz);
-		low_req_clk = DM_PP_CLOCKS_STATE_INVALID;
+		/* set max clock state for high phyclock, invalid on exceeding display clock */
+		if (clk_dce->max_clks_by_state[clk->max_clks_state].display_clk_khz
+				< req_clocks->dispclk_khz)
+			low_req_clk = DM_PP_CLOCKS_STATE_INVALID;
+		else
+			low_req_clk = clk->max_clks_state;
 	}
 
 	return low_req_clk;
 }
 
-static bool dce_clock_set_min_clocks_state(
-	struct display_clock *clk,
-	enum dm_pp_clocks_state clocks_state)
-{
-	struct dm_pp_power_level_change_request level_change_req = {
-			clocks_state };
-
-	if (clocks_state > clk->max_clks_state) {
-		/*Requested state exceeds max supported state.*/
-		DC_LOG_WARNING("Requested state exceeds max supported state");
-		return false;
-	} else if (clocks_state == clk->cur_min_clks_state) {
-		/*if we're trying to set the same state, we can just return
-		 * since nothing needs to be done*/
-		return true;
-	}
-
-	/* get max clock state from PPLIB */
-	if (dm_pp_apply_power_level_change_request(clk->ctx, &level_change_req))
-		clk->cur_min_clks_state = clocks_state;
-
-	return true;
-}
-
 static int dce_set_clock(
-	struct display_clock *clk,
+	struct dccg *clk,
 	int requested_clk_khz)
 {
-	struct dce_disp_clk *clk_dce = TO_DCE_CLOCKS(clk);
+	struct dce_dccg *clk_dce = TO_DCE_CLOCKS(clk);
 	struct bp_pixel_clock_parameters pxl_clk_params = { 0 };
 	struct dc_bios *bp = clk->ctx->dc_bios;
 	int actual_clock = requested_clk_khz;
@@ -364,10 +273,10 @@ static int dce_set_clock(
 }
 
 static int dce_psr_set_clock(
-	struct display_clock *clk,
+	struct dccg *clk,
 	int requested_clk_khz)
 {
-	struct dce_disp_clk *clk_dce = TO_DCE_CLOCKS(clk);
+	struct dce_dccg *clk_dce = TO_DCE_CLOCKS(clk);
 	struct dc_context *ctx = clk_dce->base.ctx;
 	struct dc *core_dc = ctx->dc;
 	struct dmcu *dmcu = core_dc->res_pool->dmcu;
@@ -380,10 +289,10 @@ static int dce_psr_set_clock(
 }
 
 static int dce112_set_clock(
-	struct display_clock *clk,
+	struct dccg *clk,
 	int requested_clk_khz)
 {
-	struct dce_disp_clk *clk_dce = TO_DCE_CLOCKS(clk);
+	struct dce_dccg *clk_dce = TO_DCE_CLOCKS(clk);
 	struct bp_set_dce_clock_parameters dce_clk_params;
 	struct dc_bios *bp = clk->ctx->dc_bios;
 	struct dc *core_dc = clk->ctx->dc;
@@ -432,9 +341,9 @@ static int dce112_set_clock(
 	return actual_clock;
 }
 
-static void dce_clock_read_integrated_info(struct dce_disp_clk *clk_dce)
+static void dce_clock_read_integrated_info(struct dce_dccg *clk_dce)
 {
-	struct dc_debug *debug = &clk_dce->base.ctx->dc->debug;
+	struct dc_debug_options *debug = &clk_dce->base.ctx->dc->debug;
 	struct dc_bios *bp = clk_dce->base.ctx->dc_bios;
 	struct integrated_info info = { { { 0 } } };
 	struct dc_firmware_info fw_info = { { 0 } };
@@ -488,11 +397,9 @@ static void dce_clock_read_integrated_info(struct dce_disp_clk *clk_dce)
 	if (!debug->disable_dfs_bypass && bp->integrated_info)
 		if (bp->integrated_info->gpu_cap_info & DFS_BYPASS_ENABLE)
 			clk_dce->dfs_bypass_enabled = true;
-
-	clk_dce->use_max_disp_clk = debug->max_disp_clk;
 }
 
-static void dce_clock_read_ss_info(struct dce_disp_clk *clk_dce)
+static void dce_clock_read_ss_info(struct dce_dccg *clk_dce)
 {
 	struct dc_bios *bp = clk_dce->base.ctx->dc_bios;
 	int ss_info_num = bp->funcs->get_ss_entry_number(
@@ -548,139 +455,265 @@ static void dce_clock_read_ss_info(struct dce_disp_clk *clk_dce)
 	}
 }
 
-static bool dce_apply_clock_voltage_request(
-	struct display_clock *clk,
-	enum dm_pp_clock_type clocks_type,
-	int clocks_in_khz,
-	bool pre_mode_set,
-	bool update_dp_phyclk)
+static inline bool should_set_clock(bool safe_to_lower, int calc_clk, int cur_clk)
+{
+	return ((safe_to_lower && calc_clk < cur_clk) || calc_clk > cur_clk);
+}
+
+static void dce12_update_clocks(struct dccg *dccg,
+			struct dc_clocks *new_clocks,
+			bool safe_to_lower)
 {
-	bool send_request = false;
 	struct dm_pp_clock_for_voltage_req clock_voltage_req = {0};
 
-	switch (clocks_type) {
-	case DM_PP_CLOCK_TYPE_DISPLAY_CLK:
-	case DM_PP_CLOCK_TYPE_PIXELCLK:
-	case DM_PP_CLOCK_TYPE_DISPLAYPHYCLK:
-		break;
-	default:
-		BREAK_TO_DEBUGGER();
-		return false;
+	if (should_set_clock(safe_to_lower, new_clocks->dispclk_khz, dccg->clks.dispclk_khz)) {
+		clock_voltage_req.clk_type = DM_PP_CLOCK_TYPE_DISPLAY_CLK;
+		clock_voltage_req.clocks_in_khz = new_clocks->dispclk_khz;
+		new_clocks->dispclk_khz = dccg->funcs->set_dispclk(dccg, new_clocks->dispclk_khz);
+		dccg->clks.dispclk_khz = new_clocks->dispclk_khz;
+
+		dm_pp_apply_clock_for_voltage_request(dccg->ctx, &clock_voltage_req);
 	}
 
-	clock_voltage_req.clk_type = clocks_type;
-	clock_voltage_req.clocks_in_khz = clocks_in_khz;
-
-	/* to pplib */
-	if (pre_mode_set) {
-		switch (clocks_type) {
-		case DM_PP_CLOCK_TYPE_DISPLAY_CLK:
-			if (clocks_in_khz > clk->cur_clocks_value.dispclk_in_khz) {
-				clk->cur_clocks_value.dispclk_notify_pplib_done = true;
-				send_request = true;
-			} else
-				clk->cur_clocks_value.dispclk_notify_pplib_done = false;
-			/* no matter incrase or decrase clock, update current clock value */
-			clk->cur_clocks_value.dispclk_in_khz = clocks_in_khz;
-			break;
-		case DM_PP_CLOCK_TYPE_PIXELCLK:
-			if (clocks_in_khz > clk->cur_clocks_value.max_pixelclk_in_khz) {
-				clk->cur_clocks_value.pixelclk_notify_pplib_done = true;
-				send_request = true;
-			} else
-				clk->cur_clocks_value.pixelclk_notify_pplib_done = false;
-			/* no matter incrase or decrase clock, update current clock value */
-			clk->cur_clocks_value.max_pixelclk_in_khz = clocks_in_khz;
-			break;
-		case DM_PP_CLOCK_TYPE_DISPLAYPHYCLK:
-			if (clocks_in_khz > clk->cur_clocks_value.max_non_dp_phyclk_in_khz) {
-				clk->cur_clocks_value.phyclk_notigy_pplib_done = true;
-				send_request = true;
-			} else
-				clk->cur_clocks_value.phyclk_notigy_pplib_done = false;
-			/* no matter incrase or decrase clock, update current clock value */
-			clk->cur_clocks_value.max_non_dp_phyclk_in_khz = clocks_in_khz;
-			break;
-		default:
-			ASSERT(0);
-			break;
-		}
+	if (should_set_clock(safe_to_lower, new_clocks->phyclk_khz, dccg->clks.phyclk_khz)) {
+		clock_voltage_req.clk_type = DM_PP_CLOCK_TYPE_DISPLAYPHYCLK;
+		clock_voltage_req.clocks_in_khz = new_clocks->phyclk_khz;
+		dccg->clks.phyclk_khz = new_clocks->phyclk_khz;
+
+		dm_pp_apply_clock_for_voltage_request(dccg->ctx, &clock_voltage_req);
+	}
+}
+
+#ifdef CONFIG_DRM_AMD_DC_DCN1_0
+static int dcn1_determine_dppclk_threshold(struct dccg *dccg, struct dc_clocks *new_clocks)
+{
+	bool request_dpp_div = new_clocks->dispclk_khz > new_clocks->dppclk_khz;
+	bool dispclk_increase = new_clocks->dispclk_khz > dccg->clks.dispclk_khz;
+	int disp_clk_threshold = new_clocks->max_supported_dppclk_khz;
+	bool cur_dpp_div = dccg->clks.dispclk_khz > dccg->clks.dppclk_khz;
+
+	/* increase clock, looking for div is 0 for current, request div is 1*/
+	if (dispclk_increase) {
+		/* already divided by 2, no need to reach target clk with 2 steps*/
+		if (cur_dpp_div)
+			return new_clocks->dispclk_khz;
+
+		/* request disp clk is lower than maximum supported dpp clk,
+		 * no need to reach target clk with two steps.
+		 */
+		if (new_clocks->dispclk_khz <= disp_clk_threshold)
+			return new_clocks->dispclk_khz;
+
+		/* target dpp clk not request divided by 2, still within threshold */
+		if (!request_dpp_div)
+			return new_clocks->dispclk_khz;
 
 	} else {
-		switch (clocks_type) {
-		case DM_PP_CLOCK_TYPE_DISPLAY_CLK:
-			if (!clk->cur_clocks_value.dispclk_notify_pplib_done)
-				send_request = true;
-			break;
-		case DM_PP_CLOCK_TYPE_PIXELCLK:
-			if (!clk->cur_clocks_value.pixelclk_notify_pplib_done)
-				send_request = true;
-			break;
-		case DM_PP_CLOCK_TYPE_DISPLAYPHYCLK:
-			if (!clk->cur_clocks_value.phyclk_notigy_pplib_done)
-				send_request = true;
-			break;
-		default:
-			ASSERT(0);
-			break;
-		}
+		/* decrease clock, looking for current dppclk divided by 2,
+		 * request dppclk not divided by 2.
+		 */
+
+		/* current dpp clk not divided by 2, no need to ramp*/
+		if (!cur_dpp_div)
+			return new_clocks->dispclk_khz;
+
+		/* current disp clk is lower than current maximum dpp clk,
+		 * no need to ramp
+		 */
+		if (dccg->clks.dispclk_khz <= disp_clk_threshold)
+			return new_clocks->dispclk_khz;
+
+		/* request dpp clk need to be divided by 2 */
+		if (request_dpp_div)
+			return new_clocks->dispclk_khz;
 	}
-	if (send_request) {
-#if defined(CONFIG_DRM_AMD_DC_DCN1_0)
-		if (clk->ctx->dce_version >= DCN_VERSION_1_0) {
-			struct dc *core_dc = clk->ctx->dc;
-			/*use dcfclk request voltage*/
-			clock_voltage_req.clk_type = DM_PP_CLOCK_TYPE_DCFCLK;
-			clock_voltage_req.clocks_in_khz =
-				dcn_find_dcfclk_suits_all(core_dc, &clk->cur_clocks_value);
-		}
+
+	return disp_clk_threshold;
+}
+
+static void dcn1_ramp_up_dispclk_with_dpp(struct dccg *dccg, struct dc_clocks *new_clocks)
+{
+	struct dc *dc = dccg->ctx->dc;
+	int dispclk_to_dpp_threshold = dcn1_determine_dppclk_threshold(dccg, new_clocks);
+	bool request_dpp_div = new_clocks->dispclk_khz > new_clocks->dppclk_khz;
+	int i;
+
+	/* set disp clk to dpp clk threshold */
+	dccg->funcs->set_dispclk(dccg, dispclk_to_dpp_threshold);
+
+	/* update request dpp clk division option */
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		struct pipe_ctx *pipe_ctx = &dc->current_state->res_ctx.pipe_ctx[i];
+
+		if (!pipe_ctx->plane_state)
+			continue;
+
+		pipe_ctx->plane_res.dpp->funcs->dpp_dppclk_control(
+				pipe_ctx->plane_res.dpp,
+				request_dpp_div,
+				true);
+	}
+
+	/* If target clk not same as dppclk threshold, set to target clock */
+	if (dispclk_to_dpp_threshold != new_clocks->dispclk_khz)
+		dccg->funcs->set_dispclk(dccg, new_clocks->dispclk_khz);
+
+	dccg->clks.dispclk_khz = new_clocks->dispclk_khz;
+	dccg->clks.dppclk_khz = new_clocks->dppclk_khz;
+	dccg->clks.max_supported_dppclk_khz = new_clocks->max_supported_dppclk_khz;
+}
+
+static void dcn1_update_clocks(struct dccg *dccg,
+			struct dc_clocks *new_clocks,
+			bool safe_to_lower)
+{
+	struct dc *dc = dccg->ctx->dc;
+	struct pp_smu_display_requirement_rv *smu_req_cur =
+			&dc->res_pool->pp_smu_req;
+	struct pp_smu_display_requirement_rv smu_req = *smu_req_cur;
+	struct pp_smu_funcs_rv *pp_smu = dc->res_pool->pp_smu;
+	struct dm_pp_clock_for_voltage_req clock_voltage_req = {0};
+	bool send_request_to_increase = false;
+	bool send_request_to_lower = false;
+
+	if (new_clocks->phyclk_khz)
+		smu_req.display_count = 1;
+	else
+		smu_req.display_count = 0;
+
+	if (new_clocks->dispclk_khz > dccg->clks.dispclk_khz
+			|| new_clocks->phyclk_khz > dccg->clks.phyclk_khz
+			|| new_clocks->fclk_khz > dccg->clks.fclk_khz
+			|| new_clocks->dcfclk_khz > dccg->clks.dcfclk_khz)
+		send_request_to_increase = true;
+
+	if (should_set_clock(safe_to_lower, new_clocks->phyclk_khz, dccg->clks.phyclk_khz)) {
+		dccg->clks.phyclk_khz = new_clocks->phyclk_khz;
+
+		send_request_to_lower = true;
+	}
+
+	if (should_set_clock(safe_to_lower, new_clocks->fclk_khz, dccg->clks.fclk_khz)) {
+		dccg->clks.fclk_khz = new_clocks->fclk_khz;
+		clock_voltage_req.clk_type = DM_PP_CLOCK_TYPE_FCLK;
+		clock_voltage_req.clocks_in_khz = new_clocks->fclk_khz;
+		smu_req.hard_min_fclk_khz = new_clocks->fclk_khz;
+
+		dm_pp_apply_clock_for_voltage_request(dccg->ctx, &clock_voltage_req);
+		send_request_to_lower = true;
+	}
+
+	if (should_set_clock(safe_to_lower, new_clocks->dcfclk_khz, dccg->clks.dcfclk_khz)) {
+		dccg->clks.dcfclk_khz = new_clocks->dcfclk_khz;
+		smu_req.hard_min_dcefclk_khz = new_clocks->dcfclk_khz;
+
+		send_request_to_lower = true;
+	}
+
+	if (should_set_clock(safe_to_lower,
+			new_clocks->dcfclk_deep_sleep_khz, dccg->clks.dcfclk_deep_sleep_khz)) {
+		dccg->clks.dcfclk_deep_sleep_khz = new_clocks->dcfclk_deep_sleep_khz;
+		smu_req.min_deep_sleep_dcefclk_mhz = new_clocks->dcfclk_deep_sleep_khz;
+
+		send_request_to_lower = true;
+	}
+
+	/* make sure dcf clk is before dpp clk to
+	 * make sure we have enough voltage to run dpp clk
+	 */
+	if (send_request_to_increase) {
+		/*use dcfclk to request voltage*/
+		clock_voltage_req.clk_type = DM_PP_CLOCK_TYPE_DCFCLK;
+		clock_voltage_req.clocks_in_khz = dcn_find_dcfclk_suits_all(dc, new_clocks);
+		dm_pp_apply_clock_for_voltage_request(dccg->ctx, &clock_voltage_req);
+		if (pp_smu->set_display_requirement)
+			pp_smu->set_display_requirement(&pp_smu->pp_smu, &smu_req);
+	}
+
+	/* dcn1 dppclk is tied to dispclk */
+	/* program dispclk on = as a w/a for sleep resume clock ramping issues */
+	if (should_set_clock(safe_to_lower, new_clocks->dispclk_khz, dccg->clks.dispclk_khz)
+			|| new_clocks->dispclk_khz == dccg->clks.dispclk_khz) {
+		dcn1_ramp_up_dispclk_with_dpp(dccg, new_clocks);
+		dccg->clks.dispclk_khz = new_clocks->dispclk_khz;
+
+		send_request_to_lower = true;
+	}
+
+	if (!send_request_to_increase && send_request_to_lower) {
+		/*use dcfclk to request voltage*/
+		clock_voltage_req.clk_type = DM_PP_CLOCK_TYPE_DCFCLK;
+		clock_voltage_req.clocks_in_khz = dcn_find_dcfclk_suits_all(dc, new_clocks);
+		dm_pp_apply_clock_for_voltage_request(dccg->ctx, &clock_voltage_req);
+		if (pp_smu->set_display_requirement)
+			pp_smu->set_display_requirement(&pp_smu->pp_smu, &smu_req);
+	}
+
+
+	*smu_req_cur = smu_req;
+}
 #endif
-		dm_pp_apply_clock_for_voltage_request(
-			clk->ctx, &clock_voltage_req);
+
+static void dce_update_clocks(struct dccg *dccg,
+			struct dc_clocks *new_clocks,
+			bool safe_to_lower)
+{
+	struct dm_pp_power_level_change_request level_change_req;
+
+	level_change_req.power_level = dce_get_required_clocks_state(dccg, new_clocks);
+	/* get max clock state from PPLIB */
+	if ((level_change_req.power_level < dccg->cur_min_clks_state && safe_to_lower)
+			|| level_change_req.power_level > dccg->cur_min_clks_state) {
+		if (dm_pp_apply_power_level_change_request(dccg->ctx, &level_change_req))
+			dccg->cur_min_clks_state = level_change_req.power_level;
 	}
-	if (update_dp_phyclk && (clocks_in_khz >
-	clk->cur_clocks_value.max_dp_phyclk_in_khz))
-		clk->cur_clocks_value.max_dp_phyclk_in_khz = clocks_in_khz;
 
-	return true;
+	if (should_set_clock(safe_to_lower, new_clocks->dispclk_khz, dccg->clks.dispclk_khz)) {
+		new_clocks->dispclk_khz = dccg->funcs->set_dispclk(dccg, new_clocks->dispclk_khz);
+		dccg->clks.dispclk_khz = new_clocks->dispclk_khz;
+	}
 }
 
+#ifdef CONFIG_DRM_AMD_DC_DCN1_0
+static const struct display_clock_funcs dcn1_funcs = {
+	.get_dp_ref_clk_frequency = dce12_get_dp_ref_freq_khz,
+	.set_dispclk = dce112_set_clock,
+	.update_clocks = dcn1_update_clocks
+};
+#endif
 
 static const struct display_clock_funcs dce120_funcs = {
-	.get_dp_ref_clk_frequency = dce_clocks_get_dp_ref_freq_wrkaround,
-	.apply_clock_voltage_request = dce_apply_clock_voltage_request,
-	.set_clock = dce112_set_clock
+	.get_dp_ref_clk_frequency = dce12_get_dp_ref_freq_khz,
+	.set_dispclk = dce112_set_clock,
+	.update_clocks = dce12_update_clocks
 };
 
 static const struct display_clock_funcs dce112_funcs = {
-	.get_dp_ref_clk_frequency = dce_clocks_get_dp_ref_freq,
-	.get_required_clocks_state = dce_get_required_clocks_state,
-	.set_min_clocks_state = dce_clock_set_min_clocks_state,
-	.set_clock = dce112_set_clock
+	.get_dp_ref_clk_frequency = dce_get_dp_ref_freq_khz,
+	.set_dispclk = dce112_set_clock,
+	.update_clocks = dce_update_clocks
 };
 
 static const struct display_clock_funcs dce110_funcs = {
-	.get_dp_ref_clk_frequency = dce_clocks_get_dp_ref_freq,
-	.get_required_clocks_state = dce_get_required_clocks_state,
-	.set_min_clocks_state = dce_clock_set_min_clocks_state,
-	.set_clock = dce_psr_set_clock
+	.get_dp_ref_clk_frequency = dce_get_dp_ref_freq_khz,
+	.set_dispclk = dce_psr_set_clock,
+	.update_clocks = dce_update_clocks
 };
 
 static const struct display_clock_funcs dce_funcs = {
-	.get_dp_ref_clk_frequency = dce_clocks_get_dp_ref_freq,
-	.get_required_clocks_state = dce_get_required_clocks_state,
-	.set_min_clocks_state = dce_clock_set_min_clocks_state,
-	.set_clock = dce_set_clock
+	.get_dp_ref_clk_frequency = dce_get_dp_ref_freq_khz,
+	.set_dispclk = dce_set_clock,
+	.update_clocks = dce_update_clocks
 };
 
-static void dce_disp_clk_construct(
-	struct dce_disp_clk *clk_dce,
+static void dce_dccg_construct(
+	struct dce_dccg *clk_dce,
 	struct dc_context *ctx,
-	const struct dce_disp_clk_registers *regs,
-	const struct dce_disp_clk_shift *clk_shift,
-	const struct dce_disp_clk_mask *clk_mask)
+	const struct dccg_registers *regs,
+	const struct dccg_shift *clk_shift,
+	const struct dccg_mask *clk_mask)
 {
-	struct display_clock *base = &clk_dce->base;
+	struct dccg *base = &clk_dce->base;
 
 	base->ctx = ctx;
 	base->funcs = &dce_funcs;
@@ -700,34 +733,15 @@ static void dce_disp_clk_construct(
 
 	dce_clock_read_integrated_info(clk_dce);
 	dce_clock_read_ss_info(clk_dce);
-
-	dce_divider_range_construct(
-		&clk_dce->divider_ranges[DIVIDER_RANGE_01],
-		DIVIDER_RANGE_01_START,
-		DIVIDER_RANGE_01_STEP_SIZE,
-		DIVIDER_RANGE_01_BASE_DIVIDER_ID,
-		DIVIDER_RANGE_02_BASE_DIVIDER_ID);
-	dce_divider_range_construct(
-		&clk_dce->divider_ranges[DIVIDER_RANGE_02],
-		DIVIDER_RANGE_02_START,
-		DIVIDER_RANGE_02_STEP_SIZE,
-		DIVIDER_RANGE_02_BASE_DIVIDER_ID,
-		DIVIDER_RANGE_03_BASE_DIVIDER_ID);
-	dce_divider_range_construct(
-		&clk_dce->divider_ranges[DIVIDER_RANGE_03],
-		DIVIDER_RANGE_03_START,
-		DIVIDER_RANGE_03_STEP_SIZE,
-		DIVIDER_RANGE_03_BASE_DIVIDER_ID,
-		DIVIDER_RANGE_MAX_DIVIDER_ID);
 }
 
-struct display_clock *dce_disp_clk_create(
+struct dccg *dce_dccg_create(
 	struct dc_context *ctx,
-	const struct dce_disp_clk_registers *regs,
-	const struct dce_disp_clk_shift *clk_shift,
-	const struct dce_disp_clk_mask *clk_mask)
+	const struct dccg_registers *regs,
+	const struct dccg_shift *clk_shift,
+	const struct dccg_mask *clk_mask)
 {
-	struct dce_disp_clk *clk_dce = kzalloc(sizeof(*clk_dce), GFP_KERNEL);
+	struct dce_dccg *clk_dce = kzalloc(sizeof(*clk_dce), GFP_KERNEL);
 
 	if (clk_dce == NULL) {
 		BREAK_TO_DEBUGGER();
@@ -738,19 +752,19 @@ struct display_clock *dce_disp_clk_create(
 		dce80_max_clks_by_state,
 		sizeof(dce80_max_clks_by_state));
 
-	dce_disp_clk_construct(
+	dce_dccg_construct(
 		clk_dce, ctx, regs, clk_shift, clk_mask);
 
 	return &clk_dce->base;
 }
 
-struct display_clock *dce110_disp_clk_create(
+struct dccg *dce110_dccg_create(
 	struct dc_context *ctx,
-	const struct dce_disp_clk_registers *regs,
-	const struct dce_disp_clk_shift *clk_shift,
-	const struct dce_disp_clk_mask *clk_mask)
+	const struct dccg_registers *regs,
+	const struct dccg_shift *clk_shift,
+	const struct dccg_mask *clk_mask)
 {
-	struct dce_disp_clk *clk_dce = kzalloc(sizeof(*clk_dce), GFP_KERNEL);
+	struct dce_dccg *clk_dce = kzalloc(sizeof(*clk_dce), GFP_KERNEL);
 
 	if (clk_dce == NULL) {
 		BREAK_TO_DEBUGGER();
@@ -761,7 +775,7 @@ struct display_clock *dce110_disp_clk_create(
 		dce110_max_clks_by_state,
 		sizeof(dce110_max_clks_by_state));
 
-	dce_disp_clk_construct(
+	dce_dccg_construct(
 		clk_dce, ctx, regs, clk_shift, clk_mask);
 
 	clk_dce->base.funcs = &dce110_funcs;
@@ -769,13 +783,13 @@ struct display_clock *dce110_disp_clk_create(
 	return &clk_dce->base;
 }
 
-struct display_clock *dce112_disp_clk_create(
+struct dccg *dce112_dccg_create(
 	struct dc_context *ctx,
-	const struct dce_disp_clk_registers *regs,
-	const struct dce_disp_clk_shift *clk_shift,
-	const struct dce_disp_clk_mask *clk_mask)
+	const struct dccg_registers *regs,
+	const struct dccg_shift *clk_shift,
+	const struct dccg_mask *clk_mask)
 {
-	struct dce_disp_clk *clk_dce = kzalloc(sizeof(*clk_dce), GFP_KERNEL);
+	struct dce_dccg *clk_dce = kzalloc(sizeof(*clk_dce), GFP_KERNEL);
 
 	if (clk_dce == NULL) {
 		BREAK_TO_DEBUGGER();
@@ -786,7 +800,7 @@ struct display_clock *dce112_disp_clk_create(
 		dce112_max_clks_by_state,
 		sizeof(dce112_max_clks_by_state));
 
-	dce_disp_clk_construct(
+	dce_dccg_construct(
 		clk_dce, ctx, regs, clk_shift, clk_mask);
 
 	clk_dce->base.funcs = &dce112_funcs;
@@ -794,10 +808,9 @@ struct display_clock *dce112_disp_clk_create(
 	return &clk_dce->base;
 }
 
-struct display_clock *dce120_disp_clk_create(struct dc_context *ctx)
+struct dccg *dce120_dccg_create(struct dc_context *ctx)
 {
-	struct dce_disp_clk *clk_dce = kzalloc(sizeof(*clk_dce), GFP_KERNEL);
-	struct dm_pp_clock_levels_with_voltage clk_level_info = {0};
+	struct dce_dccg *clk_dce = kzalloc(sizeof(*clk_dce), GFP_KERNEL);
 
 	if (clk_dce == NULL) {
 		BREAK_TO_DEBUGGER();
@@ -808,28 +821,59 @@ struct display_clock *dce120_disp_clk_create(struct dc_context *ctx)
 		dce120_max_clks_by_state,
 		sizeof(dce120_max_clks_by_state));
 
-	dce_disp_clk_construct(
+	dce_dccg_construct(
 		clk_dce, ctx, NULL, NULL, NULL);
 
 	clk_dce->base.funcs = &dce120_funcs;
 
-	/* new in dce120 */
-	if (!ctx->dc->debug.disable_pplib_clock_request  &&
-			dm_pp_get_clock_levels_by_type_with_voltage(
-			ctx, DM_PP_CLOCK_TYPE_DISPLAY_CLK, &clk_level_info)
-						&& clk_level_info.num_levels)
-		clk_dce->max_displ_clk_in_khz =
-			clk_level_info.data[clk_level_info.num_levels - 1].clocks_in_khz;
-	else
-		clk_dce->max_displ_clk_in_khz = 1133000;
+	return &clk_dce->base;
+}
+
+#ifdef CONFIG_DRM_AMD_DC_DCN1_0
+struct dccg *dcn1_dccg_create(struct dc_context *ctx)
+{
+	struct dc_debug_options *debug = &ctx->dc->debug;
+	struct dc_bios *bp = ctx->dc_bios;
+	struct dc_firmware_info fw_info = { { 0 } };
+	struct dce_dccg *clk_dce = kzalloc(sizeof(*clk_dce), GFP_KERNEL);
+
+	if (clk_dce == NULL) {
+		BREAK_TO_DEBUGGER();
+		return NULL;
+	}
+
+	clk_dce->base.ctx = ctx;
+	clk_dce->base.funcs = &dcn1_funcs;
+
+	clk_dce->dfs_bypass_disp_clk = 0;
+
+	clk_dce->dprefclk_ss_percentage = 0;
+	clk_dce->dprefclk_ss_divider = 1000;
+	clk_dce->ss_on_dprefclk = false;
+
+	if (bp->integrated_info)
+		clk_dce->dentist_vco_freq_khz = bp->integrated_info->dentist_vco_freq;
+	if (clk_dce->dentist_vco_freq_khz == 0) {
+		bp->funcs->get_firmware_info(bp, &fw_info);
+		clk_dce->dentist_vco_freq_khz = fw_info.smu_gpu_pll_output_freq;
+		if (clk_dce->dentist_vco_freq_khz == 0)
+			clk_dce->dentist_vco_freq_khz = 3600000;
+	}
+
+	if (!debug->disable_dfs_bypass && bp->integrated_info)
+		if (bp->integrated_info->gpu_cap_info & DFS_BYPASS_ENABLE)
+			clk_dce->dfs_bypass_enabled = true;
+
+	dce_clock_read_ss_info(clk_dce);
 
 	return &clk_dce->base;
 }
+#endif
 
-void dce_disp_clk_destroy(struct display_clock **disp_clk)
+void dce_dccg_destroy(struct dccg **dccg)
 {
-	struct dce_disp_clk *clk_dce = TO_DCE_CLOCKS(*disp_clk);
+	struct dce_dccg *clk_dce = TO_DCE_CLOCKS(*dccg);
 
 	kfree(clk_dce);
-	*disp_clk = NULL;
+	*dccg = NULL;
 }
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_clocks.h b/drivers/gpu/drm/amd/display/dc/dce/dce_clocks.h
index 0e717e0dc8f0..8a6b2d328467 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_clocks.h
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_clocks.h
@@ -33,6 +33,9 @@
 	.DPREFCLK_CNTL = mmDPREFCLK_CNTL, \
 	.DENTIST_DISPCLK_CNTL = mmDENTIST_DISPCLK_CNTL
 
+#define CLK_COMMON_REG_LIST_DCN_BASE() \
+	SR(DENTIST_DISPCLK_CNTL)
+
 #define CLK_SF(reg_name, field_name, post_fix)\
 	.field_name = reg_name ## __ ## field_name ## post_fix
 
@@ -40,58 +43,37 @@
 	CLK_SF(DPREFCLK_CNTL, DPREFCLK_SRC_SEL, mask_sh), \
 	CLK_SF(DENTIST_DISPCLK_CNTL, DENTIST_DPREFCLK_WDIVIDER, mask_sh)
 
+#define CLK_COMMON_MASK_SH_LIST_DCN_COMMON_BASE(mask_sh) \
+	CLK_SF(DENTIST_DISPCLK_CNTL, DENTIST_DISPCLK_WDIVIDER, mask_sh),\
+	CLK_SF(DENTIST_DISPCLK_CNTL, DENTIST_DISPCLK_CHG_DONE, mask_sh)
+
 #define CLK_REG_FIELD_LIST(type) \
 	type DPREFCLK_SRC_SEL; \
-	type DENTIST_DPREFCLK_WDIVIDER;
+	type DENTIST_DPREFCLK_WDIVIDER; \
+	type DENTIST_DISPCLK_WDIVIDER; \
+	type DENTIST_DISPCLK_CHG_DONE;
 
-struct dce_disp_clk_shift {
+struct dccg_shift {
 	CLK_REG_FIELD_LIST(uint8_t)
 };
 
-struct dce_disp_clk_mask {
+struct dccg_mask {
 	CLK_REG_FIELD_LIST(uint32_t)
 };
 
-struct dce_disp_clk_registers {
+struct dccg_registers {
 	uint32_t DPREFCLK_CNTL;
 	uint32_t DENTIST_DISPCLK_CNTL;
 };
 
-/* Array identifiers and count for the divider ranges.*/
-enum dce_divider_range_count {
-	DIVIDER_RANGE_01 = 0,
-	DIVIDER_RANGE_02,
-	DIVIDER_RANGE_03,
-	DIVIDER_RANGE_MAX /* == 3*/
-};
-
-enum dce_divider_error_types {
-	INVALID_DID = 0,
-	INVALID_DIVIDER = 1
-};
-
-struct dce_divider_range {
-	int div_range_start;
-	/* The end of this range of dividers.*/
-	int div_range_end;
-	/* The distance between each divider in this range.*/
-	int div_range_step;
-	/* The divider id for the lowest divider.*/
-	int did_min;
-	/* The divider id for the highest divider.*/
-	int did_max;
-};
-
-struct dce_disp_clk {
-	struct display_clock base;
-	const struct dce_disp_clk_registers *regs;
-	const struct dce_disp_clk_shift *clk_shift;
-	const struct dce_disp_clk_mask *clk_mask;
+struct dce_dccg {
+	struct dccg base;
+	const struct dccg_registers *regs;
+	const struct dccg_shift *clk_shift;
+	const struct dccg_mask *clk_mask;
 
 	struct state_dependent_clocks max_clks_by_state[DM_PP_CLOCKS_MAX_STATES];
-	struct dce_divider_range divider_ranges[DIVIDER_RANGE_MAX];
 
-	bool use_max_disp_clk;
 	int dentist_vco_freq_khz;
 
 	/* Cache the status of DFS-bypass feature*/
@@ -106,32 +88,33 @@ struct dce_disp_clk {
 	int dprefclk_ss_percentage;
 	/* DPREFCLK SS percentage Divider (100 or 1000) */
 	int dprefclk_ss_divider;
-
-	/* max disp_clk from PPLIB for max validation display clock*/
-	int max_displ_clk_in_khz;
 };
 
 
-struct display_clock *dce_disp_clk_create(
+struct dccg *dce_dccg_create(
 	struct dc_context *ctx,
-	const struct dce_disp_clk_registers *regs,
-	const struct dce_disp_clk_shift *clk_shift,
-	const struct dce_disp_clk_mask *clk_mask);
+	const struct dccg_registers *regs,
+	const struct dccg_shift *clk_shift,
+	const struct dccg_mask *clk_mask);
 
-struct display_clock *dce110_disp_clk_create(
+struct dccg *dce110_dccg_create(
 	struct dc_context *ctx,
-	const struct dce_disp_clk_registers *regs,
-	const struct dce_disp_clk_shift *clk_shift,
-	const struct dce_disp_clk_mask *clk_mask);
+	const struct dccg_registers *regs,
+	const struct dccg_shift *clk_shift,
+	const struct dccg_mask *clk_mask);
 
-struct display_clock *dce112_disp_clk_create(
+struct dccg *dce112_dccg_create(
 	struct dc_context *ctx,
-	const struct dce_disp_clk_registers *regs,
-	const struct dce_disp_clk_shift *clk_shift,
-	const struct dce_disp_clk_mask *clk_mask);
+	const struct dccg_registers *regs,
+	const struct dccg_shift *clk_shift,
+	const struct dccg_mask *clk_mask);
+
+struct dccg *dce120_dccg_create(struct dc_context *ctx);
 
-struct display_clock *dce120_disp_clk_create(struct dc_context *ctx);
+#ifdef CONFIG_DRM_AMD_DC_DCN1_0
+struct dccg *dcn1_dccg_create(struct dc_context *ctx);
+#endif
 
-void dce_disp_clk_destroy(struct display_clock **disp_clk);
+void dce_dccg_destroy(struct dccg **dccg);
 
 #endif /* _DCE_CLOCKS_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.c b/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.c
index a576b8bbb3cd..dea40b322191 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.c
@@ -150,7 +150,7 @@ static void dce_dmcu_set_psr_enable(struct dmcu *dmcu, bool enable, bool wait)
 	}
 }
 
-static void dce_dmcu_setup_psr(struct dmcu *dmcu,
+static bool dce_dmcu_setup_psr(struct dmcu *dmcu,
 		struct dc_link *link,
 		struct psr_context *psr_context)
 {
@@ -261,6 +261,8 @@ static void dce_dmcu_setup_psr(struct dmcu *dmcu,
 
 	/* notifyDMCUMsg */
 	REG_UPDATE(MASTER_COMM_CNTL_REG, MASTER_COMM_INTERRUPT, 1);
+
+	return true;
 }
 
 static bool dce_is_dmcu_initialized(struct dmcu *dmcu)
@@ -545,24 +547,25 @@ static void dcn10_dmcu_set_psr_enable(struct dmcu *dmcu, bool enable, bool wait)
 	 *  least a few frames. Should never hit the max retry assert below.
 	 */
 	if (wait == true) {
-	for (retryCount = 0; retryCount <= 1000; retryCount++) {
-		dcn10_get_dmcu_psr_state(dmcu, &psr_state);
-		if (enable) {
-			if (psr_state != 0)
-				break;
-		} else {
-			if (psr_state == 0)
-				break;
+		for (retryCount = 0; retryCount <= 1000; retryCount++) {
+			dcn10_get_dmcu_psr_state(dmcu, &psr_state);
+			if (enable) {
+				if (psr_state != 0)
+					break;
+			} else {
+				if (psr_state == 0)
+					break;
+			}
+			udelay(500);
 		}
-		udelay(500);
-	}
 
-	/* assert if max retry hit */
-	ASSERT(retryCount <= 1000);
+		/* assert if max retry hit */
+		if (retryCount >= 1000)
+			ASSERT(0);
 	}
 }
 
-static void dcn10_dmcu_setup_psr(struct dmcu *dmcu,
+static bool dcn10_dmcu_setup_psr(struct dmcu *dmcu,
 		struct dc_link *link,
 		struct psr_context *psr_context)
 {
@@ -577,7 +580,7 @@ static void dcn10_dmcu_setup_psr(struct dmcu *dmcu,
 
 	/* If microcontroller is not running, do nothing */
 	if (dmcu->dmcu_state != DMCU_RUNNING)
-		return;
+		return false;
 
 	link->link_enc->funcs->psr_program_dp_dphy_fast_training(link->link_enc,
 			psr_context->psrExitLinkTrainingRequired);
@@ -677,6 +680,11 @@ static void dcn10_dmcu_setup_psr(struct dmcu *dmcu,
 
 	/* notifyDMCUMsg */
 	REG_UPDATE(MASTER_COMM_CNTL_REG, MASTER_COMM_INTERRUPT, 1);
+
+	/* waitDMCUReadyForCmd */
+	REG_WAIT(MASTER_COMM_CNTL_REG, MASTER_COMM_INTERRUPT, 0, 1, 10000);
+
+	return true;
 }
 
 static void dcn10_psr_wait_loop(
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_hwseq.h b/drivers/gpu/drm/amd/display/dc/dce/dce_hwseq.h
index 057b8afd74bc..64dc75378541 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_hwseq.h
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_hwseq.h
@@ -147,6 +147,7 @@
 	SR(DCCG_GATE_DISABLE_CNTL2), \
 	SR(DCFCLK_CNTL),\
 	SR(DCFCLK_CNTL), \
+	SR(DC_MEM_GLOBAL_PWR_REQ_CNTL), \
 	/* todo:  get these from GVM instead of reading registers ourselves */\
 	MMHUB_SR(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32),\
 	MMHUB_SR(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32),\
@@ -249,7 +250,6 @@ struct dce_hwseq_registers {
 	uint32_t DISPCLK_FREQ_CHANGE_CNTL;
 	uint32_t RBBMIF_TIMEOUT_DIS;
 	uint32_t RBBMIF_TIMEOUT_DIS_2;
-	uint32_t DENTIST_DISPCLK_CNTL;
 	uint32_t DCHUBBUB_CRC_CTRL;
 	uint32_t DPP_TOP0_DPP_CRC_CTRL;
 	uint32_t DPP_TOP0_DPP_CRC_VAL_R_G;
@@ -276,6 +276,8 @@ struct dce_hwseq_registers {
 	uint32_t MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB;
 	uint32_t MC_VM_SYSTEM_APERTURE_LOW_ADDR;
 	uint32_t MC_VM_SYSTEM_APERTURE_HIGH_ADDR;
+	uint32_t AZALIA_AUDIO_DTO;
+	uint32_t AZALIA_CONTROLLER_CLOCK_GATING;
 };
  /* set field name */
 #define HWS_SF(blk_name, reg_name, field_name, post_fix)\
@@ -362,7 +364,8 @@ struct dce_hwseq_registers {
 	HWSEQ_PIXEL_RATE_MASK_SH_LIST(mask_sh, OTG0_),\
 	HWS_SF1(OTG0_, PHYPLL_PIXEL_RATE_CNTL, PHYPLL_PIXEL_RATE_SOURCE, mask_sh), \
 	HWS_SF(, DCHUBBUB_GLOBAL_TIMER_CNTL, DCHUBBUB_GLOBAL_TIMER_ENABLE, mask_sh), \
-	HWS_SF(, DCFCLK_CNTL, DCFCLK_GATE_DIS, mask_sh)
+	HWS_SF(, DCFCLK_CNTL, DCFCLK_GATE_DIS, mask_sh), \
+	HWS_SF(, DC_MEM_GLOBAL_PWR_REQ_CNTL, DC_MEM_GLOBAL_PWR_REQ_DIS, mask_sh)
 
 #define HWSEQ_DCN1_MASK_SH_LIST(mask_sh)\
 	HWSEQ_DCN_MASK_SH_LIST(mask_sh), \
@@ -496,14 +499,13 @@ struct dce_hwseq_registers {
 	type DOMAIN7_PGFSM_PWR_STATUS; \
 	type DCFCLK_GATE_DIS; \
 	type DCHUBBUB_GLOBAL_TIMER_REFDIV; \
-	type DENTIST_DPPCLK_WDIVIDER; \
-	type DENTIST_DISPCLK_WDIVIDER; \
 	type VGA_TEST_ENABLE; \
 	type VGA_TEST_RENDER_START; \
 	type D1VGA_MODE_ENABLE; \
 	type D2VGA_MODE_ENABLE; \
 	type D3VGA_MODE_ENABLE; \
-	type D4VGA_MODE_ENABLE;
+	type D4VGA_MODE_ENABLE; \
+	type AZALIA_AUDIO_DTO_MODULE;
 
 struct dce_hwseq_shift {
 	HWSEQ_REG_FIELD_LIST(uint8_t)
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c
index dbe3b26b6d9e..eff7d22d78fb 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c
@@ -256,6 +256,11 @@ static void setup_panel_mode(
 	enum dp_panel_mode panel_mode)
 {
 	uint32_t value;
+	struct dc_context *ctx = enc110->base.ctx;
+
+	/* if psp set panel mode, dal should be program it */
+	if (ctx->dc->caps.psp_setup_panel_mode)
+		return;
 
 	ASSERT(REG(DP_DPHY_INTERNAL_CTRL));
 	value = REG_READ(DP_DPHY_INTERNAL_CTRL);
@@ -646,6 +651,9 @@ static bool dce110_link_encoder_validate_hdmi_output(
 	if (!enc110->base.features.flags.bits.HDMI_6GB_EN &&
 		adjusted_pix_clk_khz >= 300000)
 		return false;
+	if (enc110->base.ctx->dc->debug.hdmi20_disable &&
+		crtc_timing->pixel_encoding == PIXEL_ENCODING_YCBCR420)
+		return false;
 	return true;
 }
 
@@ -773,6 +781,9 @@ void dce110_link_encoder_construct(
 				__func__,
 				result);
 	}
+	if (enc110->base.ctx->dc->debug.hdmi20_disable) {
+		enc110->base.features.flags.bits.HDMI_6GB_EN = 0;
+	}
 }
 
 bool dce110_link_encoder_validate_output_with_stream(
@@ -919,7 +930,7 @@ void dce110_link_encoder_enable_tmds_output(
 	enum bp_result result;
 
 	/* Enable the PHY */
-
+	cntl.connector_obj_id = enc110->base.connector;
 	cntl.action = TRANSMITTER_CONTROL_ENABLE;
 	cntl.engine_id = enc->preferred_engine;
 	cntl.transmitter = enc110->base.transmitter;
@@ -961,7 +972,7 @@ void dce110_link_encoder_enable_dp_output(
 	 * We need to set number of lanes manually.
 	 */
 	configure_encoder(enc110, link_settings);
-
+	cntl.connector_obj_id = enc110->base.connector;
 	cntl.action = TRANSMITTER_CONTROL_ENABLE;
 	cntl.engine_id = enc->preferred_engine;
 	cntl.transmitter = enc110->base.transmitter;
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_mem_input.c b/drivers/gpu/drm/amd/display/dc/dce/dce_mem_input.c
index b235a75355b8..85686d917636 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_mem_input.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_mem_input.c
@@ -729,7 +729,7 @@ static bool dce_mi_program_surface_flip_and_addr(
 	return true;
 }
 
-static struct mem_input_funcs dce_mi_funcs = {
+static const struct mem_input_funcs dce_mi_funcs = {
 	.mem_input_program_display_marks = dce_mi_program_display_marks,
 	.allocate_mem_input = dce_mi_allocate_dmif,
 	.free_mem_input = dce_mi_free_dmif,
@@ -741,6 +741,29 @@ static struct mem_input_funcs dce_mi_funcs = {
 	.mem_input_is_flip_pending = dce_mi_is_flip_pending
 };
 
+static const struct mem_input_funcs dce112_mi_funcs = {
+	.mem_input_program_display_marks = dce112_mi_program_display_marks,
+	.allocate_mem_input = dce_mi_allocate_dmif,
+	.free_mem_input = dce_mi_free_dmif,
+	.mem_input_program_surface_flip_and_addr =
+			dce_mi_program_surface_flip_and_addr,
+	.mem_input_program_pte_vm = dce_mi_program_pte_vm,
+	.mem_input_program_surface_config =
+			dce_mi_program_surface_config,
+	.mem_input_is_flip_pending = dce_mi_is_flip_pending
+};
+
+static const struct mem_input_funcs dce120_mi_funcs = {
+	.mem_input_program_display_marks = dce120_mi_program_display_marks,
+	.allocate_mem_input = dce_mi_allocate_dmif,
+	.free_mem_input = dce_mi_free_dmif,
+	.mem_input_program_surface_flip_and_addr =
+			dce_mi_program_surface_flip_and_addr,
+	.mem_input_program_pte_vm = dce_mi_program_pte_vm,
+	.mem_input_program_surface_config =
+			dce_mi_program_surface_config,
+	.mem_input_is_flip_pending = dce_mi_is_flip_pending
+};
 
 void dce_mem_input_construct(
 	struct dce_mem_input *dce_mi,
@@ -769,7 +792,7 @@ void dce112_mem_input_construct(
 	const struct dce_mem_input_mask *mi_mask)
 {
 	dce_mem_input_construct(dce_mi, ctx, inst, regs, mi_shift, mi_mask);
-	dce_mi->base.funcs->mem_input_program_display_marks = dce112_mi_program_display_marks;
+	dce_mi->base.funcs = &dce112_mi_funcs;
 }
 
 void dce120_mem_input_construct(
@@ -781,5 +804,5 @@ void dce120_mem_input_construct(
 	const struct dce_mem_input_mask *mi_mask)
 {
 	dce_mem_input_construct(dce_mi, ctx, inst, regs, mi_shift, mi_mask);
-	dce_mi->base.funcs->mem_input_program_display_marks = dce120_mi_program_display_marks;
+	dce_mi->base.funcs = &dce120_mi_funcs;
 }
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_transform.c b/drivers/gpu/drm/amd/display/dc/dce/dce_transform.c
index a02e719d7794..ab63d0d0304c 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_transform.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_transform.c
@@ -155,7 +155,7 @@ static void program_overscan(
 	int overscan_bottom = data->v_active
 			- data->recout.y - data->recout.height;
 
-	if (xfm_dce->base.ctx->dc->debug.surface_visual_confirm) {
+	if (xfm_dce->base.ctx->dc->debug.visual_confirm != VISUAL_CONFIRM_DISABLE) {
 		overscan_bottom += 2;
 		overscan_right += 2;
 	}
diff --git a/drivers/gpu/drm/amd/display/dc/dce100/dce100_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/dce100/dce100_hw_sequencer.c
index 41f83ecd7469..74c05e878807 100644
--- a/drivers/gpu/drm/amd/display/dc/dce100/dce100_hw_sequencer.c
+++ b/drivers/gpu/drm/amd/display/dc/dce100/dce100_hw_sequencer.c
@@ -125,17 +125,50 @@ static void dce100_pplib_apply_display_requirements(
 	dc->prev_display_config = *pp_display_cfg;
 }
 
+/* unit: in_khz before mode set, get pixel clock from context. ASIC register
+ * may not be programmed yet
+ */
+static uint32_t get_max_pixel_clock_for_all_paths(
+	struct dc *dc,
+	struct dc_state *context)
+{
+	uint32_t max_pix_clk = 0;
+	int i;
+
+	for (i = 0; i < MAX_PIPES; i++) {
+		struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[i];
+
+		if (pipe_ctx->stream == NULL)
+			continue;
+
+		/* do not check under lay */
+		if (pipe_ctx->top_pipe)
+			continue;
+
+		if (pipe_ctx->stream_res.pix_clk_params.requested_pix_clk > max_pix_clk)
+			max_pix_clk =
+				pipe_ctx->stream_res.pix_clk_params.requested_pix_clk;
+	}
+	return max_pix_clk;
+}
+
 void dce100_set_bandwidth(
 		struct dc *dc,
 		struct dc_state *context,
 		bool decrease_allowed)
 {
-	if (decrease_allowed || context->bw.dce.dispclk_khz > dc->current_state->bw.dce.dispclk_khz) {
-		dc->res_pool->display_clock->funcs->set_clock(
-				dc->res_pool->display_clock,
-				context->bw.dce.dispclk_khz * 115 / 100);
-		dc->current_state->bw.dce.dispclk_khz = context->bw.dce.dispclk_khz;
-	}
+	struct dc_clocks req_clks;
+
+	req_clks.dispclk_khz = context->bw.dce.dispclk_khz * 115 / 100;
+	req_clks.phyclk_khz = get_max_pixel_clock_for_all_paths(dc, context);
+
+	dce110_set_safe_displaymarks(&context->res_ctx, dc->res_pool);
+
+	dc->res_pool->dccg->funcs->update_clocks(
+			dc->res_pool->dccg,
+			&req_clks,
+			decrease_allowed);
+
 	dce100_pplib_apply_display_requirements(dc, context);
 }
 
diff --git a/drivers/gpu/drm/amd/display/dc/dce100/dce100_resource.c b/drivers/gpu/drm/amd/display/dc/dce100/dce100_resource.c
index 38ec0d609297..3f76e6019546 100644
--- a/drivers/gpu/drm/amd/display/dc/dce100/dce100_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/dce100/dce100_resource.c
@@ -52,6 +52,7 @@
 #include "dce/dce_10_0_sh_mask.h"
 
 #include "dce/dce_dmcu.h"
+#include "dce/dce_aux.h"
 #include "dce/dce_abm.h"
 
 #ifndef mmMC_HUB_RDREQ_DMIF_LIMIT
@@ -135,15 +136,15 @@ static const struct dce110_timing_generator_offsets dce100_tg_offsets[] = {
 	.reg_name = mm ## block ## id ## _ ## reg_name
 
 
-static const struct dce_disp_clk_registers disp_clk_regs = {
+static const struct dccg_registers disp_clk_regs = {
 		CLK_COMMON_REG_LIST_DCE_BASE()
 };
 
-static const struct dce_disp_clk_shift disp_clk_shift = {
+static const struct dccg_shift disp_clk_shift = {
 		CLK_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(__SHIFT)
 };
 
-static const struct dce_disp_clk_mask disp_clk_mask = {
+static const struct dccg_mask disp_clk_mask = {
 		CLK_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(_MASK)
 };
 
@@ -279,7 +280,20 @@ static const struct dce_opp_shift opp_shift = {
 static const struct dce_opp_mask opp_mask = {
 	OPP_COMMON_MASK_SH_LIST_DCE_100(_MASK)
 };
+#define aux_engine_regs(id)\
+[id] = {\
+	AUX_COMMON_REG_LIST(id), \
+	.AUX_RESET_MASK = 0 \
+}
 
+static const struct dce110_aux_registers aux_engine_regs[] = {
+		aux_engine_regs(0),
+		aux_engine_regs(1),
+		aux_engine_regs(2),
+		aux_engine_regs(3),
+		aux_engine_regs(4),
+		aux_engine_regs(5)
+};
 
 #define audio_regs(id)\
 [id] = {\
@@ -572,6 +586,23 @@ struct output_pixel_processor *dce100_opp_create(
 	return &opp->base;
 }
 
+struct aux_engine *dce100_aux_engine_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct aux_engine_dce110 *aux_engine =
+		kzalloc(sizeof(struct aux_engine_dce110), GFP_KERNEL);
+
+	if (!aux_engine)
+		return NULL;
+
+	dce110_aux_engine_construct(aux_engine, ctx, inst,
+				    SW_AUX_TIMEOUT_PERIOD_MULTIPLIER * AUX_TIMEOUT_PERIOD,
+				    &aux_engine_regs[inst]);
+
+	return &aux_engine->base;
+}
+
 struct clock_source *dce100_clock_source_create(
 	struct dc_context *ctx,
 	struct dc_bios *bios,
@@ -624,6 +655,10 @@ static void destruct(struct dce110_resource_pool *pool)
 			kfree(DCE110TG_FROM_TG(pool->base.timing_generators[i]));
 			pool->base.timing_generators[i] = NULL;
 		}
+
+		if (pool->base.engines[i] != NULL)
+			dce110_engine_destroy(&pool->base.engines[i]);
+
 	}
 
 	for (i = 0; i < pool->base.stream_enc_count; i++) {
@@ -644,8 +679,8 @@ static void destruct(struct dce110_resource_pool *pool)
 			dce_aud_destroy(&pool->base.audios[i]);
 	}
 
-	if (pool->base.display_clock != NULL)
-		dce_disp_clk_destroy(&pool->base.display_clock);
+	if (pool->base.dccg != NULL)
+		dce_dccg_destroy(&pool->base.dccg);
 
 	if (pool->base.abm != NULL)
 				dce_abm_destroy(&pool->base.abm);
@@ -678,9 +713,22 @@ bool dce100_validate_bandwidth(
 	struct dc  *dc,
 	struct dc_state *context)
 {
-	/* TODO implement when needed but for now hardcode max value*/
-	context->bw.dce.dispclk_khz = 681000;
-	context->bw.dce.yclk_khz = 250000 * MEMORY_TYPE_MULTIPLIER;
+	int i;
+	bool at_least_one_pipe = false;
+
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		if (context->res_ctx.pipe_ctx[i].stream)
+			at_least_one_pipe = true;
+	}
+
+	if (at_least_one_pipe) {
+		/* TODO implement when needed but for now hardcode max value*/
+		context->bw.dce.dispclk_khz = 681000;
+		context->bw.dce.yclk_khz = 250000 * MEMORY_TYPE_MULTIPLIER;
+	} else {
+		context->bw.dce.dispclk_khz = 0;
+		context->bw.dce.yclk_khz = 0;
+	}
 
 	return true;
 }
@@ -817,11 +865,11 @@ static bool construct(
 		}
 	}
 
-	pool->base.display_clock = dce_disp_clk_create(ctx,
+	pool->base.dccg = dce_dccg_create(ctx,
 			&disp_clk_regs,
 			&disp_clk_shift,
 			&disp_clk_mask);
-	if (pool->base.display_clock == NULL) {
+	if (pool->base.dccg == NULL) {
 		dm_error("DC: failed to create display clock!\n");
 		BREAK_TO_DEBUGGER();
 		goto res_create_fail;
@@ -851,7 +899,7 @@ static bool construct(
 	 * max_clock_state
 	 */
 	if (dm_pp_get_static_clocks(ctx, &static_clk_info))
-		pool->base.display_clock->max_clks_state =
+		pool->base.dccg->max_clks_state =
 					static_clk_info.max_clocks_state;
 	{
 		struct irq_service_init_data init_data;
@@ -871,7 +919,7 @@ static bool construct(
 	dc->caps.i2c_speed_in_khz = 40;
 	dc->caps.max_cursor_size = 128;
 	dc->caps.dual_link_dvi = true;
-
+	dc->caps.disable_dp_clk_share = true;
 	for (i = 0; i < pool->base.pipe_count; i++) {
 		pool->base.timing_generators[i] =
 			dce100_timing_generator_create(
@@ -915,6 +963,13 @@ static bool construct(
 				"DC: failed to create output pixel processor!\n");
 			goto res_create_fail;
 		}
+		pool->base.engines[i] = dce100_aux_engine_create(ctx, i);
+		if (pool->base.engines[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC:failed to create aux engine!!\n");
+			goto res_create_fail;
+		}
 	}
 
 	dc->caps.max_planes =  pool->base.pipe_count;
diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_compressor.c b/drivers/gpu/drm/amd/display/dc/dce110/dce110_compressor.c
index df027013e50c..1f7f25013217 100644
--- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_compressor.c
+++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_compressor.c
@@ -143,7 +143,7 @@ static void wait_for_fbc_state_changed(
 	struct dce110_compressor *cp110,
 	bool enabled)
 {
-	uint16_t counter = 0;
+	uint32_t counter = 0;
 	uint32_t addr = mmFBC_STATUS;
 	uint32_t value;
 
@@ -158,7 +158,7 @@ static void wait_for_fbc_state_changed(
 		counter++;
 	}
 
-	if (counter == 10) {
+	if (counter == 1000) {
 		DC_LOG_WARNING("%s: wait counter exceeded, changes to HW not applied",
 			__func__);
 	} else {
diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c
index 353ffcbdf5ba..14384d9675a8 100644
--- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c
+++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c
@@ -864,17 +864,22 @@ void hwss_edp_power_control(
 		if (power_up) {
 			unsigned long long current_ts = dm_get_timestamp(ctx);
 			unsigned long long duration_in_ms =
-					dm_get_elapse_time_in_ns(
+					div64_u64(dm_get_elapse_time_in_ns(
 							ctx,
 							current_ts,
-							div64_u64(link->link_trace.time_stamp.edp_poweroff, 1000000));
+							link->link_trace.time_stamp.edp_poweroff), 1000000);
 			unsigned long long wait_time_ms = 0;
 
 			/* max 500ms from LCDVDD off to on */
+			unsigned long long edp_poweroff_time_ms = 500;
+
+			if (link->local_sink != NULL)
+				edp_poweroff_time_ms =
+						500 + link->local_sink->edid_caps.panel_patch.extra_t12_ms;
 			if (link->link_trace.time_stamp.edp_poweroff == 0)
-				wait_time_ms = 500;
-			else if (duration_in_ms < 500)
-				wait_time_ms = 500 - duration_in_ms;
+				wait_time_ms = edp_poweroff_time_ms;
+			else if (duration_in_ms < edp_poweroff_time_ms)
+				wait_time_ms = edp_poweroff_time_ms - duration_in_ms;
 
 			if (wait_time_ms) {
 				msleep(wait_time_ms);
@@ -1251,7 +1256,7 @@ static void program_scaler(const struct dc *dc,
 		return;
 #endif
 
-	if (dc->debug.surface_visual_confirm)
+	if (dc->debug.visual_confirm == VISUAL_CONFIRM_SURFACE)
 		get_surface_visual_confirm_color(pipe_ctx, &color);
 	else
 		color_space_to_black_color(dc,
@@ -1475,7 +1480,7 @@ static void power_down_controllers(struct dc *dc)
 {
 	int i;
 
-	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+	for (i = 0; i < dc->res_pool->timing_generator_count; i++) {
 		dc->res_pool->timing_generators[i]->funcs->disable_crtc(
 				dc->res_pool->timing_generators[i]);
 	}
@@ -1515,12 +1520,13 @@ static void disable_vga_and_power_gate_all_controllers(
 	struct timing_generator *tg;
 	struct dc_context *ctx = dc->ctx;
 
-	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+	for (i = 0; i < dc->res_pool->timing_generator_count; i++) {
 		tg = dc->res_pool->timing_generators[i];
 
 		if (tg->funcs->disable_vga)
 			tg->funcs->disable_vga(tg);
-
+	}
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
 		/* Enable CLOCK gating for each pipe BEFORE controller
 		 * powergating. */
 		enable_display_pipe_clock_gating(ctx,
@@ -1582,7 +1588,13 @@ void dce110_enable_accelerated_mode(struct dc *dc, struct dc_state *context)
 	bool can_eDP_fast_boot_optimize = false;
 
 	if (edp_link) {
-		can_eDP_fast_boot_optimize =
+		/* this seems to cause blank screens on DCE8 */
+		if ((dc->ctx->dce_version == DCE_VERSION_8_0) ||
+		    (dc->ctx->dce_version == DCE_VERSION_8_1) ||
+		    (dc->ctx->dce_version == DCE_VERSION_8_3))
+			can_eDP_fast_boot_optimize = false;
+		else
+			can_eDP_fast_boot_optimize =
 				edp_link->link_enc->funcs->is_dig_enabled(edp_link->link_enc);
 	}
 
@@ -1663,7 +1675,7 @@ static void dce110_set_displaymarks(
 	}
 }
 
-static void set_safe_displaymarks(
+void dce110_set_safe_displaymarks(
 		struct resource_context *res_ctx,
 		const struct resource_pool *pool)
 {
@@ -1755,23 +1767,15 @@ static void set_static_screen_control(struct pipe_ctx **pipe_ctx,
 }
 
 /* unit: in_khz before mode set, get pixel clock from context. ASIC register
- * may not be programmed yet.
- * TODO: after mode set, pre_mode_set = false,
- * may read PLL register to get pixel clock
+ * may not be programmed yet
  */
 static uint32_t get_max_pixel_clock_for_all_paths(
 	struct dc *dc,
-	struct dc_state *context,
-	bool pre_mode_set)
+	struct dc_state *context)
 {
 	uint32_t max_pix_clk = 0;
 	int i;
 
-	if (!pre_mode_set) {
-		/* TODO: read ASIC register to get pixel clock */
-		ASSERT(0);
-	}
-
 	for (i = 0; i < MAX_PIPES; i++) {
 		struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[i];
 
@@ -1787,95 +1791,10 @@ static uint32_t get_max_pixel_clock_for_all_paths(
 				pipe_ctx->stream_res.pix_clk_params.requested_pix_clk;
 	}
 
-	if (max_pix_clk == 0)
-		ASSERT(0);
-
 	return max_pix_clk;
 }
 
 /*
- * Find clock state based on clock requested. if clock value is 0, simply
- * set clock state as requested without finding clock state by clock value
- */
-
-static void apply_min_clocks(
-	struct dc *dc,
-	struct dc_state *context,
-	enum dm_pp_clocks_state *clocks_state,
-	bool pre_mode_set)
-{
-	struct state_dependent_clocks req_clocks = {0};
-
-	if (!pre_mode_set) {
-		/* set clock_state without verification */
-		if (context->dis_clk->funcs->set_min_clocks_state) {
-			context->dis_clk->funcs->set_min_clocks_state(
-						context->dis_clk, *clocks_state);
-			return;
-		}
-
-		/* TODO: This is incorrect. Figure out how to fix. */
-		context->dis_clk->funcs->apply_clock_voltage_request(
-				context->dis_clk,
-				DM_PP_CLOCK_TYPE_DISPLAY_CLK,
-				context->dis_clk->cur_clocks_value.dispclk_in_khz,
-				pre_mode_set,
-				false);
-
-		context->dis_clk->funcs->apply_clock_voltage_request(
-				context->dis_clk,
-				DM_PP_CLOCK_TYPE_PIXELCLK,
-				context->dis_clk->cur_clocks_value.max_pixelclk_in_khz,
-				pre_mode_set,
-				false);
-
-		context->dis_clk->funcs->apply_clock_voltage_request(
-				context->dis_clk,
-				DM_PP_CLOCK_TYPE_DISPLAYPHYCLK,
-				context->dis_clk->cur_clocks_value.max_non_dp_phyclk_in_khz,
-				pre_mode_set,
-				false);
-		return;
-	}
-
-	/* get the required state based on state dependent clocks:
-	 * display clock and pixel clock
-	 */
-	req_clocks.display_clk_khz = context->bw.dce.dispclk_khz;
-
-	req_clocks.pixel_clk_khz = get_max_pixel_clock_for_all_paths(
-			dc, context, true);
-
-	if (context->dis_clk->funcs->get_required_clocks_state) {
-		*clocks_state = context->dis_clk->funcs->get_required_clocks_state(
-				context->dis_clk, &req_clocks);
-		context->dis_clk->funcs->set_min_clocks_state(
-			context->dis_clk, *clocks_state);
-	} else {
-		context->dis_clk->funcs->apply_clock_voltage_request(
-				context->dis_clk,
-				DM_PP_CLOCK_TYPE_DISPLAY_CLK,
-				req_clocks.display_clk_khz,
-				pre_mode_set,
-				false);
-
-		context->dis_clk->funcs->apply_clock_voltage_request(
-				context->dis_clk,
-				DM_PP_CLOCK_TYPE_PIXELCLK,
-				req_clocks.pixel_clk_khz,
-				pre_mode_set,
-				false);
-
-		context->dis_clk->funcs->apply_clock_voltage_request(
-				context->dis_clk,
-				DM_PP_CLOCK_TYPE_DISPLAYPHYCLK,
-				req_clocks.pixel_clk_khz,
-				pre_mode_set,
-				false);
-	}
-}
-
-/*
  *  Check if FBC can be enabled
  */
 static bool should_enable_fbc(struct dc *dc,
@@ -1995,7 +1914,9 @@ static void dce110_reset_hw_ctx_wrap(
 			pipe_ctx_old->plane_res.mi->funcs->free_mem_input(
 					pipe_ctx_old->plane_res.mi, dc->current_state->stream_count);
 
-			if (old_clk)
+			if (old_clk && 0 == resource_get_clock_source_reference(&context->res_ctx,
+										dc->res_pool,
+										old_clk))
 				old_clk->funcs->cs_power_down(old_clk);
 
 			dc->hwss.disable_plane(dc, pipe_ctx_old);
@@ -2093,7 +2014,6 @@ enum dc_status dce110_apply_ctx_to_hw(
 	struct dc_bios *dcb = dc->ctx->dc_bios;
 	enum dc_status status;
 	int i;
-	enum dm_pp_clocks_state clocks_state = DM_PP_CLOCKS_STATE_INVALID;
 
 	/* Reset old context */
 	/* look up the targets that have been removed since last commit */
@@ -2127,55 +2047,9 @@ enum dc_status dce110_apply_ctx_to_hw(
 				PIPE_GATING_CONTROL_DISABLE);
 	}
 
-	set_safe_displaymarks(&context->res_ctx, dc->res_pool);
-
 	if (dc->fbc_compressor)
 		dc->fbc_compressor->funcs->disable_fbc(dc->fbc_compressor);
 
-	/*TODO: when pplib works*/
-	apply_min_clocks(dc, context, &clocks_state, true);
-
-#if defined(CONFIG_DRM_AMD_DC_DCN1_0)
-	if (dc->ctx->dce_version >= DCN_VERSION_1_0) {
-		if (context->bw.dcn.calc_clk.fclk_khz
-				> dc->current_state->bw.dcn.cur_clk.fclk_khz) {
-			struct dm_pp_clock_for_voltage_req clock;
-
-			clock.clk_type = DM_PP_CLOCK_TYPE_FCLK;
-			clock.clocks_in_khz = context->bw.dcn.calc_clk.fclk_khz;
-			dm_pp_apply_clock_for_voltage_request(dc->ctx, &clock);
-			dc->current_state->bw.dcn.cur_clk.fclk_khz = clock.clocks_in_khz;
-			context->bw.dcn.cur_clk.fclk_khz = clock.clocks_in_khz;
-		}
-		if (context->bw.dcn.calc_clk.dcfclk_khz
-				> dc->current_state->bw.dcn.cur_clk.dcfclk_khz) {
-			struct dm_pp_clock_for_voltage_req clock;
-
-			clock.clk_type = DM_PP_CLOCK_TYPE_DCFCLK;
-			clock.clocks_in_khz = context->bw.dcn.calc_clk.dcfclk_khz;
-			dm_pp_apply_clock_for_voltage_request(dc->ctx, &clock);
-			dc->current_state->bw.dcn.cur_clk.dcfclk_khz = clock.clocks_in_khz;
-			context->bw.dcn.cur_clk.dcfclk_khz = clock.clocks_in_khz;
-		}
-		if (context->bw.dcn.calc_clk.dispclk_khz
-				> dc->current_state->bw.dcn.cur_clk.dispclk_khz) {
-			dc->res_pool->display_clock->funcs->set_clock(
-					dc->res_pool->display_clock,
-					context->bw.dcn.calc_clk.dispclk_khz);
-			dc->current_state->bw.dcn.cur_clk.dispclk_khz =
-					context->bw.dcn.calc_clk.dispclk_khz;
-			context->bw.dcn.cur_clk.dispclk_khz =
-					context->bw.dcn.calc_clk.dispclk_khz;
-		}
-	} else
-#endif
-	if (context->bw.dce.dispclk_khz
-			> dc->current_state->bw.dce.dispclk_khz) {
-		dc->res_pool->display_clock->funcs->set_clock(
-				dc->res_pool->display_clock,
-				context->bw.dce.dispclk_khz * 115 / 100);
-	}
-
 	dce110_setup_audio_dto(dc, context);
 
 	for (i = 0; i < dc->res_pool->pipe_count; i++) {
@@ -2204,9 +2078,6 @@ enum dc_status dce110_apply_ctx_to_hw(
 			return status;
 	}
 
-	/* to save power */
-	apply_min_clocks(dc, context, &clocks_state, false);
-
 	dcb->funcs->set_scratch_critical_state(dcb, false);
 
 	if (dc->fbc_compressor)
@@ -2667,7 +2538,7 @@ static void pplib_apply_display_requirements(
 	/* TODO: dce11.2*/
 	pp_display_cfg->avail_mclk_switch_time_in_disp_active_us = 0;
 
-	pp_display_cfg->disp_clk_khz = context->bw.dce.dispclk_khz;
+	pp_display_cfg->disp_clk_khz = dc->res_pool->dccg->clks.dispclk_khz;
 
 	dce110_fill_display_configs(context, pp_display_cfg);
 
@@ -2689,20 +2560,25 @@ static void pplib_apply_display_requirements(
 	dc->prev_display_config = *pp_display_cfg;
 }
 
-static void dce110_set_bandwidth(
+void dce110_set_bandwidth(
 		struct dc *dc,
 		struct dc_state *context,
 		bool decrease_allowed)
 {
-	dce110_set_displaymarks(dc, context);
+	struct dc_clocks req_clks;
 
-	if (decrease_allowed || context->bw.dce.dispclk_khz > dc->current_state->bw.dce.dispclk_khz) {
-		dc->res_pool->display_clock->funcs->set_clock(
-				dc->res_pool->display_clock,
-				context->bw.dce.dispclk_khz * 115 / 100);
-		dc->current_state->bw.dce.dispclk_khz = context->bw.dce.dispclk_khz;
-	}
+	req_clks.dispclk_khz = context->bw.dce.dispclk_khz;
+	req_clks.phyclk_khz = get_max_pixel_clock_for_all_paths(dc, context);
+
+	if (decrease_allowed)
+		dce110_set_displaymarks(dc, context);
+	else
+		dce110_set_safe_displaymarks(&context->res_ctx, dc->res_pool);
 
+	dc->res_pool->dccg->funcs->update_clocks(
+			dc->res_pool->dccg,
+			&req_clks,
+			decrease_allowed);
 	pplib_apply_display_requirements(dc, context);
 }
 
@@ -2938,9 +2814,11 @@ void dce110_set_cursor_position(struct pipe_ctx *pipe_ctx)
 	struct dc_cursor_mi_param param = {
 		.pixel_clk_khz = pipe_ctx->stream->timing.pix_clk_khz,
 		.ref_clk_khz = pipe_ctx->stream->ctx->dc->res_pool->ref_clock_inKhz,
-		.viewport_x_start = pipe_ctx->plane_res.scl_data.viewport.x,
-		.viewport_width = pipe_ctx->plane_res.scl_data.viewport.width,
-		.h_scale_ratio = pipe_ctx->plane_res.scl_data.ratios.horz
+		.viewport = pipe_ctx->plane_res.scl_data.viewport,
+		.h_scale_ratio = pipe_ctx->plane_res.scl_data.ratios.horz,
+		.v_scale_ratio = pipe_ctx->plane_res.scl_data.ratios.vert,
+		.rotation = pipe_ctx->plane_state->rotation,
+		.mirror = pipe_ctx->plane_state->horizontal_mirror
 	};
 
 	if (pipe_ctx->plane_state->address.type
diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.h b/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.h
index f48d5a68d238..e4c5db75c4c6 100644
--- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.h
+++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.h
@@ -60,10 +60,19 @@ void dce110_enable_accelerated_mode(struct dc *dc, struct dc_state *context);
 
 void dce110_power_down(struct dc *dc);
 
+void dce110_set_safe_displaymarks(
+		struct resource_context *res_ctx,
+		const struct resource_pool *pool);
+
 void dce110_fill_display_configs(
 	const struct dc_state *context,
 	struct dm_pp_display_configuration *pp_display_cfg);
 
+void dce110_set_bandwidth(
+		struct dc *dc,
+		struct dc_state *context,
+		bool decrease_allowed);
+
 uint32_t dce110_get_min_vblank_time_us(const struct dc_state *context);
 
 void dp_receiver_power_ctrl(struct dc_link *link, bool on);
diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_mem_input_v.c b/drivers/gpu/drm/amd/display/dc/dce110/dce110_mem_input_v.c
index 0564c8e31252..9b9fc3d96c07 100644
--- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_mem_input_v.c
+++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_mem_input_v.c
@@ -1011,7 +1011,7 @@ void dce110_free_mem_input_v(
 {
 }
 
-static struct mem_input_funcs dce110_mem_input_v_funcs = {
+static const struct mem_input_funcs dce110_mem_input_v_funcs = {
 	.mem_input_program_display_marks =
 			dce_mem_input_v_program_display_marks,
 	.mem_input_program_chroma_display_marks =
diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_resource.c b/drivers/gpu/drm/amd/display/dc/dce110/dce110_resource.c
index 20c029089551..e5e9e92521e9 100644
--- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_resource.c
@@ -49,6 +49,7 @@
 #include "dce/dce_clock_source.h"
 #include "dce/dce_hwseq.h"
 #include "dce110/dce110_hw_sequencer.h"
+#include "dce/dce_aux.h"
 #include "dce/dce_abm.h"
 #include "dce/dce_dmcu.h"
 
@@ -146,15 +147,15 @@ static const struct dce110_timing_generator_offsets dce110_tg_offsets[] = {
 #define SRI(reg_name, block, id)\
 	.reg_name = mm ## block ## id ## _ ## reg_name
 
-static const struct dce_disp_clk_registers disp_clk_regs = {
+static const struct dccg_registers disp_clk_regs = {
 		CLK_COMMON_REG_LIST_DCE_BASE()
 };
 
-static const struct dce_disp_clk_shift disp_clk_shift = {
+static const struct dccg_shift disp_clk_shift = {
 		CLK_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(__SHIFT)
 };
 
-static const struct dce_disp_clk_mask disp_clk_mask = {
+static const struct dccg_mask disp_clk_mask = {
 		CLK_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(_MASK)
 };
 
@@ -306,6 +307,21 @@ static const struct dce_opp_mask opp_mask = {
 	OPP_COMMON_MASK_SH_LIST_DCE_110(_MASK)
 };
 
+#define aux_engine_regs(id)\
+[id] = {\
+	AUX_COMMON_REG_LIST(id), \
+	.AUX_RESET_MASK = 0 \
+}
+
+static const struct dce110_aux_registers aux_engine_regs[] = {
+		aux_engine_regs(0),
+		aux_engine_regs(1),
+		aux_engine_regs(2),
+		aux_engine_regs(3),
+		aux_engine_regs(4),
+		aux_engine_regs(5)
+};
+
 #define audio_regs(id)\
 [id] = {\
 	AUD_COMMON_REG_LIST(id)\
@@ -588,6 +604,23 @@ static struct output_pixel_processor *dce110_opp_create(
 	return &opp->base;
 }
 
+struct aux_engine *dce110_aux_engine_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct aux_engine_dce110 *aux_engine =
+		kzalloc(sizeof(struct aux_engine_dce110), GFP_KERNEL);
+
+	if (!aux_engine)
+		return NULL;
+
+	dce110_aux_engine_construct(aux_engine, ctx, inst,
+				    SW_AUX_TIMEOUT_PERIOD_MULTIPLIER * AUX_TIMEOUT_PERIOD,
+				    &aux_engine_regs[inst]);
+
+	return &aux_engine->base;
+}
+
 struct clock_source *dce110_clock_source_create(
 	struct dc_context *ctx,
 	struct dc_bios *bios,
@@ -651,6 +684,10 @@ static void destruct(struct dce110_resource_pool *pool)
 			kfree(DCE110TG_FROM_TG(pool->base.timing_generators[i]));
 			pool->base.timing_generators[i] = NULL;
 		}
+
+		if (pool->base.engines[i] != NULL)
+			dce110_engine_destroy(&pool->base.engines[i]);
+
 	}
 
 	for (i = 0; i < pool->base.stream_enc_count; i++) {
@@ -679,8 +716,8 @@ static void destruct(struct dce110_resource_pool *pool)
 	if (pool->base.dmcu != NULL)
 		dce_dmcu_destroy(&pool->base.dmcu);
 
-	if (pool->base.display_clock != NULL)
-		dce_disp_clk_destroy(&pool->base.display_clock);
+	if (pool->base.dccg != NULL)
+		dce_dccg_destroy(&pool->base.dccg);
 
 	if (pool->base.irqs != NULL) {
 		dal_irq_service_destroy(&pool->base.irqs);
@@ -794,43 +831,38 @@ static bool dce110_validate_bandwidth(
 
 	if (memcmp(&dc->current_state->bw.dce,
 			&context->bw.dce, sizeof(context->bw.dce))) {
-		struct log_entry log_entry;
-		dm_logger_open(
-			dc->ctx->logger,
-			&log_entry,
-			LOG_BANDWIDTH_CALCS);
-		dm_logger_append(&log_entry, "%s: finish,\n"
+
+		DC_LOG_BANDWIDTH_CALCS(
+			"%s: finish,\n"
+			"nbpMark_b: %d nbpMark_a: %d urgentMark_b: %d urgentMark_a: %d\n"
+			"stutMark_b: %d stutMark_a: %d\n"
 			"nbpMark_b: %d nbpMark_a: %d urgentMark_b: %d urgentMark_a: %d\n"
-			"stutMark_b: %d stutMark_a: %d\n",
+			"stutMark_b: %d stutMark_a: %d\n"
+			"nbpMark_b: %d nbpMark_a: %d urgentMark_b: %d urgentMark_a: %d\n"
+			"stutMark_b: %d stutMark_a: %d stutter_mode_enable: %d\n"
+			"cstate: %d pstate: %d nbpstate: %d sync: %d dispclk: %d\n"
+			"sclk: %d sclk_sleep: %d yclk: %d blackout_recovery_time_us: %d\n"
+			,
 			__func__,
 			context->bw.dce.nbp_state_change_wm_ns[0].b_mark,
 			context->bw.dce.nbp_state_change_wm_ns[0].a_mark,
 			context->bw.dce.urgent_wm_ns[0].b_mark,
 			context->bw.dce.urgent_wm_ns[0].a_mark,
 			context->bw.dce.stutter_exit_wm_ns[0].b_mark,
-			context->bw.dce.stutter_exit_wm_ns[0].a_mark);
-		dm_logger_append(&log_entry,
-			"nbpMark_b: %d nbpMark_a: %d urgentMark_b: %d urgentMark_a: %d\n"
-			"stutMark_b: %d stutMark_a: %d\n",
+			context->bw.dce.stutter_exit_wm_ns[0].a_mark,
 			context->bw.dce.nbp_state_change_wm_ns[1].b_mark,
 			context->bw.dce.nbp_state_change_wm_ns[1].a_mark,
 			context->bw.dce.urgent_wm_ns[1].b_mark,
 			context->bw.dce.urgent_wm_ns[1].a_mark,
 			context->bw.dce.stutter_exit_wm_ns[1].b_mark,
-			context->bw.dce.stutter_exit_wm_ns[1].a_mark);
-		dm_logger_append(&log_entry,
-			"nbpMark_b: %d nbpMark_a: %d urgentMark_b: %d urgentMark_a: %d\n"
-			"stutMark_b: %d stutMark_a: %d stutter_mode_enable: %d\n",
+			context->bw.dce.stutter_exit_wm_ns[1].a_mark,
 			context->bw.dce.nbp_state_change_wm_ns[2].b_mark,
 			context->bw.dce.nbp_state_change_wm_ns[2].a_mark,
 			context->bw.dce.urgent_wm_ns[2].b_mark,
 			context->bw.dce.urgent_wm_ns[2].a_mark,
 			context->bw.dce.stutter_exit_wm_ns[2].b_mark,
 			context->bw.dce.stutter_exit_wm_ns[2].a_mark,
-			context->bw.dce.stutter_mode_enable);
-		dm_logger_append(&log_entry,
-			"cstate: %d pstate: %d nbpstate: %d sync: %d dispclk: %d\n"
-			"sclk: %d sclk_sleep: %d yclk: %d blackout_recovery_time_us: %d\n",
+			context->bw.dce.stutter_mode_enable,
 			context->bw.dce.cpuc_state_change_enable,
 			context->bw.dce.cpup_state_change_enable,
 			context->bw.dce.nbp_state_change_enable,
@@ -840,7 +872,6 @@ static bool dce110_validate_bandwidth(
 			context->bw.dce.sclk_deep_sleep_khz,
 			context->bw.dce.yclk_khz,
 			context->bw.dce.blackout_recovery_time_us);
-		dm_logger_close(&log_entry);
 	}
 	return result;
 }
@@ -1179,11 +1210,11 @@ static bool construct(
 		}
 	}
 
-	pool->base.display_clock = dce110_disp_clk_create(ctx,
+	pool->base.dccg = dce110_dccg_create(ctx,
 			&disp_clk_regs,
 			&disp_clk_shift,
 			&disp_clk_mask);
-	if (pool->base.display_clock == NULL) {
+	if (pool->base.dccg == NULL) {
 		dm_error("DC: failed to create display clock!\n");
 		BREAK_TO_DEBUGGER();
 		goto res_create_fail;
@@ -1213,7 +1244,7 @@ static bool construct(
 	 * max_clock_state
 	 */
 	if (dm_pp_get_static_clocks(ctx, &static_clk_info))
-		pool->base.display_clock->max_clks_state =
+		pool->base.dccg->max_clks_state =
 				static_clk_info.max_clocks_state;
 
 	{
@@ -1264,6 +1295,14 @@ static bool construct(
 				"DC: failed to create output pixel processor!\n");
 			goto res_create_fail;
 		}
+
+		pool->base.engines[i] = dce110_aux_engine_create(ctx, i);
+		if (pool->base.engines[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC:failed to create aux engine!!\n");
+			goto res_create_fail;
+		}
 	}
 
 	dc->fbc_compressor = dce110_compressor_create(ctx);
diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_transform_v.c b/drivers/gpu/drm/amd/display/dc/dce110/dce110_transform_v.c
index a7dce060204f..aa8d6b10d2c3 100644
--- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_transform_v.c
+++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_transform_v.c
@@ -235,7 +235,7 @@ static void program_overscan(
 	int overscan_right = data->h_active - data->recout.x - data->recout.width;
 	int overscan_bottom = data->v_active - data->recout.y - data->recout.height;
 
-	if (xfm_dce->base.ctx->dc->debug.surface_visual_confirm) {
+	if (xfm_dce->base.ctx->dc->debug.visual_confirm != VISUAL_CONFIRM_DISABLE) {
 		overscan_bottom += 2;
 		overscan_right += 2;
 	}
diff --git a/drivers/gpu/drm/amd/display/dc/dce112/dce112_resource.c b/drivers/gpu/drm/amd/display/dc/dce112/dce112_resource.c
index 00c0a1ef15eb..288129343c77 100644
--- a/drivers/gpu/drm/amd/display/dc/dce112/dce112_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/dce112/dce112_resource.c
@@ -49,6 +49,7 @@
 #include "dce112/dce112_hw_sequencer.h"
 #include "dce/dce_abm.h"
 #include "dce/dce_dmcu.h"
+#include "dce/dce_aux.h"
 
 #include "reg_helper.h"
 
@@ -146,15 +147,15 @@ static const struct dce110_timing_generator_offsets dce112_tg_offsets[] = {
 	.reg_name = mm ## block ## id ## _ ## reg_name
 
 
-static const struct dce_disp_clk_registers disp_clk_regs = {
+static const struct dccg_registers disp_clk_regs = {
 		CLK_COMMON_REG_LIST_DCE_BASE()
 };
 
-static const struct dce_disp_clk_shift disp_clk_shift = {
+static const struct dccg_shift disp_clk_shift = {
 		CLK_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(__SHIFT)
 };
 
-static const struct dce_disp_clk_mask disp_clk_mask = {
+static const struct dccg_mask disp_clk_mask = {
 		CLK_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(_MASK)
 };
 
@@ -314,6 +315,21 @@ static const struct dce_opp_mask opp_mask = {
 	OPP_COMMON_MASK_SH_LIST_DCE_112(_MASK)
 };
 
+#define aux_engine_regs(id)\
+[id] = {\
+	AUX_COMMON_REG_LIST(id), \
+	.AUX_RESET_MASK = 0 \
+}
+
+static const struct dce110_aux_registers aux_engine_regs[] = {
+		aux_engine_regs(0),
+		aux_engine_regs(1),
+		aux_engine_regs(2),
+		aux_engine_regs(3),
+		aux_engine_regs(4),
+		aux_engine_regs(5)
+};
+
 #define audio_regs(id)\
 [id] = {\
 	AUD_COMMON_REG_LIST(id)\
@@ -588,6 +604,23 @@ struct output_pixel_processor *dce112_opp_create(
 	return &opp->base;
 }
 
+struct aux_engine *dce112_aux_engine_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct aux_engine_dce110 *aux_engine =
+		kzalloc(sizeof(struct aux_engine_dce110), GFP_KERNEL);
+
+	if (!aux_engine)
+		return NULL;
+
+	dce110_aux_engine_construct(aux_engine, ctx, inst,
+				    SW_AUX_TIMEOUT_PERIOD_MULTIPLIER * AUX_TIMEOUT_PERIOD,
+				    &aux_engine_regs[inst]);
+
+	return &aux_engine->base;
+}
+
 struct clock_source *dce112_clock_source_create(
 	struct dc_context *ctx,
 	struct dc_bios *bios,
@@ -625,6 +658,9 @@ static void destruct(struct dce110_resource_pool *pool)
 		if (pool->base.opps[i] != NULL)
 			dce110_opp_destroy(&pool->base.opps[i]);
 
+		if (pool->base.engines[i] != NULL)
+			dce110_engine_destroy(&pool->base.engines[i]);
+
 		if (pool->base.transforms[i] != NULL)
 			dce112_transform_destroy(&pool->base.transforms[i]);
 
@@ -640,6 +676,7 @@ static void destruct(struct dce110_resource_pool *pool)
 			kfree(DCE110TG_FROM_TG(pool->base.timing_generators[i]));
 			pool->base.timing_generators[i] = NULL;
 		}
+
 	}
 
 	for (i = 0; i < pool->base.stream_enc_count; i++) {
@@ -668,8 +705,8 @@ static void destruct(struct dce110_resource_pool *pool)
 	if (pool->base.dmcu != NULL)
 		dce_dmcu_destroy(&pool->base.dmcu);
 
-	if (pool->base.display_clock != NULL)
-		dce_disp_clk_destroy(&pool->base.display_clock);
+	if (pool->base.dccg != NULL)
+		dce_dccg_destroy(&pool->base.dccg);
 
 	if (pool->base.irqs != NULL) {
 		dal_irq_service_destroy(&pool->base.irqs);
@@ -744,43 +781,38 @@ bool dce112_validate_bandwidth(
 
 	if (memcmp(&dc->current_state->bw.dce,
 			&context->bw.dce, sizeof(context->bw.dce))) {
-		struct log_entry log_entry;
-		dm_logger_open(
-			dc->ctx->logger,
-			&log_entry,
-			LOG_BANDWIDTH_CALCS);
-		dm_logger_append(&log_entry, "%s: finish,\n"
+
+		DC_LOG_BANDWIDTH_CALCS(
+			"%s: finish,\n"
+			"nbpMark_b: %d nbpMark_a: %d urgentMark_b: %d urgentMark_a: %d\n"
+			"stutMark_b: %d stutMark_a: %d\n"
 			"nbpMark_b: %d nbpMark_a: %d urgentMark_b: %d urgentMark_a: %d\n"
-			"stutMark_b: %d stutMark_a: %d\n",
+			"stutMark_b: %d stutMark_a: %d\n"
+			"nbpMark_b: %d nbpMark_a: %d urgentMark_b: %d urgentMark_a: %d\n"
+			"stutMark_b: %d stutMark_a: %d stutter_mode_enable: %d\n"
+			"cstate: %d pstate: %d nbpstate: %d sync: %d dispclk: %d\n"
+			"sclk: %d sclk_sleep: %d yclk: %d blackout_recovery_time_us: %d\n"
+			,
 			__func__,
 			context->bw.dce.nbp_state_change_wm_ns[0].b_mark,
 			context->bw.dce.nbp_state_change_wm_ns[0].a_mark,
 			context->bw.dce.urgent_wm_ns[0].b_mark,
 			context->bw.dce.urgent_wm_ns[0].a_mark,
 			context->bw.dce.stutter_exit_wm_ns[0].b_mark,
-			context->bw.dce.stutter_exit_wm_ns[0].a_mark);
-		dm_logger_append(&log_entry,
-			"nbpMark_b: %d nbpMark_a: %d urgentMark_b: %d urgentMark_a: %d\n"
-			"stutMark_b: %d stutMark_a: %d\n",
+			context->bw.dce.stutter_exit_wm_ns[0].a_mark,
 			context->bw.dce.nbp_state_change_wm_ns[1].b_mark,
 			context->bw.dce.nbp_state_change_wm_ns[1].a_mark,
 			context->bw.dce.urgent_wm_ns[1].b_mark,
 			context->bw.dce.urgent_wm_ns[1].a_mark,
 			context->bw.dce.stutter_exit_wm_ns[1].b_mark,
-			context->bw.dce.stutter_exit_wm_ns[1].a_mark);
-		dm_logger_append(&log_entry,
-			"nbpMark_b: %d nbpMark_a: %d urgentMark_b: %d urgentMark_a: %d\n"
-			"stutMark_b: %d stutMark_a: %d stutter_mode_enable: %d\n",
+			context->bw.dce.stutter_exit_wm_ns[1].a_mark,
 			context->bw.dce.nbp_state_change_wm_ns[2].b_mark,
 			context->bw.dce.nbp_state_change_wm_ns[2].a_mark,
 			context->bw.dce.urgent_wm_ns[2].b_mark,
 			context->bw.dce.urgent_wm_ns[2].a_mark,
 			context->bw.dce.stutter_exit_wm_ns[2].b_mark,
 			context->bw.dce.stutter_exit_wm_ns[2].a_mark,
-			context->bw.dce.stutter_mode_enable);
-		dm_logger_append(&log_entry,
-			"cstate: %d pstate: %d nbpstate: %d sync: %d dispclk: %d\n"
-			"sclk: %d sclk_sleep: %d yclk: %d blackout_recovery_time_us: %d\n",
+			context->bw.dce.stutter_mode_enable,
 			context->bw.dce.cpuc_state_change_enable,
 			context->bw.dce.cpup_state_change_enable,
 			context->bw.dce.nbp_state_change_enable,
@@ -790,7 +822,6 @@ bool dce112_validate_bandwidth(
 			context->bw.dce.sclk_deep_sleep_khz,
 			context->bw.dce.yclk_khz,
 			context->bw.dce.blackout_recovery_time_us);
-		dm_logger_close(&log_entry);
 	}
 	return result;
 }
@@ -1000,7 +1031,7 @@ static void bw_calcs_data_update_from_pplib(struct dc *dc)
 			eng_clks.data[0].clocks_in_khz;
 	clk_ranges.wm_clk_ranges[0].wm_max_eng_clk_in_khz =
 			eng_clks.data[eng_clks.num_levels*3/8].clocks_in_khz - 1;
-	clk_ranges.wm_clk_ranges[0].wm_min_memg_clk_in_khz =
+	clk_ranges.wm_clk_ranges[0].wm_min_mem_clk_in_khz =
 			mem_clks.data[0].clocks_in_khz;
 	clk_ranges.wm_clk_ranges[0].wm_max_mem_clk_in_khz =
 			mem_clks.data[mem_clks.num_levels>>1].clocks_in_khz - 1;
@@ -1010,7 +1041,7 @@ static void bw_calcs_data_update_from_pplib(struct dc *dc)
 			eng_clks.data[eng_clks.num_levels*3/8].clocks_in_khz;
 	/* 5 GHz instead of data[7].clockInKHz to cover Overdrive */
 	clk_ranges.wm_clk_ranges[1].wm_max_eng_clk_in_khz = 5000000;
-	clk_ranges.wm_clk_ranges[1].wm_min_memg_clk_in_khz =
+	clk_ranges.wm_clk_ranges[1].wm_min_mem_clk_in_khz =
 			mem_clks.data[0].clocks_in_khz;
 	clk_ranges.wm_clk_ranges[1].wm_max_mem_clk_in_khz =
 			mem_clks.data[mem_clks.num_levels>>1].clocks_in_khz - 1;
@@ -1020,7 +1051,7 @@ static void bw_calcs_data_update_from_pplib(struct dc *dc)
 			eng_clks.data[0].clocks_in_khz;
 	clk_ranges.wm_clk_ranges[2].wm_max_eng_clk_in_khz =
 			eng_clks.data[eng_clks.num_levels*3/8].clocks_in_khz - 1;
-	clk_ranges.wm_clk_ranges[2].wm_min_memg_clk_in_khz =
+	clk_ranges.wm_clk_ranges[2].wm_min_mem_clk_in_khz =
 			mem_clks.data[mem_clks.num_levels>>1].clocks_in_khz;
 	/* 5 GHz instead of data[2].clockInKHz to cover Overdrive */
 	clk_ranges.wm_clk_ranges[2].wm_max_mem_clk_in_khz = 5000000;
@@ -1030,7 +1061,7 @@ static void bw_calcs_data_update_from_pplib(struct dc *dc)
 			eng_clks.data[eng_clks.num_levels*3/8].clocks_in_khz;
 	/* 5 GHz instead of data[7].clockInKHz to cover Overdrive */
 	clk_ranges.wm_clk_ranges[3].wm_max_eng_clk_in_khz = 5000000;
-	clk_ranges.wm_clk_ranges[3].wm_min_memg_clk_in_khz =
+	clk_ranges.wm_clk_ranges[3].wm_min_mem_clk_in_khz =
 			mem_clks.data[mem_clks.num_levels>>1].clocks_in_khz;
 	/* 5 GHz instead of data[2].clockInKHz to cover Overdrive */
 	clk_ranges.wm_clk_ranges[3].wm_max_mem_clk_in_khz = 5000000;
@@ -1124,11 +1155,11 @@ static bool construct(
 		}
 	}
 
-	pool->base.display_clock = dce112_disp_clk_create(ctx,
+	pool->base.dccg = dce112_dccg_create(ctx,
 			&disp_clk_regs,
 			&disp_clk_shift,
 			&disp_clk_mask);
-	if (pool->base.display_clock == NULL) {
+	if (pool->base.dccg == NULL) {
 		dm_error("DC: failed to create display clock!\n");
 		BREAK_TO_DEBUGGER();
 		goto res_create_fail;
@@ -1158,7 +1189,7 @@ static bool construct(
 	 * max_clock_state
 	 */
 	if (dm_pp_get_static_clocks(ctx, &static_clk_info))
-		pool->base.display_clock->max_clks_state =
+		pool->base.dccg->max_clks_state =
 				static_clk_info.max_clocks_state;
 
 	{
@@ -1214,6 +1245,13 @@ static bool construct(
 				"DC:failed to create output pixel processor!\n");
 			goto res_create_fail;
 		}
+		pool->base.engines[i] = dce112_aux_engine_create(ctx, i);
+		if (pool->base.engines[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC:failed to create aux engine!!\n");
+			goto res_create_fail;
+		}
 	}
 
 	if (!resource_construct(num_virtual_links, dc, &pool->base,
diff --git a/drivers/gpu/drm/amd/display/dc/dce120/dce120_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/dce120/dce120_hw_sequencer.c
index e96ff86d2fc3..5853522a6182 100644
--- a/drivers/gpu/drm/amd/display/dc/dce120/dce120_hw_sequencer.c
+++ b/drivers/gpu/drm/amd/display/dc/dce120/dce120_hw_sequencer.c
@@ -244,7 +244,16 @@ static void dce120_update_dchub(
 	dh_data->dchub_info_valid = false;
 }
 
+static void dce120_set_bandwidth(
+		struct dc *dc,
+		struct dc_state *context,
+		bool decrease_allowed)
+{
+	if (context->stream_count <= 0)
+		return;
 
+	dce110_set_bandwidth(dc, context, decrease_allowed);
+}
 
 void dce120_hw_sequencer_construct(struct dc *dc)
 {
@@ -254,5 +263,6 @@ void dce120_hw_sequencer_construct(struct dc *dc)
 	dce110_hw_sequencer_construct(dc);
 	dc->hwss.enable_display_power_gating = dce120_enable_display_power_gating;
 	dc->hwss.update_dchub = dce120_update_dchub;
+	dc->hwss.set_bandwidth = dce120_set_bandwidth;
 }
 
diff --git a/drivers/gpu/drm/amd/display/dc/dce120/dce120_resource.c b/drivers/gpu/drm/amd/display/dc/dce120/dce120_resource.c
index 2d58daccc005..d43f37d99c7d 100644
--- a/drivers/gpu/drm/amd/display/dc/dce120/dce120_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/dce120/dce120_resource.c
@@ -53,6 +53,7 @@
 #include "dce/dce_hwseq.h"
 #include "dce/dce_abm.h"
 #include "dce/dce_dmcu.h"
+#include "dce/dce_aux.h"
 
 #include "dce/dce_12_0_offset.h"
 #include "dce/dce_12_0_sh_mask.h"
@@ -297,6 +298,20 @@ static const struct dce_opp_shift opp_shift = {
 static const struct dce_opp_mask opp_mask = {
 	OPP_COMMON_MASK_SH_LIST_DCE_120(_MASK)
 };
+ #define aux_engine_regs(id)\
+[id] = {\
+	AUX_COMMON_REG_LIST(id), \
+	.AUX_RESET_MASK = 0 \
+}
+
+static const struct dce110_aux_registers aux_engine_regs[] = {
+		aux_engine_regs(0),
+		aux_engine_regs(1),
+		aux_engine_regs(2),
+		aux_engine_regs(3),
+		aux_engine_regs(4),
+		aux_engine_regs(5)
+};
 
 #define audio_regs(id)\
 [id] = {\
@@ -361,6 +376,22 @@ struct output_pixel_processor *dce120_opp_create(
 			     ctx, inst, &opp_regs[inst], &opp_shift, &opp_mask);
 	return &opp->base;
 }
+struct aux_engine *dce120_aux_engine_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct aux_engine_dce110 *aux_engine =
+		kzalloc(sizeof(struct aux_engine_dce110), GFP_KERNEL);
+
+	if (!aux_engine)
+		return NULL;
+
+	dce110_aux_engine_construct(aux_engine, ctx, inst,
+				    SW_AUX_TIMEOUT_PERIOD_MULTIPLIER * AUX_TIMEOUT_PERIOD,
+				    &aux_engine_regs[inst]);
+
+	return &aux_engine->base;
+}
 
 static const struct bios_registers bios_regs = {
 	.BIOS_SCRATCH_6 = mmBIOS_SCRATCH_6 + NBIO_BASE(mmBIOS_SCRATCH_6_BASE_IDX)
@@ -373,7 +404,7 @@ static const struct resource_caps res_cap = {
 		.num_pll = 6,
 };
 
-static const struct dc_debug debug_defaults = {
+static const struct dc_debug_options debug_defaults = {
 		.disable_clock_gate = true,
 };
 
@@ -467,6 +498,10 @@ static void destruct(struct dce110_resource_pool *pool)
 			kfree(DCE110TG_FROM_TG(pool->base.timing_generators[i]));
 			pool->base.timing_generators[i] = NULL;
 		}
+
+		if (pool->base.engines[i] != NULL)
+			dce110_engine_destroy(&pool->base.engines[i]);
+
 	}
 
 	for (i = 0; i < pool->base.audio_count; i++) {
@@ -494,8 +529,8 @@ static void destruct(struct dce110_resource_pool *pool)
 	if (pool->base.dmcu != NULL)
 		dce_dmcu_destroy(&pool->base.dmcu);
 
-	if (pool->base.display_clock != NULL)
-		dce_disp_clk_destroy(&pool->base.display_clock);
+	if (pool->base.dccg != NULL)
+		dce_dccg_destroy(&pool->base.dccg);
 }
 
 static void read_dce_straps(
@@ -775,7 +810,7 @@ static void bw_calcs_data_update_from_pplib(struct dc *dc)
 			eng_clks.data[0].clocks_in_khz;
 	clk_ranges.wm_clk_ranges[0].wm_max_eng_clk_in_khz =
 			eng_clks.data[eng_clks.num_levels*3/8].clocks_in_khz - 1;
-	clk_ranges.wm_clk_ranges[0].wm_min_memg_clk_in_khz =
+	clk_ranges.wm_clk_ranges[0].wm_min_mem_clk_in_khz =
 			mem_clks.data[0].clocks_in_khz;
 	clk_ranges.wm_clk_ranges[0].wm_max_mem_clk_in_khz =
 			mem_clks.data[mem_clks.num_levels>>1].clocks_in_khz - 1;
@@ -785,7 +820,7 @@ static void bw_calcs_data_update_from_pplib(struct dc *dc)
 			eng_clks.data[eng_clks.num_levels*3/8].clocks_in_khz;
 	/* 5 GHz instead of data[7].clockInKHz to cover Overdrive */
 	clk_ranges.wm_clk_ranges[1].wm_max_eng_clk_in_khz = 5000000;
-	clk_ranges.wm_clk_ranges[1].wm_min_memg_clk_in_khz =
+	clk_ranges.wm_clk_ranges[1].wm_min_mem_clk_in_khz =
 			mem_clks.data[0].clocks_in_khz;
 	clk_ranges.wm_clk_ranges[1].wm_max_mem_clk_in_khz =
 			mem_clks.data[mem_clks.num_levels>>1].clocks_in_khz - 1;
@@ -795,7 +830,7 @@ static void bw_calcs_data_update_from_pplib(struct dc *dc)
 			eng_clks.data[0].clocks_in_khz;
 	clk_ranges.wm_clk_ranges[2].wm_max_eng_clk_in_khz =
 			eng_clks.data[eng_clks.num_levels*3/8].clocks_in_khz - 1;
-	clk_ranges.wm_clk_ranges[2].wm_min_memg_clk_in_khz =
+	clk_ranges.wm_clk_ranges[2].wm_min_mem_clk_in_khz =
 			mem_clks.data[mem_clks.num_levels>>1].clocks_in_khz;
 	/* 5 GHz instead of data[2].clockInKHz to cover Overdrive */
 	clk_ranges.wm_clk_ranges[2].wm_max_mem_clk_in_khz = 5000000;
@@ -805,7 +840,7 @@ static void bw_calcs_data_update_from_pplib(struct dc *dc)
 			eng_clks.data[eng_clks.num_levels*3/8].clocks_in_khz;
 	/* 5 GHz instead of data[7].clockInKHz to cover Overdrive */
 	clk_ranges.wm_clk_ranges[3].wm_max_eng_clk_in_khz = 5000000;
-	clk_ranges.wm_clk_ranges[3].wm_min_memg_clk_in_khz =
+	clk_ranges.wm_clk_ranges[3].wm_min_mem_clk_in_khz =
 			mem_clks.data[mem_clks.num_levels>>1].clocks_in_khz;
 	/* 5 GHz instead of data[2].clockInKHz to cover Overdrive */
 	clk_ranges.wm_clk_ranges[3].wm_max_mem_clk_in_khz = 5000000;
@@ -848,6 +883,7 @@ static bool construct(
 	dc->caps.i2c_speed_in_khz = 100;
 	dc->caps.max_cursor_size = 128;
 	dc->caps.dual_link_dvi = true;
+	dc->caps.psp_setup_panel_mode = true;
 
 	dc->debug = debug_defaults;
 
@@ -894,11 +930,11 @@ static bool construct(
 		}
 	}
 
-	pool->base.display_clock = dce120_disp_clk_create(ctx);
-	if (pool->base.display_clock == NULL) {
+	pool->base.dccg = dce120_dccg_create(ctx);
+	if (pool->base.dccg == NULL) {
 		dm_error("DC: failed to create display clock!\n");
 		BREAK_TO_DEBUGGER();
-		goto disp_clk_create_fail;
+		goto dccg_create_fail;
 	}
 
 	pool->base.dmcu = dce_dmcu_create(ctx,
@@ -984,6 +1020,13 @@ static bool construct(
 			dm_error(
 				"DC: failed to create output pixel processor!\n");
 		}
+		pool->base.engines[i] = dce120_aux_engine_create(ctx, i);
+				if (pool->base.engines[i] == NULL) {
+					BREAK_TO_DEBUGGER();
+					dm_error(
+						"DC:failed to create aux engine!!\n");
+					goto res_create_fail;
+				}
 
 		/* check next valid pipe */
 		j++;
@@ -1011,7 +1054,7 @@ static bool construct(
 
 irqs_create_fail:
 controller_create_fail:
-disp_clk_create_fail:
+dccg_create_fail:
 clk_src_create_fail:
 res_create_fail:
 
diff --git a/drivers/gpu/drm/amd/display/dc/dce120/dce120_timing_generator.c b/drivers/gpu/drm/amd/display/dc/dce120/dce120_timing_generator.c
index 2ea490f8482e..04b866f0fa1f 100644
--- a/drivers/gpu/drm/amd/display/dc/dce120/dce120_timing_generator.c
+++ b/drivers/gpu/drm/amd/display/dc/dce120/dce120_timing_generator.c
@@ -772,7 +772,7 @@ void dce120_tg_set_blank(struct timing_generator *tg,
 
 	CRTC_REG_SET(
 		CRTC0_CRTC_DOUBLE_BUFFER_CONTROL,
-		CRTC_BLANK_DATA_DOUBLE_BUFFER_EN, 0);
+		CRTC_BLANK_DATA_DOUBLE_BUFFER_EN, 1);
 
 	if (enable_blanking)
 		CRTC_REG_SET(CRTC0_CRTC_BLANK_CONTROL, CRTC_BLANK_DATA_EN, 1);
diff --git a/drivers/gpu/drm/amd/display/dc/dce80/dce80_resource.c b/drivers/gpu/drm/amd/display/dc/dce80/dce80_resource.c
index 48a068964722..604c62969ead 100644
--- a/drivers/gpu/drm/amd/display/dc/dce80/dce80_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/dce80/dce80_resource.c
@@ -54,6 +54,7 @@
 #include "reg_helper.h"
 
 #include "dce/dce_dmcu.h"
+#include "dce/dce_aux.h"
 #include "dce/dce_abm.h"
 /* TODO remove this include */
 
@@ -153,15 +154,15 @@ static const struct dce110_timing_generator_offsets dce80_tg_offsets[] = {
 	.reg_name = mm ## block ## id ## _ ## reg_name
 
 
-static const struct dce_disp_clk_registers disp_clk_regs = {
+static const struct dccg_registers disp_clk_regs = {
 		CLK_COMMON_REG_LIST_DCE_BASE()
 };
 
-static const struct dce_disp_clk_shift disp_clk_shift = {
+static const struct dccg_shift disp_clk_shift = {
 		CLK_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(__SHIFT)
 };
 
-static const struct dce_disp_clk_mask disp_clk_mask = {
+static const struct dccg_mask disp_clk_mask = {
 		CLK_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(_MASK)
 };
 
@@ -298,6 +299,21 @@ static const struct dce_opp_mask opp_mask = {
 	OPP_COMMON_MASK_SH_LIST_DCE_80(_MASK)
 };
 
+#define aux_engine_regs(id)\
+[id] = {\
+	AUX_COMMON_REG_LIST(id), \
+	.AUX_RESET_MASK = 0 \
+}
+
+static const struct dce110_aux_registers aux_engine_regs[] = {
+		aux_engine_regs(0),
+		aux_engine_regs(1),
+		aux_engine_regs(2),
+		aux_engine_regs(3),
+		aux_engine_regs(4),
+		aux_engine_regs(5)
+};
+
 #define audio_regs(id)\
 [id] = {\
 	AUD_COMMON_REG_LIST(id)\
@@ -448,6 +464,23 @@ static struct output_pixel_processor *dce80_opp_create(
 	return &opp->base;
 }
 
+struct aux_engine *dce80_aux_engine_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct aux_engine_dce110 *aux_engine =
+		kzalloc(sizeof(struct aux_engine_dce110), GFP_KERNEL);
+
+	if (!aux_engine)
+		return NULL;
+
+	dce110_aux_engine_construct(aux_engine, ctx, inst,
+				    SW_AUX_TIMEOUT_PERIOD_MULTIPLIER * AUX_TIMEOUT_PERIOD,
+				    &aux_engine_regs[inst]);
+
+	return &aux_engine->base;
+}
+
 static struct stream_encoder *dce80_stream_encoder_create(
 	enum engine_id eng_id,
 	struct dc_context *ctx)
@@ -655,6 +688,9 @@ static void destruct(struct dce110_resource_pool *pool)
 			kfree(DCE110TG_FROM_TG(pool->base.timing_generators[i]));
 			pool->base.timing_generators[i] = NULL;
 		}
+
+		if (pool->base.engines[i] != NULL)
+			dce110_engine_destroy(&pool->base.engines[i]);
 	}
 
 	for (i = 0; i < pool->base.stream_enc_count; i++) {
@@ -683,8 +719,8 @@ static void destruct(struct dce110_resource_pool *pool)
 		}
 	}
 
-	if (pool->base.display_clock != NULL)
-		dce_disp_clk_destroy(&pool->base.display_clock);
+	if (pool->base.dccg != NULL)
+		dce_dccg_destroy(&pool->base.dccg);
 
 	if (pool->base.irqs != NULL) {
 		dal_irq_service_destroy(&pool->base.irqs);
@@ -822,11 +858,11 @@ static bool dce80_construct(
 		}
 	}
 
-	pool->base.display_clock = dce_disp_clk_create(ctx,
+	pool->base.dccg = dce_dccg_create(ctx,
 			&disp_clk_regs,
 			&disp_clk_shift,
 			&disp_clk_mask);
-	if (pool->base.display_clock == NULL) {
+	if (pool->base.dccg == NULL) {
 		dm_error("DC: failed to create display clock!\n");
 		BREAK_TO_DEBUGGER();
 		goto res_create_fail;
@@ -852,7 +888,7 @@ static bool dce80_construct(
 		goto res_create_fail;
 	}
 	if (dm_pp_get_static_clocks(ctx, &static_clk_info))
-		pool->base.display_clock->max_clks_state =
+		pool->base.dccg->max_clks_state =
 					static_clk_info.max_clocks_state;
 
 	{
@@ -899,9 +935,18 @@ static bool dce80_construct(
 			dm_error("DC: failed to create output pixel processor!\n");
 			goto res_create_fail;
 		}
+
+		pool->base.engines[i] = dce80_aux_engine_create(ctx, i);
+		if (pool->base.engines[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC:failed to create aux engine!!\n");
+			goto res_create_fail;
+		}
 	}
 
 	dc->caps.max_planes =  pool->base.pipe_count;
+	dc->caps.disable_dp_clk_share = true;
 
 	if (!resource_construct(num_virtual_links, dc, &pool->base,
 			&res_create_funcs))
@@ -1006,11 +1051,11 @@ static bool dce81_construct(
 		}
 	}
 
-	pool->base.display_clock = dce_disp_clk_create(ctx,
+	pool->base.dccg = dce_dccg_create(ctx,
 			&disp_clk_regs,
 			&disp_clk_shift,
 			&disp_clk_mask);
-	if (pool->base.display_clock == NULL) {
+	if (pool->base.dccg == NULL) {
 		dm_error("DC: failed to create display clock!\n");
 		BREAK_TO_DEBUGGER();
 		goto res_create_fail;
@@ -1037,7 +1082,7 @@ static bool dce81_construct(
 	}
 
 	if (dm_pp_get_static_clocks(ctx, &static_clk_info))
-		pool->base.display_clock->max_clks_state =
+		pool->base.dccg->max_clks_state =
 					static_clk_info.max_clocks_state;
 
 	{
@@ -1087,6 +1132,7 @@ static bool dce81_construct(
 	}
 
 	dc->caps.max_planes =  pool->base.pipe_count;
+	dc->caps.disable_dp_clk_share = true;
 
 	if (!resource_construct(num_virtual_links, dc, &pool->base,
 			&res_create_funcs))
@@ -1187,11 +1233,11 @@ static bool dce83_construct(
 		}
 	}
 
-	pool->base.display_clock = dce_disp_clk_create(ctx,
+	pool->base.dccg = dce_dccg_create(ctx,
 			&disp_clk_regs,
 			&disp_clk_shift,
 			&disp_clk_mask);
-	if (pool->base.display_clock == NULL) {
+	if (pool->base.dccg == NULL) {
 		dm_error("DC: failed to create display clock!\n");
 		BREAK_TO_DEBUGGER();
 		goto res_create_fail;
@@ -1218,7 +1264,7 @@ static bool dce83_construct(
 	}
 
 	if (dm_pp_get_static_clocks(ctx, &static_clk_info))
-		pool->base.display_clock->max_clks_state =
+		pool->base.dccg->max_clks_state =
 					static_clk_info.max_clocks_state;
 
 	{
@@ -1268,6 +1314,7 @@ static bool dce83_construct(
 	}
 
 	dc->caps.max_planes =  pool->base.pipe_count;
+	dc->caps.disable_dp_clk_share = true;
 
 	if (!resource_construct(num_virtual_links, dc, &pool->base,
 			&res_create_funcs))
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.c
index c69fa4bfab0a..bf8b68f8db4f 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.c
@@ -145,10 +145,10 @@ static bool dpp_get_optimal_number_of_taps(
 		pixel_width = scl_data->viewport.width;
 
 	/* Some ASICs does not support  FP16 scaling, so we reject modes require this*/
-	if (scl_data->viewport.width  != scl_data->h_active &&
-		scl_data->viewport.height != scl_data->v_active &&
+	if (scl_data->format == PIXEL_FORMAT_FP16 &&
 		dpp->caps->dscl_data_proc_format == DSCL_DATA_PRCESSING_FIXED_FORMAT &&
-		scl_data->format == PIXEL_FORMAT_FP16)
+		scl_data->ratios.horz.value != dc_fixpt_one.value &&
+		scl_data->ratios.vert.value != dc_fixpt_one.value)
 		return false;
 
 	if (scl_data->viewport.width > scl_data->h_active &&
@@ -445,10 +445,10 @@ void dpp1_set_cursor_position(
 		uint32_t width)
 {
 	struct dcn10_dpp *dpp = TO_DCN10_DPP(dpp_base);
-	int src_x_offset = pos->x - pos->x_hotspot - param->viewport_x_start;
+	int src_x_offset = pos->x - pos->x_hotspot - param->viewport.x;
 	uint32_t cur_en = pos->enable ? 1 : 0;
 
-	if (src_x_offset >= (int)param->viewport_width)
+	if (src_x_offset >= (int)param->viewport.width)
 		cur_en = 0;  /* not visible beyond right edge*/
 
 	if (src_x_offset + (int)width <= 0)
@@ -459,6 +459,18 @@ void dpp1_set_cursor_position(
 
 }
 
+void dpp1_cnv_set_optional_cursor_attributes(
+		struct dpp *dpp_base,
+		struct dpp_cursor_attributes *attr)
+{
+	struct dcn10_dpp *dpp = TO_DCN10_DPP(dpp_base);
+
+	if (attr) {
+		REG_UPDATE(CURSOR0_FP_SCALE_BIAS,  CUR0_FP_BIAS,  attr->bias);
+		REG_UPDATE(CURSOR0_FP_SCALE_BIAS,  CUR0_FP_SCALE, attr->scale);
+	}
+}
+
 void dpp1_dppclk_control(
 		struct dpp *dpp_base,
 		bool dppclk_div,
@@ -499,6 +511,7 @@ static const struct dpp_funcs dcn10_dpp_funcs = {
 		.dpp_full_bypass		= dpp1_full_bypass,
 		.set_cursor_attributes = dpp1_set_cursor_attributes,
 		.set_cursor_position = dpp1_set_cursor_position,
+		.set_optional_cursor_attributes = dpp1_cnv_set_optional_cursor_attributes,
 		.dpp_dppclk_control = dpp1_dppclk_control,
 		.dpp_set_hdr_multiplier = dpp1_set_hdr_multiplier,
 };
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.h
index e862cafa6501..e2889e61b18c 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.h
@@ -119,6 +119,7 @@
 	SRI(CURSOR0_CONTROL, CNVC_CUR, id), \
 	SRI(CURSOR0_COLOR0, CNVC_CUR, id), \
 	SRI(CURSOR0_COLOR1, CNVC_CUR, id), \
+	SRI(CURSOR0_FP_SCALE_BIAS, CNVC_CUR, id), \
 	SRI(DPP_CONTROL, DPP_TOP, id), \
 	SRI(CM_HDR_MULT_COEF, CM, id)
 
@@ -324,6 +325,8 @@
 	TF_SF(CNVC_CUR0_CURSOR0_CONTROL, CUR0_ENABLE, mask_sh), \
 	TF_SF(CNVC_CUR0_CURSOR0_COLOR0, CUR0_COLOR0, mask_sh), \
 	TF_SF(CNVC_CUR0_CURSOR0_COLOR1, CUR0_COLOR1, mask_sh), \
+	TF_SF(CNVC_CUR0_CURSOR0_FP_SCALE_BIAS, CUR0_FP_BIAS, mask_sh), \
+	TF_SF(CNVC_CUR0_CURSOR0_FP_SCALE_BIAS, CUR0_FP_SCALE, mask_sh), \
 	TF_SF(DPP_TOP0_DPP_CONTROL, DPP_CLOCK_ENABLE, mask_sh), \
 	TF_SF(CM0_CM_HDR_MULT_COEF, CM_HDR_MULT_COEF, mask_sh)
 
@@ -1076,7 +1079,9 @@
 	type CUR0_COLOR1; \
 	type DPPCLK_RATE_CONTROL; \
 	type DPP_CLOCK_ENABLE; \
-	type CM_HDR_MULT_COEF;
+	type CM_HDR_MULT_COEF; \
+	type CUR0_FP_BIAS; \
+	type CUR0_FP_SCALE;
 
 struct dcn_dpp_shift {
 	TF_REG_FIELD_LIST(uint8_t)
@@ -1329,7 +1334,8 @@ struct dcn_dpp_mask {
 	uint32_t CURSOR0_COLOR0; \
 	uint32_t CURSOR0_COLOR1; \
 	uint32_t DPP_CONTROL; \
-	uint32_t CM_HDR_MULT_COEF;
+	uint32_t CM_HDR_MULT_COEF; \
+	uint32_t CURSOR0_FP_SCALE_BIAS;
 
 struct dcn_dpp_registers {
 	DPP_COMMON_REG_VARIABLE_LIST
@@ -1370,6 +1376,10 @@ void dpp1_set_cursor_position(
 		const struct dc_cursor_mi_param *param,
 		uint32_t width);
 
+void dpp1_cnv_set_optional_cursor_attributes(
+			struct dpp *dpp_base,
+			struct dpp_cursor_attributes *attr);
+
 bool dpp1_dscl_is_lb_conf_valid(
 		int ceil_vratio,
 		int num_partitions,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_dscl.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_dscl.c
index f862fd148cca..4a863a5dab41 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_dscl.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_dscl.c
@@ -621,6 +621,10 @@ static void dpp1_dscl_set_manual_ratio_init(
 static void dpp1_dscl_set_recout(
 			struct dcn10_dpp *dpp, const struct rect *recout)
 {
+	int visual_confirm_on = 0;
+	if (dpp->base.ctx->dc->debug.visual_confirm != VISUAL_CONFIRM_DISABLE)
+		visual_confirm_on = 1;
+
 	REG_SET_2(RECOUT_START, 0,
 		/* First pixel of RECOUT */
 			 RECOUT_START_X, recout->x,
@@ -632,8 +636,7 @@ static void dpp1_dscl_set_recout(
 			 RECOUT_WIDTH, recout->width,
 		/* Number of RECOUT vertical lines */
 			 RECOUT_HEIGHT, recout->height
-			 - dpp->base.ctx->dc->debug.surface_visual_confirm * 4 *
-			 (dpp->base.inst + 1));
+			 - visual_confirm_on * 4 * (dpp->base.inst + 1));
 }
 
 /* Main function to program scaler and line buffer in manual scaling mode */
@@ -655,6 +658,12 @@ void dpp1_dscl_set_scaler_manual_scale(
 
 	dpp->scl_data = *scl_data;
 
+	/* Autocal off */
+	REG_SET_3(DSCL_AUTOCAL, 0,
+		AUTOCAL_MODE, AUTOCAL_MODE_OFF,
+		AUTOCAL_NUM_PIPE, 0,
+		AUTOCAL_PIPE_ID, 0);
+
 	/* Recout */
 	dpp1_dscl_set_recout(dpp, &scl_data->recout);
 
@@ -678,12 +687,6 @@ void dpp1_dscl_set_scaler_manual_scale(
 	if (dscl_mode == DSCL_MODE_SCALING_444_BYPASS)
 		return;
 
-	/* Autocal off */
-	REG_SET_3(DSCL_AUTOCAL, 0,
-		AUTOCAL_MODE, AUTOCAL_MODE_OFF,
-		AUTOCAL_NUM_PIPE, 0,
-		AUTOCAL_PIPE_ID, 0);
-
 	/* Black offsets */
 	if (ycbcr)
 		REG_SET_2(SCL_BLACK_OFFSET, 0,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.c
index 943143efbb82..1ea91e153d3a 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.c
@@ -190,10 +190,17 @@ static uint32_t convert_and_clamp(
 }
 
 
+void hubbub1_wm_change_req_wa(struct hubbub *hubbub)
+{
+	REG_UPDATE_SEQ(DCHUBBUB_ARB_WATERMARK_CHANGE_CNTL,
+			DCHUBBUB_ARB_WATERMARK_CHANGE_REQUEST, 0, 1);
+}
+
 void hubbub1_program_watermarks(
 		struct hubbub *hubbub,
 		struct dcn_watermark_set *watermarks,
-		unsigned int refclk_mhz)
+		unsigned int refclk_mhz,
+		bool safe_to_lower)
 {
 	uint32_t force_en = hubbub->ctx->dc->debug.disable_stutter ? 1 : 0;
 	/*
@@ -202,191 +209,259 @@ void hubbub1_program_watermarks(
 	 */
 	uint32_t prog_wm_value;
 
-	REG_UPDATE(DCHUBBUB_ARB_WATERMARK_CHANGE_CNTL,
-			DCHUBBUB_ARB_WATERMARK_CHANGE_REQUEST, 0);
 
 	/* Repeat for water mark set A, B, C and D. */
 	/* clock state A */
-	prog_wm_value = convert_and_clamp(watermarks->a.urgent_ns,
-			refclk_mhz, 0x1fffff);
-	REG_WRITE(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_A, prog_wm_value);
-
-	DC_LOG_BANDWIDTH_CALCS("URGENCY_WATERMARK_A calculated =%d\n"
-		"HW register value = 0x%x\n",
-		watermarks->a.urgent_ns, prog_wm_value);
+	if (safe_to_lower || watermarks->a.urgent_ns > hubbub->watermarks.a.urgent_ns) {
+		hubbub->watermarks.a.urgent_ns = watermarks->a.urgent_ns;
+		prog_wm_value = convert_and_clamp(watermarks->a.urgent_ns,
+				refclk_mhz, 0x1fffff);
+		REG_WRITE(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_A, prog_wm_value);
 
-	prog_wm_value = convert_and_clamp(watermarks->a.pte_meta_urgent_ns,
-			refclk_mhz, 0x1fffff);
-	REG_WRITE(DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_A, prog_wm_value);
-	DC_LOG_BANDWIDTH_CALCS("PTE_META_URGENCY_WATERMARK_A calculated =%d\n"
-		"HW register value = 0x%x\n",
-		watermarks->a.pte_meta_urgent_ns, prog_wm_value);
+		DC_LOG_BANDWIDTH_CALCS("URGENCY_WATERMARK_A calculated =%d\n"
+			"HW register value = 0x%x\n",
+			watermarks->a.urgent_ns, prog_wm_value);
+	}
 
-	if (REG(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_A)) {
-		prog_wm_value = convert_and_clamp(
-				watermarks->a.cstate_pstate.cstate_enter_plus_exit_ns,
+	if (safe_to_lower || watermarks->a.pte_meta_urgent_ns > hubbub->watermarks.a.pte_meta_urgent_ns) {
+		hubbub->watermarks.a.pte_meta_urgent_ns = watermarks->a.pte_meta_urgent_ns;
+		prog_wm_value = convert_and_clamp(watermarks->a.pte_meta_urgent_ns,
 				refclk_mhz, 0x1fffff);
-		REG_WRITE(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_A, prog_wm_value);
-		DC_LOG_BANDWIDTH_CALCS("SR_ENTER_EXIT_WATERMARK_A calculated =%d\n"
+		REG_WRITE(DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_A, prog_wm_value);
+		DC_LOG_BANDWIDTH_CALCS("PTE_META_URGENCY_WATERMARK_A calculated =%d\n"
 			"HW register value = 0x%x\n",
-			watermarks->a.cstate_pstate.cstate_enter_plus_exit_ns, prog_wm_value);
+			watermarks->a.pte_meta_urgent_ns, prog_wm_value);
+	}
+
+	if (REG(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_A)) {
+		if (safe_to_lower || watermarks->a.cstate_pstate.cstate_enter_plus_exit_ns
+				> hubbub->watermarks.a.cstate_pstate.cstate_enter_plus_exit_ns) {
+			hubbub->watermarks.a.cstate_pstate.cstate_enter_plus_exit_ns =
+					watermarks->a.cstate_pstate.cstate_enter_plus_exit_ns;
+			prog_wm_value = convert_and_clamp(
+					watermarks->a.cstate_pstate.cstate_enter_plus_exit_ns,
+					refclk_mhz, 0x1fffff);
+			REG_WRITE(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_A, prog_wm_value);
+			DC_LOG_BANDWIDTH_CALCS("SR_ENTER_EXIT_WATERMARK_A calculated =%d\n"
+				"HW register value = 0x%x\n",
+				watermarks->a.cstate_pstate.cstate_enter_plus_exit_ns, prog_wm_value);
+		}
 
+		if (safe_to_lower || watermarks->a.cstate_pstate.cstate_exit_ns
+				> hubbub->watermarks.a.cstate_pstate.cstate_exit_ns) {
+			hubbub->watermarks.a.cstate_pstate.cstate_exit_ns =
+					watermarks->a.cstate_pstate.cstate_exit_ns;
+			prog_wm_value = convert_and_clamp(
+					watermarks->a.cstate_pstate.cstate_exit_ns,
+					refclk_mhz, 0x1fffff);
+			REG_WRITE(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_A, prog_wm_value);
+			DC_LOG_BANDWIDTH_CALCS("SR_EXIT_WATERMARK_A calculated =%d\n"
+				"HW register value = 0x%x\n",
+				watermarks->a.cstate_pstate.cstate_exit_ns, prog_wm_value);
+		}
+	}
 
+	if (safe_to_lower || watermarks->a.cstate_pstate.pstate_change_ns
+			> hubbub->watermarks.a.cstate_pstate.pstate_change_ns) {
+		hubbub->watermarks.a.cstate_pstate.pstate_change_ns =
+				watermarks->a.cstate_pstate.pstate_change_ns;
 		prog_wm_value = convert_and_clamp(
-				watermarks->a.cstate_pstate.cstate_exit_ns,
+				watermarks->a.cstate_pstate.pstate_change_ns,
 				refclk_mhz, 0x1fffff);
-		REG_WRITE(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_A, prog_wm_value);
-		DC_LOG_BANDWIDTH_CALCS("SR_EXIT_WATERMARK_A calculated =%d\n"
-			"HW register value = 0x%x\n",
-			watermarks->a.cstate_pstate.cstate_exit_ns, prog_wm_value);
+		REG_WRITE(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_A, prog_wm_value);
+		DC_LOG_BANDWIDTH_CALCS("DRAM_CLK_CHANGE_WATERMARK_A calculated =%d\n"
+			"HW register value = 0x%x\n\n",
+			watermarks->a.cstate_pstate.pstate_change_ns, prog_wm_value);
 	}
 
-	prog_wm_value = convert_and_clamp(
-			watermarks->a.cstate_pstate.pstate_change_ns,
-			refclk_mhz, 0x1fffff);
-	REG_WRITE(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_A, prog_wm_value);
-	DC_LOG_BANDWIDTH_CALCS("DRAM_CLK_CHANGE_WATERMARK_A calculated =%d\n"
-		"HW register value = 0x%x\n\n",
-		watermarks->a.cstate_pstate.pstate_change_ns, prog_wm_value);
-
-
 	/* clock state B */
-	prog_wm_value = convert_and_clamp(
-			watermarks->b.urgent_ns, refclk_mhz, 0x1fffff);
-	REG_WRITE(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_B, prog_wm_value);
-	DC_LOG_BANDWIDTH_CALCS("URGENCY_WATERMARK_B calculated =%d\n"
-		"HW register value = 0x%x\n",
-		watermarks->b.urgent_ns, prog_wm_value);
-
-
-	prog_wm_value = convert_and_clamp(
-			watermarks->b.pte_meta_urgent_ns,
-			refclk_mhz, 0x1fffff);
-	REG_WRITE(DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_B, prog_wm_value);
-	DC_LOG_BANDWIDTH_CALCS("PTE_META_URGENCY_WATERMARK_B calculated =%d\n"
-		"HW register value = 0x%x\n",
-		watermarks->b.pte_meta_urgent_ns, prog_wm_value);
+	if (safe_to_lower || watermarks->b.urgent_ns > hubbub->watermarks.b.urgent_ns) {
+		hubbub->watermarks.b.urgent_ns = watermarks->b.urgent_ns;
+		prog_wm_value = convert_and_clamp(watermarks->b.urgent_ns,
+				refclk_mhz, 0x1fffff);
+		REG_WRITE(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_B, prog_wm_value);
 
+		DC_LOG_BANDWIDTH_CALCS("URGENCY_WATERMARK_B calculated =%d\n"
+			"HW register value = 0x%x\n",
+			watermarks->b.urgent_ns, prog_wm_value);
+	}
 
-	if (REG(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_B)) {
-		prog_wm_value = convert_and_clamp(
-				watermarks->b.cstate_pstate.cstate_enter_plus_exit_ns,
+	if (safe_to_lower || watermarks->b.pte_meta_urgent_ns > hubbub->watermarks.b.pte_meta_urgent_ns) {
+		hubbub->watermarks.b.pte_meta_urgent_ns = watermarks->b.pte_meta_urgent_ns;
+		prog_wm_value = convert_and_clamp(watermarks->b.pte_meta_urgent_ns,
 				refclk_mhz, 0x1fffff);
-		REG_WRITE(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_B, prog_wm_value);
-		DC_LOG_BANDWIDTH_CALCS("SR_ENTER_WATERMARK_B calculated =%d\n"
+		REG_WRITE(DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_B, prog_wm_value);
+		DC_LOG_BANDWIDTH_CALCS("PTE_META_URGENCY_WATERMARK_B calculated =%d\n"
 			"HW register value = 0x%x\n",
-			watermarks->b.cstate_pstate.cstate_enter_plus_exit_ns, prog_wm_value);
+			watermarks->b.pte_meta_urgent_ns, prog_wm_value);
+	}
+
+	if (REG(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_B)) {
+		if (safe_to_lower || watermarks->b.cstate_pstate.cstate_enter_plus_exit_ns
+				> hubbub->watermarks.b.cstate_pstate.cstate_enter_plus_exit_ns) {
+			hubbub->watermarks.b.cstate_pstate.cstate_enter_plus_exit_ns =
+					watermarks->b.cstate_pstate.cstate_enter_plus_exit_ns;
+			prog_wm_value = convert_and_clamp(
+					watermarks->b.cstate_pstate.cstate_enter_plus_exit_ns,
+					refclk_mhz, 0x1fffff);
+			REG_WRITE(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_B, prog_wm_value);
+			DC_LOG_BANDWIDTH_CALCS("SR_ENTER_EXIT_WATERMARK_B calculated =%d\n"
+				"HW register value = 0x%x\n",
+				watermarks->b.cstate_pstate.cstate_enter_plus_exit_ns, prog_wm_value);
+		}
 
+		if (safe_to_lower || watermarks->b.cstate_pstate.cstate_exit_ns
+				> hubbub->watermarks.b.cstate_pstate.cstate_exit_ns) {
+			hubbub->watermarks.b.cstate_pstate.cstate_exit_ns =
+					watermarks->b.cstate_pstate.cstate_exit_ns;
+			prog_wm_value = convert_and_clamp(
+					watermarks->b.cstate_pstate.cstate_exit_ns,
+					refclk_mhz, 0x1fffff);
+			REG_WRITE(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_B, prog_wm_value);
+			DC_LOG_BANDWIDTH_CALCS("SR_EXIT_WATERMARK_B calculated =%d\n"
+				"HW register value = 0x%x\n",
+				watermarks->b.cstate_pstate.cstate_exit_ns, prog_wm_value);
+		}
+	}
 
+	if (safe_to_lower || watermarks->b.cstate_pstate.pstate_change_ns
+			> hubbub->watermarks.b.cstate_pstate.pstate_change_ns) {
+		hubbub->watermarks.b.cstate_pstate.pstate_change_ns =
+				watermarks->b.cstate_pstate.pstate_change_ns;
 		prog_wm_value = convert_and_clamp(
-				watermarks->b.cstate_pstate.cstate_exit_ns,
+				watermarks->b.cstate_pstate.pstate_change_ns,
 				refclk_mhz, 0x1fffff);
-		REG_WRITE(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_B, prog_wm_value);
-		DC_LOG_BANDWIDTH_CALCS("SR_EXIT_WATERMARK_B calculated =%d\n"
-			"HW register value = 0x%x\n",
-			watermarks->b.cstate_pstate.cstate_exit_ns, prog_wm_value);
+		REG_WRITE(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_B, prog_wm_value);
+		DC_LOG_BANDWIDTH_CALCS("DRAM_CLK_CHANGE_WATERMARK_B calculated =%d\n"
+			"HW register value = 0x%x\n\n",
+			watermarks->b.cstate_pstate.pstate_change_ns, prog_wm_value);
 	}
 
-	prog_wm_value = convert_and_clamp(
-			watermarks->b.cstate_pstate.pstate_change_ns,
-			refclk_mhz, 0x1fffff);
-	REG_WRITE(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_B, prog_wm_value);
-	DC_LOG_BANDWIDTH_CALCS("DRAM_CLK_CHANGE_WATERMARK_B calculated =%d\n\n"
-		"HW register value = 0x%x\n",
-		watermarks->b.cstate_pstate.pstate_change_ns, prog_wm_value);
-
 	/* clock state C */
-	prog_wm_value = convert_and_clamp(
-			watermarks->c.urgent_ns, refclk_mhz, 0x1fffff);
-	REG_WRITE(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_C, prog_wm_value);
-	DC_LOG_BANDWIDTH_CALCS("URGENCY_WATERMARK_C calculated =%d\n"
-		"HW register value = 0x%x\n",
-		watermarks->c.urgent_ns, prog_wm_value);
-
-
-	prog_wm_value = convert_and_clamp(
-			watermarks->c.pte_meta_urgent_ns,
-			refclk_mhz, 0x1fffff);
-	REG_WRITE(DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_C, prog_wm_value);
-	DC_LOG_BANDWIDTH_CALCS("PTE_META_URGENCY_WATERMARK_C calculated =%d\n"
-		"HW register value = 0x%x\n",
-		watermarks->c.pte_meta_urgent_ns, prog_wm_value);
+	if (safe_to_lower || watermarks->c.urgent_ns > hubbub->watermarks.c.urgent_ns) {
+		hubbub->watermarks.c.urgent_ns = watermarks->c.urgent_ns;
+		prog_wm_value = convert_and_clamp(watermarks->c.urgent_ns,
+				refclk_mhz, 0x1fffff);
+		REG_WRITE(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_C, prog_wm_value);
 
+		DC_LOG_BANDWIDTH_CALCS("URGENCY_WATERMARK_C calculated =%d\n"
+			"HW register value = 0x%x\n",
+			watermarks->c.urgent_ns, prog_wm_value);
+	}
 
-	if (REG(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_C)) {
-		prog_wm_value = convert_and_clamp(
-				watermarks->c.cstate_pstate.cstate_enter_plus_exit_ns,
+	if (safe_to_lower || watermarks->c.pte_meta_urgent_ns > hubbub->watermarks.c.pte_meta_urgent_ns) {
+		hubbub->watermarks.c.pte_meta_urgent_ns = watermarks->c.pte_meta_urgent_ns;
+		prog_wm_value = convert_and_clamp(watermarks->c.pte_meta_urgent_ns,
 				refclk_mhz, 0x1fffff);
-		REG_WRITE(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_C, prog_wm_value);
-		DC_LOG_BANDWIDTH_CALCS("SR_ENTER_WATERMARK_C calculated =%d\n"
+		REG_WRITE(DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_C, prog_wm_value);
+		DC_LOG_BANDWIDTH_CALCS("PTE_META_URGENCY_WATERMARK_C calculated =%d\n"
 			"HW register value = 0x%x\n",
-			watermarks->c.cstate_pstate.cstate_enter_plus_exit_ns, prog_wm_value);
+			watermarks->c.pte_meta_urgent_ns, prog_wm_value);
+	}
 
+	if (REG(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_C)) {
+		if (safe_to_lower || watermarks->c.cstate_pstate.cstate_enter_plus_exit_ns
+				> hubbub->watermarks.c.cstate_pstate.cstate_enter_plus_exit_ns) {
+			hubbub->watermarks.c.cstate_pstate.cstate_enter_plus_exit_ns =
+					watermarks->c.cstate_pstate.cstate_enter_plus_exit_ns;
+			prog_wm_value = convert_and_clamp(
+					watermarks->c.cstate_pstate.cstate_enter_plus_exit_ns,
+					refclk_mhz, 0x1fffff);
+			REG_WRITE(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_C, prog_wm_value);
+			DC_LOG_BANDWIDTH_CALCS("SR_ENTER_EXIT_WATERMARK_C calculated =%d\n"
+				"HW register value = 0x%x\n",
+				watermarks->c.cstate_pstate.cstate_enter_plus_exit_ns, prog_wm_value);
+		}
+
+		if (safe_to_lower || watermarks->c.cstate_pstate.cstate_exit_ns
+				> hubbub->watermarks.c.cstate_pstate.cstate_exit_ns) {
+			hubbub->watermarks.c.cstate_pstate.cstate_exit_ns =
+					watermarks->c.cstate_pstate.cstate_exit_ns;
+			prog_wm_value = convert_and_clamp(
+					watermarks->c.cstate_pstate.cstate_exit_ns,
+					refclk_mhz, 0x1fffff);
+			REG_WRITE(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_C, prog_wm_value);
+			DC_LOG_BANDWIDTH_CALCS("SR_EXIT_WATERMARK_C calculated =%d\n"
+				"HW register value = 0x%x\n",
+				watermarks->c.cstate_pstate.cstate_exit_ns, prog_wm_value);
+		}
+	}
 
+	if (safe_to_lower || watermarks->c.cstate_pstate.pstate_change_ns
+			> hubbub->watermarks.c.cstate_pstate.pstate_change_ns) {
+		hubbub->watermarks.c.cstate_pstate.pstate_change_ns =
+				watermarks->c.cstate_pstate.pstate_change_ns;
 		prog_wm_value = convert_and_clamp(
-				watermarks->c.cstate_pstate.cstate_exit_ns,
+				watermarks->c.cstate_pstate.pstate_change_ns,
 				refclk_mhz, 0x1fffff);
-		REG_WRITE(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_C, prog_wm_value);
-		DC_LOG_BANDWIDTH_CALCS("SR_EXIT_WATERMARK_C calculated =%d\n"
-			"HW register value = 0x%x\n",
-			watermarks->c.cstate_pstate.cstate_exit_ns, prog_wm_value);
+		REG_WRITE(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_C, prog_wm_value);
+		DC_LOG_BANDWIDTH_CALCS("DRAM_CLK_CHANGE_WATERMARK_C calculated =%d\n"
+			"HW register value = 0x%x\n\n",
+			watermarks->c.cstate_pstate.pstate_change_ns, prog_wm_value);
 	}
 
-	prog_wm_value = convert_and_clamp(
-			watermarks->c.cstate_pstate.pstate_change_ns,
-			refclk_mhz, 0x1fffff);
-	REG_WRITE(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_C, prog_wm_value);
-	DC_LOG_BANDWIDTH_CALCS("DRAM_CLK_CHANGE_WATERMARK_C calculated =%d\n\n"
-		"HW register value = 0x%x\n",
-		watermarks->c.cstate_pstate.pstate_change_ns, prog_wm_value);
-
 	/* clock state D */
-	prog_wm_value = convert_and_clamp(
-			watermarks->d.urgent_ns, refclk_mhz, 0x1fffff);
-	REG_WRITE(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_D, prog_wm_value);
-	DC_LOG_BANDWIDTH_CALCS("URGENCY_WATERMARK_D calculated =%d\n"
-		"HW register value = 0x%x\n",
-		watermarks->d.urgent_ns, prog_wm_value);
-
-	prog_wm_value = convert_and_clamp(
-			watermarks->d.pte_meta_urgent_ns,
-			refclk_mhz, 0x1fffff);
-	REG_WRITE(DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_D, prog_wm_value);
-	DC_LOG_BANDWIDTH_CALCS("PTE_META_URGENCY_WATERMARK_D calculated =%d\n"
-		"HW register value = 0x%x\n",
-		watermarks->d.pte_meta_urgent_ns, prog_wm_value);
-
-
-	if (REG(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_D)) {
-		prog_wm_value = convert_and_clamp(
-				watermarks->d.cstate_pstate.cstate_enter_plus_exit_ns,
+	if (safe_to_lower || watermarks->d.urgent_ns > hubbub->watermarks.d.urgent_ns) {
+		hubbub->watermarks.d.urgent_ns = watermarks->d.urgent_ns;
+		prog_wm_value = convert_and_clamp(watermarks->d.urgent_ns,
 				refclk_mhz, 0x1fffff);
-		REG_WRITE(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_D, prog_wm_value);
-		DC_LOG_BANDWIDTH_CALCS("SR_ENTER_WATERMARK_D calculated =%d\n"
-			"HW register value = 0x%x\n",
-			watermarks->d.cstate_pstate.cstate_enter_plus_exit_ns, prog_wm_value);
+		REG_WRITE(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_D, prog_wm_value);
 
+		DC_LOG_BANDWIDTH_CALCS("URGENCY_WATERMARK_D calculated =%d\n"
+			"HW register value = 0x%x\n",
+			watermarks->d.urgent_ns, prog_wm_value);
+	}
 
-		prog_wm_value = convert_and_clamp(
-				watermarks->d.cstate_pstate.cstate_exit_ns,
+	if (safe_to_lower || watermarks->d.pte_meta_urgent_ns > hubbub->watermarks.d.pte_meta_urgent_ns) {
+		hubbub->watermarks.d.pte_meta_urgent_ns = watermarks->d.pte_meta_urgent_ns;
+		prog_wm_value = convert_and_clamp(watermarks->d.pte_meta_urgent_ns,
 				refclk_mhz, 0x1fffff);
-		REG_WRITE(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_D, prog_wm_value);
-		DC_LOG_BANDWIDTH_CALCS("SR_EXIT_WATERMARK_D calculated =%d\n"
+		REG_WRITE(DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_D, prog_wm_value);
+		DC_LOG_BANDWIDTH_CALCS("PTE_META_URGENCY_WATERMARK_D calculated =%d\n"
 			"HW register value = 0x%x\n",
-			watermarks->d.cstate_pstate.cstate_exit_ns, prog_wm_value);
+			watermarks->d.pte_meta_urgent_ns, prog_wm_value);
 	}
 
+	if (REG(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_D)) {
+		if (safe_to_lower || watermarks->d.cstate_pstate.cstate_enter_plus_exit_ns
+				> hubbub->watermarks.d.cstate_pstate.cstate_enter_plus_exit_ns) {
+			hubbub->watermarks.d.cstate_pstate.cstate_enter_plus_exit_ns =
+					watermarks->d.cstate_pstate.cstate_enter_plus_exit_ns;
+			prog_wm_value = convert_and_clamp(
+					watermarks->d.cstate_pstate.cstate_enter_plus_exit_ns,
+					refclk_mhz, 0x1fffff);
+			REG_WRITE(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_D, prog_wm_value);
+			DC_LOG_BANDWIDTH_CALCS("SR_ENTER_EXIT_WATERMARK_D calculated =%d\n"
+				"HW register value = 0x%x\n",
+				watermarks->d.cstate_pstate.cstate_enter_plus_exit_ns, prog_wm_value);
+		}
 
-	prog_wm_value = convert_and_clamp(
-			watermarks->d.cstate_pstate.pstate_change_ns,
-			refclk_mhz, 0x1fffff);
-	REG_WRITE(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_D, prog_wm_value);
-	DC_LOG_BANDWIDTH_CALCS("DRAM_CLK_CHANGE_WATERMARK_D calculated =%d\n"
-		"HW register value = 0x%x\n\n",
-		watermarks->d.cstate_pstate.pstate_change_ns, prog_wm_value);
+		if (safe_to_lower || watermarks->d.cstate_pstate.cstate_exit_ns
+				> hubbub->watermarks.d.cstate_pstate.cstate_exit_ns) {
+			hubbub->watermarks.d.cstate_pstate.cstate_exit_ns =
+					watermarks->d.cstate_pstate.cstate_exit_ns;
+			prog_wm_value = convert_and_clamp(
+					watermarks->d.cstate_pstate.cstate_exit_ns,
+					refclk_mhz, 0x1fffff);
+			REG_WRITE(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_D, prog_wm_value);
+			DC_LOG_BANDWIDTH_CALCS("SR_EXIT_WATERMARK_D calculated =%d\n"
+				"HW register value = 0x%x\n",
+				watermarks->d.cstate_pstate.cstate_exit_ns, prog_wm_value);
+		}
+	}
 
-	REG_UPDATE(DCHUBBUB_ARB_WATERMARK_CHANGE_CNTL,
-			DCHUBBUB_ARB_WATERMARK_CHANGE_REQUEST, 1);
+	if (safe_to_lower || watermarks->d.cstate_pstate.pstate_change_ns
+			> hubbub->watermarks.d.cstate_pstate.pstate_change_ns) {
+		hubbub->watermarks.d.cstate_pstate.pstate_change_ns =
+				watermarks->d.cstate_pstate.pstate_change_ns;
+		prog_wm_value = convert_and_clamp(
+				watermarks->d.cstate_pstate.pstate_change_ns,
+				refclk_mhz, 0x1fffff);
+		REG_WRITE(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_D, prog_wm_value);
+		DC_LOG_BANDWIDTH_CALCS("DRAM_CLK_CHANGE_WATERMARK_D calculated =%d\n"
+			"HW register value = 0x%x\n\n",
+			watermarks->d.cstate_pstate.pstate_change_ns, prog_wm_value);
+	}
 
 	REG_UPDATE(DCHUBBUB_ARB_SAT_LEVEL,
 			DCHUBBUB_ARB_SAT_LEVEL, 60 * refclk_mhz);
@@ -408,6 +483,11 @@ void hubbub1_update_dchub(
 	struct hubbub *hubbub,
 	struct dchub_init_data *dh_data)
 {
+	if (REG(DCHUBBUB_SDPIF_FB_TOP) == 0) {
+		ASSERT(false);
+		/*should not come here*/
+		return;
+	}
 	/* TODO: port code from dal2 */
 	switch (dh_data->fb_mode) {
 	case FRAME_BUFFER_MODE_ZFB_ONLY:
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.h
index 6315a0e6b0d6..d6e596eef4c5 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.h
@@ -185,6 +185,7 @@ struct hubbub {
 	const struct dcn_hubbub_shift *shifts;
 	const struct dcn_hubbub_mask *masks;
 	unsigned int debug_test_index_pstate;
+	struct dcn_watermark_set watermarks;
 };
 
 void hubbub1_update_dchub(
@@ -194,10 +195,13 @@ void hubbub1_update_dchub(
 bool hubbub1_verify_allow_pstate_change_high(
 	struct hubbub *hubbub);
 
+void hubbub1_wm_change_req_wa(struct hubbub *hubbub);
+
 void hubbub1_program_watermarks(
 		struct hubbub *hubbub,
 		struct dcn_watermark_set *watermarks,
-		unsigned int refclk_mhz);
+		unsigned int refclk_mhz,
+		bool safe_to_lower);
 
 void hubbub1_toggle_watermark_change_req(
 		struct hubbub *hubbub);
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.c
index c28085be39ff..2138cd3c5d1d 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.c
@@ -152,21 +152,19 @@ void hubp1_program_tiling(
 			PIPE_ALIGNED, info->gfx9.pipe_aligned);
 }
 
-void hubp1_program_size_and_rotation(
+void hubp1_program_size(
 	struct hubp *hubp,
-	enum dc_rotation_angle rotation,
 	enum surface_pixel_format format,
 	const union plane_size *plane_size,
-	struct dc_plane_dcc_param *dcc,
-	bool horizontal_mirror)
+	struct dc_plane_dcc_param *dcc)
 {
 	struct dcn10_hubp *hubp1 = TO_DCN10_HUBP(hubp);
-	uint32_t pitch, meta_pitch, pitch_c, meta_pitch_c, mirror;
+	uint32_t pitch, meta_pitch, pitch_c, meta_pitch_c;
 
 	/* Program data and meta surface pitch (calculation from addrlib)
 	 * 444 or 420 luma
 	 */
-	if (format >= SURFACE_PIXEL_FORMAT_VIDEO_BEGIN) {
+	if (format >= SURFACE_PIXEL_FORMAT_VIDEO_BEGIN && format < SURFACE_PIXEL_FORMAT_SUBSAMPLE_END) {
 		ASSERT(plane_size->video.chroma_pitch != 0);
 		/* Chroma pitch zero can cause system hang! */
 
@@ -192,13 +190,22 @@ void hubp1_program_size_and_rotation(
 	if (format >= SURFACE_PIXEL_FORMAT_VIDEO_BEGIN)
 		REG_UPDATE_2(DCSURF_SURFACE_PITCH_C,
 			PITCH_C, pitch_c, META_PITCH_C, meta_pitch_c);
+}
+
+void hubp1_program_rotation(
+	struct hubp *hubp,
+	enum dc_rotation_angle rotation,
+	bool horizontal_mirror)
+{
+	struct dcn10_hubp *hubp1 = TO_DCN10_HUBP(hubp);
+	uint32_t mirror;
+
 
 	if (horizontal_mirror)
 		mirror = 1;
 	else
 		mirror = 0;
 
-
 	/* Program rotation angle and horz mirror - no mirror */
 	if (rotation == ROTATION_ANGLE_0)
 		REG_UPDATE_2(DCSURF_SURFACE_CONFIG,
@@ -287,6 +294,10 @@ void hubp1_program_pixel_format(
 		REG_UPDATE(DCSURF_SURFACE_CONFIG,
 				SURFACE_PIXEL_FORMAT, 66);
 		break;
+	case SURFACE_PIXEL_FORMAT_VIDEO_AYCrCb8888:
+		REG_UPDATE(DCSURF_SURFACE_CONFIG,
+				SURFACE_PIXEL_FORMAT, 12);
+		break;
 	default:
 		BREAK_TO_DEBUGGER();
 		break;
@@ -450,9 +461,6 @@ bool hubp1_program_surface_flip_and_addr(
 
 	hubp->request_address = *address;
 
-	if (flip_immediate)
-		hubp->current_address = *address;
-
 	return true;
 }
 
@@ -481,8 +489,8 @@ void hubp1_program_surface_config(
 {
 	hubp1_dcc_control(hubp, dcc->enable, dcc->grph.independent_64b_blks);
 	hubp1_program_tiling(hubp, tiling_info, format);
-	hubp1_program_size_and_rotation(
-			hubp, rotation, format, plane_size, dcc, horizontal_mirror);
+	hubp1_program_size(hubp, format, plane_size, dcc);
+	hubp1_program_rotation(hubp, rotation, horizontal_mirror);
 	hubp1_program_pixel_format(hubp, format);
 }
 
@@ -688,7 +696,6 @@ bool hubp1_is_flip_pending(struct hubp *hubp)
 	if (earliest_inuse_address.grph.addr.quad_part != hubp->request_address.grph.addr.quad_part)
 		return true;
 
-	hubp->current_address = hubp->request_address;
 	return false;
 }
 
@@ -1061,9 +1068,11 @@ void hubp1_cursor_set_position(
 		const struct dc_cursor_mi_param *param)
 {
 	struct dcn10_hubp *hubp1 = TO_DCN10_HUBP(hubp);
-	int src_x_offset = pos->x - pos->x_hotspot - param->viewport_x_start;
+	int src_x_offset = pos->x - pos->x_hotspot - param->viewport.x;
+	int x_hotspot = pos->x_hotspot;
+	int y_hotspot = pos->y_hotspot;
+	uint32_t dst_x_offset;
 	uint32_t cur_en = pos->enable ? 1 : 0;
-	uint32_t dst_x_offset = (src_x_offset >= 0) ? src_x_offset : 0;
 
 	/*
 	 * Guard aganst cursor_set_position() from being called with invalid
@@ -1075,6 +1084,18 @@ void hubp1_cursor_set_position(
 	if (hubp->curs_attr.address.quad_part == 0)
 		return;
 
+	if (param->rotation == ROTATION_ANGLE_90 || param->rotation == ROTATION_ANGLE_270) {
+		src_x_offset = pos->y - pos->y_hotspot - param->viewport.x;
+		y_hotspot = pos->x_hotspot;
+		x_hotspot = pos->y_hotspot;
+	}
+
+	if (param->mirror) {
+		x_hotspot = param->viewport.width - x_hotspot;
+		src_x_offset = param->viewport.x + param->viewport.width - src_x_offset;
+	}
+
+	dst_x_offset = (src_x_offset >= 0) ? src_x_offset : 0;
 	dst_x_offset *= param->ref_clk_khz;
 	dst_x_offset /= param->pixel_clk_khz;
 
@@ -1085,7 +1106,7 @@ void hubp1_cursor_set_position(
 				dc_fixpt_from_int(dst_x_offset),
 				param->h_scale_ratio));
 
-	if (src_x_offset >= (int)param->viewport_width)
+	if (src_x_offset >= (int)param->viewport.width)
 		cur_en = 0;  /* not visible beyond right edge*/
 
 	if (src_x_offset + (int)hubp->curs_attr.width <= 0)
@@ -1102,8 +1123,8 @@ void hubp1_cursor_set_position(
 			CURSOR_Y_POSITION, pos->y);
 
 	REG_SET_2(CURSOR_HOT_SPOT, 0,
-			CURSOR_HOT_SPOT_X, pos->x_hotspot,
-			CURSOR_HOT_SPOT_Y, pos->y_hotspot);
+			CURSOR_HOT_SPOT_X, x_hotspot,
+			CURSOR_HOT_SPOT_Y, y_hotspot);
 
 	REG_SET(CURSOR_DST_OFFSET, 0,
 			CURSOR_DST_X_OFFSET, dst_x_offset);
@@ -1125,7 +1146,7 @@ void hubp1_vtg_sel(struct hubp *hubp, uint32_t otg_inst)
 	REG_UPDATE(DCHUBP_CNTL, HUBP_VTG_SEL, otg_inst);
 }
 
-static struct hubp_funcs dcn10_hubp_funcs = {
+static const struct hubp_funcs dcn10_hubp_funcs = {
 	.hubp_program_surface_flip_and_addr =
 			hubp1_program_surface_flip_and_addr,
 	.hubp_program_surface_config =
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.h
index d901d5092969..f689feace82d 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.h
@@ -268,8 +268,6 @@
 	HUBP_SF(HUBPREQ0_DCSURF_SURFACE_PITCH, META_PITCH, mask_sh),\
 	HUBP_SF(HUBPREQ0_DCSURF_SURFACE_PITCH_C, PITCH_C, mask_sh),\
 	HUBP_SF(HUBPREQ0_DCSURF_SURFACE_PITCH_C, META_PITCH_C, mask_sh),\
-	HUBP_SF(HUBP0_DCSURF_SURFACE_CONFIG, ROTATION_ANGLE, mask_sh),\
-	HUBP_SF(HUBP0_DCSURF_SURFACE_CONFIG, H_MIRROR_EN, mask_sh),\
 	HUBP_SF(HUBP0_DCSURF_SURFACE_CONFIG, SURFACE_PIXEL_FORMAT, mask_sh),\
 	HUBP_SF(HUBPREQ0_DCSURF_FLIP_CONTROL, SURFACE_FLIP_TYPE, mask_sh),\
 	HUBP_SF(HUBPREQ0_DCSURF_FLIP_CONTROL, SURFACE_FLIP_PENDING, mask_sh),\
@@ -388,6 +386,8 @@
 #define HUBP_MASK_SH_LIST_DCN10(mask_sh)\
 	HUBP_MASK_SH_LIST_DCN(mask_sh),\
 	HUBP_MASK_SH_LIST_DCN_VM(mask_sh),\
+	HUBP_SF(HUBP0_DCSURF_SURFACE_CONFIG, ROTATION_ANGLE, mask_sh),\
+	HUBP_SF(HUBP0_DCSURF_SURFACE_CONFIG, H_MIRROR_EN, mask_sh),\
 	HUBP_SF(HUBPREQ0_PREFETCH_SETTINS, DST_Y_PREFETCH, mask_sh),\
 	HUBP_SF(HUBPREQ0_PREFETCH_SETTINS, VRATIO_PREFETCH, mask_sh),\
 	HUBP_SF(HUBPREQ0_PREFETCH_SETTINS_C, VRATIO_PREFETCH_C, mask_sh),\
@@ -679,12 +679,15 @@ void hubp1_program_pixel_format(
 	struct hubp *hubp,
 	enum surface_pixel_format format);
 
-void hubp1_program_size_and_rotation(
+void hubp1_program_size(
 	struct hubp *hubp,
-	enum dc_rotation_angle rotation,
 	enum surface_pixel_format format,
 	const union plane_size *plane_size,
-	struct dc_plane_dcc_param *dcc,
+	struct dc_plane_dcc_param *dcc);
+
+void hubp1_program_rotation(
+	struct hubp *hubp,
+	enum dc_rotation_angle rotation,
 	bool horizontal_mirror);
 
 void hubp1_program_tiling(
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c
index 03eb736a312f..cfcc54f2ce65 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c
@@ -337,13 +337,13 @@ void dcn10_log_hw_state(struct dc *dc)
 
 	DTN_INFO("\nCALCULATED Clocks: dcfclk_khz:%d  dcfclk_deep_sleep_khz:%d  dispclk_khz:%d\n"
 		"dppclk_khz:%d  max_supported_dppclk_khz:%d  fclk_khz:%d  socclk_khz:%d\n\n",
-			dc->current_state->bw.dcn.calc_clk.dcfclk_khz,
-			dc->current_state->bw.dcn.calc_clk.dcfclk_deep_sleep_khz,
-			dc->current_state->bw.dcn.calc_clk.dispclk_khz,
-			dc->current_state->bw.dcn.calc_clk.dppclk_khz,
-			dc->current_state->bw.dcn.calc_clk.max_supported_dppclk_khz,
-			dc->current_state->bw.dcn.calc_clk.fclk_khz,
-			dc->current_state->bw.dcn.calc_clk.socclk_khz);
+			dc->current_state->bw.dcn.clk.dcfclk_khz,
+			dc->current_state->bw.dcn.clk.dcfclk_deep_sleep_khz,
+			dc->current_state->bw.dcn.clk.dispclk_khz,
+			dc->current_state->bw.dcn.clk.dppclk_khz,
+			dc->current_state->bw.dcn.clk.max_supported_dppclk_khz,
+			dc->current_state->bw.dcn.clk.fclk_khz,
+			dc->current_state->bw.dcn.clk.socclk_khz);
 
 	log_mpc_crc(dc);
 
@@ -415,6 +415,8 @@ static void dpp_pg_control(
 
 	if (hws->ctx->dc->debug.disable_dpp_power_gate)
 		return;
+	if (REG(DOMAIN1_PG_CONFIG) == 0)
+		return;
 
 	switch (dpp_inst) {
 	case 0: /* DPP0 */
@@ -465,6 +467,8 @@ static void hubp_pg_control(
 
 	if (hws->ctx->dc->debug.disable_hubp_power_gate)
 		return;
+	if (REG(DOMAIN0_PG_CONFIG) == 0)
+		return;
 
 	switch (hubp_inst) {
 	case 0: /* DCHUBP0 */
@@ -830,7 +834,7 @@ static bool dcn10_hw_wa_force_recovery(struct dc *dc)
 }
 
 
-static void dcn10_verify_allow_pstate_change_high(struct dc *dc)
+void dcn10_verify_allow_pstate_change_high(struct dc *dc)
 {
 	static bool should_log_hw_state; /* prevent hw state log by default */
 
@@ -865,7 +869,8 @@ void hwss1_plane_atomic_disconnect(struct dc *dc, struct pipe_ctx *pipe_ctx)
 		return;
 
 	mpc->funcs->remove_mpcc(mpc, mpc_tree_params, mpcc_to_remove);
-	opp->mpcc_disconnect_pending[pipe_ctx->plane_res.mpcc_inst] = true;
+	if (opp != NULL)
+		opp->mpcc_disconnect_pending[pipe_ctx->plane_res.mpcc_inst] = true;
 
 	dc->optimized_required = true;
 
@@ -1010,7 +1015,7 @@ static void dcn10_init_hw(struct dc *dc)
 	/* Reset all MPCC muxes */
 	dc->res_pool->mpc->funcs->mpc_init(dc->res_pool->mpc);
 
-	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+	for (i = 0; i < dc->res_pool->timing_generator_count; i++) {
 		struct timing_generator *tg = dc->res_pool->timing_generators[i];
 		struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[i];
 		struct hubp *hubp = dc->res_pool->hubps[i];
@@ -1084,6 +1089,8 @@ static void dcn10_init_hw(struct dc *dc)
 	}
 
 	enable_power_gating_plane(dc->hwseq, true);
+
+	memset(&dc->res_pool->dccg->clks, 0, sizeof(dc->res_pool->dccg->clks));
 }
 
 static void reset_hw_ctx_wrap(
@@ -1152,12 +1159,19 @@ static void dcn10_update_plane_addr(const struct dc *dc, struct pipe_ctx *pipe_c
 
 	if (plane_state == NULL)
 		return;
+
 	addr_patched = patch_address_for_sbs_tb_stereo(pipe_ctx, &addr);
+
 	pipe_ctx->plane_res.hubp->funcs->hubp_program_surface_flip_and_addr(
 			pipe_ctx->plane_res.hubp,
 			&plane_state->address,
 			plane_state->flip_immediate);
+
 	plane_state->status.requested_address = plane_state->address;
+
+	if (plane_state->flip_immediate)
+		plane_state->status.current_address = plane_state->address;
+
 	if (addr_patched)
 		pipe_ctx->plane_state->address.grph_stereo.left_addr = addr;
 }
@@ -1201,8 +1215,11 @@ static bool dcn10_set_input_transfer_func(struct pipe_ctx *pipe_ctx,
 	} else if (tf->type == TF_TYPE_BYPASS) {
 		dpp_base->funcs->dpp_set_degamma(dpp_base, IPP_DEGAMMA_MODE_BYPASS);
 	} else {
-		/*TF_TYPE_DISTRIBUTED_POINTS*/
-		result = false;
+		cm_helper_translate_curve_to_degamma_hw_format(tf,
+					&dpp_base->degamma_params);
+		dpp_base->funcs->dpp_program_degamma_pwl(dpp_base,
+				&dpp_base->degamma_params);
+		result = true;
 	}
 
 	return result;
@@ -1343,10 +1360,11 @@ static void dcn10_enable_per_frame_crtc_position_reset(
 
 	DC_SYNC_INFO("Setting up\n");
 	for (i = 0; i < group_size; i++)
-		grouped_pipes[i]->stream_res.tg->funcs->enable_crtc_reset(
-				grouped_pipes[i]->stream_res.tg,
-				grouped_pipes[i]->stream->triggered_crtc_reset.event_source->status.primary_otg_inst,
-				&grouped_pipes[i]->stream->triggered_crtc_reset);
+		if (grouped_pipes[i]->stream_res.tg->funcs->enable_crtc_reset)
+			grouped_pipes[i]->stream_res.tg->funcs->enable_crtc_reset(
+					grouped_pipes[i]->stream_res.tg,
+					grouped_pipes[i]->stream->triggered_crtc_reset.event_source->status.primary_otg_inst,
+					&grouped_pipes[i]->stream->triggered_crtc_reset);
 
 	DC_SYNC_INFO("Waiting for trigger\n");
 
@@ -1762,6 +1780,43 @@ static void dcn10_get_surface_visual_confirm_color(
 	}
 }
 
+static void dcn10_get_hdr_visual_confirm_color(
+		struct pipe_ctx *pipe_ctx,
+		struct tg_color *color)
+{
+	uint32_t color_value = MAX_TG_COLOR_VALUE;
+
+	// Determine the overscan color based on the top-most (desktop) plane's context
+	struct pipe_ctx *top_pipe_ctx  = pipe_ctx;
+
+	while (top_pipe_ctx->top_pipe != NULL)
+		top_pipe_ctx = top_pipe_ctx->top_pipe;
+
+	switch (top_pipe_ctx->plane_res.scl_data.format) {
+	case PIXEL_FORMAT_ARGB2101010:
+		if (top_pipe_ctx->stream->out_transfer_func->tf == TRANSFER_FUNCTION_UNITY) {
+			/* HDR10, ARGB2101010 - set boarder color to red */
+			color->color_r_cr = color_value;
+		}
+		break;
+	case PIXEL_FORMAT_FP16:
+		if (top_pipe_ctx->stream->out_transfer_func->tf == TRANSFER_FUNCTION_PQ) {
+			/* HDR10, FP16 - set boarder color to blue */
+			color->color_b_cb = color_value;
+		} else if (top_pipe_ctx->stream->out_transfer_func->tf == TRANSFER_FUNCTION_GAMMA22) {
+			/* FreeSync 2 HDR - set boarder color to green */
+			color->color_g_y = color_value;
+		}
+		break;
+	default:
+		/* SDR - set boarder color to Gray */
+		color->color_r_cr = color_value/2;
+		color->color_b_cb = color_value/2;
+		color->color_g_y = color_value/2;
+		break;
+	}
+}
+
 static uint16_t fixed_point_to_int_frac(
 	struct fixed31_32 arg,
 	uint8_t integer_bits,
@@ -1842,11 +1897,10 @@ static void update_dpp(struct dpp *dpp, struct dc_plane_state *plane_state)
 		dpp->funcs->dpp_program_bias_and_scale(dpp, &bns_params);
 }
 
-
-static void update_mpcc(struct dc *dc, struct pipe_ctx *pipe_ctx)
+static void dcn10_update_mpcc(struct dc *dc, struct pipe_ctx *pipe_ctx)
 {
 	struct hubp *hubp = pipe_ctx->plane_res.hubp;
-	struct mpcc_blnd_cfg blnd_cfg;
+	struct mpcc_blnd_cfg blnd_cfg = {0};
 	bool per_pixel_alpha = pipe_ctx->plane_state->per_pixel_alpha && pipe_ctx->bottom_pipe;
 	int mpcc_id;
 	struct mpcc *new_mpcc;
@@ -1857,13 +1911,17 @@ static void update_mpcc(struct dc *dc, struct pipe_ctx *pipe_ctx)
 
 	/* TODO: proper fix once fpga works */
 
-	if (dc->debug.surface_visual_confirm)
+	if (dc->debug.visual_confirm == VISUAL_CONFIRM_HDR) {
+		dcn10_get_hdr_visual_confirm_color(
+				pipe_ctx, &blnd_cfg.black_color);
+	} else if (dc->debug.visual_confirm == VISUAL_CONFIRM_SURFACE) {
 		dcn10_get_surface_visual_confirm_color(
 				pipe_ctx, &blnd_cfg.black_color);
-	else
+	} else {
 		color_space_to_black_color(
-			dc, pipe_ctx->stream->output_color_space,
-			&blnd_cfg.black_color);
+				dc, pipe_ctx->stream->output_color_space,
+				&blnd_cfg.black_color);
+	}
 
 	if (per_pixel_alpha)
 		blnd_cfg.alpha_mode = MPCC_ALPHA_BLEND_MODE_PER_PIXEL_ALPHA;
@@ -1952,18 +2010,17 @@ static void update_dchubp_dpp(
 	 * divided by 2
 	 */
 	if (plane_state->update_flags.bits.full_update) {
-		bool should_divided_by_2 = context->bw.dcn.calc_clk.dppclk_khz <=
-				context->bw.dcn.cur_clk.dispclk_khz / 2;
+		bool should_divided_by_2 = context->bw.dcn.clk.dppclk_khz <=
+				dc->res_pool->dccg->clks.dispclk_khz / 2;
 
 		dpp->funcs->dpp_dppclk_control(
 				dpp,
 				should_divided_by_2,
 				true);
 
-		dc->current_state->bw.dcn.cur_clk.dppclk_khz =
-				should_divided_by_2 ?
-				context->bw.dcn.cur_clk.dispclk_khz / 2 :
-				context->bw.dcn.cur_clk.dispclk_khz;
+		dc->res_pool->dccg->clks.dppclk_khz = should_divided_by_2 ?
+						dc->res_pool->dccg->clks.dispclk_khz / 2 :
+							dc->res_pool->dccg->clks.dispclk_khz;
 	}
 
 	/* TODO: Need input parameter to tell current DCHUB pipe tie to which OTG
@@ -1989,7 +2046,7 @@ static void update_dchubp_dpp(
 
 	if (plane_state->update_flags.bits.full_update ||
 		plane_state->update_flags.bits.per_pixel_alpha_change)
-		update_mpcc(dc, pipe_ctx);
+		dc->hwss.update_mpcc(dc, pipe_ctx);
 
 	if (plane_state->update_flags.bits.full_update ||
 		plane_state->update_flags.bits.per_pixel_alpha_change ||
@@ -2099,6 +2156,33 @@ static void set_hdr_multiplier(struct pipe_ctx *pipe_ctx)
 			pipe_ctx->plane_res.dpp, hw_mult);
 }
 
+void dcn10_program_pipe(
+		struct dc *dc,
+		struct pipe_ctx *pipe_ctx,
+		struct dc_state *context)
+{
+	if (pipe_ctx->plane_state->update_flags.bits.full_update)
+		dcn10_enable_plane(dc, pipe_ctx, context);
+
+	update_dchubp_dpp(dc, pipe_ctx, context);
+
+	set_hdr_multiplier(pipe_ctx);
+
+	if (pipe_ctx->plane_state->update_flags.bits.full_update ||
+			pipe_ctx->plane_state->update_flags.bits.in_transfer_func_change ||
+			pipe_ctx->plane_state->update_flags.bits.gamma_change)
+		dc->hwss.set_input_transfer_func(pipe_ctx, pipe_ctx->plane_state);
+
+	/* dcn10_translate_regamma_to_hw_format takes 750us to finish
+	 * only do gamma programming for full update.
+	 * TODO: This can be further optimized/cleaned up
+	 * Always call this for now since it does memcmp inside before
+	 * doing heavy calculation and programming
+	 */
+	if (pipe_ctx->plane_state->update_flags.bits.full_update)
+		dc->hwss.set_output_transfer_func(pipe_ctx, pipe_ctx->stream);
+}
+
 static void program_all_pipe_in_tree(
 		struct dc *dc,
 		struct pipe_ctx *pipe_ctx,
@@ -2117,29 +2201,11 @@ static void program_all_pipe_in_tree(
 				pipe_ctx->stream_res.tg);
 
 		dc->hwss.blank_pixel_data(dc, pipe_ctx, blank);
+
 	}
 
 	if (pipe_ctx->plane_state != NULL) {
-		if (pipe_ctx->plane_state->update_flags.bits.full_update)
-			dcn10_enable_plane(dc, pipe_ctx, context);
-
-		update_dchubp_dpp(dc, pipe_ctx, context);
-
-		set_hdr_multiplier(pipe_ctx);
-
-		if (pipe_ctx->plane_state->update_flags.bits.full_update ||
-				pipe_ctx->plane_state->update_flags.bits.in_transfer_func_change ||
-				pipe_ctx->plane_state->update_flags.bits.gamma_change)
-			dc->hwss.set_input_transfer_func(pipe_ctx, pipe_ctx->plane_state);
-
-		/* dcn10_translate_regamma_to_hw_format takes 750us to finish
-		 * only do gamma programming for full update.
-		 * TODO: This can be further optimized/cleaned up
-		 * Always call this for now since it does memcmp inside before
-		 * doing heavy calculation and programming
-		 */
-		if (pipe_ctx->plane_state->update_flags.bits.full_update)
-			dc->hwss.set_output_transfer_func(pipe_ctx, pipe_ctx->stream);
+		dcn10_program_pipe(dc, pipe_ctx, context);
 	}
 
 	if (pipe_ctx->bottom_pipe != NULL && pipe_ctx->bottom_pipe != pipe_ctx) {
@@ -2153,12 +2219,12 @@ static void dcn10_pplib_apply_display_requirements(
 {
 	struct dm_pp_display_configuration *pp_display_cfg = &context->pp_display_cfg;
 
-	pp_display_cfg->min_engine_clock_khz = context->bw.dcn.cur_clk.dcfclk_khz;
-	pp_display_cfg->min_memory_clock_khz = context->bw.dcn.cur_clk.fclk_khz;
-	pp_display_cfg->min_engine_clock_deep_sleep_khz = context->bw.dcn.cur_clk.dcfclk_deep_sleep_khz;
-	pp_display_cfg->min_dcfc_deep_sleep_clock_khz = context->bw.dcn.cur_clk.dcfclk_deep_sleep_khz;
-	pp_display_cfg->min_dcfclock_khz = context->bw.dcn.cur_clk.dcfclk_khz;
-	pp_display_cfg->disp_clk_khz = context->bw.dcn.cur_clk.dispclk_khz;
+	pp_display_cfg->min_engine_clock_khz = dc->res_pool->dccg->clks.dcfclk_khz;
+	pp_display_cfg->min_memory_clock_khz = dc->res_pool->dccg->clks.fclk_khz;
+	pp_display_cfg->min_engine_clock_deep_sleep_khz = dc->res_pool->dccg->clks.dcfclk_deep_sleep_khz;
+	pp_display_cfg->min_dcfc_deep_sleep_clock_khz = dc->res_pool->dccg->clks.dcfclk_deep_sleep_khz;
+	pp_display_cfg->min_dcfclock_khz = dc->res_pool->dccg->clks.dcfclk_khz;
+	pp_display_cfg->disp_clk_khz = dc->res_pool->dccg->clks.dispclk_khz;
 	dce110_fill_display_configs(context, pp_display_cfg);
 
 	if (memcmp(&dc->prev_display_config, pp_display_cfg, sizeof(
@@ -2220,8 +2286,6 @@ static void dcn10_apply_ctx_for_surface(
 	int i;
 	struct timing_generator *tg;
 	bool removed_pipe[4] = { false };
-	unsigned int ref_clk_mhz = dc->res_pool->ref_clock_inKhz/1000;
-	bool program_water_mark = false;
 	struct pipe_ctx *top_pipe_to_program =
 			find_top_pipe_for_stream(dc, context, stream);
 	DC_LOGGER_INIT(dc->ctx->logger);
@@ -2266,11 +2330,10 @@ static void dcn10_apply_ctx_for_surface(
 			old_pipe_ctx->plane_state &&
 			old_pipe_ctx->stream_res.tg == tg) {
 
-			hwss1_plane_atomic_disconnect(dc, old_pipe_ctx);
+			dc->hwss.plane_atomic_disconnect(dc, old_pipe_ctx);
 			removed_pipe[i] = true;
 
-			DC_LOG_DC(
-					"Reset mpcc for pipe %d\n",
+			DC_LOG_DC("Reset mpcc for pipe %d\n",
 					old_pipe_ctx->pipe_idx);
 		}
 	}
@@ -2283,248 +2346,41 @@ static void dcn10_apply_ctx_for_surface(
 	if (num_planes == 0)
 		false_optc_underflow_wa(dc, stream, tg);
 
-	for (i = 0; i < dc->res_pool->pipe_count; i++) {
-		struct pipe_ctx *old_pipe_ctx =
-				&dc->current_state->res_ctx.pipe_ctx[i];
-		struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[i];
-
-		if (pipe_ctx->stream == stream &&
-				pipe_ctx->plane_state &&
-			pipe_ctx->plane_state->update_flags.bits.full_update)
-			program_water_mark = true;
-
+	for (i = 0; i < dc->res_pool->pipe_count; i++)
 		if (removed_pipe[i])
-			dcn10_disable_plane(dc, old_pipe_ctx);
-	}
+			dcn10_disable_plane(dc, &dc->current_state->res_ctx.pipe_ctx[i]);
 
-	if (program_water_mark) {
-		if (dc->debug.sanity_checks) {
-			/* pstate stuck check after watermark update */
-			dcn10_verify_allow_pstate_change_high(dc);
-		}
-
-		/* watermark is for all pipes */
-		hubbub1_program_watermarks(dc->res_pool->hubbub,
-				&context->bw.dcn.watermarks, ref_clk_mhz);
-
-		if (dc->debug.sanity_checks) {
-			/* pstate stuck check after watermark update */
-			dcn10_verify_allow_pstate_change_high(dc);
-		}
-	}
-/*	DC_LOG_BANDWIDTH_CALCS(dc->ctx->logger,
-			"\n============== Watermark parameters ==============\n"
-			"a.urgent_ns: %d \n"
-			"a.cstate_enter_plus_exit: %d \n"
-			"a.cstate_exit: %d \n"
-			"a.pstate_change: %d \n"
-			"a.pte_meta_urgent: %d \n"
-			"b.urgent_ns: %d \n"
-			"b.cstate_enter_plus_exit: %d \n"
-			"b.cstate_exit: %d \n"
-			"b.pstate_change: %d \n"
-			"b.pte_meta_urgent: %d \n",
-			context->bw.dcn.watermarks.a.urgent_ns,
-			context->bw.dcn.watermarks.a.cstate_pstate.cstate_enter_plus_exit_ns,
-			context->bw.dcn.watermarks.a.cstate_pstate.cstate_exit_ns,
-			context->bw.dcn.watermarks.a.cstate_pstate.pstate_change_ns,
-			context->bw.dcn.watermarks.a.pte_meta_urgent_ns,
-			context->bw.dcn.watermarks.b.urgent_ns,
-			context->bw.dcn.watermarks.b.cstate_pstate.cstate_enter_plus_exit_ns,
-			context->bw.dcn.watermarks.b.cstate_pstate.cstate_exit_ns,
-			context->bw.dcn.watermarks.b.cstate_pstate.pstate_change_ns,
-			context->bw.dcn.watermarks.b.pte_meta_urgent_ns
-			);
-	DC_LOG_BANDWIDTH_CALCS(dc->ctx->logger,
-			"\nc.urgent_ns: %d \n"
-			"c.cstate_enter_plus_exit: %d \n"
-			"c.cstate_exit: %d \n"
-			"c.pstate_change: %d \n"
-			"c.pte_meta_urgent: %d \n"
-			"d.urgent_ns: %d \n"
-			"d.cstate_enter_plus_exit: %d \n"
-			"d.cstate_exit: %d \n"
-			"d.pstate_change: %d \n"
-			"d.pte_meta_urgent: %d \n"
-			"========================================================\n",
-			context->bw.dcn.watermarks.c.urgent_ns,
-			context->bw.dcn.watermarks.c.cstate_pstate.cstate_enter_plus_exit_ns,
-			context->bw.dcn.watermarks.c.cstate_pstate.cstate_exit_ns,
-			context->bw.dcn.watermarks.c.cstate_pstate.pstate_change_ns,
-			context->bw.dcn.watermarks.c.pte_meta_urgent_ns,
-			context->bw.dcn.watermarks.d.urgent_ns,
-			context->bw.dcn.watermarks.d.cstate_pstate.cstate_enter_plus_exit_ns,
-			context->bw.dcn.watermarks.d.cstate_pstate.cstate_exit_ns,
-			context->bw.dcn.watermarks.d.cstate_pstate.pstate_change_ns,
-			context->bw.dcn.watermarks.d.pte_meta_urgent_ns
-			);
-*/
-}
-
-static inline bool should_set_clock(bool decrease_allowed, int calc_clk, int cur_clk)
-{
-	return ((decrease_allowed && calc_clk < cur_clk) || calc_clk > cur_clk);
-}
-
-static int determine_dppclk_threshold(struct dc *dc, struct dc_state *context)
-{
-	bool request_dpp_div = context->bw.dcn.calc_clk.dispclk_khz >
-			context->bw.dcn.calc_clk.dppclk_khz;
-	bool dispclk_increase = context->bw.dcn.calc_clk.dispclk_khz >
-			context->bw.dcn.cur_clk.dispclk_khz;
-	int disp_clk_threshold = context->bw.dcn.calc_clk.max_supported_dppclk_khz;
-	bool cur_dpp_div = context->bw.dcn.cur_clk.dispclk_khz >
-			context->bw.dcn.cur_clk.dppclk_khz;
-
-	/* increase clock, looking for div is 0 for current, request div is 1*/
-	if (dispclk_increase) {
-		/* already divided by 2, no need to reach target clk with 2 steps*/
-		if (cur_dpp_div)
-			return context->bw.dcn.calc_clk.dispclk_khz;
-
-		/* request disp clk is lower than maximum supported dpp clk,
-		 * no need to reach target clk with two steps.
-		 */
-		if (context->bw.dcn.calc_clk.dispclk_khz <= disp_clk_threshold)
-			return context->bw.dcn.calc_clk.dispclk_khz;
-
-		/* target dpp clk not request divided by 2, still within threshold */
-		if (!request_dpp_div)
-			return context->bw.dcn.calc_clk.dispclk_khz;
-
-	} else {
-		/* decrease clock, looking for current dppclk divided by 2,
-		 * request dppclk not divided by 2.
-		 */
-
-		/* current dpp clk not divided by 2, no need to ramp*/
-		if (!cur_dpp_div)
-			return context->bw.dcn.calc_clk.dispclk_khz;
-
-		/* current disp clk is lower than current maximum dpp clk,
-		 * no need to ramp
-		 */
-		if (context->bw.dcn.cur_clk.dispclk_khz <= disp_clk_threshold)
-			return context->bw.dcn.calc_clk.dispclk_khz;
-
-		/* request dpp clk need to be divided by 2 */
-		if (request_dpp_div)
-			return context->bw.dcn.calc_clk.dispclk_khz;
-	}
-
-	return disp_clk_threshold;
-}
-
-static void ramp_up_dispclk_with_dpp(struct dc *dc, struct dc_state *context)
-{
-	int i;
-	bool request_dpp_div = context->bw.dcn.calc_clk.dispclk_khz >
-				context->bw.dcn.calc_clk.dppclk_khz;
-
-	int dispclk_to_dpp_threshold = determine_dppclk_threshold(dc, context);
-
-	/* set disp clk to dpp clk threshold */
-	dc->res_pool->display_clock->funcs->set_clock(
-			dc->res_pool->display_clock,
-			dispclk_to_dpp_threshold);
-
-	/* update request dpp clk division option */
-	for (i = 0; i < dc->res_pool->pipe_count; i++) {
-		struct pipe_ctx *pipe_ctx = &dc->current_state->res_ctx.pipe_ctx[i];
-
-		if (!pipe_ctx->plane_state)
-			continue;
-
-		pipe_ctx->plane_res.dpp->funcs->dpp_dppclk_control(
-				pipe_ctx->plane_res.dpp,
-				request_dpp_div,
-				true);
-	}
-
-	/* If target clk not same as dppclk threshold, set to target clock */
-	if (dispclk_to_dpp_threshold != context->bw.dcn.calc_clk.dispclk_khz) {
-		dc->res_pool->display_clock->funcs->set_clock(
-				dc->res_pool->display_clock,
-				context->bw.dcn.calc_clk.dispclk_khz);
-	}
-
-	context->bw.dcn.cur_clk.dispclk_khz =
-			context->bw.dcn.calc_clk.dispclk_khz;
-	context->bw.dcn.cur_clk.dppclk_khz =
-			context->bw.dcn.calc_clk.dppclk_khz;
-	context->bw.dcn.cur_clk.max_supported_dppclk_khz =
-			context->bw.dcn.calc_clk.max_supported_dppclk_khz;
+	if (dc->hwseq->wa.DEGVIDCN10_254)
+		hubbub1_wm_change_req_wa(dc->res_pool->hubbub);
 }
 
 static void dcn10_set_bandwidth(
 		struct dc *dc,
 		struct dc_state *context,
-		bool decrease_allowed)
+		bool safe_to_lower)
 {
-	struct pp_smu_display_requirement_rv *smu_req_cur =
-			&dc->res_pool->pp_smu_req;
-	struct pp_smu_display_requirement_rv smu_req = *smu_req_cur;
-	struct pp_smu_funcs_rv *pp_smu = dc->res_pool->pp_smu;
-
-	if (dc->debug.sanity_checks) {
+	if (dc->debug.sanity_checks)
 		dcn10_verify_allow_pstate_change_high(dc);
-	}
 
-	if (IS_FPGA_MAXIMUS_DC(dc->ctx->dce_environment))
-		return;
-
-	if (should_set_clock(
-			decrease_allowed,
-			context->bw.dcn.calc_clk.dcfclk_khz,
-			dc->current_state->bw.dcn.cur_clk.dcfclk_khz)) {
-		context->bw.dcn.cur_clk.dcfclk_khz =
-				context->bw.dcn.calc_clk.dcfclk_khz;
-		smu_req.hard_min_dcefclk_khz =
-				context->bw.dcn.calc_clk.dcfclk_khz;
-	}
-
-	if (should_set_clock(
-			decrease_allowed,
-			context->bw.dcn.calc_clk.dcfclk_deep_sleep_khz,
-			dc->current_state->bw.dcn.cur_clk.dcfclk_deep_sleep_khz)) {
-		context->bw.dcn.cur_clk.dcfclk_deep_sleep_khz =
-				context->bw.dcn.calc_clk.dcfclk_deep_sleep_khz;
-	}
-
-	if (should_set_clock(
-			decrease_allowed,
-			context->bw.dcn.calc_clk.fclk_khz,
-			dc->current_state->bw.dcn.cur_clk.fclk_khz)) {
-		context->bw.dcn.cur_clk.fclk_khz =
-				context->bw.dcn.calc_clk.fclk_khz;
-		smu_req.hard_min_fclk_khz = context->bw.dcn.calc_clk.fclk_khz;
-	}
-
-	smu_req.display_count = context->stream_count;
-
-	if (pp_smu->set_display_requirement)
-		pp_smu->set_display_requirement(&pp_smu->pp_smu, &smu_req);
+	if (!IS_FPGA_MAXIMUS_DC(dc->ctx->dce_environment)) {
+		if (context->stream_count == 0)
+			context->bw.dcn.clk.phyclk_khz = 0;
 
-	*smu_req_cur = smu_req;
+		dc->res_pool->dccg->funcs->update_clocks(
+				dc->res_pool->dccg,
+				&context->bw.dcn.clk,
+				safe_to_lower);
 
-	/* make sure dcf clk is before dpp clk to
-	 * make sure we have enough voltage to run dpp clk
-	 */
-	if (should_set_clock(
-			decrease_allowed,
-			context->bw.dcn.calc_clk.dispclk_khz,
-			dc->current_state->bw.dcn.cur_clk.dispclk_khz)) {
-
-		ramp_up_dispclk_with_dpp(dc, context);
+		dcn10_pplib_apply_display_requirements(dc, context);
 	}
 
-	dcn10_pplib_apply_display_requirements(dc, context);
+	hubbub1_program_watermarks(dc->res_pool->hubbub,
+			&context->bw.dcn.watermarks,
+			dc->res_pool->ref_clock_inKhz / 1000,
+			true);
 
-	if (dc->debug.sanity_checks) {
+	if (dc->debug.sanity_checks)
 		dcn10_verify_allow_pstate_change_high(dc);
-	}
-
-	/* need to fix this function.  not doing the right thing here */
 }
 
 static void set_drr(struct pipe_ctx **pipe_ctx,
@@ -2689,16 +2545,20 @@ static void dcn10_update_pending_status(struct pipe_ctx *pipe_ctx)
 {
 	struct dc_plane_state *plane_state = pipe_ctx->plane_state;
 	struct timing_generator *tg = pipe_ctx->stream_res.tg;
+	bool flip_pending;
 
 	if (plane_state == NULL)
 		return;
 
-	plane_state->status.is_flip_pending =
-			pipe_ctx->plane_res.hubp->funcs->hubp_is_flip_pending(
+	flip_pending = pipe_ctx->plane_res.hubp->funcs->hubp_is_flip_pending(
 					pipe_ctx->plane_res.hubp);
 
-	plane_state->status.current_address = pipe_ctx->plane_res.hubp->current_address;
-	if (pipe_ctx->plane_res.hubp->current_address.type == PLN_ADDR_TYPE_GRPH_STEREO &&
+	plane_state->status.is_flip_pending = flip_pending;
+
+	if (!flip_pending)
+		plane_state->status.current_address = plane_state->status.requested_address;
+
+	if (plane_state->status.current_address.type == PLN_ADDR_TYPE_GRPH_STEREO &&
 			tg->funcs->is_stereo_left_eye) {
 		plane_state->status.is_right_eye =
 				!tg->funcs->is_stereo_left_eye(pipe_ctx->stream_res.tg);
@@ -2707,8 +2567,14 @@ static void dcn10_update_pending_status(struct pipe_ctx *pipe_ctx)
 
 static void dcn10_update_dchub(struct dce_hwseq *hws, struct dchub_init_data *dh_data)
 {
-	if (hws->ctx->dc->res_pool->hubbub != NULL)
-		hubbub1_update_dchub(hws->ctx->dc->res_pool->hubbub, dh_data);
+	if (hws->ctx->dc->res_pool->hubbub != NULL) {
+		struct hubp *hubp = hws->ctx->dc->res_pool->hubps[0];
+
+		if (hubp->funcs->hubp_update_dchub)
+			hubp->funcs->hubp_update_dchub(hubp, dh_data);
+		else
+			hubbub1_update_dchub(hws->ctx->dc->res_pool->hubbub, dh_data);
+	}
 }
 
 static void dcn10_set_cursor_position(struct pipe_ctx *pipe_ctx)
@@ -2719,9 +2585,11 @@ static void dcn10_set_cursor_position(struct pipe_ctx *pipe_ctx)
 	struct dc_cursor_mi_param param = {
 		.pixel_clk_khz = pipe_ctx->stream->timing.pix_clk_khz,
 		.ref_clk_khz = pipe_ctx->stream->ctx->dc->res_pool->ref_clock_inKhz,
-		.viewport_x_start = pipe_ctx->plane_res.scl_data.viewport.x,
-		.viewport_width = pipe_ctx->plane_res.scl_data.viewport.width,
-		.h_scale_ratio = pipe_ctx->plane_res.scl_data.ratios.horz
+		.viewport = pipe_ctx->plane_res.scl_data.viewport,
+		.h_scale_ratio = pipe_ctx->plane_res.scl_data.ratios.horz,
+		.v_scale_ratio = pipe_ctx->plane_res.scl_data.ratios.vert,
+		.rotation = pipe_ctx->plane_state->rotation,
+		.mirror = pipe_ctx->plane_state->horizontal_mirror
 	};
 
 	if (pipe_ctx->plane_state->address.type
@@ -2745,6 +2613,33 @@ static void dcn10_set_cursor_attribute(struct pipe_ctx *pipe_ctx)
 		pipe_ctx->plane_res.dpp, attributes->color_format);
 }
 
+static void dcn10_set_cursor_sdr_white_level(struct pipe_ctx *pipe_ctx)
+{
+	uint32_t sdr_white_level = pipe_ctx->stream->cursor_attributes.sdr_white_level;
+	struct fixed31_32 multiplier;
+	struct dpp_cursor_attributes opt_attr = { 0 };
+	uint32_t hw_scale = 0x3c00; // 1.0 default multiplier
+	struct custom_float_format fmt;
+
+	if (!pipe_ctx->plane_res.dpp->funcs->set_optional_cursor_attributes)
+		return;
+
+	fmt.exponenta_bits = 5;
+	fmt.mantissa_bits = 10;
+	fmt.sign = true;
+
+	if (sdr_white_level > 80) {
+		multiplier = dc_fixpt_from_fraction(sdr_white_level, 80);
+		convert_to_custom_float_format(multiplier, &fmt, &hw_scale);
+	}
+
+	opt_attr.scale = hw_scale;
+	opt_attr.bias = 0;
+
+	pipe_ctx->plane_res.dpp->funcs->set_optional_cursor_attributes(
+			pipe_ctx->plane_res.dpp, &opt_attr);
+}
+
 static const struct hw_sequencer_funcs dcn10_funcs = {
 	.program_gamut_remap = program_gamut_remap,
 	.program_csc_matrix = program_csc_matrix,
@@ -2752,7 +2647,9 @@ static const struct hw_sequencer_funcs dcn10_funcs = {
 	.apply_ctx_to_hw = dce110_apply_ctx_to_hw,
 	.apply_ctx_for_surface = dcn10_apply_ctx_for_surface,
 	.update_plane_addr = dcn10_update_plane_addr,
+	.plane_atomic_disconnect = hwss1_plane_atomic_disconnect,
 	.update_dchub = dcn10_update_dchub,
+	.update_mpcc = dcn10_update_mpcc,
 	.update_pending_status = dcn10_update_pending_status,
 	.set_input_transfer_func = dcn10_set_input_transfer_func,
 	.set_output_transfer_func = dcn10_set_output_transfer_func,
@@ -2790,7 +2687,8 @@ static const struct hw_sequencer_funcs dcn10_funcs = {
 	.edp_power_control = hwss_edp_power_control,
 	.edp_wait_for_hpd_ready = hwss_edp_wait_for_hpd_ready,
 	.set_cursor_position = dcn10_set_cursor_position,
-	.set_cursor_attribute = dcn10_set_cursor_attribute
+	.set_cursor_attribute = dcn10_set_cursor_attribute,
+	.set_cursor_sdr_white_level = dcn10_set_cursor_sdr_white_level
 };
 
 
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.h
index 44f734b73f9e..7139fb73e966 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.h
@@ -39,4 +39,11 @@ bool is_rgb_cspace(enum dc_color_space output_color_space);
 
 void hwss1_plane_atomic_disconnect(struct dc *dc, struct pipe_ctx *pipe_ctx);
 
+void dcn10_verify_allow_pstate_change_high(struct dc *dc);
+
+void dcn10_program_pipe(
+		struct dc *dc,
+		struct pipe_ctx *pipe_ctx,
+		struct dc_state *context);
+
 #endif /* __DC_HWSS_DCN10_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.c
index 21fa40ac0786..6f675206a136 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.c
@@ -65,11 +65,6 @@ enum {
 	DP_MST_UPDATE_MAX_RETRY = 50
 };
 
-
-
-static void aux_initialize(struct dcn10_link_encoder *enc10);
-
-
 static const struct link_encoder_funcs dcn10_lnk_enc_funcs = {
 	.validate_output_with_stream =
 		dcn10_link_encoder_validate_output_with_stream,
@@ -445,12 +440,11 @@ static uint8_t get_frontend_source(
 	}
 }
 
-static void configure_encoder(
+void configure_encoder(
 	struct dcn10_link_encoder *enc10,
 	const struct dc_link_settings *link_settings)
 {
 	/* set number of lanes */
-
 	REG_SET(DP_CONFIG, 0,
 			DP_UDI_LANES, link_settings->lane_count - LANE_COUNT_ONE);
 
@@ -602,6 +596,9 @@ static bool dcn10_link_encoder_validate_hdmi_output(
 	if (!enc10->base.features.flags.bits.HDMI_6GB_EN &&
 		adjusted_pix_clk_khz >= 300000)
 		return false;
+	if (enc10->base.ctx->dc->debug.hdmi20_disable &&
+		crtc_timing->pixel_encoding == PIXEL_ENCODING_YCBCR420)
+		return false;
 	return true;
 }
 
@@ -734,6 +731,9 @@ void dcn10_link_encoder_construct(
 				__func__,
 				result);
 	}
+	if (enc10->base.ctx->dc->debug.hdmi20_disable) {
+		enc10->base.features.flags.bits.HDMI_6GB_EN = 0;
+	}
 }
 
 bool dcn10_link_encoder_validate_output_with_stream(
@@ -812,7 +812,7 @@ void dcn10_link_encoder_hw_init(
 		ASSERT(result == BP_RESULT_OK);
 
 	}
-	aux_initialize(enc10);
+	dcn10_aux_initialize(enc10);
 
 	/* reinitialize HPD.
 	 * hpd_initialize() will pass DIG_FE id to HW context.
@@ -995,6 +995,8 @@ void dcn10_link_encoder_disable_output(
 
 	if (!dcn10_is_dig_enabled(enc)) {
 		/* OF_SKIP_POWER_DOWN_INACTIVE_ENCODER */
+	/*in DP_Alt_No_Connect case, we turn off the dig already,
+	after excuation the PHY w/a sequence, not allow touch PHY any more*/
 		return;
 	}
 	/* Power-down RX and disable GPU PHY should be paired.
@@ -1347,8 +1349,7 @@ void dcn10_link_encoder_disable_hpd(struct link_encoder *enc)
 				FN(reg, f1), v1,\
 				FN(reg, f2), v2)
 
-static void aux_initialize(
-	struct dcn10_link_encoder *enc10)
+void dcn10_aux_initialize(struct dcn10_link_encoder *enc10)
 {
 	enum hpd_source_id hpd_source = enc10->base.hpd_source;
 
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.h
index 2a97cdb2cfbb..49ead12b2532 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.h
@@ -42,6 +42,7 @@
 #define LE_DCN_COMMON_REG_LIST(id) \
 	SRI(DIG_BE_CNTL, DIG, id), \
 	SRI(DIG_BE_EN_CNTL, DIG, id), \
+	SRI(TMDS_CTL_BITS, DIG, id), \
 	SRI(DP_CONFIG, DP, id), \
 	SRI(DP_DPHY_CNTL, DP, id), \
 	SRI(DP_DPHY_PRBS_CNTL, DP, id), \
@@ -64,6 +65,7 @@
 	SRI(DP_DPHY_INTERNAL_CTRL, DP, id), \
 	SRI(DP_DPHY_HBR2_PATTERN_CONTROL, DP, id)
 
+
 #define LE_DCN10_REG_LIST(id)\
 	LE_DCN_COMMON_REG_LIST(id)
 
@@ -100,6 +102,7 @@ struct dcn10_link_enc_registers {
 	uint32_t DP_DPHY_BS_SR_SWAP_CNTL;
 	uint32_t DP_DPHY_HBR2_PATTERN_CONTROL;
 	uint32_t DP_SEC_CNTL1;
+	uint32_t TMDS_CTL_BITS;
 };
 
 #define LE_SF(reg_name, field_name, post_fix)\
@@ -110,6 +113,7 @@ struct dcn10_link_enc_registers {
 	LE_SF(DIG0_DIG_BE_CNTL, DIG_HPD_SELECT, mask_sh),\
 	LE_SF(DIG0_DIG_BE_CNTL, DIG_MODE, mask_sh),\
 	LE_SF(DIG0_DIG_BE_CNTL, DIG_FE_SOURCE_SELECT, mask_sh),\
+	LE_SF(DIG0_TMDS_CTL_BITS, TMDS_CTL0, mask_sh), \
 	LE_SF(DP0_DP_DPHY_CNTL, DPHY_BYPASS, mask_sh),\
 	LE_SF(DP0_DP_DPHY_CNTL, DPHY_ATEST_SEL_LANE0, mask_sh),\
 	LE_SF(DP0_DP_DPHY_CNTL, DPHY_ATEST_SEL_LANE1, mask_sh),\
@@ -198,10 +202,11 @@ struct dcn10_link_enc_registers {
 	type DP_MSE_SAT_SLOT_COUNT3;\
 	type DP_MSE_SAT_UPDATE;\
 	type DP_MSE_16_MTP_KEEPOUT;\
+	type DC_HPD_EN;\
+	type TMDS_CTL0;\
 	type AUX_HPD_SEL;\
 	type AUX_LS_READ_EN;\
-	type AUX_RX_RECEIVE_WINDOW;\
-	type DC_HPD_EN
+	type AUX_RX_RECEIVE_WINDOW
 
 struct dcn10_link_enc_shift {
 	DCN_LINK_ENCODER_REG_FIELD_LIST(uint8_t);
@@ -266,6 +271,10 @@ void dcn10_link_encoder_setup(
 	struct link_encoder *enc,
 	enum signal_type signal);
 
+void configure_encoder(
+	struct dcn10_link_encoder *enc10,
+	const struct dc_link_settings *link_settings);
+
 /* enables TMDS PHY output */
 /* TODO: still need depth or just pass in adjusted pixel clock? */
 void dcn10_link_encoder_enable_tmds_output(
@@ -327,4 +336,6 @@ void dcn10_psr_program_secondary_packet(struct link_encoder *enc,
 
 bool dcn10_is_dig_enabled(struct link_encoder *enc);
 
+void dcn10_aux_initialize(struct dcn10_link_encoder *enc10);
+
 #endif /* __DC_LINK_ENCODER__DCN10_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_mpc.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_mpc.c
index 9ca51ae46de7..958994edf2c4 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_mpc.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_mpc.c
@@ -428,7 +428,7 @@ void mpc1_read_mpcc_state(
 			MPCC_BUSY, &s->busy);
 }
 
-const struct mpc_funcs dcn10_mpc_funcs = {
+static const struct mpc_funcs dcn10_mpc_funcs = {
 	.read_mpcc_state = mpc1_read_mpcc_state,
 	.insert_plane = mpc1_insert_plane,
 	.remove_mpcc = mpc1_remove_mpcc,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_opp.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_opp.c
index 77a1a9d541a4..ab958cff3b76 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_opp.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_opp.c
@@ -385,7 +385,7 @@ void opp1_destroy(struct output_pixel_processor **opp)
 	*opp = NULL;
 }
 
-static struct opp_funcs dcn10_opp_funcs = {
+static const struct opp_funcs dcn10_opp_funcs = {
 		.opp_set_dyn_expansion = opp1_set_dyn_expansion,
 		.opp_program_fmt = opp1_program_fmt,
 		.opp_program_bit_depth_reduction = opp1_program_bit_depth_reduction,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.c
index e6a3ade154b9..411f89218e01 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.c
@@ -1324,6 +1324,72 @@ bool optc1_is_optc_underflow_occurred(struct timing_generator *optc)
 	return (underflow_occurred == 1);
 }
 
+bool optc1_configure_crc(struct timing_generator *optc,
+			  const struct crc_params *params)
+{
+	struct optc *optc1 = DCN10TG_FROM_TG(optc);
+
+	/* Cannot configure crc on a CRTC that is disabled */
+	if (!optc1_is_tg_enabled(optc))
+		return false;
+
+	REG_WRITE(OTG_CRC_CNTL, 0);
+
+	if (!params->enable)
+		return true;
+
+	/* Program frame boundaries */
+	/* Window A x axis start and end. */
+	REG_UPDATE_2(OTG_CRC0_WINDOWA_X_CONTROL,
+			OTG_CRC0_WINDOWA_X_START, params->windowa_x_start,
+			OTG_CRC0_WINDOWA_X_END, params->windowa_x_end);
+
+	/* Window A y axis start and end. */
+	REG_UPDATE_2(OTG_CRC0_WINDOWA_Y_CONTROL,
+			OTG_CRC0_WINDOWA_Y_START, params->windowa_y_start,
+			OTG_CRC0_WINDOWA_Y_END, params->windowa_y_end);
+
+	/* Window B x axis start and end. */
+	REG_UPDATE_2(OTG_CRC0_WINDOWB_X_CONTROL,
+			OTG_CRC0_WINDOWB_X_START, params->windowb_x_start,
+			OTG_CRC0_WINDOWB_X_END, params->windowb_x_end);
+
+	/* Window B y axis start and end. */
+	REG_UPDATE_2(OTG_CRC0_WINDOWB_Y_CONTROL,
+			OTG_CRC0_WINDOWB_Y_START, params->windowb_y_start,
+			OTG_CRC0_WINDOWB_Y_END, params->windowb_y_end);
+
+	/* Set crc mode and selection, and enable. Only using CRC0*/
+	REG_UPDATE_3(OTG_CRC_CNTL,
+			OTG_CRC_CONT_EN, params->continuous_mode ? 1 : 0,
+			OTG_CRC0_SELECT, params->selection,
+			OTG_CRC_EN, 1);
+
+	return true;
+}
+
+bool optc1_get_crc(struct timing_generator *optc,
+		    uint32_t *r_cr, uint32_t *g_y, uint32_t *b_cb)
+{
+	uint32_t field = 0;
+	struct optc *optc1 = DCN10TG_FROM_TG(optc);
+
+	REG_GET(OTG_CRC_CNTL, OTG_CRC_EN, &field);
+
+	/* Early return if CRC is not enabled for this CRTC */
+	if (!field)
+		return false;
+
+	REG_GET_2(OTG_CRC0_DATA_RG,
+			CRC0_R_CR, r_cr,
+			CRC0_G_Y, g_y);
+
+	REG_GET(OTG_CRC0_DATA_B,
+			CRC0_B_CB, b_cb);
+
+	return true;
+}
+
 static const struct timing_generator_funcs dcn10_tg_funcs = {
 		.validate_timing = optc1_validate_timing,
 		.program_timing = optc1_program_timing,
@@ -1360,6 +1426,8 @@ static const struct timing_generator_funcs dcn10_tg_funcs = {
 		.is_tg_enabled = optc1_is_tg_enabled,
 		.is_optc_underflow_occurred = optc1_is_optc_underflow_occurred,
 		.clear_optc_underflow = optc1_clear_optc_underflow,
+		.get_crc = optc1_get_crc,
+		.configure_crc = optc1_configure_crc,
 };
 
 void dcn10_timing_generator_init(struct optc *optc1)
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.h
index 59ed272e0c49..c1b114209fe8 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.h
@@ -75,7 +75,14 @@
 	SRI(CONTROL, VTG, inst),\
 	SRI(OTG_VERT_SYNC_CONTROL, OTG, inst),\
 	SRI(OTG_MASTER_UPDATE_MODE, OTG, inst),\
-	SRI(OTG_GSL_CONTROL, OTG, inst)
+	SRI(OTG_GSL_CONTROL, OTG, inst),\
+	SRI(OTG_CRC_CNTL, OTG, inst),\
+	SRI(OTG_CRC0_DATA_RG, OTG, inst),\
+	SRI(OTG_CRC0_DATA_B, OTG, inst),\
+	SRI(OTG_CRC0_WINDOWA_X_CONTROL, OTG, inst),\
+	SRI(OTG_CRC0_WINDOWA_Y_CONTROL, OTG, inst),\
+	SRI(OTG_CRC0_WINDOWB_X_CONTROL, OTG, inst),\
+	SRI(OTG_CRC0_WINDOWB_Y_CONTROL, OTG, inst)
 
 #define TG_COMMON_REG_LIST_DCN1_0(inst) \
 	TG_COMMON_REG_LIST_DCN(inst),\
@@ -138,6 +145,13 @@ struct dcn_optc_registers {
 	uint32_t OTG_GSL_WINDOW_X;
 	uint32_t OTG_GSL_WINDOW_Y;
 	uint32_t OTG_VUPDATE_KEEPOUT;
+	uint32_t OTG_CRC_CNTL;
+	uint32_t OTG_CRC0_DATA_RG;
+	uint32_t OTG_CRC0_DATA_B;
+	uint32_t OTG_CRC0_WINDOWA_X_CONTROL;
+	uint32_t OTG_CRC0_WINDOWA_Y_CONTROL;
+	uint32_t OTG_CRC0_WINDOWB_X_CONTROL;
+	uint32_t OTG_CRC0_WINDOWB_Y_CONTROL;
 };
 
 #define TG_COMMON_MASK_SH_LIST_DCN(mask_sh)\
@@ -232,7 +246,21 @@ struct dcn_optc_registers {
 	SF(OTG0_OTG_GSL_CONTROL, OTG_GSL2_EN, mask_sh),\
 	SF(OTG0_OTG_GSL_CONTROL, OTG_GSL_MASTER_EN, mask_sh),\
 	SF(OTG0_OTG_GSL_CONTROL, OTG_GSL_FORCE_DELAY, mask_sh),\
-	SF(OTG0_OTG_GSL_CONTROL, OTG_GSL_CHECK_ALL_FIELDS, mask_sh)
+	SF(OTG0_OTG_GSL_CONTROL, OTG_GSL_CHECK_ALL_FIELDS, mask_sh),\
+	SF(OTG0_OTG_CRC_CNTL, OTG_CRC_CONT_EN, mask_sh),\
+	SF(OTG0_OTG_CRC_CNTL, OTG_CRC0_SELECT, mask_sh),\
+	SF(OTG0_OTG_CRC_CNTL, OTG_CRC_EN, mask_sh),\
+	SF(OTG0_OTG_CRC0_DATA_RG, CRC0_R_CR, mask_sh),\
+	SF(OTG0_OTG_CRC0_DATA_RG, CRC0_G_Y, mask_sh),\
+	SF(OTG0_OTG_CRC0_DATA_B, CRC0_B_CB, mask_sh),\
+	SF(OTG0_OTG_CRC0_WINDOWA_X_CONTROL, OTG_CRC0_WINDOWA_X_START, mask_sh),\
+	SF(OTG0_OTG_CRC0_WINDOWA_X_CONTROL, OTG_CRC0_WINDOWA_X_END, mask_sh),\
+	SF(OTG0_OTG_CRC0_WINDOWA_Y_CONTROL, OTG_CRC0_WINDOWA_Y_START, mask_sh),\
+	SF(OTG0_OTG_CRC0_WINDOWA_Y_CONTROL, OTG_CRC0_WINDOWA_Y_END, mask_sh),\
+	SF(OTG0_OTG_CRC0_WINDOWB_X_CONTROL, OTG_CRC0_WINDOWB_X_START, mask_sh),\
+	SF(OTG0_OTG_CRC0_WINDOWB_X_CONTROL, OTG_CRC0_WINDOWB_X_END, mask_sh),\
+	SF(OTG0_OTG_CRC0_WINDOWB_Y_CONTROL, OTG_CRC0_WINDOWB_Y_START, mask_sh),\
+	SF(OTG0_OTG_CRC0_WINDOWB_Y_CONTROL, OTG_CRC0_WINDOWB_Y_END, mask_sh)
 
 
 #define TG_COMMON_MASK_SH_LIST_DCN1_0(mask_sh)\
@@ -363,7 +391,22 @@ struct dcn_optc_registers {
 	type OTG_MASTER_UPDATE_LOCK_GSL_EN;\
 	type MASTER_UPDATE_LOCK_VUPDATE_KEEPOUT_START_OFFSET;\
 	type MASTER_UPDATE_LOCK_VUPDATE_KEEPOUT_END_OFFSET;\
-	type OTG_MASTER_UPDATE_LOCK_VUPDATE_KEEPOUT_EN;
+	type OTG_MASTER_UPDATE_LOCK_VUPDATE_KEEPOUT_EN;\
+	type OTG_CRC_CONT_EN;\
+	type OTG_CRC0_SELECT;\
+	type OTG_CRC_EN;\
+	type CRC0_R_CR;\
+	type CRC0_G_Y;\
+	type CRC0_B_CB;\
+	type OTG_CRC0_WINDOWA_X_START;\
+	type OTG_CRC0_WINDOWA_X_END;\
+	type OTG_CRC0_WINDOWA_Y_START;\
+	type OTG_CRC0_WINDOWA_Y_END;\
+	type OTG_CRC0_WINDOWB_X_START;\
+	type OTG_CRC0_WINDOWB_X_END;\
+	type OTG_CRC0_WINDOWB_Y_START;\
+	type OTG_CRC0_WINDOWB_Y_END;
+
 
 #define TG_REG_FIELD_LIST(type) \
 	TG_REG_FIELD_LIST_DCN1_0(type)
@@ -511,4 +554,15 @@ bool optc1_get_otg_active_size(struct timing_generator *optc,
 		uint32_t *otg_active_width,
 		uint32_t *otg_active_height);
 
+void optc1_enable_crtc_reset(
+		struct timing_generator *optc,
+		int source_tg_inst,
+		struct crtc_trigger_info *crtc_tp);
+
+bool optc1_configure_crc(struct timing_generator *optc,
+			  const struct crc_params *params);
+
+bool optc1_get_crc(struct timing_generator *optc,
+		    uint32_t *r_cr, uint32_t *g_y, uint32_t *b_cb);
+
 #endif /* __DC_TIMING_GENERATOR_DCN10_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.c
index 2da325ce781b..6b44ed3697a4 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.c
@@ -64,6 +64,69 @@
 #include "reg_helper.h"
 #include "dce/dce_abm.h"
 #include "dce/dce_dmcu.h"
+#include "dce/dce_aux.h"
+
+const struct _vcs_dpi_ip_params_st dcn1_0_ip = {
+	.rob_buffer_size_kbytes = 64,
+	.det_buffer_size_kbytes = 164,
+	.dpte_buffer_size_in_pte_reqs = 42,
+	.dpp_output_buffer_pixels = 2560,
+	.opp_output_buffer_lines = 1,
+	.pixel_chunk_size_kbytes = 8,
+	.pte_enable = 1,
+	.pte_chunk_size_kbytes = 2,
+	.meta_chunk_size_kbytes = 2,
+	.writeback_chunk_size_kbytes = 2,
+	.line_buffer_size_bits = 589824,
+	.max_line_buffer_lines = 12,
+	.IsLineBufferBppFixed = 0,
+	.LineBufferFixedBpp = -1,
+	.writeback_luma_buffer_size_kbytes = 12,
+	.writeback_chroma_buffer_size_kbytes = 8,
+	.max_num_dpp = 4,
+	.max_num_wb = 2,
+	.max_dchub_pscl_bw_pix_per_clk = 4,
+	.max_pscl_lb_bw_pix_per_clk = 2,
+	.max_lb_vscl_bw_pix_per_clk = 4,
+	.max_vscl_hscl_bw_pix_per_clk = 4,
+	.max_hscl_ratio = 4,
+	.max_vscl_ratio = 4,
+	.hscl_mults = 4,
+	.vscl_mults = 4,
+	.max_hscl_taps = 8,
+	.max_vscl_taps = 8,
+	.dispclk_ramp_margin_percent = 1,
+	.underscan_factor = 1.10,
+	.min_vblank_lines = 14,
+	.dppclk_delay_subtotal = 90,
+	.dispclk_delay_subtotal = 42,
+	.dcfclk_cstate_latency = 10,
+	.max_inter_dcn_tile_repeaters = 8,
+	.can_vstartup_lines_exceed_vsync_plus_back_porch_lines_minus_one = 0,
+	.bug_forcing_LC_req_same_size_fixed = 0,
+};
+
+const struct _vcs_dpi_soc_bounding_box_st dcn1_0_soc = {
+	.sr_exit_time_us = 9.0,
+	.sr_enter_plus_exit_time_us = 11.0,
+	.urgent_latency_us = 4.0,
+	.writeback_latency_us = 12.0,
+	.ideal_dram_bw_after_urgent_percent = 80.0,
+	.max_request_size_bytes = 256,
+	.downspread_percent = 0.5,
+	.dram_page_open_time_ns = 50.0,
+	.dram_rw_turnaround_time_ns = 17.5,
+	.dram_return_buffer_per_channel_bytes = 8192,
+	.round_trip_ping_latency_dcfclk_cycles = 128,
+	.urgent_out_of_order_return_per_channel_bytes = 256,
+	.channel_interleave_bytes = 256,
+	.num_banks = 8,
+	.num_chans = 2,
+	.vmm_page_size_bytes = 4096,
+	.dram_clock_change_latency_us = 17.0,
+	.writeback_dram_clock_change_latency_us = 23.0,
+	.return_bus_width_bytes = 64,
+};
 
 #ifndef mmDP0_DP_DPHY_INTERNAL_CTRL
 	#define mmDP0_DP_DPHY_INTERNAL_CTRL		0x210f
@@ -294,6 +357,21 @@ static const struct dcn10_opp_mask opp_mask = {
 		OPP_MASK_SH_LIST_DCN10(_MASK),
 };
 
+#define aux_engine_regs(id)\
+[id] = {\
+	AUX_COMMON_REG_LIST(id), \
+	.AUX_RESET_MASK = 0 \
+}
+
+static const struct dce110_aux_registers aux_engine_regs[] = {
+		aux_engine_regs(0),
+		aux_engine_regs(1),
+		aux_engine_regs(2),
+		aux_engine_regs(3),
+		aux_engine_regs(4),
+		aux_engine_regs(5)
+};
+
 #define tf_regs(id)\
 [id] = {\
 	TF_REG_LIST_DCN10(id),\
@@ -424,7 +502,7 @@ static const struct resource_caps res_cap = {
 		.num_pll = 4,
 };
 
-static const struct dc_debug debug_defaults_drv = {
+static const struct dc_debug_options debug_defaults_drv = {
 		.sanity_checks = true,
 		.disable_dmcu = true,
 		.force_abm_enable = false,
@@ -437,7 +515,7 @@ static const struct dc_debug debug_defaults_drv = {
 		 */
 		.min_disp_clk_khz = 100000,
 
-		.disable_pplib_clock_request = true,
+		.disable_pplib_clock_request = false,
 		.disable_pplib_wm_range = false,
 		.pplib_wm_report_mode = WM_REPORT_DEFAULT,
 		.pipe_split_policy = MPC_SPLIT_AVOID_MULT_DISP,
@@ -452,7 +530,7 @@ static const struct dc_debug debug_defaults_drv = {
 		.max_downscale_src_width = 3840,
 };
 
-static const struct dc_debug debug_defaults_diags = {
+static const struct dc_debug_options debug_defaults_diags = {
 		.disable_dmcu = true,
 		.force_abm_enable = false,
 		.timing_trace = true,
@@ -516,6 +594,23 @@ static struct output_pixel_processor *dcn10_opp_create(
 	return &opp->base;
 }
 
+struct aux_engine *dcn10_aux_engine_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct aux_engine_dce110 *aux_engine =
+		kzalloc(sizeof(struct aux_engine_dce110), GFP_KERNEL);
+
+	if (!aux_engine)
+		return NULL;
+
+	dce110_aux_engine_construct(aux_engine, ctx, inst,
+				    SW_AUX_TIMEOUT_PERIOD_MULTIPLIER * AUX_TIMEOUT_PERIOD,
+				    &aux_engine_regs[inst]);
+
+	return &aux_engine->base;
+}
+
 static struct mpc *dcn10_mpc_create(struct dc_context *ctx)
 {
 	struct dcn10_mpc *mpc10 = kzalloc(sizeof(struct dcn10_mpc),
@@ -681,6 +776,7 @@ static struct dce_hwseq *dcn10_hwseq_create(
 		hws->masks = &hwseq_mask;
 		hws->wa.DEGVIDCN10_253 = true;
 		hws->wa.false_optc_underflow = true;
+		hws->wa.DEGVIDCN10_254 = true;
 	}
 	return hws;
 }
@@ -763,6 +859,9 @@ static void destruct(struct dcn10_resource_pool *pool)
 			kfree(DCN10TG_FROM_TG(pool->base.timing_generators[i]));
 			pool->base.timing_generators[i] = NULL;
 		}
+
+		if (pool->base.engines[i] != NULL)
+			pool->base.engines[i]->funcs->destroy_engine(&pool->base.engines[i]);
 	}
 
 	for (i = 0; i < pool->base.stream_enc_count; i++)
@@ -791,8 +890,8 @@ static void destruct(struct dcn10_resource_pool *pool)
 	if (pool->base.dmcu != NULL)
 		dce_dmcu_destroy(&pool->base.dmcu);
 
-	if (pool->base.display_clock != NULL)
-		dce_disp_clk_destroy(&pool->base.display_clock);
+	if (pool->base.dccg != NULL)
+		dce_dccg_destroy(&pool->base.dccg);
 
 	kfree(pool->base.pp_smu);
 }
@@ -972,11 +1071,11 @@ static enum dc_status dcn10_validate_plane(const struct dc_plane_state *plane_st
 	return DC_OK;
 }
 
-static struct dc_cap_funcs cap_funcs = {
+static const struct dc_cap_funcs cap_funcs = {
 	.get_dcc_compression_cap = dcn10_get_dcc_compression_cap
 };
 
-static struct resource_funcs dcn10_res_pool_funcs = {
+static const struct resource_funcs dcn10_res_pool_funcs = {
 	.destroy = dcn10_destroy_resource_pool,
 	.link_enc_create = dcn10_link_encoder_create,
 	.validate_bandwidth = dcn_validate_bandwidth,
@@ -1005,8 +1104,7 @@ static bool construct(
 
 	ctx->dc_bios->regs = &bios_regs;
 
-		pool->base.res_cap = &res_cap;
-
+	pool->base.res_cap = &res_cap;
 	pool->base.funcs = &dcn10_res_pool_funcs;
 
 	/*
@@ -1029,6 +1127,8 @@ static bool construct(
 	dc->caps.max_slave_planes = 1;
 	dc->caps.is_apu = true;
 	dc->caps.post_blend_color_processing = false;
+	/* Raven DP PHY HBR2 eye diagram pattern is not stable. Use TP4 */
+	dc->caps.force_dp_tps4_for_cp2520 = true;
 
 	if (dc->ctx->dce_environment == DCE_ENV_PRODUCTION_DRV)
 		dc->debug = debug_defaults_drv;
@@ -1072,8 +1172,8 @@ static bool construct(
 		}
 	}
 
-	pool->base.display_clock = dce120_disp_clk_create(ctx);
-	if (pool->base.display_clock == NULL) {
+	pool->base.dccg = dcn1_dccg_create(ctx);
+	if (pool->base.dccg == NULL) {
 		dm_error("DC: failed to create display clock!\n");
 		BREAK_TO_DEBUGGER();
 		goto fail;
@@ -1193,6 +1293,14 @@ static bool construct(
 			goto fail;
 		}
 
+		pool->base.engines[i] = dcn10_aux_engine_create(ctx, i);
+		if (pool->base.engines[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC:failed to create aux engine!!\n");
+			goto fail;
+		}
+
 		/* check next valid pipe */
 		j++;
 	}
diff --git a/drivers/gpu/drm/amd/display/dc/dm_helpers.h b/drivers/gpu/drm/amd/display/dc/dm_helpers.h
index 034369fbb9e2..5d4527d03045 100644
--- a/drivers/gpu/drm/amd/display/dc/dm_helpers.h
+++ b/drivers/gpu/drm/amd/display/dc/dm_helpers.h
@@ -40,6 +40,14 @@ enum dc_edid_status dm_helpers_parse_edid_caps(
 	const struct dc_edid *edid,
 	struct dc_edid_caps *edid_caps);
 
+
+/*
+ * Update DP branch info
+ */
+void dm_helpers_dp_update_branch_info(
+		struct dc_context *ctx,
+		const struct dc_link *link);
+
 /*
  * Writes payload allocation table in immediate downstream device.
  */
@@ -103,6 +111,9 @@ bool dm_helpers_submit_i2c(
 		const struct dc_link *link,
 		struct i2c_command *cmd);
 
+bool dm_helpers_is_dp_sink_present(
+		struct dc_link *link);
+
 enum dc_edid_status dm_helpers_read_local_edid(
 		struct dc_context *ctx,
 		struct dc_link *link,
diff --git a/drivers/gpu/drm/amd/display/dc/dm_pp_smu.h b/drivers/gpu/drm/amd/display/dc/dm_pp_smu.h
index eac4bfe12257..58ed2055ef9f 100644
--- a/drivers/gpu/drm/amd/display/dc/dm_pp_smu.h
+++ b/drivers/gpu/drm/amd/display/dc/dm_pp_smu.h
@@ -40,7 +40,7 @@ enum wm_set_id {
 	WM_B,
 	WM_C,
 	WM_D,
-	WM_COUNT,
+	WM_SET_COUNT,
 };
 
 struct pp_smu_wm_set_range {
@@ -53,10 +53,10 @@ struct pp_smu_wm_set_range {
 
 struct pp_smu_wm_range_sets {
 	uint32_t num_reader_wm_sets;
-	struct pp_smu_wm_set_range reader_wm_sets[WM_COUNT];
+	struct pp_smu_wm_set_range reader_wm_sets[WM_SET_COUNT];
 
 	uint32_t num_writer_wm_sets;
-	struct pp_smu_wm_set_range writer_wm_sets[WM_COUNT];
+	struct pp_smu_wm_set_range writer_wm_sets[WM_SET_COUNT];
 };
 
 struct pp_smu_display_requirement_rv {
diff --git a/drivers/gpu/drm/amd/display/dc/dm_services_types.h b/drivers/gpu/drm/amd/display/dc/dm_services_types.h
index ab8c77d4e6df..2b83f922ac02 100644
--- a/drivers/gpu/drm/amd/display/dc/dm_services_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dm_services_types.h
@@ -137,7 +137,7 @@ struct dm_pp_clock_range_for_wm_set {
 	enum dm_pp_wm_set_id wm_set_id;
 	uint32_t wm_min_eng_clk_in_khz;
 	uint32_t wm_max_eng_clk_in_khz;
-	uint32_t wm_min_memg_clk_in_khz;
+	uint32_t wm_min_mem_clk_in_khz;
 	uint32_t wm_max_mem_clk_in_khz;
 };
 
@@ -150,7 +150,7 @@ struct dm_pp_clock_range_for_dmif_wm_set_soc15 {
 	enum dm_pp_wm_set_id wm_set_id;
 	uint32_t wm_min_dcfclk_clk_in_khz;
 	uint32_t wm_max_dcfclk_clk_in_khz;
-	uint32_t wm_min_memg_clk_in_khz;
+	uint32_t wm_min_mem_clk_in_khz;
 	uint32_t wm_max_mem_clk_in_khz;
 };
 
@@ -158,7 +158,7 @@ struct dm_pp_clock_range_for_mcif_wm_set_soc15 {
 	enum dm_pp_wm_set_id wm_set_id;
 	uint32_t wm_min_socclk_clk_in_khz;
 	uint32_t wm_max_socclk_clk_in_khz;
-	uint32_t wm_min_memg_clk_in_khz;
+	uint32_t wm_min_mem_clk_in_khz;
 	uint32_t wm_max_mem_clk_in_khz;
 };
 
diff --git a/drivers/gpu/drm/amd/display/dc/dml/Makefile b/drivers/gpu/drm/amd/display/dc/dml/Makefile
index f83a608f93e9..d97ca6528f9d 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/dml/Makefile
@@ -36,11 +36,10 @@ CFLAGS_display_mode_lib.o := $(dml_ccflags)
 CFLAGS_display_pipe_clocks.o := $(dml_ccflags)
 CFLAGS_dml1_display_rq_dlg_calc.o := $(dml_ccflags)
 CFLAGS_display_rq_dlg_helpers.o := $(dml_ccflags)
-CFLAGS_soc_bounding_box.o := $(dml_ccflags)
 CFLAGS_dml_common_defs.o := $(dml_ccflags)
 
 DML = display_mode_lib.o display_rq_dlg_helpers.o dml1_display_rq_dlg_calc.o \
-	  soc_bounding_box.o dml_common_defs.o
+	dml_common_defs.o
 
 AMD_DAL_DML = $(addprefix $(AMDDALPATH)/dc/dml/,$(DML))
 
diff --git a/drivers/gpu/drm/amd/display/dc/dml/display_mode_lib.c b/drivers/gpu/drm/amd/display/dc/dml/display_mode_lib.c
index fd9d97aab071..dddeb0d4db8f 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/display_mode_lib.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/display_mode_lib.c
@@ -26,67 +26,8 @@
 #include "display_mode_lib.h"
 #include "dc_features.h"
 
-static const struct _vcs_dpi_ip_params_st dcn1_0_ip = {
-	.rob_buffer_size_kbytes = 64,
-	.det_buffer_size_kbytes = 164,
-	.dpte_buffer_size_in_pte_reqs = 42,
-	.dpp_output_buffer_pixels = 2560,
-	.opp_output_buffer_lines = 1,
-	.pixel_chunk_size_kbytes = 8,
-	.pte_enable = 1,
-	.pte_chunk_size_kbytes = 2,
-	.meta_chunk_size_kbytes = 2,
-	.writeback_chunk_size_kbytes = 2,
-	.line_buffer_size_bits = 589824,
-	.max_line_buffer_lines = 12,
-	.IsLineBufferBppFixed = 0,
-	.LineBufferFixedBpp = -1,
-	.writeback_luma_buffer_size_kbytes = 12,
-	.writeback_chroma_buffer_size_kbytes = 8,
-	.max_num_dpp = 4,
-	.max_num_wb = 2,
-	.max_dchub_pscl_bw_pix_per_clk = 4,
-	.max_pscl_lb_bw_pix_per_clk = 2,
-	.max_lb_vscl_bw_pix_per_clk = 4,
-	.max_vscl_hscl_bw_pix_per_clk = 4,
-	.max_hscl_ratio = 4,
-	.max_vscl_ratio = 4,
-	.hscl_mults = 4,
-	.vscl_mults = 4,
-	.max_hscl_taps = 8,
-	.max_vscl_taps = 8,
-	.dispclk_ramp_margin_percent = 1,
-	.underscan_factor = 1.10,
-	.min_vblank_lines = 14,
-	.dppclk_delay_subtotal = 90,
-	.dispclk_delay_subtotal = 42,
-	.dcfclk_cstate_latency = 10,
-	.max_inter_dcn_tile_repeaters = 8,
-	.can_vstartup_lines_exceed_vsync_plus_back_porch_lines_minus_one = 0,
-	.bug_forcing_LC_req_same_size_fixed = 0,
-};
-
-static const struct _vcs_dpi_soc_bounding_box_st dcn1_0_soc = {
-	.sr_exit_time_us = 9.0,
-	.sr_enter_plus_exit_time_us = 11.0,
-	.urgent_latency_us = 4.0,
-	.writeback_latency_us = 12.0,
-	.ideal_dram_bw_after_urgent_percent = 80.0,
-	.max_request_size_bytes = 256,
-	.downspread_percent = 0.5,
-	.dram_page_open_time_ns = 50.0,
-	.dram_rw_turnaround_time_ns = 17.5,
-	.dram_return_buffer_per_channel_bytes = 8192,
-	.round_trip_ping_latency_dcfclk_cycles = 128,
-	.urgent_out_of_order_return_per_channel_bytes = 256,
-	.channel_interleave_bytes = 256,
-	.num_banks = 8,
-	.num_chans = 2,
-	.vmm_page_size_bytes = 4096,
-	.dram_clock_change_latency_us = 17.0,
-	.writeback_dram_clock_change_latency_us = 23.0,
-	.return_bus_width_bytes = 64,
-};
+extern const struct _vcs_dpi_ip_params_st dcn1_0_ip;
+extern const struct _vcs_dpi_soc_bounding_box_st dcn1_0_soc;
 
 static void set_soc_bounding_box(struct _vcs_dpi_soc_bounding_box_st *soc, enum dml_project project)
 {
diff --git a/drivers/gpu/drm/amd/display/dc/dml/display_mode_lib.h b/drivers/gpu/drm/amd/display/dc/dml/display_mode_lib.h
index 3c2abcb8a1b0..635206248889 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/display_mode_lib.h
+++ b/drivers/gpu/drm/amd/display/dc/dml/display_mode_lib.h
@@ -27,7 +27,6 @@
 
 
 #include "dml_common_defs.h"
-#include "soc_bounding_box.h"
 #include "dml1_display_rq_dlg_calc.h"
 
 enum dml_project {
diff --git a/drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h b/drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h
index 7fa0375939ae..cbafce649e33 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h
+++ b/drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h
@@ -64,10 +64,9 @@ struct _vcs_dpi_voltage_scaling_st {
 	double dscclk_mhz;
 	double dcfclk_mhz;
 	double socclk_mhz;
-	double dram_speed_mhz;
+	double dram_speed_mts;
 	double fabricclk_mhz;
 	double dispclk_mhz;
-	double dram_bw_per_chan_gbps;
 	double phyclk_mhz;
 	double dppclk_mhz;
 };
@@ -112,6 +111,8 @@ struct _vcs_dpi_soc_bounding_box_st {
 	double xfc_bus_transport_time_us;
 	double xfc_xbuf_latency_tolerance_us;
 	int use_urgent_burst_bw;
+	double max_hscl_ratio;
+	double max_vscl_ratio;
 	struct _vcs_dpi_voltage_scaling_st clock_limits[7];
 };
 
@@ -304,6 +305,7 @@ struct _vcs_dpi_display_pipe_dest_params_st {
 	unsigned char otg_inst;
 	unsigned char odm_split_cnt;
 	unsigned char odm_combine;
+	unsigned char use_maximum_vstartup;
 };
 
 struct _vcs_dpi_display_pipe_params_st {
diff --git a/drivers/gpu/drm/amd/display/dc/dml/soc_bounding_box.c b/drivers/gpu/drm/amd/display/dc/dml/soc_bounding_box.c
deleted file mode 100644
index 324239c77958..000000000000
--- a/drivers/gpu/drm/amd/display/dc/dml/soc_bounding_box.c
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Copyright 2017 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: AMD
- *
- */
-#include "soc_bounding_box.h"
-#include "display_mode_lib.h"
-#include "dc_features.h"
-
-#include "dml_inline_defs.h"
-
-/*
- * NOTE:
- *   This file is gcc-parseable HW gospel, coming straight from HW engineers.
- *
- * It doesn't adhere to Linux kernel style and sometimes will do things in odd
- * ways. Unless there is something clearly wrong with it the code should
- * remain as-is as it provides us with a guarantee from HW that it is correct.
- */
-
-void dml_socbb_set_latencies(soc_bounding_box_st *to_box, soc_bounding_box_st *from_box)
-{
-	to_box->dram_clock_change_latency_us = from_box->dram_clock_change_latency_us;
-	to_box->sr_exit_time_us = from_box->sr_exit_time_us;
-	to_box->sr_enter_plus_exit_time_us = from_box->sr_enter_plus_exit_time_us;
-	to_box->urgent_latency_us = from_box->urgent_latency_us;
-	to_box->writeback_latency_us = from_box->writeback_latency_us;
-}
-
-voltage_scaling_st dml_socbb_voltage_scaling(
-		const soc_bounding_box_st *soc,
-		enum voltage_state voltage)
-{
-	const voltage_scaling_st *voltage_state;
-	const voltage_scaling_st * const voltage_end = soc->clock_limits + DC__VOLTAGE_STATES;
-
-	for (voltage_state = soc->clock_limits;
-			voltage_state < voltage_end && voltage_state->state != voltage;
-			voltage_state++) {
-	}
-
-	if (voltage_state < voltage_end)
-		return *voltage_state;
-	return soc->clock_limits[DC__VOLTAGE_STATES - 1];
-}
-
-double dml_socbb_return_bw_mhz(soc_bounding_box_st *box, enum voltage_state voltage)
-{
-	double return_bw;
-
-	voltage_scaling_st state = dml_socbb_voltage_scaling(box, voltage);
-
-	return_bw = dml_min((double) box->return_bus_width_bytes * state.dcfclk_mhz,
-			state.dram_bw_per_chan_gbps * 1000.0 * (double) box->num_chans
-					* box->ideal_dram_bw_after_urgent_percent / 100.0);
-
-	return_bw = dml_min((double) box->return_bus_width_bytes * state.fabricclk_mhz, return_bw);
-
-	return return_bw;
-}
diff --git a/drivers/gpu/drm/amd/display/dc/gpio/ddc_regs.h b/drivers/gpu/drm/amd/display/dc/gpio/ddc_regs.h
index 9c4a56c738c0..bf40725f982f 100644
--- a/drivers/gpu/drm/amd/display/dc/gpio/ddc_regs.h
+++ b/drivers/gpu/drm/amd/display/dc/gpio/ddc_regs.h
@@ -82,13 +82,16 @@
 	DDC_GPIO_I2C_REG_LIST(cd),\
 	.ddc_setup = 0
 
-#define DDC_MASK_SH_LIST(mask_sh) \
+#define DDC_MASK_SH_LIST_COMMON(mask_sh) \
 		SF_DDC(DC_I2C_DDC1_SETUP, DC_I2C_DDC1_ENABLE, mask_sh),\
 		SF_DDC(DC_I2C_DDC1_SETUP, DC_I2C_DDC1_EDID_DETECT_ENABLE, mask_sh),\
 		SF_DDC(DC_I2C_DDC1_SETUP, DC_I2C_DDC1_EDID_DETECT_MODE, mask_sh),\
 		SF_DDC(DC_GPIO_DDC1_MASK, DC_GPIO_DDC1DATA_PD_EN, mask_sh),\
 		SF_DDC(DC_GPIO_DDC1_MASK, DC_GPIO_DDC1CLK_PD_EN, mask_sh),\
-		SF_DDC(DC_GPIO_DDC1_MASK, AUX_PAD1_MODE, mask_sh),\
+		SF_DDC(DC_GPIO_DDC1_MASK, AUX_PAD1_MODE, mask_sh)
+
+#define DDC_MASK_SH_LIST(mask_sh) \
+		DDC_MASK_SH_LIST_COMMON(mask_sh),\
 		SF_DDC(DC_GPIO_I2CPAD_MASK, DC_GPIO_SDA_PD_DIS, mask_sh),\
 		SF_DDC(DC_GPIO_I2CPAD_MASK, DC_GPIO_SCL_PD_DIS, mask_sh)
 
diff --git a/drivers/gpu/drm/amd/display/dc/gpio/gpio_service.c b/drivers/gpu/drm/amd/display/dc/gpio/gpio_service.c
index ab5483c0c502..f20161c5706d 100644
--- a/drivers/gpu/drm/amd/display/dc/gpio/gpio_service.c
+++ b/drivers/gpu/drm/amd/display/dc/gpio/gpio_service.c
@@ -375,6 +375,7 @@ struct gpio *dal_gpio_create_irq(
 	case GPIO_ID_GPIO_PAD:
 	break;
 	default:
+		id = GPIO_ID_HPD;
 		ASSERT_CRITICAL(false);
 		return NULL;
 	}
diff --git a/drivers/gpu/drm/amd/display/dc/i2caux/aux_engine.c b/drivers/gpu/drm/amd/display/dc/i2caux/aux_engine.c
index 1d7309611978..0afd2fa57bbe 100644
--- a/drivers/gpu/drm/amd/display/dc/i2caux/aux_engine.c
+++ b/drivers/gpu/drm/amd/display/dc/i2caux/aux_engine.c
@@ -128,8 +128,20 @@ static void process_read_reply(
 			ctx->status =
 				I2CAUX_TRANSACTION_STATUS_FAILED_PROTOCOL_ERROR;
 			ctx->operation_succeeded = false;
+		} else if (ctx->returned_byte < ctx->current_read_length) {
+			ctx->current_read_length -= ctx->returned_byte;
+
+			ctx->offset += ctx->returned_byte;
+
+			++ctx->invalid_reply_retry_aux_on_ack;
+
+			if (ctx->invalid_reply_retry_aux_on_ack >
+				AUX_INVALID_REPLY_RETRY_COUNTER) {
+				ctx->status =
+				I2CAUX_TRANSACTION_STATUS_FAILED_PROTOCOL_ERROR;
+				ctx->operation_succeeded = false;
+			}
 		} else {
-			ctx->current_read_length = ctx->returned_byte;
 			ctx->status = I2CAUX_TRANSACTION_STATUS_SUCCEEDED;
 			ctx->transaction_complete = true;
 			ctx->operation_succeeded = true;
@@ -290,7 +302,6 @@ static bool read_command(
 				ctx.operation_succeeded);
 	}
 
-	request->payload.length = ctx.reply.length;
 	return ctx.operation_succeeded;
 }
 
diff --git a/drivers/gpu/drm/amd/display/dc/i2caux/aux_engine.h b/drivers/gpu/drm/amd/display/dc/i2caux/aux_engine.h
index b01488f710d5..c33a2898d967 100644
--- a/drivers/gpu/drm/amd/display/dc/i2caux/aux_engine.h
+++ b/drivers/gpu/drm/amd/display/dc/i2caux/aux_engine.h
@@ -44,6 +44,12 @@ struct aux_engine_funcs {
 	void (*process_channel_reply)(
 		struct aux_engine *engine,
 		struct aux_reply_transaction_data *reply);
+	int (*read_channel_reply)(
+		struct aux_engine *engine,
+		uint32_t size,
+		uint8_t *buffer,
+		uint8_t *reply_result,
+		uint32_t *sw_status);
 	enum aux_channel_operation_result (*get_channel_status)(
 		struct aux_engine *engine,
 		uint8_t *returned_bytes);
diff --git a/drivers/gpu/drm/amd/display/dc/i2caux/dce100/i2caux_dce100.c b/drivers/gpu/drm/amd/display/dc/i2caux/dce100/i2caux_dce100.c
index e8d3781deaed..8b704ab0471c 100644
--- a/drivers/gpu/drm/amd/display/dc/i2caux/dce100/i2caux_dce100.c
+++ b/drivers/gpu/drm/amd/display/dc/i2caux/dce100/i2caux_dce100.c
@@ -97,6 +97,7 @@ struct i2caux *dal_i2caux_dce100_create(
 
 	dal_i2caux_dce110_construct(i2caux_dce110,
 				    ctx,
+				    ARRAY_SIZE(dce100_aux_regs),
 				    dce100_aux_regs,
 				    dce100_hw_engine_regs,
 				    &i2c_shift,
diff --git a/drivers/gpu/drm/amd/display/dc/i2caux/dce110/aux_engine_dce110.c b/drivers/gpu/drm/amd/display/dc/i2caux/dce110/aux_engine_dce110.c
index 2b927f25937b..ae5caa97caca 100644
--- a/drivers/gpu/drm/amd/display/dc/i2caux/dce110/aux_engine_dce110.c
+++ b/drivers/gpu/drm/amd/display/dc/i2caux/dce110/aux_engine_dce110.c
@@ -275,61 +275,92 @@ static void submit_channel_request(
 	REG_UPDATE(AUX_SW_CONTROL, AUX_SW_GO, 1);
 }
 
-static void process_channel_reply(
-	struct aux_engine *engine,
-	struct aux_reply_transaction_data *reply)
+static int read_channel_reply(struct aux_engine *engine, uint32_t size,
+			      uint8_t *buffer, uint8_t *reply_result,
+			      uint32_t *sw_status)
 {
 	struct aux_engine_dce110 *aux110 = FROM_AUX_ENGINE(engine);
+	uint32_t bytes_replied;
+	uint32_t reply_result_32;
 
-	/* Need to do a read to get the number of bytes to process
-	 * Alternatively, this information can be passed -
-	 * but that causes coupling which isn't good either. */
+	*sw_status = REG_GET(AUX_SW_STATUS, AUX_SW_REPLY_BYTE_COUNT,
+			     &bytes_replied);
 
-	uint32_t bytes_replied;
-	uint32_t value;
+	/* In case HPD is LOW, exit AUX transaction */
+	if ((*sw_status & AUX_SW_STATUS__AUX_SW_HPD_DISCON_MASK))
+		return -1;
 
-	value = REG_GET(AUX_SW_STATUS,
-			AUX_SW_REPLY_BYTE_COUNT, &bytes_replied);
+	/* Need at least the status byte */
+	if (!bytes_replied)
+		return -1;
 
-	/* in case HPD is LOW, exit AUX transaction */
-	if ((value & AUX_SW_STATUS__AUX_SW_HPD_DISCON_MASK)) {
-		reply->status = AUX_TRANSACTION_REPLY_HPD_DISCON;
-		return;
-	}
+	REG_UPDATE_1BY1_3(AUX_SW_DATA,
+			  AUX_SW_INDEX, 0,
+			  AUX_SW_AUTOINCREMENT_DISABLE, 1,
+			  AUX_SW_DATA_RW, 1);
 
-	if (bytes_replied) {
-		uint32_t reply_result;
+	REG_GET(AUX_SW_DATA, AUX_SW_DATA, &reply_result_32);
+	reply_result_32 = reply_result_32 >> 4;
+	*reply_result = (uint8_t)reply_result_32;
 
-		REG_UPDATE_1BY1_3(AUX_SW_DATA,
-				AUX_SW_INDEX, 0,
-				AUX_SW_AUTOINCREMENT_DISABLE, 1,
-				AUX_SW_DATA_RW, 1);
+	if (reply_result_32 == 0) { /* ACK */
+		uint32_t i = 0;
 
-		REG_GET(AUX_SW_DATA,
-				AUX_SW_DATA, &reply_result);
+		/* First byte was already used to get the command status */
+		--bytes_replied;
 
-		reply_result = reply_result >> 4;
+		/* Do not overflow buffer */
+		if (bytes_replied > size)
+			return -1;
 
-		switch (reply_result) {
-		case 0: /* ACK */ {
-			uint32_t i = 0;
+		while (i < bytes_replied) {
+			uint32_t aux_sw_data_val;
 
-			/* first byte was already used
-			 * to get the command status */
-			--bytes_replied;
+			REG_GET(AUX_SW_DATA, AUX_SW_DATA, &aux_sw_data_val);
+			buffer[i] = aux_sw_data_val;
+			++i;
+		}
+
+		return i;
+	}
+
+	return 0;
+}
+
+static void process_channel_reply(
+	struct aux_engine *engine,
+	struct aux_reply_transaction_data *reply)
+{
+	int bytes_replied;
+	uint8_t reply_result;
+	uint32_t sw_status;
 
-			while (i < bytes_replied) {
-				uint32_t aux_sw_data_val;
+	bytes_replied = read_channel_reply(engine, reply->length, reply->data,
+					   &reply_result, &sw_status);
 
-				REG_GET(AUX_SW_DATA,
-						AUX_SW_DATA, &aux_sw_data_val);
+	/* in case HPD is LOW, exit AUX transaction */
+	if ((sw_status & AUX_SW_STATUS__AUX_SW_HPD_DISCON_MASK)) {
+		reply->status = AUX_CHANNEL_OPERATION_FAILED_HPD_DISCON;
+		return;
+	}
 
-				reply->data[i] = aux_sw_data_val;
-				++i;
-			}
+	if (bytes_replied < 0) {
+		/* Need to handle an error case...
+		 * Hopefully, upper layer function won't call this function if
+		 * the number of bytes in the reply was 0, because there was
+		 * surely an error that was asserted that should have been
+		 * handled for hot plug case, this could happens
+		 */
+		if (!(sw_status & AUX_SW_STATUS__AUX_SW_HPD_DISCON_MASK)) {
+			reply->status = AUX_TRANSACTION_REPLY_INVALID;
+			ASSERT_CRITICAL(false);
+			return;
+		}
+	} else {
 
+		switch (reply_result) {
+		case 0: /* ACK */
 			reply->status = AUX_TRANSACTION_REPLY_AUX_ACK;
-		}
 		break;
 		case 1: /* NACK */
 			reply->status = AUX_TRANSACTION_REPLY_AUX_NACK;
@@ -346,17 +377,6 @@ static void process_channel_reply(
 		default:
 			reply->status = AUX_TRANSACTION_REPLY_INVALID;
 		}
-	} else {
-		/* Need to handle an error case...
-		 * hopefully, upper layer function won't call this function
-		 * if the number of bytes in the reply was 0
-		 * because there was surely an error that was asserted
-		 * that should have been handled
-		 * for hot plug case, this could happens*/
-		if (!(value & AUX_SW_STATUS__AUX_SW_HPD_DISCON_MASK)) {
-			reply->status = AUX_TRANSACTION_REPLY_INVALID;
-			ASSERT_CRITICAL(false);
-		}
 	}
 }
 
@@ -427,6 +447,7 @@ static const struct aux_engine_funcs aux_engine_funcs = {
 	.acquire_engine = acquire_engine,
 	.submit_channel_request = submit_channel_request,
 	.process_channel_reply = process_channel_reply,
+	.read_channel_reply = read_channel_reply,
 	.get_channel_status = get_channel_status,
 	.is_engine_available = is_engine_available,
 };
diff --git a/drivers/gpu/drm/amd/display/dc/i2caux/dce110/i2c_hw_engine_dce110.c b/drivers/gpu/drm/amd/display/dc/i2caux/dce110/i2c_hw_engine_dce110.c
index b7256f595052..9cbe1a7a6bcb 100644
--- a/drivers/gpu/drm/amd/display/dc/i2caux/dce110/i2c_hw_engine_dce110.c
+++ b/drivers/gpu/drm/amd/display/dc/i2caux/dce110/i2c_hw_engine_dce110.c
@@ -62,12 +62,7 @@ enum dc_i2c_arbitration {
 	DC_I2C_ARBITRATION__DC_I2C_SW_PRIORITY_HIGH
 };
 
-enum {
-	/* No timeout in HW
-	 * (timeout implemented in SW by querying status) */
-	I2C_SETUP_TIME_LIMIT = 255,
-	I2C_HW_BUFFER_SIZE = 538
-};
+
 
 /*
  * @brief
@@ -152,6 +147,11 @@ static bool setup_engine(
 	struct i2c_engine *i2c_engine)
 {
 	struct i2c_hw_engine_dce110 *hw_engine = FROM_I2C_ENGINE(i2c_engine);
+	uint32_t i2c_setup_limit = I2C_SETUP_TIME_LIMIT_DCE;
+	uint32_t  reset_length = 0;
+
+	if (hw_engine->base.base.setup_limit != 0)
+		i2c_setup_limit = hw_engine->base.base.setup_limit;
 
 	/* Program pin select */
 	REG_UPDATE_6(
@@ -164,11 +164,15 @@ static bool setup_engine(
 			DC_I2C_DDC_SELECT, hw_engine->engine_id);
 
 	/* Program time limit */
-	REG_UPDATE_N(
-			SETUP, 2,
-			FN(DC_I2C_DDC1_SETUP, DC_I2C_DDC1_TIME_LIMIT), I2C_SETUP_TIME_LIMIT,
-			FN(DC_I2C_DDC1_SETUP, DC_I2C_DDC1_ENABLE), 1);
-
+	if (hw_engine->base.base.send_reset_length == 0) {
+		/*pre-dcn*/
+		REG_UPDATE_N(
+				SETUP, 2,
+				FN(DC_I2C_DDC1_SETUP, DC_I2C_DDC1_TIME_LIMIT), i2c_setup_limit,
+				FN(DC_I2C_DDC1_SETUP, DC_I2C_DDC1_ENABLE), 1);
+	} else {
+		reset_length = hw_engine->base.base.send_reset_length;
+	}
 	/* Program HW priority
 	 * set to High - interrupt software I2C at any time
 	 * Enable restart of SW I2C that was interrupted by HW
diff --git a/drivers/gpu/drm/amd/display/dc/i2caux/dce110/i2c_hw_engine_dce110.h b/drivers/gpu/drm/amd/display/dc/i2caux/dce110/i2c_hw_engine_dce110.h
index 5bb04085f670..fea2946906ed 100644
--- a/drivers/gpu/drm/amd/display/dc/i2caux/dce110/i2c_hw_engine_dce110.h
+++ b/drivers/gpu/drm/amd/display/dc/i2caux/dce110/i2c_hw_engine_dce110.h
@@ -192,6 +192,7 @@ struct i2c_hw_engine_dce110 {
 	/* number of pending transactions (before GO) */
 	uint32_t transaction_count;
 	uint32_t engine_keep_power_up_count;
+	uint32_t i2_setup_time_limit;
 };
 
 struct i2c_hw_engine_dce110_create_arg {
@@ -207,4 +208,11 @@ struct i2c_hw_engine_dce110_create_arg {
 struct i2c_engine *dal_i2c_hw_engine_dce110_create(
 	const struct i2c_hw_engine_dce110_create_arg *arg);
 
+enum {
+	I2C_SETUP_TIME_LIMIT_DCE = 255,
+	I2C_SETUP_TIME_LIMIT_DCN = 3,
+	I2C_HW_BUFFER_SIZE = 538,
+	I2C_SEND_RESET_LENGTH_9 = 9,
+	I2C_SEND_RESET_LENGTH_10 = 10,
+};
 #endif
diff --git a/drivers/gpu/drm/amd/display/dc/i2caux/dce110/i2caux_dce110.c b/drivers/gpu/drm/amd/display/dc/i2caux/dce110/i2caux_dce110.c
index 2a047f8ca0e9..1d748ac1d6d6 100644
--- a/drivers/gpu/drm/amd/display/dc/i2caux/dce110/i2caux_dce110.c
+++ b/drivers/gpu/drm/amd/display/dc/i2caux/dce110/i2caux_dce110.c
@@ -43,6 +43,9 @@
 #include "i2c_sw_engine_dce110.h"
 #include "i2c_hw_engine_dce110.h"
 #include "aux_engine_dce110.h"
+#include "../../dc.h"
+#include "dc_types.h"
+
 
 /*
  * Post-requisites: headers required by this unit
@@ -199,6 +202,7 @@ static const struct dce110_i2c_hw_engine_mask i2c_mask = {
 void dal_i2caux_dce110_construct(
 	struct i2caux_dce110 *i2caux_dce110,
 	struct dc_context *ctx,
+	unsigned int num_i2caux_inst,
 	const struct dce110_aux_registers aux_regs[],
 	const struct dce110_i2c_hw_engine_registers i2c_hw_engine_regs[],
 	const struct dce110_i2c_hw_engine_shift *i2c_shift,
@@ -249,9 +253,22 @@ void dal_i2caux_dce110_construct(
 
 		base->i2c_hw_engines[line_id] =
 			dal_i2c_hw_engine_dce110_create(&hw_arg_dce110);
-
+		if (base->i2c_hw_engines[line_id] != NULL) {
+			switch (ctx->dce_version) {
+			case DCN_VERSION_1_0:
+				base->i2c_hw_engines[line_id]->setup_limit =
+					I2C_SETUP_TIME_LIMIT_DCN;
+				base->i2c_hw_engines[line_id]->send_reset_length  = 0;
+			break;
+			default:
+				base->i2c_hw_engines[line_id]->setup_limit =
+					I2C_SETUP_TIME_LIMIT_DCE;
+				base->i2c_hw_engines[line_id]->send_reset_length  = 0;
+				break;
+			}
+		}
 		++i;
-	} while (i < ARRAY_SIZE(hw_ddc_lines));
+	} while (i < num_i2caux_inst);
 
 	/* Create AUX engines for all lines which has assisted HW AUX
 	 * 'i' (loop counter) used as DDC/AUX engine_id */
@@ -272,7 +289,7 @@ void dal_i2caux_dce110_construct(
 			dal_aux_engine_dce110_create(&aux_init_data);
 
 		++i;
-	} while (i < ARRAY_SIZE(hw_aux_lines));
+	} while (i < num_i2caux_inst);
 
 	/*TODO Generic I2C SW and HW*/
 }
@@ -303,6 +320,7 @@ struct i2caux *dal_i2caux_dce110_create(
 
 	dal_i2caux_dce110_construct(i2caux_dce110,
 				    ctx,
+				    ARRAY_SIZE(dce110_aux_regs),
 				    dce110_aux_regs,
 				    i2c_hw_engine_regs,
 				    &i2c_shift,
diff --git a/drivers/gpu/drm/amd/display/dc/i2caux/dce110/i2caux_dce110.h b/drivers/gpu/drm/amd/display/dc/i2caux/dce110/i2caux_dce110.h
index 1b1f71c60ac9..d3d8cc58666a 100644
--- a/drivers/gpu/drm/amd/display/dc/i2caux/dce110/i2caux_dce110.h
+++ b/drivers/gpu/drm/amd/display/dc/i2caux/dce110/i2caux_dce110.h
@@ -45,6 +45,7 @@ struct i2caux *dal_i2caux_dce110_create(
 void dal_i2caux_dce110_construct(
 	struct i2caux_dce110 *i2caux_dce110,
 	struct dc_context *ctx,
+	unsigned int num_i2caux_inst,
 	const struct dce110_aux_registers *aux_regs,
 	const struct dce110_i2c_hw_engine_registers *i2c_hw_engine_regs,
 	const struct dce110_i2c_hw_engine_shift *i2c_shift,
diff --git a/drivers/gpu/drm/amd/display/dc/i2caux/dce112/i2caux_dce112.c b/drivers/gpu/drm/amd/display/dc/i2caux/dce112/i2caux_dce112.c
index dafc1a727f7f..a9db04738724 100644
--- a/drivers/gpu/drm/amd/display/dc/i2caux/dce112/i2caux_dce112.c
+++ b/drivers/gpu/drm/amd/display/dc/i2caux/dce112/i2caux_dce112.c
@@ -93,6 +93,7 @@ static void construct(
 {
 	dal_i2caux_dce110_construct(i2caux_dce110,
 				    ctx,
+				    ARRAY_SIZE(dce112_aux_regs),
 				    dce112_aux_regs,
 				    dce112_hw_engine_regs,
 				    &i2c_shift,
diff --git a/drivers/gpu/drm/amd/display/dc/i2caux/dce120/i2caux_dce120.c b/drivers/gpu/drm/amd/display/dc/i2caux/dce120/i2caux_dce120.c
index 0e7b18260027..6a4f344c1db4 100644
--- a/drivers/gpu/drm/amd/display/dc/i2caux/dce120/i2caux_dce120.c
+++ b/drivers/gpu/drm/amd/display/dc/i2caux/dce120/i2caux_dce120.c
@@ -111,6 +111,7 @@ struct i2caux *dal_i2caux_dce120_create(
 
 	dal_i2caux_dce110_construct(i2caux_dce110,
 				    ctx,
+				    ARRAY_SIZE(dce120_aux_regs),
 				    dce120_aux_regs,
 				    dce120_hw_engine_regs,
 				    &i2c_shift,
diff --git a/drivers/gpu/drm/amd/display/dc/i2caux/dcn10/i2caux_dcn10.c b/drivers/gpu/drm/amd/display/dc/i2caux/dcn10/i2caux_dcn10.c
index e44a8901f38b..a59c1f50c1e8 100644
--- a/drivers/gpu/drm/amd/display/dc/i2caux/dcn10/i2caux_dcn10.c
+++ b/drivers/gpu/drm/amd/display/dc/i2caux/dcn10/i2caux_dcn10.c
@@ -111,6 +111,7 @@ struct i2caux *dal_i2caux_dcn10_create(
 
 	dal_i2caux_dce110_construct(i2caux_dce110,
 				    ctx,
+				    ARRAY_SIZE(dcn10_aux_regs),
 				    dcn10_aux_regs,
 				    dcn10_hw_engine_regs,
 				    &i2c_shift,
diff --git a/drivers/gpu/drm/amd/display/dc/i2caux/engine.h b/drivers/gpu/drm/amd/display/dc/i2caux/engine.h
index 1e8a1585e401..b16fb1ff687d 100644
--- a/drivers/gpu/drm/amd/display/dc/i2caux/engine.h
+++ b/drivers/gpu/drm/amd/display/dc/i2caux/engine.h
@@ -96,6 +96,7 @@ struct engine_funcs {
 
 struct engine {
 	const struct engine_funcs *funcs;
+	uint32_t inst;
 	struct ddc *ddc;
 	struct dc_context *ctx;
 };
diff --git a/drivers/gpu/drm/amd/display/dc/i2caux/i2c_engine.h b/drivers/gpu/drm/amd/display/dc/i2caux/i2c_engine.h
index 58fc0f25eceb..ded6ea34b714 100644
--- a/drivers/gpu/drm/amd/display/dc/i2caux/i2c_engine.h
+++ b/drivers/gpu/drm/amd/display/dc/i2caux/i2c_engine.h
@@ -86,6 +86,8 @@ struct i2c_engine {
 	struct engine base;
 	const struct i2c_engine_funcs *funcs;
 	uint32_t timeout_delay;
+	uint32_t setup_limit;
+	uint32_t send_reset_length;
 };
 
 void dal_i2c_engine_construct(
diff --git a/drivers/gpu/drm/amd/display/dc/i2caux/i2caux.c b/drivers/gpu/drm/amd/display/dc/i2caux/i2caux.c
index 14dc8c94d862..9b0bcc6b769b 100644
--- a/drivers/gpu/drm/amd/display/dc/i2caux/i2caux.c
+++ b/drivers/gpu/drm/amd/display/dc/i2caux/i2caux.c
@@ -254,7 +254,6 @@ bool dal_i2caux_submit_aux_command(
 			break;
 		}
 
-		cmd->payloads->length = request.payload.length;
 		++index_of_payload;
 	}
 
diff --git a/drivers/gpu/drm/amd/display/dc/inc/core_types.h b/drivers/gpu/drm/amd/display/dc/inc/core_types.h
index 4beddca0180c..c0b9ca13393b 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/core_types.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/core_types.h
@@ -92,6 +92,7 @@ struct resource_context;
 
 struct resource_funcs {
 	void (*destroy)(struct resource_pool **pool);
+	void (*link_init)(struct dc_link *link);
 	struct link_encoder *(*link_enc_create)(
 			const struct encoder_init_data *init);
 
@@ -138,7 +139,7 @@ struct resource_pool {
 	struct output_pixel_processor *opps[MAX_PIPES];
 	struct timing_generator *timing_generators[MAX_PIPES];
 	struct stream_encoder *stream_enc[MAX_PIPES * 2];
-
+	struct aux_engine *engines[MAX_PIPES];
 	struct hubbub *hubbub;
 	struct mpc *mpc;
 	struct pp_smu_funcs_rv *pp_smu;
@@ -148,7 +149,6 @@ struct resource_pool {
 	unsigned int underlay_pipe_index;
 	unsigned int stream_enc_count;
 	unsigned int ref_clock_inKhz;
-	unsigned int dentist_vco_freq_khz;
 	unsigned int timing_generator_count;
 
 	/*
@@ -163,7 +163,7 @@ struct resource_pool {
 	unsigned int audio_count;
 	struct audio_support audio_support;
 
-	struct display_clock *display_clock;
+	struct dccg *dccg;
 	struct irq_service *irqs;
 
 	struct abm *abm;
@@ -256,8 +256,7 @@ struct dce_bw_output {
 };
 
 struct dcn_bw_output {
-	struct dc_clocks cur_clk;
-	struct dc_clocks calc_clk;
+	struct dc_clocks clk;
 	struct dcn_watermark_set watermarks;
 };
 
@@ -282,7 +281,7 @@ struct dc_state {
 	struct dcn_bw_internal_vars dcn_bw_vars;
 #endif
 
-	struct display_clock *dis_clk;
+	struct dccg *dis_clk;
 
 	struct kref refcount;
 };
diff --git a/drivers/gpu/drm/amd/display/dc/inc/dc_link_ddc.h b/drivers/gpu/drm/amd/display/dc/inc/dc_link_ddc.h
index 30b3a08b91be..538b83303b86 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/dc_link_ddc.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/dc_link_ddc.h
@@ -102,22 +102,13 @@ bool dal_ddc_service_query_ddc_data(
 		uint8_t *read_buf,
 		uint32_t read_size);
 
-enum ddc_result dal_ddc_service_read_dpcd_data(
-		struct ddc_service *ddc,
-		bool i2c,
-		enum i2c_mot_mode mot,
-		uint32_t address,
-		uint8_t *data,
-		uint32_t len,
-		uint32_t *read);
-
-enum ddc_result dal_ddc_service_write_dpcd_data(
-		struct ddc_service *ddc,
-		bool i2c,
-		enum i2c_mot_mode mot,
-		uint32_t address,
-		const uint8_t *data,
-		uint32_t len);
+int dc_link_aux_transfer(struct ddc_service *ddc,
+			     unsigned int address,
+			     uint8_t *reply,
+			     void *buffer,
+			     unsigned int size,
+			     enum aux_transaction_type type,
+			     enum i2caux_transaction_action action);
 
 void dal_ddc_service_write_scdc_data(
 		struct ddc_service *ddc_service,
diff --git a/drivers/gpu/drm/amd/display/dc/inc/dc_link_dp.h b/drivers/gpu/drm/amd/display/dc/inc/dc_link_dp.h
index 2f783c650084..a37255c757e0 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/dc_link_dp.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/dc_link_dp.h
@@ -33,9 +33,10 @@ struct dc_link;
 struct dc_stream_state;
 struct dc_link_settings;
 
-bool dp_hbr_verify_link_cap(
+bool dp_verify_link_cap(
 	struct dc_link *link,
-	struct dc_link_settings *known_limit_link_setting);
+	struct dc_link_settings *known_limit_link_setting,
+	int *fail_count);
 
 bool dp_validate_mode_timing(
 	struct dc_link *link,
diff --git a/drivers/gpu/drm/amd/display/dc/inc/dcn_calcs.h b/drivers/gpu/drm/amd/display/dc/inc/dcn_calcs.h
index 132d18d4b293..ddbb673caa08 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/dcn_calcs.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/dcn_calcs.h
@@ -625,7 +625,7 @@ bool dcn_validate_bandwidth(
 
 unsigned int dcn_find_dcfclk_suits_all(
 	const struct dc *dc,
-	struct clocks_value *clocks);
+	struct dc_clocks *clocks);
 
 void dcn_bw_update_from_pplib(struct dc *dc);
 void dcn_bw_notify_pplib_of_wm_ranges(struct dc *dc);
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/aux_engine.h b/drivers/gpu/drm/amd/display/dc/inc/hw/aux_engine.h
new file mode 100644
index 000000000000..e79cd4e92919
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/aux_engine.h
@@ -0,0 +1,180 @@
+/*
+ * Copyright 2012-15 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef __DAL_AUX_ENGINE_H__
+#define __DAL_AUX_ENGINE_H__
+
+#include "dc_ddc_types.h"
+#include "include/i2caux_interface.h"
+
+enum i2caux_transaction_operation {
+	I2CAUX_TRANSACTION_READ,
+	I2CAUX_TRANSACTION_WRITE
+};
+
+enum i2caux_transaction_address_space {
+	I2CAUX_TRANSACTION_ADDRESS_SPACE_I2C = 1,
+	I2CAUX_TRANSACTION_ADDRESS_SPACE_DPCD
+};
+
+struct i2caux_transaction_payload {
+	enum i2caux_transaction_address_space address_space;
+	uint32_t address;
+	uint32_t length;
+	uint8_t *data;
+};
+
+enum i2caux_transaction_status {
+	I2CAUX_TRANSACTION_STATUS_UNKNOWN = (-1L),
+	I2CAUX_TRANSACTION_STATUS_SUCCEEDED,
+	I2CAUX_TRANSACTION_STATUS_FAILED_CHANNEL_BUSY,
+	I2CAUX_TRANSACTION_STATUS_FAILED_TIMEOUT,
+	I2CAUX_TRANSACTION_STATUS_FAILED_PROTOCOL_ERROR,
+	I2CAUX_TRANSACTION_STATUS_FAILED_NACK,
+	I2CAUX_TRANSACTION_STATUS_FAILED_INCOMPLETE,
+	I2CAUX_TRANSACTION_STATUS_FAILED_OPERATION,
+	I2CAUX_TRANSACTION_STATUS_FAILED_INVALID_OPERATION,
+	I2CAUX_TRANSACTION_STATUS_FAILED_BUFFER_OVERFLOW,
+	I2CAUX_TRANSACTION_STATUS_FAILED_HPD_DISCON
+};
+
+struct i2caux_transaction_request {
+	enum i2caux_transaction_operation operation;
+	struct i2caux_transaction_payload payload;
+	enum i2caux_transaction_status status;
+};
+
+enum i2caux_engine_type {
+	I2CAUX_ENGINE_TYPE_UNKNOWN = (-1L),
+	I2CAUX_ENGINE_TYPE_AUX,
+	I2CAUX_ENGINE_TYPE_I2C_DDC_HW,
+	I2CAUX_ENGINE_TYPE_I2C_GENERIC_HW,
+	I2CAUX_ENGINE_TYPE_I2C_SW
+};
+
+enum i2c_default_speed {
+	I2CAUX_DEFAULT_I2C_HW_SPEED = 50,
+	I2CAUX_DEFAULT_I2C_SW_SPEED = 50
+};
+
+union aux_config;
+
+struct aux_engine {
+	uint32_t inst;
+	struct ddc *ddc;
+	struct dc_context *ctx;
+	const struct aux_engine_funcs *funcs;
+	/* following values are expressed in milliseconds */
+	uint32_t delay;
+	uint32_t max_defer_write_retry;
+	bool acquire_reset;
+};
+
+struct read_command_context {
+	uint8_t *buffer;
+	uint32_t current_read_length;
+	uint32_t offset;
+	enum i2caux_transaction_status status;
+
+	struct aux_request_transaction_data request;
+	struct aux_reply_transaction_data reply;
+
+	uint8_t returned_byte;
+
+	uint32_t timed_out_retry_aux;
+	uint32_t invalid_reply_retry_aux;
+	uint32_t defer_retry_aux;
+	uint32_t defer_retry_i2c;
+	uint32_t invalid_reply_retry_aux_on_ack;
+
+	bool transaction_complete;
+	bool operation_succeeded;
+};
+
+struct write_command_context {
+	bool mot;
+
+	uint8_t *buffer;
+	uint32_t current_write_length;
+	enum i2caux_transaction_status status;
+
+	struct aux_request_transaction_data request;
+	struct aux_reply_transaction_data reply;
+
+	uint8_t returned_byte;
+
+	uint32_t timed_out_retry_aux;
+	uint32_t invalid_reply_retry_aux;
+	uint32_t defer_retry_aux;
+	uint32_t defer_retry_i2c;
+	uint32_t max_defer_retry;
+	uint32_t ack_m_retry;
+
+	uint8_t reply_data[DEFAULT_AUX_MAX_DATA_SIZE];
+
+	bool transaction_complete;
+	bool operation_succeeded;
+};
+
+
+struct aux_engine_funcs {
+	void (*destroy)(
+		struct aux_engine **ptr);
+	bool (*acquire_engine)(
+		struct aux_engine *engine);
+	void (*configure)(
+		struct aux_engine *engine,
+		union aux_config cfg);
+	void (*submit_channel_request)(
+		struct aux_engine *engine,
+		struct aux_request_transaction_data *request);
+	void (*process_channel_reply)(
+		struct aux_engine *engine,
+		struct aux_reply_transaction_data *reply);
+	int (*read_channel_reply)(
+		struct aux_engine *engine,
+		uint32_t size,
+		uint8_t *buffer,
+		uint8_t *reply_result,
+		uint32_t *sw_status);
+	enum aux_channel_operation_result (*get_channel_status)(
+		struct aux_engine *engine,
+		uint8_t *returned_bytes);
+	bool (*is_engine_available)(struct aux_engine *engine);
+	enum i2caux_engine_type (*get_engine_type)(
+		const struct aux_engine *engine);
+	bool (*acquire)(
+		struct aux_engine *engine,
+		struct ddc *ddc);
+	bool (*submit_request)(
+		struct aux_engine *engine,
+		struct i2caux_transaction_request *request,
+		bool middle_of_transaction);
+	void (*release_engine)(
+		struct aux_engine *engine);
+	void (*destroy_engine)(
+		struct aux_engine **engine);
+};
+#endif
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/display_clock.h b/drivers/gpu/drm/amd/display/dc/inc/hw/display_clock.h
index f5f69cd81f6f..3c7ccb68ecdb 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/display_clock.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/display_clock.h
@@ -27,23 +27,7 @@
 #define __DISPLAY_CLOCK_H__
 
 #include "dm_services_types.h"
-
-
-struct clocks_value {
-	int dispclk_in_khz;
-	int max_pixelclk_in_khz;
-	int max_non_dp_phyclk_in_khz;
-	int max_dp_phyclk_in_khz;
-	bool dispclk_notify_pplib_done;
-	bool pixelclk_notify_pplib_done;
-	bool phyclk_notigy_pplib_done;
-	int dcfclock_in_khz;
-	int dppclk_in_khz;
-	int mclk_in_khz;
-	int phyclk_in_khz;
-	int common_vdd_level;
-};
-
+#include "dc.h"
 
 /* Structure containing all state-dependent clocks
  * (dependent on "enum clocks_state") */
@@ -52,34 +36,23 @@ struct state_dependent_clocks {
 	int pixel_clk_khz;
 };
 
-struct display_clock {
+struct dccg {
 	struct dc_context *ctx;
 	const struct display_clock_funcs *funcs;
 
 	enum dm_pp_clocks_state max_clks_state;
 	enum dm_pp_clocks_state cur_min_clks_state;
-	struct clocks_value cur_clocks_value;
+	struct dc_clocks clks;
 };
 
 struct display_clock_funcs {
-	int (*set_clock)(struct display_clock *disp_clk,
+	void (*update_clocks)(struct dccg *dccg,
+			struct dc_clocks *new_clocks,
+			bool safe_to_lower);
+	int (*set_dispclk)(struct dccg *dccg,
 		int requested_clock_khz);
 
-	enum dm_pp_clocks_state (*get_required_clocks_state)(
-		struct display_clock *disp_clk,
-		struct state_dependent_clocks *req_clocks);
-
-	bool (*set_min_clocks_state)(struct display_clock *disp_clk,
-		enum dm_pp_clocks_state dm_pp_clocks_state);
-
-	int (*get_dp_ref_clk_frequency)(struct display_clock *disp_clk);
-
-	bool (*apply_clock_voltage_request)(
-		struct display_clock *disp_clk,
-		enum dm_pp_clock_type clocks_type,
-		int clocks_in_khz,
-		bool pre_mode_set,
-		bool update_dp_phyclk);
+	int (*get_dp_ref_clk_frequency)(struct dccg *dccg);
 };
 
 #endif /* __DISPLAY_CLOCK_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/dmcu.h b/drivers/gpu/drm/amd/display/dc/inc/hw/dmcu.h
index de60f940030d..4550747fb61c 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/dmcu.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/dmcu.h
@@ -48,7 +48,7 @@ struct dmcu_funcs {
 			const char *src,
 			unsigned int bytes);
 	void (*set_psr_enable)(struct dmcu *dmcu, bool enable, bool wait);
-	void (*setup_psr)(struct dmcu *dmcu,
+	bool (*setup_psr)(struct dmcu *dmcu,
 			struct dc_link *link,
 			struct psr_context *psr_context);
 	void (*get_psr_state)(struct dmcu *dmcu, uint32_t *psr_state);
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/dpp.h b/drivers/gpu/drm/amd/display/dc/inc/hw/dpp.h
index 582458f028f8..74ad94b0e4f0 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/dpp.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/dpp.h
@@ -151,6 +151,9 @@ struct dpp_funcs {
 	void (*dpp_set_hdr_multiplier)(
 			struct dpp *dpp_base,
 			uint32_t multiplier);
+	void (*set_optional_cursor_attributes)(
+			struct dpp *dpp_base,
+			struct dpp_cursor_attributes *attr);
 
 	void (*dpp_dppclk_control)(
 			struct dpp *dpp_base,
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/hubp.h b/drivers/gpu/drm/amd/display/dc/inc/hw/hubp.h
index 97df82cddf82..4f3f9e68ccfa 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/hubp.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/hubp.h
@@ -43,10 +43,9 @@ enum cursor_lines_per_chunk {
 };
 
 struct hubp {
-	struct hubp_funcs *funcs;
+	const struct hubp_funcs *funcs;
 	struct dc_context *ctx;
 	struct dc_plane_address request_address;
-	struct dc_plane_address current_address;
 	int inst;
 
 	/* run time states */
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/mem_input.h b/drivers/gpu/drm/amd/display/dc/inc/hw/mem_input.h
index 47f1dc5a43b7..da89c2edb07c 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/mem_input.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/mem_input.h
@@ -64,7 +64,7 @@ struct stutter_modes {
 };
 
 struct mem_input {
-	struct mem_input_funcs *funcs;
+	const struct mem_input_funcs *funcs;
 	struct dc_context *ctx;
 	struct dc_plane_address request_address;
 	struct dc_plane_address current_address;
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer.h b/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer.h
index a71770ed4b9f..a14ce4de80b2 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer.h
@@ -44,6 +44,7 @@ struct dce_hwseq_wa {
 	bool blnd_crtc_trigger;
 	bool DEGVIDCN10_253;
 	bool false_optc_underflow;
+	bool DEGVIDCN10_254;
 };
 
 struct hwseq_wa_state {
@@ -101,10 +102,18 @@ struct hw_sequencer_funcs {
 		const struct dc *dc,
 		struct pipe_ctx *pipe_ctx);
 
+	void (*plane_atomic_disconnect)(
+		struct dc *dc,
+		struct pipe_ctx *pipe_ctx);
+
 	void (*update_dchub)(
 		struct dce_hwseq *hws,
 		struct dchub_init_data *dh_data);
 
+	void (*update_mpcc)(
+		struct dc *dc,
+		struct pipe_ctx *pipe_ctx);
+
 	void (*update_pending_status)(
 			struct pipe_ctx *pipe_ctx);
 
@@ -171,7 +180,7 @@ struct hw_sequencer_funcs {
 	void (*set_bandwidth)(
 			struct dc *dc,
 			struct dc_state *context,
-			bool decrease_allowed);
+			bool safe_to_lower);
 
 	void (*set_drr)(struct pipe_ctx **pipe_ctx, int num_pipes,
 			int vmin, int vmax);
@@ -214,6 +223,7 @@ struct hw_sequencer_funcs {
 
 	void (*set_cursor_position)(struct pipe_ctx *pipe);
 	void (*set_cursor_attribute)(struct pipe_ctx *pipe);
+	void (*set_cursor_sdr_white_level)(struct pipe_ctx *pipe);
 
 };
 
diff --git a/drivers/gpu/drm/amd/display/dc/inc/reg_helper.h b/drivers/gpu/drm/amd/display/dc/inc/reg_helper.h
index 3306e7b0b3e3..cf5a84b9e27c 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/reg_helper.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/reg_helper.h
@@ -445,4 +445,50 @@ uint32_t generic_reg_get8(const struct dc_context *ctx, uint32_t addr,
 		uint8_t shift6, uint32_t mask6, uint32_t *field_value6,
 		uint8_t shift7, uint32_t mask7, uint32_t *field_value7,
 		uint8_t shift8, uint32_t mask8, uint32_t *field_value8);
+
+
+/* indirect register access */
+
+#define IX_REG_SET_N(index_reg_name, data_reg_name, index, n, initial_val, ...)	\
+		generic_indirect_reg_update_ex(CTX, \
+				REG(index_reg_name), REG(data_reg_name), IND_REG(index), \
+				initial_val, \
+				n, __VA_ARGS__)
+
+#define IX_REG_SET_2(index_reg_name, data_reg_name, index, init_value, f1, v1, f2, v2)	\
+		IX_REG_SET_N(index_reg_name, data_reg_name, index, 2, init_value, \
+				FN(reg, f1), v1,\
+				FN(reg, f2), v2)
+
+
+#define IX_REG_READ(index_reg_name, data_reg_name, index) \
+		generic_read_indirect_reg(CTX, REG(index_reg_name), REG(data_reg_name), IND_REG(index))
+
+
+
+#define IX_REG_UPDATE_N(index_reg_name, data_reg_name, index, n, ...)	\
+		generic_indirect_reg_update_ex(CTX, \
+				REG(index_reg_name), REG(data_reg_name), IND_REG(index), \
+				IX_REG_READ(index_reg_name, data_reg_name, index), \
+				n, __VA_ARGS__)
+
+#define IX_REG_UPDATE_2(index_reg_name, data_reg_name, index, f1, v1, f2, v2)	\
+		IX_REG_UPDATE_N(index_reg_name, data_reg_name, index, 2,\
+				FN(reg, f1), v1,\
+				FN(reg, f2), v2)
+
+void generic_write_indirect_reg(const struct dc_context *ctx,
+		uint32_t addr_index, uint32_t addr_data,
+		uint32_t index, uint32_t data);
+
+uint32_t generic_read_indirect_reg(const struct dc_context *ctx,
+		uint32_t addr_index, uint32_t addr_data,
+		uint32_t index);
+
+uint32_t generic_indirect_reg_update_ex(const struct dc_context *ctx,
+		uint32_t addr_index, uint32_t addr_data,
+		uint32_t index, uint32_t reg_val, int n,
+		uint8_t shift1, uint32_t mask1, uint32_t field_value1,
+		...);
+
 #endif /* DRIVERS_GPU_DRM_AMD_DC_DEV_DC_INC_REG_HELPER_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/inc/resource.h b/drivers/gpu/drm/amd/display/dc/inc/resource.h
index e92facbd038f..5b321008b0b5 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/resource.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/resource.h
@@ -103,6 +103,11 @@ void resource_reference_clock_source(
 		const struct resource_pool *pool,
 		struct clock_source *clock_source);
 
+int resource_get_clock_source_reference(
+		struct resource_context *res_ctx,
+		const struct resource_pool *pool,
+		struct clock_source *clock_source);
+
 bool resource_are_streams_timing_synchronizable(
 		struct dc_stream_state *stream1,
 		struct dc_stream_state *stream2);
diff --git a/drivers/gpu/drm/amd/display/dc/irq/irq_service.c b/drivers/gpu/drm/amd/display/dc/irq/irq_service.c
index dcdfa0f01551..604bea01fc13 100644
--- a/drivers/gpu/drm/amd/display/dc/irq/irq_service.c
+++ b/drivers/gpu/drm/amd/display/dc/irq/irq_service.c
@@ -78,7 +78,7 @@ const struct irq_source_info *find_irq_source_info(
 	struct irq_service *irq_service,
 	enum dc_irq_source source)
 {
-	if (source > DAL_IRQ_SOURCES_NUMBER || source < DC_IRQ_SOURCE_INVALID)
+	if (source >= DAL_IRQ_SOURCES_NUMBER || source < DC_IRQ_SOURCE_INVALID)
 		return NULL;
 
 	return &irq_service->info[source];
diff --git a/drivers/gpu/drm/amd/display/include/dpcd_defs.h b/drivers/gpu/drm/amd/display/include/dpcd_defs.h
index d8e52e3b8e3c..1c66166d0a94 100644
--- a/drivers/gpu/drm/amd/display/include/dpcd_defs.h
+++ b/drivers/gpu/drm/amd/display/include/dpcd_defs.h
@@ -27,6 +27,9 @@
 #define __DAL_DPCD_DEFS_H__
 
 #include <drm/drm_dp_helper.h>
+#ifndef DP_SINK_HW_REVISION_START // can remove this once the define gets into linux drm_dp_helper.h
+#define DP_SINK_HW_REVISION_START 0x409
+#endif
 
 enum dpcd_revision {
 	DPCD_REV_10 = 0x10,
diff --git a/drivers/gpu/drm/amd/display/include/grph_object_ctrl_defs.h b/drivers/gpu/drm/amd/display/include/grph_object_ctrl_defs.h
index 36bbad594267..f312834fef50 100644
--- a/drivers/gpu/drm/amd/display/include/grph_object_ctrl_defs.h
+++ b/drivers/gpu/drm/amd/display/include/grph_object_ctrl_defs.h
@@ -395,6 +395,8 @@ struct integrated_info {
 	struct i2c_reg_info dp3_ext_hdmi_reg_settings[9];
 	unsigned char dp3_ext_hdmi_6g_reg_num;
 	struct i2c_reg_info dp3_ext_hdmi_6g_reg_settings[3];
+	/* V11 */
+	uint32_t dp_ss_control;
 };
 
 /**
diff --git a/drivers/gpu/drm/amd/display/include/grph_object_defs.h b/drivers/gpu/drm/amd/display/include/grph_object_defs.h
index 2941b882b0b6..58bb42ed85ca 100644
--- a/drivers/gpu/drm/amd/display/include/grph_object_defs.h
+++ b/drivers/gpu/drm/amd/display/include/grph_object_defs.h
@@ -37,6 +37,10 @@
  * ********************************************************************
  */
 
+#define MAX_CONNECTOR_NUMBER_PER_SLOT	(16)
+#define MAX_BOARD_SLOTS					(4)
+#define INVALID_CONNECTOR_INDEX			((unsigned int)(-1))
+
 /* HPD unit id - HW direct translation */
 enum hpd_source_id {
 	HPD_SOURCEID1 = 0,
@@ -136,5 +140,47 @@ enum sync_source {
 	SYNC_SOURCE_DUAL_GPU_PIN
 };
 
+/* connector sizes in millimeters - from BiosParserTypes.hpp */
+#define CONNECTOR_SIZE_DVI			40
+#define CONNECTOR_SIZE_VGA			32
+#define CONNECTOR_SIZE_HDMI			16
+#define CONNECTOR_SIZE_DP			16
+#define CONNECTOR_SIZE_MINI_DP			9
+#define CONNECTOR_SIZE_UNKNOWN			30
+
+enum connector_layout_type {
+	CONNECTOR_LAYOUT_TYPE_UNKNOWN,
+	CONNECTOR_LAYOUT_TYPE_DVI_D,
+	CONNECTOR_LAYOUT_TYPE_DVI_I,
+	CONNECTOR_LAYOUT_TYPE_VGA,
+	CONNECTOR_LAYOUT_TYPE_HDMI,
+	CONNECTOR_LAYOUT_TYPE_DP,
+	CONNECTOR_LAYOUT_TYPE_MINI_DP,
+};
+struct connector_layout_info {
+	struct graphics_object_id connector_id;
+	enum connector_layout_type connector_type;
+	unsigned int length;
+	unsigned int position;  /* offset in mm from right side of the board */
+};
+
+/* length and width in mm */
+struct slot_layout_info {
+	unsigned int length;
+	unsigned int width;
+	unsigned int num_of_connectors;
+	struct connector_layout_info connectors[MAX_CONNECTOR_NUMBER_PER_SLOT];
+};
+
+struct board_layout_info {
+	unsigned int num_of_slots;
 
+	/* indicates valid information in bracket layout structure. */
+	unsigned int is_number_of_slots_valid : 1;
+	unsigned int is_slots_size_valid : 1;
+	unsigned int is_connector_offsets_valid : 1;
+	unsigned int is_connector_lengths_valid : 1;
+
+	struct slot_layout_info slots[MAX_BOARD_SLOTS];
+};
 #endif
diff --git a/drivers/gpu/drm/amd/display/include/grph_object_id.h b/drivers/gpu/drm/amd/display/include/grph_object_id.h
index c4197432eb7c..33b3d755fe65 100644
--- a/drivers/gpu/drm/amd/display/include/grph_object_id.h
+++ b/drivers/gpu/drm/amd/display/include/grph_object_id.h
@@ -197,6 +197,11 @@ enum transmitter_color_depth {
 	TRANSMITTER_COLOR_DEPTH_48       /* 16 bits */
 };
 
+enum dp_alt_mode {
+	DP_Alt_mode__Unknown = 0,
+	DP_Alt_mode__Connect,
+	DP_Alt_mode__NoConnect,
+};
 /*
  *****************************************************************************
  * graphics_object_id struct
@@ -287,4 +292,15 @@ static inline enum engine_id dal_graphics_object_id_get_engine_id(
 		return (enum engine_id) id.id;
 	return ENGINE_ID_UNKNOWN;
 }
+
+static inline bool dal_graphics_object_id_equal(
+	struct graphics_object_id id_1,
+	struct graphics_object_id id_2)
+{
+	if ((id_1.id == id_2.id) && (id_1.enum_id == id_2.enum_id) &&
+		(id_1.type == id_2.type)) {
+		return true;
+	}
+	return false;
+}
 #endif
diff --git a/drivers/gpu/drm/amd/display/include/logger_interface.h b/drivers/gpu/drm/amd/display/include/logger_interface.h
index 0f10ed710e0d..e3c79616682d 100644
--- a/drivers/gpu/drm/amd/display/include/logger_interface.h
+++ b/drivers/gpu/drm/amd/display/include/logger_interface.h
@@ -40,49 +40,7 @@ struct dc_state;
  *
  */
 
-struct dal_logger *dal_logger_create(struct dc_context *ctx, uint32_t log_mask);
-
-uint32_t dal_logger_destroy(struct dal_logger **logger);
-
-void dm_logger_flush_buffer(struct dal_logger *logger, bool should_warn);
-
-void dm_logger_write(
-		struct dal_logger *logger,
-		enum dc_log_type log_type,
-		const char *msg,
-		...);
-
-void dm_logger_append(
-		struct log_entry *entry,
-		const char *msg,
-		...);
-
-void dm_logger_append_va(
-		struct log_entry *entry,
-		const char *msg,
-		va_list args);
-
-void dm_logger_append_heading(struct log_entry *entry);
-
-void dm_logger_open(
-		struct dal_logger *logger,
-		struct log_entry *entry,
-		enum dc_log_type log_type);
-
-void dm_logger_close(struct log_entry *entry);
-
-void dc_conn_log(struct dc_context *ctx,
-		const struct dc_link *link,
-		uint8_t *hex_data,
-		int hex_data_count,
-		enum dc_log_type event,
-		const char *msg,
-		...);
-
-void logger_write(struct dal_logger *logger,
-		enum dc_log_type log_type,
-		const char *msg,
-		void *paralist);
+void dc_conn_log_hex_linux(const uint8_t *hex_data, int hex_data_count);
 
 void pre_surface_trace(
 		struct dc *dc,
@@ -108,28 +66,31 @@ void context_clock_trace(
  * marked by this macro.
  * Note that the message will be printed exactly once for every function
  * it is used in order to avoid repeating of the same message. */
+
 #define DAL_LOGGER_NOT_IMPL(fmt, ...) \
-{ \
-	static bool print_not_impl = true; \
-\
-	if (print_not_impl == true) { \
-		print_not_impl = false; \
-		dm_logger_write(ctx->logger, LOG_WARNING, \
-		"DAL_NOT_IMPL: " fmt, ##__VA_ARGS__); \
-	} \
-}
+	do { \
+		static bool print_not_impl = true; \
+		if (print_not_impl == true) { \
+			print_not_impl = false; \
+			DRM_WARN("DAL_NOT_IMPL: " fmt, ##__VA_ARGS__); \
+		} \
+	} while (0)
 
 /******************************************************************************
  * Convenience macros to save on typing.
  *****************************************************************************/
 
 #define DC_ERROR(...) \
-	dm_logger_write(dc_ctx->logger, LOG_ERROR, \
-		__VA_ARGS__)
+		do { \
+			(void)(dc_ctx); \
+			DC_LOG_ERROR(__VA_ARGS__); \
+		} while (0)
 
 #define DC_SYNC_INFO(...) \
-	dm_logger_write(dc_ctx->logger, LOG_SYNC, \
-		__VA_ARGS__)
+		do { \
+			(void)(dc_ctx); \
+			DC_LOG_SYNC(__VA_ARGS__); \
+		} while (0)
 
 /* Connectivity log format:
  * [time stamp]   [drm] [Major_minor] [connector name] message.....
@@ -139,20 +100,30 @@ void context_clock_trace(
  */
 
 #define CONN_DATA_DETECT(link, hex_data, hex_len, ...) \
-		dc_conn_log(link->ctx, link, hex_data, hex_len, \
-				LOG_EVENT_DETECTION, ##__VA_ARGS__)
+		do { \
+			(void)(link); \
+			dc_conn_log_hex_linux(hex_data, hex_len); \
+			DC_LOG_EVENT_DETECTION(__VA_ARGS__); \
+		} while (0)
 
 #define CONN_DATA_LINK_LOSS(link, hex_data, hex_len, ...) \
-		dc_conn_log(link->ctx, link, hex_data, hex_len, \
-				LOG_EVENT_LINK_LOSS, ##__VA_ARGS__)
+		do { \
+			(void)(link); \
+			dc_conn_log_hex_linux(hex_data, hex_len); \
+			DC_LOG_EVENT_LINK_LOSS(__VA_ARGS__); \
+		} while (0)
 
 #define CONN_MSG_LT(link, ...) \
-		dc_conn_log(link->ctx, link, NULL, 0, \
-				LOG_EVENT_LINK_TRAINING, ##__VA_ARGS__)
+		do { \
+			(void)(link); \
+			DC_LOG_EVENT_LINK_TRAINING(__VA_ARGS__); \
+		} while (0)
 
 #define CONN_MSG_MODE(link, ...) \
-		dc_conn_log(link->ctx, link, NULL, 0, \
-				LOG_EVENT_MODE_SET, ##__VA_ARGS__)
+		do { \
+			(void)(link); \
+			DC_LOG_EVENT_MODE_SET(__VA_ARGS__); \
+		} while (0)
 
 /*
  * Display Test Next logging
@@ -167,38 +138,21 @@ void context_clock_trace(
 	dm_dtn_log_end(dc_ctx)
 
 #define PERFORMANCE_TRACE_START() \
-	unsigned long long perf_trc_start_stmp = dm_get_timestamp(dc->ctx); \
-	unsigned long long perf_trc_start_log_msk = dc->ctx->logger->mask; \
-	unsigned int perf_trc_start_log_flags = dc->ctx->logger->flags.value; \
-	if (dc->debug.performance_trace) {\
-		dm_logger_flush_buffer(dc->ctx->logger, false);\
-		dc->ctx->logger->mask = 1<<LOG_PERF_TRACE;\
-		dc->ctx->logger->flags.bits.ENABLE_CONSOLE = 0;\
-		dc->ctx->logger->flags.bits.ENABLE_BUFFER = 1;\
-	}
-
-#define PERFORMANCE_TRACE_END() do {\
-	unsigned long long perf_trc_end_stmp = dm_get_timestamp(dc->ctx);\
-	if (dc->debug.performance_trace) {\
-		dm_logger_write(dc->ctx->logger, \
-				LOG_PERF_TRACE, \
-				"%s duration: %d ticks\n", __func__,\
+	unsigned long long perf_trc_start_stmp = dm_get_timestamp(dc->ctx)
+
+#define PERFORMANCE_TRACE_END() \
+	do { \
+		unsigned long long perf_trc_end_stmp = dm_get_timestamp(dc->ctx); \
+		if (dc->debug.performance_trace) { \
+			DC_LOG_PERF_TRACE("%s duration: %lld ticks\n", __func__, \
 				perf_trc_end_stmp - perf_trc_start_stmp); \
-		if (perf_trc_start_log_msk != 1<<LOG_PERF_TRACE) {\
-			dc->ctx->logger->mask = perf_trc_start_log_msk;\
-			dc->ctx->logger->flags.value = perf_trc_start_log_flags;\
-			dm_logger_flush_buffer(dc->ctx->logger, false);\
 		} \
-	} \
-} while (0)
+	} while (0)
 
-#define DISPLAY_STATS_BEGIN(entry) \
-	dm_logger_open(dc->ctx->logger, &entry, LOG_DISPLAYSTATS)
+#define DISPLAY_STATS_BEGIN(entry) (void)(entry)
 
-#define DISPLAY_STATS(msg, ...) \
-	dm_logger_append(&log_entry, msg, ##__VA_ARGS__)
+#define DISPLAY_STATS(msg, ...) DC_LOG_PERF_TRACE(msg, __VA_ARGS__)
 
-#define DISPLAY_STATS_END(entry) \
-	dm_logger_close(&entry)
+#define DISPLAY_STATS_END(entry) (void)(entry)
 
 #endif /* __DAL_LOGGER_INTERFACE_H__ */
diff --git a/drivers/gpu/drm/amd/display/include/logger_types.h b/drivers/gpu/drm/amd/display/include/logger_types.h
index 0a540b9897a6..ad3695e67b76 100644
--- a/drivers/gpu/drm/amd/display/include/logger_types.h
+++ b/drivers/gpu/drm/amd/display/include/logger_types.h
@@ -138,63 +138,4 @@ enum dc_log_type {
 		(1 << LOG_HW_AUDIO)| \
 		(1 << LOG_BANDWIDTH_CALCS)*/
 
-union logger_flags {
-	struct {
-		uint32_t ENABLE_CONSOLE:1; /* Print to console */
-		uint32_t ENABLE_BUFFER:1; /* Print to buffer */
-		uint32_t RESERVED:30;
-	} bits;
-	uint32_t value;
-};
-
-struct log_entry {
-	struct dal_logger *logger;
-	enum dc_log_type type;
-
-	char *buf;
-	uint32_t buf_offset;
-	uint32_t max_buf_bytes;
-};
-
-/**
-* Structure for enumerating log types
-*/
-struct dc_log_type_info {
-	enum dc_log_type type;
-	char name[MAX_NAME_LEN];
-};
-
-/* Structure for keeping track of offsets, buffer, etc */
-
-#define DAL_LOGGER_BUFFER_MAX_SIZE 2048
-
-/*Connectivity log needs to output EDID, which needs at lease 256x3 bytes,
- * change log line size to 896 to meet the request.
- */
-#define LOG_MAX_LINE_SIZE 896
-
-struct dal_logger {
-
-	/* How far into the circular buffer has been read by dsat
-	 * Read offset should never cross write offset. Write \0's to
-	 * read data just to be sure?
-	 */
-	uint32_t buffer_read_offset;
-
-	/* How far into the circular buffer we have written
-	 * Write offset should never cross read offset
-	 */
-	uint32_t buffer_write_offset;
-
-	uint32_t open_count;
-
-	char *log_buffer;	/* Pointer to malloc'ed buffer */
-	uint32_t log_buffer_size; /* Size of circular buffer */
-
-	uint32_t mask; /*array of masks for major elements*/
-
-	union logger_flags flags;
-	struct dc_context *ctx;
-};
-
 #endif /* __DAL_LOGGER_TYPES_H__ */
diff --git a/drivers/gpu/drm/amd/display/modules/color/color_gamma.c b/drivers/gpu/drm/amd/display/modules/color/color_gamma.c
index 98edaefa2b47..bf29733958c3 100644
--- a/drivers/gpu/drm/amd/display/modules/color/color_gamma.c
+++ b/drivers/gpu/drm/amd/display/modules/color/color_gamma.c
@@ -997,7 +997,9 @@ static void scale_user_regamma_ramp(struct pwl_float_data *pwl_rgb,
  * norm_y = 4095*regamma_y, and index is just truncating to nearest integer
  * lut1 = lut1D[index], lut2 = lut1D[index+1]
  *
- *adjustedY is then linearly interpolating regamma Y between lut1 and lut2
+ * adjustedY is then linearly interpolating regamma Y between lut1 and lut2
+ *
+ * Custom degamma on Linux uses the same interpolation math, so is handled here
  */
 static void apply_lut_1d(
 		const struct dc_gamma *ramp,
@@ -1018,7 +1020,7 @@ static void apply_lut_1d(
 	struct fixed31_32 delta_lut;
 	struct fixed31_32 delta_index;
 
-	if (ramp->type != GAMMA_CS_TFM_1D)
+	if (ramp->type != GAMMA_CS_TFM_1D && ramp->type != GAMMA_CUSTOM)
 		return; // this is not expected
 
 	for (i = 0; i < num_hw_points; i++) {
@@ -1636,7 +1638,9 @@ bool mod_color_calculate_degamma_params(struct dc_transfer_func *input_tf,
 	map_regamma_hw_to_x_user(ramp, coeff, rgb_user,
 			coordinates_x, axix_x, curve,
 			MAX_HW_POINTS, tf_pts,
-			mapUserRamp);
+			mapUserRamp && ramp->type != GAMMA_CUSTOM);
+	if (ramp->type == GAMMA_CUSTOM)
+		apply_lut_1d(ramp, MAX_HW_POINTS, tf_pts);
 
 	ret = true;
 
@@ -1723,8 +1727,8 @@ bool  mod_color_calculate_curve(enum dc_transfer_func_predefined trans,
 		kvfree(rgb_regamma);
 	} else if (trans == TRANSFER_FUNCTION_HLG ||
 		trans == TRANSFER_FUNCTION_HLG12) {
-		rgb_regamma = kvzalloc(sizeof(*rgb_regamma) *
-				       (MAX_HW_POINTS + _EXTRA_POINTS),
+		rgb_regamma = kvcalloc(MAX_HW_POINTS + _EXTRA_POINTS,
+				       sizeof(*rgb_regamma),
 				       GFP_KERNEL);
 		if (!rgb_regamma)
 			goto rgb_regamma_alloc_fail;
@@ -1802,8 +1806,8 @@ bool  mod_color_calculate_degamma_curve(enum dc_transfer_func_predefined trans,
 		kvfree(rgb_degamma);
 	} else if (trans == TRANSFER_FUNCTION_HLG ||
 		trans == TRANSFER_FUNCTION_HLG12) {
-		rgb_degamma = kvzalloc(sizeof(*rgb_degamma) *
-				       (MAX_HW_POINTS + _EXTRA_POINTS),
+		rgb_degamma = kvcalloc(MAX_HW_POINTS + _EXTRA_POINTS,
+				       sizeof(*rgb_degamma),
 				       GFP_KERNEL);
 		if (!rgb_degamma)
 			goto rgb_degamma_alloc_fail;
diff --git a/drivers/gpu/drm/amd/display/modules/color/luts_1d.h b/drivers/gpu/drm/amd/display/modules/color/luts_1d.h
new file mode 100644
index 000000000000..66b1fad572ac
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/modules/color/luts_1d.h
@@ -0,0 +1,51 @@
+/*
+ * Copyright 2016 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+#ifndef LUTS_1D_H
+#define LUTS_1D_H
+
+#include "hw_shared.h"
+
+struct point_config {
+	uint32_t custom_float_x;
+	uint32_t custom_float_y;
+	uint32_t custom_float_slope;
+};
+
+struct lut_point {
+	uint32_t red;
+	uint32_t green;
+	uint32_t blue;
+	uint32_t delta_red;
+	uint32_t delta_green;
+	uint32_t delta_blue;
+};
+
+struct pwl_1dlut_parameter {
+	struct gamma_curve	arr_curve_points[34];
+	struct point_config	arr_points[2];
+	struct lut_point rgb_resulted[256];
+	uint32_t hw_points_num;
+};
+#endif // LUTS_1D_H
diff --git a/drivers/gpu/drm/amd/include/amd_pcie.h b/drivers/gpu/drm/amd/include/amd_pcie.h
index 5eb895fd98bf..9cb9ceb4d74d 100644
--- a/drivers/gpu/drm/amd/include/amd_pcie.h
+++ b/drivers/gpu/drm/amd/include/amd_pcie.h
@@ -27,6 +27,7 @@
 #define CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1        0x00010000
 #define CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2        0x00020000
 #define CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3        0x00040000
+#define CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4        0x00080000
 #define CAIL_PCIE_LINK_SPEED_SUPPORT_MASK        0xFFFF0000
 #define CAIL_PCIE_LINK_SPEED_SUPPORT_SHIFT       16
 
@@ -34,6 +35,7 @@
 #define CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1   0x00000001
 #define CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2   0x00000002
 #define CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3   0x00000004
+#define CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4   0x00000008
 #define CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_MASK   0x0000FFFF
 #define CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_SHIFT  0
 
diff --git a/drivers/gpu/drm/amd/include/amd_shared.h b/drivers/gpu/drm/amd/include/amd_shared.h
index b178176b72ac..265621d8945c 100644
--- a/drivers/gpu/drm/amd/include/amd_shared.h
+++ b/drivers/gpu/drm/amd/include/amd_shared.h
@@ -128,47 +128,57 @@ enum PP_FEATURE_MASK {
 	PP_OVERDRIVE_MASK = 0x4000,
 	PP_GFXOFF_MASK = 0x8000,
 	PP_ACG_MASK = 0x10000,
+	PP_STUTTER_MODE = 0x20000,
 };
 
+/**
+ * struct amd_ip_funcs - general hooks for managing amdgpu IP Blocks
+ */
 struct amd_ip_funcs {
-	/* Name of IP block */
+	/** @name: Name of IP block */
 	char *name;
-	/* sets up early driver state (pre sw_init), does not configure hw - Optional */
+	/**
+	 * @early_init:
+	 *
+	 * sets up early driver state (pre sw_init),
+	 * does not configure hw - Optional
+	 */
 	int (*early_init)(void *handle);
-	/* sets up late driver/hw state (post hw_init) - Optional */
+	/** @late_init: sets up late driver/hw state (post hw_init) - Optional */
 	int (*late_init)(void *handle);
-	/* sets up driver state, does not configure hw */
+	/** @sw_init: sets up driver state, does not configure hw */
 	int (*sw_init)(void *handle);
-	/* tears down driver state, does not configure hw */
+	/** @sw_fini: tears down driver state, does not configure hw */
 	int (*sw_fini)(void *handle);
-	/* sets up the hw state */
+	/** @hw_init: sets up the hw state */
 	int (*hw_init)(void *handle);
-	/* tears down the hw state */
+	/** @hw_fini: tears down the hw state */
 	int (*hw_fini)(void *handle);
+	/** @late_fini: final cleanup */
 	void (*late_fini)(void *handle);
-	/* handles IP specific hw/sw changes for suspend */
+	/** @suspend: handles IP specific hw/sw changes for suspend */
 	int (*suspend)(void *handle);
-	/* handles IP specific hw/sw changes for resume */
+	/** @resume: handles IP specific hw/sw changes for resume */
 	int (*resume)(void *handle);
-	/* returns current IP block idle status */
+	/** @is_idle: returns current IP block idle status */
 	bool (*is_idle)(void *handle);
-	/* poll for idle */
+	/** @wait_for_idle: poll for idle */
 	int (*wait_for_idle)(void *handle);
-	/* check soft reset the IP block */
+	/** @check_soft_reset: check soft reset the IP block */
 	bool (*check_soft_reset)(void *handle);
-	/* pre soft reset the IP block */
+	/** @pre_soft_reset: pre soft reset the IP block */
 	int (*pre_soft_reset)(void *handle);
-	/* soft reset the IP block */
+	/** @soft_reset: soft reset the IP block */
 	int (*soft_reset)(void *handle);
-	/* post soft reset the IP block */
+	/** @post_soft_reset: post soft reset the IP block */
 	int (*post_soft_reset)(void *handle);
-	/* enable/disable cg for the IP block */
+	/** @set_clockgating_state: enable/disable cg for the IP block */
 	int (*set_clockgating_state)(void *handle,
 				     enum amd_clockgating_state state);
-	/* enable/disable pg for the IP block */
+	/** @set_powergating_state: enable/disable pg for the IP block */
 	int (*set_powergating_state)(void *handle,
 				     enum amd_powergating_state state);
-	/* get current clockgating status */
+	/** @get_clockgating_state: get current clockgating status */
 	void (*get_clockgating_state)(void *handle, u32 *flags);
 };
 
diff --git a/drivers/gpu/drm/amd/include/atomfirmware.h b/drivers/gpu/drm/amd/include/atomfirmware.h
index 092d800b703a..4bc118df3bc4 100644
--- a/drivers/gpu/drm/amd/include/atomfirmware.h
+++ b/drivers/gpu/drm/amd/include/atomfirmware.h
@@ -1074,7 +1074,7 @@ struct atom_integrated_system_info_v1_11
   uint16_t  dpphy_override;                   // bit vector, enum of atom_sysinfo_dpphy_override_def
   uint16_t  lvds_misc;                        // enum of atom_sys_info_lvds_misc_def
   uint16_t  backlight_pwm_hz;                 // pwm frequency in hz
-  uint8_t   memorytype;                       // enum of atom_sys_mem_type
+  uint8_t   memorytype;                       // enum of atom_dmi_t17_mem_type_def, APU memory type indication.
   uint8_t   umachannelnumber;                 // number of memory channels
   uint8_t   pwr_on_digon_to_de;               /* all pwr sequence numbers below are in uint of 4ms */
   uint8_t   pwr_on_de_to_vary_bl;
@@ -1084,18 +1084,25 @@ struct atom_integrated_system_info_v1_11
   uint8_t   pwr_on_vary_bl_to_blon;
   uint8_t   pwr_down_bloff_to_vary_bloff;
   uint8_t   min_allowed_bl_level;
+  uint8_t   htc_hyst_limit;
+  uint8_t   htc_tmp_limit;
+  uint8_t   reserved1;
+  uint8_t   reserved2;
   struct atom_external_display_connection_info extdispconninfo;
   struct atom_14nm_dpphy_dvihdmi_tuningset dvi_tuningset;
   struct atom_14nm_dpphy_dvihdmi_tuningset hdmi_tuningset;
   struct atom_14nm_dpphy_dvihdmi_tuningset hdmi6g_tuningset;
-  struct atom_14nm_dpphy_dp_tuningset dp_tuningset;
-  struct atom_14nm_dpphy_dp_tuningset dp_hbr3_tuningset;
+  struct atom_14nm_dpphy_dp_tuningset dp_tuningset;        // rbr 1.62G dp tuning set
+  struct atom_14nm_dpphy_dp_tuningset dp_hbr3_tuningset;   // HBR3 dp tuning set
   struct atom_camera_data  camera_info;
   struct atom_hdmi_retimer_redriver_set dp0_retimer_set;   //for DP0
   struct atom_hdmi_retimer_redriver_set dp1_retimer_set;   //for DP1
   struct atom_hdmi_retimer_redriver_set dp2_retimer_set;   //for DP2
   struct atom_hdmi_retimer_redriver_set dp3_retimer_set;   //for DP3
-  uint32_t  reserved[108];
+  struct atom_14nm_dpphy_dp_tuningset dp_hbr_tuningset;    //hbr 2.7G dp tuning set
+  struct atom_14nm_dpphy_dp_tuningset dp_hbr2_tuningset;   //hbr2 5.4G dp turnig set
+  struct atom_14nm_dpphy_dp_tuningset edp_tuningset;       //edp tuning set
+  uint32_t  reserved[66];
 };
 
 
@@ -1433,7 +1440,10 @@ struct atom_smc_dpm_info_v4_1
 	uint8_t  acggfxclkspreadpercent;
 	uint16_t acggfxclkspreadfreq;
 
-	uint32_t boardreserved[10];
+	uint8_t Vr2_I2C_address;
+	uint8_t padding_vr2[3];
+
+	uint32_t boardreserved[9];
 };
 
 /* 
diff --git a/drivers/gpu/drm/amd/include/dm_pp_interface.h b/drivers/gpu/drm/amd/include/dm_pp_interface.h
index 7852952d1fde..1d93a0c574c9 100644
--- a/drivers/gpu/drm/amd/include/dm_pp_interface.h
+++ b/drivers/gpu/drm/amd/include/dm_pp_interface.h
@@ -23,6 +23,8 @@
 #ifndef _DM_PP_INTERFACE_
 #define _DM_PP_INTERFACE_
 
+#include "dm_services_types.h"
+
 #define PP_MAX_CLOCK_LEVELS 16
 
 enum amd_pp_display_config_type{
@@ -189,39 +191,4 @@ struct pp_display_clock_request {
 	uint32_t clock_freq_in_khz;
 };
 
-#define PP_MAX_WM_SETS 4
-
-enum pp_wm_set_id {
-	DC_WM_SET_A = 0,
-	DC_WM_SET_B,
-	DC_WM_SET_C,
-	DC_WM_SET_D,
-	DC_WM_SET_INVALID = 0xffff,
-};
-
-struct pp_wm_set_with_dmif_clock_range_soc15 {
-	enum pp_wm_set_id wm_set_id;
-	uint32_t wm_min_dcefclk_in_khz;
-	uint32_t wm_max_dcefclk_in_khz;
-	uint32_t wm_min_memclk_in_khz;
-	uint32_t wm_max_memclk_in_khz;
-};
-
-struct pp_wm_set_with_mcif_clock_range_soc15 {
-	enum pp_wm_set_id wm_set_id;
-	uint32_t wm_min_socclk_in_khz;
-	uint32_t wm_max_socclk_in_khz;
-	uint32_t wm_min_memclk_in_khz;
-	uint32_t wm_max_memclk_in_khz;
-};
-
-struct pp_wm_sets_with_clock_ranges_soc15 {
-	uint32_t num_wm_sets_dmif;
-	uint32_t num_wm_sets_mcif;
-	struct pp_wm_set_with_dmif_clock_range_soc15
-		wm_sets_dmif[PP_MAX_WM_SETS];
-	struct pp_wm_set_with_mcif_clock_range_soc15
-		wm_sets_mcif[PP_MAX_WM_SETS];
-};
-
 #endif /* _DM_PP_INTERFACE_ */
diff --git a/drivers/gpu/drm/amd/include/ivsrcid/gfx/irqsrcs_gfx_9_0.h b/drivers/gpu/drm/amd/include/ivsrcid/gfx/irqsrcs_gfx_9_0.h
new file mode 100644
index 000000000000..36306c57a2b4
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/ivsrcid/gfx/irqsrcs_gfx_9_0.h
@@ -0,0 +1,55 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef __IRQSRCS_GFX_9_0_H__
+#define __IRQSRCS_GFX_9_0_H__
+
+
+#define GFX_9_0__SRCID__CP_RB_INTERRUPT_PKT					176		/* B0 CP_INTERRUPT pkt in RB */
+#define GFX_9_0__SRCID__CP_IB1_INTERRUPT_PKT				177		/* B1 CP_INTERRUPT pkt in IB1 */
+#define GFX_9_0__SRCID__CP_IB2_INTERRUPT_PKT				178		/* B2 CP_INTERRUPT pkt in IB2 */
+#define GFX_9_0__SRCID__CP_PM4_PKT_RSVD_BIT_ERROR			180		/* B4 PM4 Pkt Rsvd Bits Error */
+#define GFX_9_0__SRCID__CP_EOP_INTERRUPT					181		/* B5 End-of-Pipe Interrupt */
+#define GFX_9_0__SRCID__CP_BAD_OPCODE_ERROR					183		/* B7 Bad Opcode Error */
+#define GFX_9_0__SRCID__CP_PRIV_REG_FAULT					184		/* B8 Privileged Register Fault */
+#define GFX_9_0__SRCID__CP_PRIV_INSTR_FAULT					185		/* B9 Privileged Instr Fault */
+#define GFX_9_0__SRCID__CP_WAIT_MEM_SEM_FAULT				186		/* BA Wait Memory Semaphore Fault (Synchronization Object Fault) */
+#define GFX_9_0__SRCID__CP_CTX_EMPTY_INTERRUPT				187		/* BB Context Empty Interrupt */
+#define GFX_9_0__SRCID__CP_CTX_BUSY_INTERRUPT				188		/* BC Context Busy Interrupt */
+#define GFX_9_0__SRCID__CP_ME_WAIT_REG_MEM_POLL_TIMEOUT		192		/* C0 CP.ME Wait_Reg_Mem Poll Timeout */
+#define GFX_9_0__SRCID__CP_SIG_INCOMPLETE					193		/* C1 "Surface Probe Fault Signal Incomplete" */
+#define GFX_9_0__SRCID__CP_PREEMPT_ACK					    194		/* C2 Preemption Ack-wledge */
+#define GFX_9_0__SRCID__CP_GPF					            195		/* C3 General Protection Fault (GPF) */
+#define GFX_9_0__SRCID__CP_GDS_ALLOC_ERROR					196		/* C4 GDS Alloc Error */
+#define GFX_9_0__SRCID__CP_ECC_ERROR					    197		/* C5 ECC  Error */
+#define GFX_9_0__SRCID__CP_COMPUTE_QUERY_STATUS             199     /* C7 Compute query status */
+#define GFX_9_0__SRCID__CP_VM_DOORBELL					    200		/* C8 Unattached VM Doorbell Received */
+#define GFX_9_0__SRCID__CP_FUE_ERROR					    201		/* C9 ECC FUE Error */
+#define GFX_9_0__SRCID__RLC_STRM_PERF_MONITOR_INTERRUPT		202		/* CA Streaming Perf Monitor Interrupt */
+#define GFX_9_0__SRCID__GRBM_RD_TIMEOUT_ERROR				232		/* E8 CRead timeout error */
+#define GFX_9_0__SRCID__GRBM_REG_GUI_IDLE					233		/* E9 Register GUI Idle */
+#define GFX_9_0__SRCID__SQ_INTERRUPT_ID					    239		/* EF SQ Interrupt (ttrace wrap, errors) */
+
+#endif /* __IRQSRCS_GFX_9_0_H__ */
diff --git a/drivers/gpu/drm/amd/include/ivsrcid/ivsrcid_vislands30.h b/drivers/gpu/drm/amd/include/ivsrcid/ivsrcid_vislands30.h
index c6b6f97de9de..aaed7f59e0e2 100644
--- a/drivers/gpu/drm/amd/include/ivsrcid/ivsrcid_vislands30.h
+++ b/drivers/gpu/drm/amd/include/ivsrcid/ivsrcid_vislands30.h
@@ -198,4 +198,102 @@
 #define VISLANDS30_IV_SRCID_HPD_RX_F		                    42	    // 0x2a		
 #define VISLANDS30_IV_EXTID_HPD_RX_F                         11
 
+#define VISLANDS30_IV_SRCID_GPIO_19                            0x00000053  /* 83 */
+
+#define VISLANDS30_IV_SRCID_SRBM_READ_TIMEOUT_ERR              0x00000060  /* 96 */
+#define VISLANDS30_IV_SRCID_SRBM_CTX_SWITCH                    0x00000061  /* 97 */
+
+#define VISLANDS30_IV_SRBM_REG_ACCESS_ERROR                    0x00000062  /* 98 */
+
+
+#define VISLANDS30_IV_SRCID_UVD_ENC_GEN_PURP                   0x00000077  /* 119 */
+#define VISLANDS30_IV_SRCID_UVD_SYSTEM_MESSAGE                 0x0000007c  /* 124 */
+
+#define VISLANDS30_IV_SRCID_BIF_PF_VF_MSGBUF_VALID             0x00000087  /* 135 */
+
+#define VISLANDS30_IV_SRCID_BIF_VF_PF_MSGBUF_ACK               0x0000008a  /* 138 */
+
+#define VISLANDS30_IV_SRCID_SYS_PAGE_INV_FAULT                 0x0000008c  /* 140 */
+#define VISLANDS30_IV_SRCID_SYS_MEM_PROT_FAULT                 0x0000008d  /* 141 */
+
+#define VISLANDS30_IV_SRCID_SEM_PAGE_INV_FAULT                 0x00000090  /* 144 */
+#define VISLANDS30_IV_SRCID_SEM_MEM_PROT_FAULT                 0x00000091  /* 145 */
+
+#define VISLANDS30_IV_SRCID_GFX_PAGE_INV_FAULT                 0x00000092  /* 146 */
+#define VISLANDS30_IV_SRCID_GFX_MEM_PROT_FAULT                 0x00000093  /* 147 */
+
+#define VISLANDS30_IV_SRCID_ACP                                0x000000a2  /* 162 */
+
+#define VISLANDS30_IV_SRCID_VCE_TRAP                           0x000000a7  /* 167 */
+#define VISLANDS30_IV_EXTID_VCE_TRAP_GENERAL_PURPOSE           0
+#define VISLANDS30_IV_EXTID_VCE_TRAP_LOW_LATENCY               1
+#define VISLANDS30_IV_EXTID_VCE_TRAP_REAL_TIME                 2
+
+#define VISLANDS30_IV_SRCID_CP_INT_RB                          0x000000b0  /* 176 */
+#define VISLANDS30_IV_SRCID_CP_INT_IB1                         0x000000b1  /* 177 */
+#define VISLANDS30_IV_SRCID_CP_INT_IB2                         0x000000b2  /* 178 */
+#define VISLANDS30_IV_SRCID_CP_PM4_RES_BITS_ERR                0x000000b4  /* 180 */
+#define VISLANDS30_IV_SRCID_CP_END_OF_PIPE                     0x000000b5  /* 181 */
+#define VISLANDS30_IV_SRCID_CP_BAD_OPCODE                      0x000000b7  /* 183 */
+#define VISLANDS30_IV_SRCID_CP_PRIV_REG_FAULT                  0x000000b8  /* 184 */
+#define VISLANDS30_IV_SRCID_CP_PRIV_INSTR_FAULT                0x000000b9  /* 185 */
+#define VISLANDS30_IV_SRCID_CP_WAIT_MEM_SEM_FAULT              0x000000ba  /* 186 */
+#define VISLANDS30_IV_SRCID_CP_GUI_IDLE                        0x000000bb  /* 187 */
+#define VISLANDS30_IV_SRCID_CP_GUI_BUSY                        0x000000bc  /* 188 */
+
+#define VISLANDS30_IV_SRCID_CP_COMPUTE_QUERY_STATUS            0x000000bf  /* 191 */
+#define VISLANDS30_IV_SRCID_CP_ECC_ERROR                       0x000000c5  /* 197 */
+
+#define CARRIZO_IV_SRCID_CP_COMPUTE_QUERY_STATUS               0x000000c7  /* 199 */
+
+#define VISLANDS30_IV_SRCID_CP_WAIT_REG_MEM_POLL_TIMEOUT       0x000000c0  /* 192 */
+#define VISLANDS30_IV_SRCID_CP_SEM_SIG_INCOMPL                 0x000000c1  /* 193 */
+#define VISLANDS30_IV_SRCID_CP_PREEMPT_ACK                     0x000000c2  /* 194 */
+#define VISLANDS30_IV_SRCID_CP_GENERAL_PROT_FAULT              0x000000c3  /* 195 */
+#define VISLANDS30_IV_SRCID_CP_GDS_ALLOC_ERROR                 0x000000c4  /* 196 */
+#define VISLANDS30_IV_SRCID_CP_ECC_ERROR                       0x000000c5  /* 197 */
+
+#define VISLANDS30_IV_SRCID_RLC_STRM_PERF_MONITOR              0x000000ca  /* 202 */
+
+#define VISLANDS30_IV_SDMA_ATOMIC_SRC_ID                       0x000000da  /* 218 */
+
+#define VISLANDS30_IV_SRCID_SDMA_ECC_ERROR                     0x000000dc  /* 220 */
+
+#define VISLANDS30_IV_SRCID_SDMA_TRAP          	               0x000000e0  /* 224 */
+#define VISLANDS30_IV_SRCID_SDMA_SEM_INCOMPLETE                0x000000e1  /* 225 */
+#define VISLANDS30_IV_SRCID_SDMA_SEM_WAIT                      0x000000e2  /* 226 */
+
+
+#define VISLANDS30_IV_SRCID_SMU_DISP_TIMER2_TRIGGER            0x000000e5  /* 229 */
+
+#define VISLANDS30_IV_SRCID_CG_TSS_THERMAL_LOW_TO_HIGH         0x000000e6  /* 230 */
+#define VISLANDS30_IV_SRCID_CG_TSS_THERMAL_HIGH_TO_LOW         0x000000e7  /* 231 */
+
+#define VISLANDS30_IV_SRCID_GRBM_READ_TIMEOUT_ERR              0x000000e8  /* 232 */
+#define VISLANDS30_IV_SRCID_GRBM_REG_GUI_IDLE                  0x000000e9  /* 233 */
+
+#define VISLANDS30_IV_SRCID_SQ_INTERRUPT_MSG                   0x000000ef  /* 239 */
+
+#define VISLANDS30_IV_SRCID_SDMA_PREEMPT                       0x000000f0  /* 240 */
+#define VISLANDS30_IV_SRCID_SDMA_VM_HOLE                       0x000000f2  /* 242 */
+#define VISLANDS30_IV_SRCID_SDMA_CTXEMPTY                      0x000000f3  /* 243 */
+#define VISLANDS30_IV_SRCID_SDMA_DOORBELL_INVALID              0x000000f4  /* 244 */
+#define VISLANDS30_IV_SRCID_SDMA_FROZEN                        0x000000f5  /* 245 */
+#define VISLANDS30_IV_SRCID_SDMA_POLL_TIMEOUT                  0x000000f6  /* 246 */
+#define VISLANDS30_IV_SRCID_SDMA_SRBM_WRITE                    0x000000f7  /* 247 */
+
+#define VISLANDS30_IV_SRCID_CG_THERMAL_TRIG                    0x000000f8  /* 248 */
+
+#define VISLANDS30_IV_SRCID_SMU_DISP_TIMER_TRIGGER             0x000000fd  /* 253 */
+
+/* These are not "real" source ids defined by HW */
+#define VISLANDS30_IV_SRCID_VM_CONTEXT_ALL                     0x00000100  /* 256 */
+#define VISLANDS30_IV_EXTID_VM_CONTEXT0_ALL                    0
+#define VISLANDS30_IV_EXTID_VM_CONTEXT1_ALL                    1
+
+
+/* IV Extended IDs */
+#define VISLANDS30_IV_EXTID_NONE                               0x00000000
+#define VISLANDS30_IV_EXTID_INVALID                            0xffffffff
+
 #endif // _IVSRCID_VISLANDS30_H_
diff --git a/drivers/gpu/drm/amd/include/ivsrcid/sdma0/irqsrcs_sdma0_4_0.h b/drivers/gpu/drm/amd/include/ivsrcid/sdma0/irqsrcs_sdma0_4_0.h
new file mode 100644
index 000000000000..802413832fe8
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/ivsrcid/sdma0/irqsrcs_sdma0_4_0.h
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef __IRQSRCS_SDMA0_4_0_H__
+#define __IRQSRCS_SDMA0_4_0_H__
+
+#define SDMA0_4_0__SRCID__SDMA_ATOMIC_RTN_DONE                         217             /* 0xD9 SDMA atomic*_rtn ops complete  */
+#define SDMA0_4_0__SRCID__SDMA_ATOMIC_TIMEOUT                          218             /* 0xDA SDMA atomic CMPSWAP loop timeout  */
+#define SDMA0_4_0__SRCID__SDMA_IB_PREEMPT                                      219             /* 0xDB sdma mid-command buffer preempt interrupt  */
+#define SDMA0_4_0__SRCID__SDMA_ECC                                             220             /* 0xDC ECC  Error  */
+#define SDMA0_4_0__SRCID__SDMA_PAGE_FAULT                                      221             /* 0xDD Page Fault Error from UTCL2 when nack=3  */
+#define SDMA0_4_0__SRCID__SDMA_PAGE_NULL                                       222             /* 0xDE Page Null from UTCL2 when nack=2  */
+#define SDMA0_4_0__SRCID__SDMA_XNACK                                       223         /* 0xDF Page retry  timeout after UTCL2 return nack=1  */
+#define SDMA0_4_0__SRCID__SDMA_TRAP                                            224             /* 0xE0 Trap  */
+#define SDMA0_4_0__SRCID__SDMA_SEM_INCOMPLETE_TIMEOUT          225             /* 0xE1 0xDAGPF (Sem incomplete timeout)  */
+#define SDMA0_4_0__SRCID__SDMA_SEM_WAIT_FAIL_TIMEOUT           226             /* 0xE2 Semaphore wait fail timeout  */
+#define SDMA0_4_0__SRCID__SDMA_SRAM_ECC                                            228         /* 0xE4 SRAM ECC Error  */
+#define SDMA0_4_0__SRCID__SDMA_PREEMPT                                     240         /* 0xF0 SDMA New Run List  */
+#define SDMA0_4_0__SRCID__SDMA_VM_HOLE                                     242         /* 0xF2 MC or SEM address in VM hole  */
+#define SDMA0_4_0__SRCID__SDMA_CTXEMPTY                                            243         /* 0xF3 Context Empty  */
+#define SDMA0_4_0__SRCID__SDMA_DOORBELL_INVALID                                244             /* 0xF4 Doorbell BE invalid  */
+#define SDMA0_4_0__SRCID__SDMA_FROZEN                                      245         /* 0xF5 SDMA Frozen  */
+#define SDMA0_4_0__SRCID__SDMA_POLL_TIMEOUT                                    246             /* 0xF6 SRBM read poll timeout  */
+#define SDMA0_4_0__SRCID__SDMA_SRBMWRITE                                       247             /* 0xF7 SRBM write Protection  */
+
+#endif /* __IRQSRCS_SDMA_4_0_H__ */
+
+
diff --git a/drivers/gpu/drm/amd/include/ivsrcid/sdma1/irqsrcs_sdma1_4_0.h b/drivers/gpu/drm/amd/include/ivsrcid/sdma1/irqsrcs_sdma1_4_0.h
new file mode 100644
index 000000000000..d12a35619f9a
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/ivsrcid/sdma1/irqsrcs_sdma1_4_0.h
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef __IRQSRCS_SDMA1_4_0_H__
+#define __IRQSRCS_SDMA1_4_0_H__
+
+#define SDMA1_4_0__SRCID__SDMA_ATOMIC_RTN_DONE                         217             /* 0xD9 SDMA atomic*_rtn ops complete  */
+#define SDMA1_4_0__SRCID__SDMA_ATOMIC_TIMEOUT                          218             /* 0xDA SDMA atomic CMPSWAP loop timeout  */
+#define SDMA1_4_0__SRCID__SDMA_IB_PREEMPT                                      219             /* 0xDB sdma mid-command buffer preempt interrupt  */
+#define SDMA1_4_0__SRCID__SDMA_ECC                                             220             /* 0xDC ECC  Error  */
+#define SDMA1_4_0__SRCID__SDMA_PAGE_FAULT                                      221             /* 0xDD Page Fault Error from UTCL2 when nack=3  */
+#define SDMA1_4_0__SRCID__SDMA_PAGE_NULL                                       222             /* 0xDE Page Null from UTCL2 when nack=2  */
+#define SDMA1_4_0__SRCID__SDMA_XNACK                                       223         /* 0xDF Page retry  timeout after UTCL2 return nack=1  */
+#define SDMA1_4_0__SRCID__SDMA_TRAP                                            224             /* 0xE0 Trap  */
+#define SDMA1_4_0__SRCID__SDMA_SEM_INCOMPLETE_TIMEOUT          225             /* 0xE1 0xDAGPF (Sem incomplete timeout)  */
+#define SDMA1_4_0__SRCID__SDMA_SEM_WAIT_FAIL_TIMEOUT           226             /* 0xE2 Semaphore wait fail timeout  */
+#define SDMA1_4_0__SRCID__SDMA_SRAM_ECC                                            228         /* 0xE4 SRAM ECC Error  */
+#define SDMA1_4_0__SRCID__SDMA_PREEMPT                                     240         /* 0xF0 SDMA New Run List  */
+#define SDMA1_4_0__SRCID__SDMA_VM_HOLE                                     242         /* 0xF2 MC or SEM address in VM hole  */
+#define SDMA1_4_0__SRCID__SDMA_CTXEMPTY                                            243         /* 0xF3 Context Empty  */
+#define SDMA1_4_0__SRCID__SDMA_DOORBELL_INVALID                                244             /* 0xF4 Doorbell BE invalid  */
+#define SDMA1_4_0__SRCID__SDMA_FROZEN                                      245         /* 0xF5 SDMA Frozen  */
+#define SDMA1_4_0__SRCID__SDMA_POLL_TIMEOUT                                    246             /* 0xF6 SRBM read poll timeout  */
+#define SDMA1_4_0__SRCID__SDMA_SRBMWRITE                                       247             /* 0xF7 SRBM write Protection  */
+
+#endif /* __IRQSRCS_SDMA1_4_0_H__ */
+
+
diff --git a/drivers/gpu/drm/amd/display/dc/dml/soc_bounding_box.h b/drivers/gpu/drm/amd/include/ivsrcid/smuio/irqsrcs_smuio_9_0.h
index 7a65206a6d21..02bab4673cd4 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/soc_bounding_box.h
+++ b/drivers/gpu/drm/amd/include/ivsrcid/smuio/irqsrcs_smuio_9_0.h
@@ -23,13 +23,10 @@
  *
  */
 
-#ifndef __SOC_BOUNDING_BOX_H__
-#define __SOC_BOUNDING_BOX_H__
+#ifndef __IRQSRCS_SMUIO_9_0_H__
+#define __IRQSRCS_SMUIO_9_0_H__
 
-#include "dml_common_defs.h"
+#define SMUIO_9_0__SRCID__SMUIO_GPIO19			83		/* GPIO19 interrupt  */
 
-void dml_socbb_set_latencies(soc_bounding_box_st *to_box, soc_bounding_box_st *from_box);
-voltage_scaling_st dml_socbb_voltage_scaling(const soc_bounding_box_st *box, enum voltage_state voltage);
-double dml_socbb_return_bw_mhz(soc_bounding_box_st *box, enum voltage_state voltage);
+#endif /* __IRQSRCS_SMUIO_9_0_H__ */
 
-#endif
diff --git a/drivers/gpu/drm/amd/include/ivsrcid/thm/irqsrcs_thm_9_0.h b/drivers/gpu/drm/amd/include/ivsrcid/thm/irqsrcs_thm_9_0.h
new file mode 100644
index 000000000000..5218bc53fb2d
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/ivsrcid/thm/irqsrcs_thm_9_0.h
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef __IRQSRCS_THM_9_0_H__
+#define __IRQSRCS_THM_9_0_H__
+
+#define THM_9_0__SRCID__THM_DIG_THERM_L2H		0		/* ASIC_TEMP > CG_THERMAL_INT.DIG_THERM_INTH  */
+#define THM_9_0__SRCID__THM_DIG_THERM_H2L		1		/* ASIC_TEMP < CG_THERMAL_INT.DIG_THERM_INTL  */
+
+#endif /* __IRQSRCS_THM_9_0_H__ */
+
diff --git a/drivers/gpu/drm/amd/include/ivsrcid/uvd/irqsrcs_uvd_7_0.h b/drivers/gpu/drm/amd/include/ivsrcid/uvd/irqsrcs_uvd_7_0.h
new file mode 100644
index 000000000000..fb041aee6c66
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/ivsrcid/uvd/irqsrcs_uvd_7_0.h
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef __IRQSRCS_UVD_7_0_H__
+#define __IRQSRCS_UVD_7_0_H__
+
+#define UVD_7_0__SRCID__UVD_ENC_GEN_PURP                119
+#define UVD_7_0__SRCID__UVD_ENC_LOW_LATENCY             120
+#define UVD_7_0__SRCID__UVD_SYSTEM_MESSAGE_INTERRUPT	124		/* UVD system message interrupt  */
+
+#endif /* __IRQSRCS_UVD_7_0_H__ */
+
diff --git a/drivers/gpu/drm/amd/include/ivsrcid/vce/irqsrcs_vce_4_0.h b/drivers/gpu/drm/amd/include/ivsrcid/vce/irqsrcs_vce_4_0.h
new file mode 100644
index 000000000000..3440bab565af
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/ivsrcid/vce/irqsrcs_vce_4_0.h
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef __IRQSRCS_VCE_4_0_H__
+#define __IRQSRCS_VCE_4_0_H__
+
+#define VCE_4_0__CTXID__VCE_TRAP_GENERAL_PURPOSE               0
+#define VCE_4_0__CTXID__VCE_TRAP_LOW_LATENCY                   1
+#define VCE_4_0__CTXID__VCE_TRAP_REAL_TIME                     2
+
+#endif /* __IRQSRCS_VCE_4_0_H__ */
+
diff --git a/drivers/gpu/drm/amd/include/ivsrcid/vcn/irqsrcs_vcn_1_0.h b/drivers/gpu/drm/amd/include/ivsrcid/vcn/irqsrcs_vcn_1_0.h
new file mode 100644
index 000000000000..e5951709bfc3
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/ivsrcid/vcn/irqsrcs_vcn_1_0.h
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef __IRQSRCS_VCN_1_0_H__
+#define __IRQSRCS_VCN_1_0_H__
+
+#define VCN_1_0__SRCID__UVD_ENC_GENERAL_PURPOSE                 119     /* 0x77 Encoder General Purpose  */
+#define VCN_1_0__SRCID__UVD_ENC_LOW_LATENCY                     120     /* 0x78 Encoder Low Latency  */
+#define VCN_1_0__SRCID__UVD_SYSTEM_MESSAGE_INTERRUPT	        124		/* 0x7c UVD system message interrupt  */
+
+#endif /* __IRQSRCS_VCN_1_0_H__ */
+
diff --git a/drivers/gpu/drm/amd/include/ivsrcid/vmc/irqsrcs_vmc_1_0.h b/drivers/gpu/drm/amd/include/ivsrcid/vmc/irqsrcs_vmc_1_0.h
new file mode 100644
index 000000000000..d130936c9989
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/ivsrcid/vmc/irqsrcs_vmc_1_0.h
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef __IRQSRCS_VMC_1_0_H__
+#define __IRQSRCS_VMC_1_0_H__
+
+
+#define VMC_1_0__SRCID__VM_FAULT                            0
+#define VMC_1_0__SRCID__VM_CONTEXT0_ALL                     256
+#define VMC_1_0__SRCID__VM_CONTEXT1_ALL                     257
+
+#define UTCL2_1_0__SRCID__FAULT                             0       /* UTC L2 has encountered a fault or retry scenario */
+
+
+#endif /* __IRQSRCS_VMC_1_0_H__ */
diff --git a/drivers/gpu/drm/amd/include/kgd_kfd_interface.h b/drivers/gpu/drm/amd/include/kgd_kfd_interface.h
index 5733fbee07f7..14391b06080c 100644
--- a/drivers/gpu/drm/amd/include/kgd_kfd_interface.h
+++ b/drivers/gpu/drm/amd/include/kgd_kfd_interface.h
@@ -47,6 +47,17 @@ enum kfd_preempt_type {
 	KFD_PREEMPT_TYPE_WAVEFRONT_RESET,
 };
 
+struct kfd_vm_fault_info {
+	uint64_t	page_addr;
+	uint32_t	vmid;
+	uint32_t	mc_id;
+	uint32_t	status;
+	bool		prot_valid;
+	bool		prot_read;
+	bool		prot_write;
+	bool		prot_exec;
+};
+
 struct kfd_cu_info {
 	uint32_t num_shader_engines;
 	uint32_t num_shader_arrays_per_engine;
@@ -259,6 +270,21 @@ struct tile_config {
  * IB to the corresponding ring (ring type). The IB is executed with the
  * specified VMID in a user mode context.
  *
+ * @get_vm_fault_info: Return information about a recent VM fault on
+ * GFXv7 and v8. If multiple VM faults occurred since the last call of
+ * this function, it will return information about the first of those
+ * faults. On GFXv9 VM fault information is fully contained in the IH
+ * packet and this function is not needed.
+ *
+ * @read_vmid_from_vmfault_reg: On Hawaii the VMID is not set in the
+ * IH ring entry. This function allows the KFD ISR to get the VMID
+ * from the fault status register as early as possible.
+ *
+ * @gpu_recover: let kgd reset gpu after kfd detect CPC hang
+ *
+ * @set_compute_idle: Indicates that compute is idle on a device. This
+ * can be used to change power profiles depending on compute activity.
+ *
  * This structure contains function pointers to services that the kgd driver
  * provides to amdkfd driver.
  *
@@ -374,6 +400,14 @@ struct kfd2kgd_calls {
 	int (*submit_ib)(struct kgd_dev *kgd, enum kgd_engine_type engine,
 			uint32_t vmid, uint64_t gpu_addr,
 			uint32_t *ib_cmd, uint32_t ib_len);
+
+	int (*get_vm_fault_info)(struct kgd_dev *kgd,
+			struct kfd_vm_fault_info *info);
+	uint32_t (*read_vmid_from_vmfault_reg)(struct kgd_dev *kgd);
+
+	void (*gpu_recover)(struct kgd_dev *kgd);
+
+	void (*set_compute_idle)(struct kgd_dev *kgd, bool idle);
 };
 
 /**
@@ -399,6 +433,10 @@ struct kfd2kgd_calls {
  * @schedule_evict_and_restore_process: Schedules work queue that will prepare
  * for safe eviction of KFD BOs that belong to the specified process.
  *
+ * @pre_reset: Notifies amdkfd that amdgpu about to reset the gpu
+ *
+ * @post_reset: Notify amdkfd that amgpu successfully reseted the gpu
+ *
  * This structure contains function callback pointers so the kgd driver
  * will notify to the amdkfd about certain status changes.
  *
@@ -417,6 +455,8 @@ struct kgd2kfd_calls {
 	int (*resume_mm)(struct mm_struct *mm);
 	int (*schedule_evict_and_restore_process)(struct mm_struct *mm,
 			struct dma_fence *fence);
+	int  (*pre_reset)(struct kfd_dev *kfd);
+	int  (*post_reset)(struct kfd_dev *kfd);
 };
 
 int kgd2kfd_init(unsigned interface_version,
diff --git a/drivers/gpu/drm/amd/include/kgd_pp_interface.h b/drivers/gpu/drm/amd/include/kgd_pp_interface.h
index 06f08f34a110..6a41b81c7325 100644
--- a/drivers/gpu/drm/amd/include/kgd_pp_interface.h
+++ b/drivers/gpu/drm/amd/include/kgd_pp_interface.h
@@ -192,7 +192,6 @@ struct amd_pp_simple_clock_info;
 struct amd_pp_display_configuration;
 struct amd_pp_clock_info;
 struct pp_display_clock_request;
-struct pp_wm_sets_with_clock_ranges_soc15;
 struct pp_clock_levels_with_voltage;
 struct pp_clock_levels_with_latency;
 struct amd_pp_clocks;
@@ -232,16 +231,19 @@ struct amd_pm_funcs {
 	void (*debugfs_print_current_performance_level)(void *handle, struct seq_file *m);
 	int (*switch_power_profile)(void *handle, enum PP_SMC_POWER_PROFILE type, bool en);
 /* export to amdgpu */
-	void (*powergate_uvd)(void *handle, bool gate);
-	void (*powergate_vce)(void *handle, bool gate);
 	struct amd_vce_state *(*get_vce_clock_state)(void *handle, u32 idx);
 	int (*dispatch_tasks)(void *handle, enum amd_pp_task task_id,
 			enum amd_pm_state_type *user_state);
 	int (*load_firmware)(void *handle);
 	int (*wait_for_fw_loading_complete)(void *handle);
+	int (*set_powergating_by_smu)(void *handle,
+				uint32_t block_type, bool gate);
 	int (*set_clockgating_by_smu)(void *handle, uint32_t msg_id);
 	int (*set_power_limit)(void *handle, uint32_t n);
 	int (*get_power_limit)(void *handle, uint32_t *limit, bool default_limit);
+	int (*get_power_profile_mode)(void *handle, char *buf);
+	int (*set_power_profile_mode)(void *handle, long *input, uint32_t size);
+	int (*odn_edit_dpm_table)(void *handle, uint32_t type, long *input, uint32_t size);
 /* export to DC */
 	u32 (*get_sclk)(void *handle, bool low);
 	u32 (*get_mclk)(void *handle, bool low);
@@ -261,15 +263,12 @@ struct amd_pm_funcs {
 		enum amd_pp_clock_type type,
 		struct pp_clock_levels_with_voltage *clocks);
 	int (*set_watermarks_for_clocks_ranges)(void *handle,
-		struct pp_wm_sets_with_clock_ranges_soc15 *wm_with_clock_ranges);
+						void *clock_ranges);
 	int (*display_clock_voltage_request)(void *handle,
 				struct pp_display_clock_request *clock);
 	int (*get_display_mode_validation_clocks)(void *handle,
 		struct amd_pp_simple_clock_info *clocks);
-	int (*get_power_profile_mode)(void *handle, char *buf);
-	int (*set_power_profile_mode)(void *handle, long *input, uint32_t size);
-	int (*odn_edit_dpm_table)(void *handle, uint32_t type, long *input, uint32_t size);
-	int (*set_mmhub_powergating_by_smu)(void *handle);
+	int (*notify_smu_enable_pwe)(void *handle);
 };
 
 #endif
diff --git a/drivers/gpu/drm/amd/powerplay/amd_powerplay.c b/drivers/gpu/drm/amd/powerplay/amd_powerplay.c
index d567be49c31b..7a646f94b478 100644
--- a/drivers/gpu/drm/amd/powerplay/amd_powerplay.c
+++ b/drivers/gpu/drm/amd/powerplay/amd_powerplay.c
@@ -221,29 +221,7 @@ static int pp_sw_reset(void *handle)
 static int pp_set_powergating_state(void *handle,
 				    enum amd_powergating_state state)
 {
-	struct amdgpu_device *adev = handle;
-	struct pp_hwmgr *hwmgr = adev->powerplay.pp_handle;
-	int ret;
-
-	if (!hwmgr || !hwmgr->pm_en)
-		return 0;
-
-	if (hwmgr->hwmgr_func->gfx_off_control) {
-		/* Enable/disable GFX off through SMU */
-		ret = hwmgr->hwmgr_func->gfx_off_control(hwmgr,
-							 state == AMD_PG_STATE_GATE);
-		if (ret)
-			pr_err("gfx off control failed!\n");
-	}
-
-	if (hwmgr->hwmgr_func->enable_per_cu_power_gating == NULL) {
-		pr_debug("%s was not implemented.\n", __func__);
-		return 0;
-	}
-
-	/* Enable/disable GFX per cu powergating through SMU */
-	return hwmgr->hwmgr_func->enable_per_cu_power_gating(hwmgr,
-			state == AMD_PG_STATE_GATE);
+	return 0;
 }
 
 static int pp_suspend(void *handle)
@@ -1020,7 +998,7 @@ static int pp_get_display_power_level(void *handle,
 static int pp_get_current_clocks(void *handle,
 		struct amd_pp_clock_info *clocks)
 {
-	struct amd_pp_simple_clock_info simple_clocks;
+	struct amd_pp_simple_clock_info simple_clocks = { 0 };
 	struct pp_clock_info hw_clocks;
 	struct pp_hwmgr *hwmgr = handle;
 	int ret = 0;
@@ -1056,7 +1034,10 @@ static int pp_get_current_clocks(void *handle,
 	clocks->max_engine_clock_in_sr = hw_clocks.max_eng_clk;
 	clocks->min_engine_clock_in_sr = hw_clocks.min_eng_clk;
 
-	clocks->max_clocks_state = simple_clocks.level;
+	if (simple_clocks.level == 0)
+		clocks->max_clocks_state = PP_DAL_POWERLEVEL_7;
+	else
+		clocks->max_clocks_state = simple_clocks.level;
 
 	if (0 == phm_get_current_shallow_sleep_clocks(hwmgr, &hwmgr->current_ps->hardware, &hw_clocks)) {
 		clocks->max_engine_clock_in_sr = hw_clocks.max_eng_clk;
@@ -1118,17 +1099,17 @@ static int pp_get_clock_by_type_with_voltage(void *handle,
 }
 
 static int pp_set_watermarks_for_clocks_ranges(void *handle,
-		struct pp_wm_sets_with_clock_ranges_soc15 *wm_with_clock_ranges)
+		void *clock_ranges)
 {
 	struct pp_hwmgr *hwmgr = handle;
 	int ret = 0;
 
-	if (!hwmgr || !hwmgr->pm_en ||!wm_with_clock_ranges)
+	if (!hwmgr || !hwmgr->pm_en || !clock_ranges)
 		return -EINVAL;
 
 	mutex_lock(&hwmgr->smu_lock);
 	ret = phm_set_watermarks_for_clocks_ranges(hwmgr,
-			wm_with_clock_ranges);
+			clock_ranges);
 	mutex_unlock(&hwmgr->smu_lock);
 
 	return ret;
@@ -1159,6 +1140,8 @@ static int pp_get_display_mode_validation_clocks(void *handle,
 	if (!hwmgr || !hwmgr->pm_en ||!clocks)
 		return -EINVAL;
 
+	clocks->level = PP_DAL_POWERLEVEL_7;
+
 	mutex_lock(&hwmgr->smu_lock);
 
 	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_DynamicPatchPowerState))
@@ -1168,19 +1151,78 @@ static int pp_get_display_mode_validation_clocks(void *handle,
 	return ret;
 }
 
-static int pp_set_mmhub_powergating_by_smu(void *handle)
+static int pp_dpm_powergate_mmhub(void *handle)
 {
 	struct pp_hwmgr *hwmgr = handle;
 
 	if (!hwmgr || !hwmgr->pm_en)
 		return -EINVAL;
 
-	if (hwmgr->hwmgr_func->set_mmhub_powergating_by_smu == NULL) {
+	if (hwmgr->hwmgr_func->powergate_mmhub == NULL) {
+		pr_info("%s was not implemented.\n", __func__);
+		return 0;
+	}
+
+	return hwmgr->hwmgr_func->powergate_mmhub(hwmgr);
+}
+
+static int pp_dpm_powergate_gfx(void *handle, bool gate)
+{
+	struct pp_hwmgr *hwmgr = handle;
+
+	if (!hwmgr || !hwmgr->pm_en)
+		return 0;
+
+	if (hwmgr->hwmgr_func->powergate_gfx == NULL) {
 		pr_info("%s was not implemented.\n", __func__);
 		return 0;
 	}
 
-	return hwmgr->hwmgr_func->set_mmhub_powergating_by_smu(hwmgr);
+	return hwmgr->hwmgr_func->powergate_gfx(hwmgr, gate);
+}
+
+static int pp_set_powergating_by_smu(void *handle,
+				uint32_t block_type, bool gate)
+{
+	int ret = 0;
+
+	switch (block_type) {
+	case AMD_IP_BLOCK_TYPE_UVD:
+	case AMD_IP_BLOCK_TYPE_VCN:
+		pp_dpm_powergate_uvd(handle, gate);
+		break;
+	case AMD_IP_BLOCK_TYPE_VCE:
+		pp_dpm_powergate_vce(handle, gate);
+		break;
+	case AMD_IP_BLOCK_TYPE_GMC:
+		pp_dpm_powergate_mmhub(handle);
+		break;
+	case AMD_IP_BLOCK_TYPE_GFX:
+		ret = pp_dpm_powergate_gfx(handle, gate);
+		break;
+	default:
+		break;
+	}
+	return ret;
+}
+
+static int pp_notify_smu_enable_pwe(void *handle)
+{
+	struct pp_hwmgr *hwmgr = handle;
+
+	if (!hwmgr || !hwmgr->pm_en)
+		return -EINVAL;
+
+	if (hwmgr->hwmgr_func->smus_notify_pwe == NULL) {
+		pr_info("%s was not implemented.\n", __func__);
+		return -EINVAL;;
+	}
+
+	mutex_lock(&hwmgr->smu_lock);
+	hwmgr->hwmgr_func->smus_notify_pwe(hwmgr);
+	mutex_unlock(&hwmgr->smu_lock);
+
+	return 0;
 }
 
 static const struct amd_pm_funcs pp_dpm_funcs = {
@@ -1189,8 +1231,6 @@ static const struct amd_pm_funcs pp_dpm_funcs = {
 	.force_performance_level = pp_dpm_force_performance_level,
 	.get_performance_level = pp_dpm_get_performance_level,
 	.get_current_power_state = pp_dpm_get_current_power_state,
-	.powergate_vce = pp_dpm_powergate_vce,
-	.powergate_uvd = pp_dpm_powergate_uvd,
 	.dispatch_tasks = pp_dpm_dispatch_tasks,
 	.set_fan_control_mode = pp_dpm_set_fan_control_mode,
 	.get_fan_control_mode = pp_dpm_get_fan_control_mode,
@@ -1210,6 +1250,7 @@ static const struct amd_pm_funcs pp_dpm_funcs = {
 	.get_vce_clock_state = pp_dpm_get_vce_clock_state,
 	.switch_power_profile = pp_dpm_switch_power_profile,
 	.set_clockgating_by_smu = pp_set_clockgating_by_smu,
+	.set_powergating_by_smu = pp_set_powergating_by_smu,
 	.get_power_profile_mode = pp_get_power_profile_mode,
 	.set_power_profile_mode = pp_set_power_profile_mode,
 	.odn_edit_dpm_table = pp_odn_edit_dpm_table,
@@ -1227,5 +1268,5 @@ static const struct amd_pm_funcs pp_dpm_funcs = {
 	.set_watermarks_for_clocks_ranges = pp_set_watermarks_for_clocks_ranges,
 	.display_clock_voltage_request = pp_display_clock_voltage_request,
 	.get_display_mode_validation_clocks = pp_get_display_mode_validation_clocks,
-	.set_mmhub_powergating_by_smu = pp_set_mmhub_powergating_by_smu,
+	.notify_smu_enable_pwe = pp_notify_smu_enable_pwe,
 };
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/hardwaremanager.c b/drivers/gpu/drm/amd/powerplay/hwmgr/hardwaremanager.c
index a0bb921fac22..6ef3c875fedd 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/hardwaremanager.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/hardwaremanager.c
@@ -75,10 +75,12 @@ int phm_set_power_state(struct pp_hwmgr *hwmgr,
 
 int phm_enable_dynamic_state_management(struct pp_hwmgr *hwmgr)
 {
+	struct amdgpu_device *adev = NULL;
 	int ret = -EINVAL;;
 	PHM_FUNC_CHECK(hwmgr);
+	adev = hwmgr->adev;
 
-	if (smum_is_dpm_running(hwmgr)) {
+	if (smum_is_dpm_running(hwmgr) && !amdgpu_passthrough(adev)) {
 		pr_info("dpm has been enabled\n");
 		return 0;
 	}
@@ -435,7 +437,7 @@ int phm_get_clock_by_type_with_voltage(struct pp_hwmgr *hwmgr,
 }
 
 int phm_set_watermarks_for_clocks_ranges(struct pp_hwmgr *hwmgr,
-		struct pp_wm_sets_with_clock_ranges_soc15 *wm_with_clock_ranges)
+					void *clock_ranges)
 {
 	PHM_FUNC_CHECK(hwmgr);
 
@@ -443,7 +445,7 @@ int phm_set_watermarks_for_clocks_ranges(struct pp_hwmgr *hwmgr,
 		return -EINVAL;
 
 	return hwmgr->hwmgr_func->set_watermarks_for_clocks_ranges(hwmgr,
-			wm_with_clock_ranges);
+								clock_ranges);
 }
 
 int phm_display_clock_voltage_request(struct pp_hwmgr *hwmgr,
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr.c
index 9b675d9bd162..8994aa5c8cf8 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr.c
@@ -147,10 +147,10 @@ int hwmgr_early_init(struct pp_hwmgr *hwmgr)
 		smu7_init_function_pointers(hwmgr);
 		break;
 	case AMDGPU_FAMILY_AI:
-		hwmgr->feature_mask &= ~PP_GFXOFF_MASK;
 		switch (hwmgr->chip_id) {
 		case CHIP_VEGA10:
 		case CHIP_VEGA20:
+			hwmgr->feature_mask &= ~PP_GFXOFF_MASK;
 			hwmgr->smumgr_funcs = &vega10_smu_funcs;
 			vega10_hwmgr_init(hwmgr);
 			break;
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.c b/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.c
index 7047e29755c3..01dc46dc9c8a 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.c
@@ -1544,14 +1544,14 @@ void atomctrl_get_voltage_range(struct pp_hwmgr *hwmgr, uint32_t *max_vddc,
 		switch (hwmgr->chip_id) {
 		case CHIP_TONGA:
 		case CHIP_FIJI:
-			*max_vddc = le32_to_cpu(((ATOM_ASIC_PROFILING_INFO_V3_3 *)profile)->ulMaxVddc/4);
-			*min_vddc = le32_to_cpu(((ATOM_ASIC_PROFILING_INFO_V3_3 *)profile)->ulMinVddc/4);
+			*max_vddc = le32_to_cpu(((ATOM_ASIC_PROFILING_INFO_V3_3 *)profile)->ulMaxVddc) / 4;
+			*min_vddc = le32_to_cpu(((ATOM_ASIC_PROFILING_INFO_V3_3 *)profile)->ulMinVddc) / 4;
 			return;
 		case CHIP_POLARIS11:
 		case CHIP_POLARIS10:
 		case CHIP_POLARIS12:
-			*max_vddc = le32_to_cpu(((ATOM_ASIC_PROFILING_INFO_V3_6 *)profile)->ulMaxVddc/100);
-			*min_vddc = le32_to_cpu(((ATOM_ASIC_PROFILING_INFO_V3_6 *)profile)->ulMinVddc/100);
+			*max_vddc = le32_to_cpu(((ATOM_ASIC_PROFILING_INFO_V3_6 *)profile)->ulMaxVddc) / 100;
+			*min_vddc = le32_to_cpu(((ATOM_ASIC_PROFILING_INFO_V3_6 *)profile)->ulMinVddc) / 100;
 			return;
 		default:
 			break;
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomfwctrl.c b/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomfwctrl.c
index 5325661fedff..d27c1c9df286 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomfwctrl.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomfwctrl.c
@@ -512,14 +512,82 @@ int pp_atomfwctrl_get_clk_information_by_clkid(struct pp_hwmgr *hwmgr, BIOS_CLKI
 	return 0;
 }
 
+static void pp_atomfwctrl_copy_vbios_bootup_values_3_2(struct pp_hwmgr *hwmgr,
+			struct pp_atomfwctrl_bios_boot_up_values *boot_values,
+			struct atom_firmware_info_v3_2 *fw_info)
+{
+	uint32_t frequency = 0;
+
+	boot_values->ulRevision = fw_info->firmware_revision;
+	boot_values->ulGfxClk   = fw_info->bootup_sclk_in10khz;
+	boot_values->ulUClk     = fw_info->bootup_mclk_in10khz;
+	boot_values->usVddc     = fw_info->bootup_vddc_mv;
+	boot_values->usVddci    = fw_info->bootup_vddci_mv;
+	boot_values->usMvddc    = fw_info->bootup_mvddc_mv;
+	boot_values->usVddGfx   = fw_info->bootup_vddgfx_mv;
+	boot_values->ucCoolingID = fw_info->coolingsolution_id;
+	boot_values->ulSocClk   = 0;
+	boot_values->ulDCEFClk   = 0;
+
+	if (!pp_atomfwctrl_get_clk_information_by_clkid(hwmgr, SMU11_SYSPLL0_SOCCLK_ID, &frequency))
+		boot_values->ulSocClk   = frequency;
+
+	if (!pp_atomfwctrl_get_clk_information_by_clkid(hwmgr, SMU11_SYSPLL0_DCEFCLK_ID, &frequency))
+		boot_values->ulDCEFClk  = frequency;
+
+	if (!pp_atomfwctrl_get_clk_information_by_clkid(hwmgr, SMU11_SYSPLL0_ECLK_ID, &frequency))
+		boot_values->ulEClk     = frequency;
+
+	if (!pp_atomfwctrl_get_clk_information_by_clkid(hwmgr, SMU11_SYSPLL0_VCLK_ID, &frequency))
+		boot_values->ulVClk     = frequency;
+
+	if (!pp_atomfwctrl_get_clk_information_by_clkid(hwmgr, SMU11_SYSPLL0_DCLK_ID, &frequency))
+		boot_values->ulDClk     = frequency;
+}
+
+static void pp_atomfwctrl_copy_vbios_bootup_values_3_1(struct pp_hwmgr *hwmgr,
+			struct pp_atomfwctrl_bios_boot_up_values *boot_values,
+			struct atom_firmware_info_v3_1 *fw_info)
+{
+	uint32_t frequency = 0;
+
+	boot_values->ulRevision = fw_info->firmware_revision;
+	boot_values->ulGfxClk   = fw_info->bootup_sclk_in10khz;
+	boot_values->ulUClk     = fw_info->bootup_mclk_in10khz;
+	boot_values->usVddc     = fw_info->bootup_vddc_mv;
+	boot_values->usVddci    = fw_info->bootup_vddci_mv;
+	boot_values->usMvddc    = fw_info->bootup_mvddc_mv;
+	boot_values->usVddGfx   = fw_info->bootup_vddgfx_mv;
+	boot_values->ucCoolingID = fw_info->coolingsolution_id;
+	boot_values->ulSocClk   = 0;
+	boot_values->ulDCEFClk   = 0;
+
+	if (!pp_atomfwctrl_get_clk_information_by_clkid(hwmgr, SMU9_SYSPLL0_SOCCLK_ID, &frequency))
+		boot_values->ulSocClk   = frequency;
+
+	if (!pp_atomfwctrl_get_clk_information_by_clkid(hwmgr, SMU9_SYSPLL0_DCEFCLK_ID, &frequency))
+		boot_values->ulDCEFClk  = frequency;
+
+	if (!pp_atomfwctrl_get_clk_information_by_clkid(hwmgr, SMU9_SYSPLL0_ECLK_ID, &frequency))
+		boot_values->ulEClk     = frequency;
+
+	if (!pp_atomfwctrl_get_clk_information_by_clkid(hwmgr, SMU9_SYSPLL0_VCLK_ID, &frequency))
+		boot_values->ulVClk     = frequency;
+
+	if (!pp_atomfwctrl_get_clk_information_by_clkid(hwmgr, SMU9_SYSPLL0_DCLK_ID, &frequency))
+		boot_values->ulDClk     = frequency;
+}
+
 int pp_atomfwctrl_get_vbios_bootup_values(struct pp_hwmgr *hwmgr,
 			struct pp_atomfwctrl_bios_boot_up_values *boot_values)
 {
-	struct atom_firmware_info_v3_1 *info = NULL;
+	struct atom_firmware_info_v3_2 *fwinfo_3_2;
+	struct atom_firmware_info_v3_1 *fwinfo_3_1;
+	struct atom_common_table_header *info = NULL;
 	uint16_t ix;
 
 	ix = GetIndexIntoMasterDataTable(firmwareinfo);
-	info = (struct atom_firmware_info_v3_1 *)
+	info = (struct atom_common_table_header *)
 		smu_atom_get_data_table(hwmgr->adev,
 				ix, NULL, NULL, NULL);
 
@@ -528,16 +596,18 @@ int pp_atomfwctrl_get_vbios_bootup_values(struct pp_hwmgr *hwmgr,
 		return -EINVAL;
 	}
 
-	boot_values->ulRevision = info->firmware_revision;
-	boot_values->ulGfxClk   = info->bootup_sclk_in10khz;
-	boot_values->ulUClk     = info->bootup_mclk_in10khz;
-	boot_values->usVddc     = info->bootup_vddc_mv;
-	boot_values->usVddci    = info->bootup_vddci_mv;
-	boot_values->usMvddc    = info->bootup_mvddc_mv;
-	boot_values->usVddGfx   = info->bootup_vddgfx_mv;
-	boot_values->ucCoolingID = info->coolingsolution_id;
-	boot_values->ulSocClk   = 0;
-	boot_values->ulDCEFClk   = 0;
+	if ((info->format_revision == 3) && (info->content_revision == 2)) {
+		fwinfo_3_2 = (struct atom_firmware_info_v3_2 *)info;
+		pp_atomfwctrl_copy_vbios_bootup_values_3_2(hwmgr,
+				boot_values, fwinfo_3_2);
+	} else if ((info->format_revision == 3) && (info->content_revision == 1)) {
+		fwinfo_3_1 = (struct atom_firmware_info_v3_1 *)info;
+		pp_atomfwctrl_copy_vbios_bootup_values_3_1(hwmgr,
+				boot_values, fwinfo_3_1);
+	} else {
+		pr_info("Fw info table revision does not match!");
+		return -EINVAL;
+	}
 
 	return 0;
 }
@@ -629,5 +699,7 @@ int pp_atomfwctrl_get_smc_dpm_information(struct pp_hwmgr *hwmgr,
 	param->acggfxclkspreadpercent = info->acggfxclkspreadpercent;
 	param->acggfxclkspreadfreq = info->acggfxclkspreadfreq;
 
+	param->Vr2_I2C_address = info->Vr2_I2C_address;
+
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomfwctrl.h b/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomfwctrl.h
index fe10aa4db5e6..22e21668c93a 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomfwctrl.h
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomfwctrl.h
@@ -136,6 +136,9 @@ struct pp_atomfwctrl_bios_boot_up_values {
 	uint32_t   ulUClk;
 	uint32_t   ulSocClk;
 	uint32_t   ulDCEFClk;
+	uint32_t   ulEClk;
+	uint32_t   ulVClk;
+	uint32_t   ulDClk;
 	uint16_t   usVddc;
 	uint16_t   usVddci;
 	uint16_t   usMvddc;
@@ -207,6 +210,8 @@ struct pp_atomfwctrl_smc_dpm_parameters
 	uint8_t  acggfxclkspreadenabled;
 	uint8_t  acggfxclkspreadpercent;
 	uint16_t acggfxclkspreadfreq;
+
+	uint8_t Vr2_I2C_address;
 };
 
 int pp_atomfwctrl_get_gpu_pll_dividers_vega10(struct pp_hwmgr *hwmgr,
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/process_pptables_v1_0.c b/drivers/gpu/drm/amd/powerplay/hwmgr/process_pptables_v1_0.c
index 35bd9870ab10..4e1fd5393845 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/process_pptables_v1_0.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/process_pptables_v1_0.c
@@ -183,10 +183,10 @@ static int get_vddc_lookup_table(
 					ATOM_Tonga_Voltage_Lookup_Record,
 					entries, vddc_lookup_pp_tables, i);
 		record->us_calculated = 0;
-		record->us_vdd = atom_record->usVdd;
-		record->us_cac_low = atom_record->usCACLow;
-		record->us_cac_mid = atom_record->usCACMid;
-		record->us_cac_high = atom_record->usCACHigh;
+		record->us_vdd = le16_to_cpu(atom_record->usVdd);
+		record->us_cac_low = le16_to_cpu(atom_record->usCACLow);
+		record->us_cac_mid = le16_to_cpu(atom_record->usCACMid);
+		record->us_cac_high = le16_to_cpu(atom_record->usCACHigh);
 	}
 
 	*lookup_table = table;
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu10_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/smu10_hwmgr.c
index d4bc83e81389..a63e00653324 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu10_hwmgr.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/smu10_hwmgr.c
@@ -993,7 +993,7 @@ static int smu10_get_clock_by_type_with_latency(struct pp_hwmgr *hwmgr,
 
 	clocks->num_levels = 0;
 	for (i = 0; i < pclk_vol_table->count; i++) {
-		clocks->data[i].clocks_in_khz = pclk_vol_table->entries[i].clk;
+		clocks->data[i].clocks_in_khz = pclk_vol_table->entries[i].clk * 10;
 		clocks->data[i].latency_in_us = latency_required ?
 						smu10_get_mem_latency(hwmgr,
 						pclk_vol_table->entries[i].clk) :
@@ -1044,7 +1044,7 @@ static int smu10_get_clock_by_type_with_voltage(struct pp_hwmgr *hwmgr,
 
 	clocks->num_levels = 0;
 	for (i = 0; i < pclk_vol_table->count; i++) {
-		clocks->data[i].clocks_in_khz = pclk_vol_table->entries[i].clk;
+		clocks->data[i].clocks_in_khz = pclk_vol_table->entries[i].clk  * 10;
 		clocks->data[i].voltage_in_mv = pclk_vol_table->entries[i].vol;
 		clocks->num_levels++;
 	}
@@ -1108,9 +1108,10 @@ static int smu10_read_sensor(struct pp_hwmgr *hwmgr, int idx,
 }
 
 static int smu10_set_watermarks_for_clocks_ranges(struct pp_hwmgr *hwmgr,
-		struct pp_wm_sets_with_clock_ranges_soc15 *wm_with_clock_ranges)
+		void *clock_ranges)
 {
 	struct smu10_hwmgr *data = hwmgr->backend;
+	struct dm_pp_wm_sets_with_clock_ranges_soc15 *wm_with_clock_ranges = clock_ranges;
 	Watermarks_t *table = &(data->water_marks_table);
 	int result = 0;
 
@@ -1126,7 +1127,7 @@ static int smu10_smus_notify_pwe(struct pp_hwmgr *hwmgr)
 	return smum_send_msg_to_smc(hwmgr, PPSMC_MSG_SetRccPfcPmeRestoreRegister);
 }
 
-static int smu10_set_mmhub_powergating_by_smu(struct pp_hwmgr *hwmgr)
+static int smu10_powergate_mmhub(struct pp_hwmgr *hwmgr)
 {
 	return smum_send_msg_to_smc(hwmgr, PPSMC_MSG_PowerGateMmHub);
 }
@@ -1182,10 +1183,11 @@ static const struct pp_hwmgr_func smu10_hwmgr_funcs = {
 	.asic_setup = smu10_setup_asic_task,
 	.power_state_set = smu10_set_power_state_tasks,
 	.dynamic_state_management_disable = smu10_disable_dpm_tasks,
-	.set_mmhub_powergating_by_smu = smu10_set_mmhub_powergating_by_smu,
+	.powergate_mmhub = smu10_powergate_mmhub,
 	.smus_notify_pwe = smu10_smus_notify_pwe,
 	.gfx_off_control = smu10_gfx_off_control,
 	.display_clock_voltage_request = smu10_display_clock_voltage_request,
+	.powergate_gfx = smu10_gfx_off_control,
 };
 
 int smu10_init_function_pointers(struct pp_hwmgr *hwmgr)
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_clockpowergating.c b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_clockpowergating.c
index 41495621d94a..683b29a99366 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_clockpowergating.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_clockpowergating.c
@@ -416,7 +416,7 @@ int smu7_update_clock_gatings(struct pp_hwmgr *hwmgr,
  * Powerplay will only control the static per CU Power Gating.
  * Dynamic per CU Power Gating will be done in gfx.
  */
-int smu7_enable_per_cu_power_gating(struct pp_hwmgr *hwmgr, bool enable)
+int smu7_powergate_gfx(struct pp_hwmgr *hwmgr, bool enable)
 {
 	struct amdgpu_device *adev = hwmgr->adev;
 
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_clockpowergating.h b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_clockpowergating.h
index be7f66d2b234..fc8f8a6acc72 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_clockpowergating.h
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_clockpowergating.h
@@ -33,6 +33,6 @@ int smu7_powergate_acp(struct pp_hwmgr *hwmgr, bool bgate);
 int smu7_disable_clock_power_gating(struct pp_hwmgr *hwmgr);
 int smu7_update_clock_gatings(struct pp_hwmgr *hwmgr,
 					const uint32_t *msg_id);
-int smu7_enable_per_cu_power_gating(struct pp_hwmgr *hwmgr, bool enable);
+int smu7_powergate_gfx(struct pp_hwmgr *hwmgr, bool enable);
 
 #endif
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c
index b89d6fb8559b..052e60dfaf9f 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c
@@ -48,6 +48,8 @@
 #include "processpptables.h"
 #include "pp_thermal.h"
 
+#include "ivsrcid/ivsrcid_vislands30.h"
+
 #define MC_CG_ARB_FREQ_F0           0x0a
 #define MC_CG_ARB_FREQ_F1           0x0b
 #define MC_CG_ARB_FREQ_F2           0x0c
@@ -1578,7 +1580,7 @@ static void smu7_init_dpm_defaults(struct pp_hwmgr *hwmgr)
 	data->current_profile_setting.sclk_up_hyst = 0;
 	data->current_profile_setting.sclk_down_hyst = 100;
 	data->current_profile_setting.sclk_activity = SMU7_SCLK_TARGETACTIVITY_DFLT;
-	data->current_profile_setting.bupdate_sclk = 1;
+	data->current_profile_setting.bupdate_mclk = 1;
 	data->current_profile_setting.mclk_up_hyst = 0;
 	data->current_profile_setting.mclk_down_hyst = 100;
 	data->current_profile_setting.mclk_activity = SMU7_MCLK_TARGETACTIVITY_DFLT;
@@ -3183,7 +3185,7 @@ static int smu7_get_pp_table_entry_callback_func_v1(struct pp_hwmgr *hwmgr,
 	performance_level->pcie_gen = get_pcie_gen_support(data->pcie_gen_cap,
 			state_entry->ucPCIEGenLow);
 	performance_level->pcie_lane = get_pcie_lane_support(data->pcie_lane_cap,
-			state_entry->ucPCIELaneHigh);
+			state_entry->ucPCIELaneLow);
 
 	performance_level = &(smu7_power_state->performance_levels
 			[smu7_power_state->performance_level_count++]);
@@ -4105,17 +4107,17 @@ static int smu7_register_irq_handlers(struct pp_hwmgr *hwmgr)
 
 	amdgpu_irq_add_id((struct amdgpu_device *)(hwmgr->adev),
 			AMDGPU_IH_CLIENTID_LEGACY,
-			230,
+			VISLANDS30_IV_SRCID_CG_TSS_THERMAL_LOW_TO_HIGH,
 			source);
 	amdgpu_irq_add_id((struct amdgpu_device *)(hwmgr->adev),
 			AMDGPU_IH_CLIENTID_LEGACY,
-			231,
+			VISLANDS30_IV_SRCID_CG_TSS_THERMAL_HIGH_TO_LOW,
 			source);
 
 	/* Register CTF(GPIO_19) interrupt */
 	amdgpu_irq_add_id((struct amdgpu_device *)(hwmgr->adev),
 			AMDGPU_IH_CLIENTID_LEGACY,
-			83,
+			VISLANDS30_IV_SRCID_GPIO_19,
 			source);
 
 	return 0;
@@ -4610,12 +4612,12 @@ static int smu7_get_sclks(struct pp_hwmgr *hwmgr, struct amd_pp_clocks *clocks)
 			return -EINVAL;
 		dep_sclk_table = table_info->vdd_dep_on_sclk;
 		for (i = 0; i < dep_sclk_table->count; i++)
-			clocks->clock[i] = dep_sclk_table->entries[i].clk;
+			clocks->clock[i] = dep_sclk_table->entries[i].clk * 10;
 		clocks->count = dep_sclk_table->count;
 	} else if (hwmgr->pp_table_version == PP_TABLE_V0) {
 		sclk_table = hwmgr->dyn_state.vddc_dependency_on_sclk;
 		for (i = 0; i < sclk_table->count; i++)
-			clocks->clock[i] = sclk_table->entries[i].clk;
+			clocks->clock[i] = sclk_table->entries[i].clk * 10;
 		clocks->count = sclk_table->count;
 	}
 
@@ -4647,7 +4649,7 @@ static int smu7_get_mclks(struct pp_hwmgr *hwmgr, struct amd_pp_clocks *clocks)
 			return -EINVAL;
 		dep_mclk_table = table_info->vdd_dep_on_mclk;
 		for (i = 0; i < dep_mclk_table->count; i++) {
-			clocks->clock[i] = dep_mclk_table->entries[i].clk;
+			clocks->clock[i] = dep_mclk_table->entries[i].clk * 10;
 			clocks->latency[i] = smu7_get_mem_latency(hwmgr,
 						dep_mclk_table->entries[i].clk);
 		}
@@ -4655,7 +4657,7 @@ static int smu7_get_mclks(struct pp_hwmgr *hwmgr, struct amd_pp_clocks *clocks)
 	} else if (hwmgr->pp_table_version == PP_TABLE_V0) {
 		mclk_table = hwmgr->dyn_state.vddc_dependency_on_mclk;
 		for (i = 0; i < mclk_table->count; i++)
-			clocks->clock[i] = mclk_table->entries[i].clk;
+			clocks->clock[i] = mclk_table->entries[i].clk * 10;
 		clocks->count = mclk_table->count;
 	}
 	return 0;
@@ -5044,7 +5046,7 @@ static const struct pp_hwmgr_func smu7_hwmgr_funcs = {
 	.get_fan_control_mode = smu7_get_fan_control_mode,
 	.force_clock_level = smu7_force_clock_level,
 	.print_clock_levels = smu7_print_clock_levels,
-	.enable_per_cu_power_gating = smu7_enable_per_cu_power_gating,
+	.powergate_gfx = smu7_powergate_gfx,
 	.get_sclk_od = smu7_get_sclk_od,
 	.set_sclk_od = smu7_set_sclk_od,
 	.get_mclk_od = smu7_get_mclk_od,
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_powertune.c b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_powertune.c
index c952845833d7..5e19f5977eb1 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_powertune.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_powertune.c
@@ -403,6 +403,49 @@ static const struct gpu_pt_config_reg DIDTConfig_Polaris12[] = {
 	{   ixDIDT_SQ_CTRL1,                   DIDT_SQ_CTRL1__MAX_POWER_MASK,                      DIDT_SQ_CTRL1__MAX_POWER__SHIFT,                    0xffff,     GPU_CONFIGREG_DIDT_IND },
 
 	{   ixDIDT_SQ_CTRL_OCP,                DIDT_SQ_CTRL_OCP__UNUSED_0_MASK,                    DIDT_SQ_CTRL_OCP__UNUSED_0__SHIFT,                  0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_CTRL_OCP,                DIDT_SQ_CTRL_OCP__OCP_MAX_POWER_MASK,               DIDT_SQ_CTRL_OCP__OCP_MAX_POWER__SHIFT,             0xffff,     GPU_CONFIGREG_DIDT_IND },
+
+	{   ixDIDT_SQ_CTRL2,                   DIDT_SQ_CTRL2__MAX_POWER_DELTA_MASK,                DIDT_SQ_CTRL2__MAX_POWER_DELTA__SHIFT,              0x3853,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_CTRL2,                   DIDT_SQ_CTRL2__UNUSED_0_MASK,                       DIDT_SQ_CTRL2__UNUSED_0__SHIFT,                     0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_CTRL2,                   DIDT_SQ_CTRL2__SHORT_TERM_INTERVAL_SIZE_MASK,       DIDT_SQ_CTRL2__SHORT_TERM_INTERVAL_SIZE__SHIFT,     0x005a,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_CTRL2,                   DIDT_SQ_CTRL2__UNUSED_1_MASK,                       DIDT_SQ_CTRL2__UNUSED_1__SHIFT,                     0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_CTRL2,                   DIDT_SQ_CTRL2__LONG_TERM_INTERVAL_RATIO_MASK,       DIDT_SQ_CTRL2__LONG_TERM_INTERVAL_RATIO__SHIFT,     0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_CTRL2,                   DIDT_SQ_CTRL2__UNUSED_2_MASK,                       DIDT_SQ_CTRL2__UNUSED_2__SHIFT,                     0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+	{   ixDIDT_SQ_STALL_CTRL,              DIDT_SQ_STALL_CTRL__DIDT_STALL_CTRL_ENABLE_MASK,    DIDT_SQ_STALL_CTRL__DIDT_STALL_CTRL_ENABLE__SHIFT,  0x0001,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_STALL_CTRL,              DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_HI_MASK,       DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_HI__SHIFT,     0x0001,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_STALL_CTRL,              DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_LO_MASK,       DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_LO__SHIFT,     0x0001,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_STALL_CTRL,              DIDT_SQ_STALL_CTRL__DIDT_HI_POWER_THRESHOLD_MASK,   DIDT_SQ_STALL_CTRL__DIDT_HI_POWER_THRESHOLD__SHIFT, 0x0ebb,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_STALL_CTRL,              DIDT_SQ_STALL_CTRL__UNUSED_0_MASK,                  DIDT_SQ_STALL_CTRL__UNUSED_0__SHIFT,                0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+	{   ixDIDT_SQ_TUNING_CTRL,             DIDT_SQ_TUNING_CTRL__DIDT_TUNING_ENABLE_MASK,       DIDT_SQ_TUNING_CTRL__DIDT_TUNING_ENABLE__SHIFT,     0x0001,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_TUNING_CTRL,             DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_HI_MASK,       DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_HI__SHIFT,     0x3853,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_TUNING_CTRL,             DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_LO_MASK,       DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_LO__SHIFT,     0x3153,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_TUNING_CTRL,             DIDT_SQ_TUNING_CTRL__UNUSED_0_MASK,                 DIDT_SQ_TUNING_CTRL__UNUSED_0__SHIFT,               0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+	{   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_CTRL_EN_MASK,                   DIDT_SQ_CTRL0__DIDT_CTRL_EN__SHIFT,                 0x0001,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__USE_REF_CLOCK_MASK,                  DIDT_SQ_CTRL0__USE_REF_CLOCK__SHIFT,                0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__PHASE_OFFSET_MASK,                   DIDT_SQ_CTRL0__PHASE_OFFSET__SHIFT,                 0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_CTRL_RST_MASK,                  DIDT_SQ_CTRL0__DIDT_CTRL_RST__SHIFT,                0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK,           DIDT_SQ_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT,         0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI_MASK,     DIDT_SQ_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI__SHIFT,   0x0010,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO_MASK,     DIDT_SQ_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO__SHIFT,   0x0010,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__UNUSED_0_MASK,                       DIDT_SQ_CTRL0__UNUSED_0__SHIFT,                     0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+	{   ixDIDT_TD_WEIGHT0_3,               DIDT_TD_WEIGHT0_3__WEIGHT0_MASK,                    DIDT_TD_WEIGHT0_3__WEIGHT0__SHIFT,                  0x000a,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_WEIGHT0_3,               DIDT_TD_WEIGHT0_3__WEIGHT1_MASK,                    DIDT_TD_WEIGHT0_3__WEIGHT1__SHIFT,                  0x0010,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_WEIGHT0_3,               DIDT_TD_WEIGHT0_3__WEIGHT2_MASK,                    DIDT_TD_WEIGHT0_3__WEIGHT2__SHIFT,                  0x0017,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_WEIGHT0_3,               DIDT_TD_WEIGHT0_3__WEIGHT3_MASK,                    DIDT_TD_WEIGHT0_3__WEIGHT3__SHIFT,                  0x002f,     GPU_CONFIGREG_DIDT_IND },
+
+	{   ixDIDT_TD_WEIGHT4_7,               DIDT_TD_WEIGHT4_7__WEIGHT4_MASK,                    DIDT_TD_WEIGHT4_7__WEIGHT4__SHIFT,                  0x0046,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_WEIGHT4_7,               DIDT_TD_WEIGHT4_7__WEIGHT5_MASK,                    DIDT_TD_WEIGHT4_7__WEIGHT5__SHIFT,                  0x005d,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_WEIGHT4_7,               DIDT_TD_WEIGHT4_7__WEIGHT6_MASK,                    DIDT_TD_WEIGHT4_7__WEIGHT6__SHIFT,                  0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_WEIGHT4_7,               DIDT_TD_WEIGHT4_7__WEIGHT7_MASK,                    DIDT_TD_WEIGHT4_7__WEIGHT7__SHIFT,                  0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+	{   ixDIDT_TD_CTRL1,                   DIDT_TD_CTRL1__MIN_POWER_MASK,                      DIDT_TD_CTRL1__MIN_POWER__SHIFT,                    0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_CTRL1,                   DIDT_TD_CTRL1__MAX_POWER_MASK,                      DIDT_TD_CTRL1__MAX_POWER__SHIFT,                    0xffff,     GPU_CONFIGREG_DIDT_IND },
+
+	{   ixDIDT_TD_CTRL_OCP,                DIDT_TD_CTRL_OCP__UNUSED_0_MASK,                    DIDT_TD_CTRL_OCP__UNUSED_0__SHIFT,                  0x0000,     GPU_CONFIGREG_DIDT_IND },
 	{   ixDIDT_TD_CTRL_OCP,                DIDT_TD_CTRL_OCP__OCP_MAX_POWER_MASK,               DIDT_TD_CTRL_OCP__OCP_MAX_POWER__SHIFT,             0x00ff,     GPU_CONFIGREG_DIDT_IND },
 
 	{   ixDIDT_TD_CTRL2,                   DIDT_TD_CTRL2__MAX_POWER_DELTA_MASK,                DIDT_TD_CTRL2__MAX_POWER_DELTA__SHIFT,              0x3fff,     GPU_CONFIGREG_DIDT_IND },
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu8_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/smu8_hwmgr.c
index 50690c72b2ea..0adfc5392cd3 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu8_hwmgr.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/smu8_hwmgr.c
@@ -244,6 +244,7 @@ static int smu8_initialize_dpm_defaults(struct pp_hwmgr *hwmgr)
 	return 0;
 }
 
+/* convert form 8bit vid to real voltage in mV*4 */
 static uint32_t smu8_convert_8Bit_index_to_voltage(
 			struct pp_hwmgr *hwmgr, uint16_t voltage)
 {
@@ -1604,17 +1605,17 @@ static int smu8_get_clock_by_type(struct pp_hwmgr *hwmgr, enum amd_pp_clock_type
 	switch (type) {
 	case amd_pp_disp_clock:
 		for (i = 0; i < clocks->count; i++)
-			clocks->clock[i] = data->sys_info.display_clock[i];
+			clocks->clock[i] = data->sys_info.display_clock[i] * 10;
 		break;
 	case amd_pp_sys_clock:
 		table = hwmgr->dyn_state.vddc_dependency_on_sclk;
 		for (i = 0; i < clocks->count; i++)
-			clocks->clock[i] = table->entries[i].clk;
+			clocks->clock[i] = table->entries[i].clk * 10;
 		break;
 	case amd_pp_mem_clock:
 		clocks->count = SMU8_NUM_NBPMEMORYCLOCK;
 		for (i = 0; i < clocks->count; i++)
-			clocks->clock[i] = data->sys_info.nbp_memory_clock[clocks->count - 1 - i];
+			clocks->clock[i] = data->sys_info.nbp_memory_clock[clocks->count - 1 - i] * 10;
 		break;
 	default:
 		return -1;
@@ -1702,13 +1703,13 @@ static int smu8_read_sensor(struct pp_hwmgr *hwmgr, int idx,
 	case AMDGPU_PP_SENSOR_VDDNB:
 		tmp = (cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixSMUSVI_NB_CURRENTVID) &
 			CURRENT_NB_VID_MASK) >> CURRENT_NB_VID__SHIFT;
-		vddnb = smu8_convert_8Bit_index_to_voltage(hwmgr, tmp);
+		vddnb = smu8_convert_8Bit_index_to_voltage(hwmgr, tmp) / 4;
 		*((uint32_t *)value) = vddnb;
 		return 0;
 	case AMDGPU_PP_SENSOR_VDDGFX:
 		tmp = (cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixSMUSVI_GFX_CURRENTVID) &
 			CURRENT_GFX_VID_MASK) >> CURRENT_GFX_VID__SHIFT;
-		vddgfx = smu8_convert_8Bit_index_to_voltage(hwmgr, (u16)tmp);
+		vddgfx = smu8_convert_8Bit_index_to_voltage(hwmgr, (u16)tmp) / 4;
 		*((uint32_t *)value) = vddgfx;
 		return 0;
 	case AMDGPU_PP_SENSOR_UVD_VCLK:
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu_helper.c b/drivers/gpu/drm/amd/powerplay/hwmgr/smu_helper.c
index 93a3d022ba47..2aab1b475945 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu_helper.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/smu_helper.c
@@ -25,6 +25,9 @@
 #include "ppatomctrl.h"
 #include "ppsmc.h"
 #include "atom.h"
+#include "ivsrcid/thm/irqsrcs_thm_9_0.h"
+#include "ivsrcid/smuio/irqsrcs_smuio_9_0.h"
+#include "ivsrcid/ivsrcid_vislands30.h"
 
 uint8_t convert_to_vid(uint16_t vddc)
 {
@@ -543,17 +546,17 @@ int phm_irq_process(struct amdgpu_device *adev,
 	uint32_t src_id = entry->src_id;
 
 	if (client_id == AMDGPU_IH_CLIENTID_LEGACY) {
-		if (src_id == 230)
+		if (src_id == VISLANDS30_IV_SRCID_CG_TSS_THERMAL_LOW_TO_HIGH)
 			pr_warn("GPU over temperature range detected on PCIe %d:%d.%d!\n",
 						PCI_BUS_NUM(adev->pdev->devfn),
 						PCI_SLOT(adev->pdev->devfn),
 						PCI_FUNC(adev->pdev->devfn));
-		else if (src_id == 231)
+		else if (src_id == VISLANDS30_IV_SRCID_CG_TSS_THERMAL_HIGH_TO_LOW)
 			pr_warn("GPU under temperature range detected on PCIe %d:%d.%d!\n",
 					PCI_BUS_NUM(adev->pdev->devfn),
 					PCI_SLOT(adev->pdev->devfn),
 					PCI_FUNC(adev->pdev->devfn));
-		else if (src_id == 83)
+		else if (src_id == VISLANDS30_IV_SRCID_GPIO_19)
 			pr_warn("GPU Critical Temperature Fault detected on PCIe %d:%d.%d!\n",
 					PCI_BUS_NUM(adev->pdev->devfn),
 					PCI_SLOT(adev->pdev->devfn),
@@ -594,17 +597,17 @@ int smu9_register_irq_handlers(struct pp_hwmgr *hwmgr)
 
 	amdgpu_irq_add_id((struct amdgpu_device *)(hwmgr->adev),
 			SOC15_IH_CLIENTID_THM,
-			0,
+			THM_9_0__SRCID__THM_DIG_THERM_L2H,
 			source);
 	amdgpu_irq_add_id((struct amdgpu_device *)(hwmgr->adev),
 			SOC15_IH_CLIENTID_THM,
-			1,
+			THM_9_0__SRCID__THM_DIG_THERM_H2L,
 			source);
 
 	/* Register CTF(GPIO_19) interrupt */
 	amdgpu_irq_add_id((struct amdgpu_device *)(hwmgr->adev),
 			SOC15_IH_CLIENTID_ROM_SMUIO,
-			83,
+			SMUIO_9_0__SRCID__SMUIO_GPIO19,
 			source);
 
 	return 0;
@@ -652,7 +655,7 @@ int smu_get_voltage_dependency_table_ppt_v1(
 }
 
 int smu_set_watermarks_for_clocks_ranges(void *wt_table,
-		struct pp_wm_sets_with_clock_ranges_soc15 *wm_with_clock_ranges)
+		struct dm_pp_wm_sets_with_clock_ranges_soc15 *wm_with_clock_ranges)
 {
 	uint32_t i;
 	struct watermarks *table = wt_table;
@@ -660,49 +663,49 @@ int smu_set_watermarks_for_clocks_ranges(void *wt_table,
 	if (!table || !wm_with_clock_ranges)
 		return -EINVAL;
 
-	if (wm_with_clock_ranges->num_wm_sets_dmif > 4 || wm_with_clock_ranges->num_wm_sets_mcif > 4)
+	if (wm_with_clock_ranges->num_wm_dmif_sets > 4 || wm_with_clock_ranges->num_wm_mcif_sets > 4)
 		return -EINVAL;
 
-	for (i = 0; i < wm_with_clock_ranges->num_wm_sets_dmif; i++) {
+	for (i = 0; i < wm_with_clock_ranges->num_wm_dmif_sets; i++) {
 		table->WatermarkRow[1][i].MinClock =
 			cpu_to_le16((uint16_t)
-			(wm_with_clock_ranges->wm_sets_dmif[i].wm_min_dcefclk_in_khz) /
-			100);
+			(wm_with_clock_ranges->wm_dmif_clocks_ranges[i].wm_min_dcfclk_clk_in_khz) /
+			1000);
 		table->WatermarkRow[1][i].MaxClock =
 			cpu_to_le16((uint16_t)
-			(wm_with_clock_ranges->wm_sets_dmif[i].wm_max_dcefclk_in_khz) /
+			(wm_with_clock_ranges->wm_dmif_clocks_ranges[i].wm_max_dcfclk_clk_in_khz) /
 			100);
 		table->WatermarkRow[1][i].MinUclk =
 			cpu_to_le16((uint16_t)
-			(wm_with_clock_ranges->wm_sets_dmif[i].wm_min_memclk_in_khz) /
-			100);
+			(wm_with_clock_ranges->wm_dmif_clocks_ranges[i].wm_min_mem_clk_in_khz) /
+			1000);
 		table->WatermarkRow[1][i].MaxUclk =
 			cpu_to_le16((uint16_t)
-			(wm_with_clock_ranges->wm_sets_dmif[i].wm_max_memclk_in_khz) /
-			100);
+			(wm_with_clock_ranges->wm_dmif_clocks_ranges[i].wm_max_mem_clk_in_khz) /
+			1000);
 		table->WatermarkRow[1][i].WmSetting = (uint8_t)
-				wm_with_clock_ranges->wm_sets_dmif[i].wm_set_id;
+				wm_with_clock_ranges->wm_dmif_clocks_ranges[i].wm_set_id;
 	}
 
-	for (i = 0; i < wm_with_clock_ranges->num_wm_sets_mcif; i++) {
+	for (i = 0; i < wm_with_clock_ranges->num_wm_mcif_sets; i++) {
 		table->WatermarkRow[0][i].MinClock =
 			cpu_to_le16((uint16_t)
-			(wm_with_clock_ranges->wm_sets_mcif[i].wm_min_socclk_in_khz) /
-			100);
+			(wm_with_clock_ranges->wm_mcif_clocks_ranges[i].wm_min_socclk_clk_in_khz) /
+			1000);
 		table->WatermarkRow[0][i].MaxClock =
 			cpu_to_le16((uint16_t)
-			(wm_with_clock_ranges->wm_sets_mcif[i].wm_max_socclk_in_khz) /
-			100);
+			(wm_with_clock_ranges->wm_mcif_clocks_ranges[i].wm_max_socclk_clk_in_khz) /
+			1000);
 		table->WatermarkRow[0][i].MinUclk =
 			cpu_to_le16((uint16_t)
-			(wm_with_clock_ranges->wm_sets_mcif[i].wm_min_memclk_in_khz) /
-			100);
+			(wm_with_clock_ranges->wm_mcif_clocks_ranges[i].wm_min_mem_clk_in_khz) /
+			1000);
 		table->WatermarkRow[0][i].MaxUclk =
 			cpu_to_le16((uint16_t)
-			(wm_with_clock_ranges->wm_sets_mcif[i].wm_max_memclk_in_khz) /
-			100);
+			(wm_with_clock_ranges->wm_mcif_clocks_ranges[i].wm_max_mem_clk_in_khz) /
+			1000);
 		table->WatermarkRow[0][i].WmSetting = (uint8_t)
-				wm_with_clock_ranges->wm_sets_mcif[i].wm_set_id;
+				wm_with_clock_ranges->wm_mcif_clocks_ranges[i].wm_set_id;
 	}
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu_helper.h b/drivers/gpu/drm/amd/powerplay/hwmgr/smu_helper.h
index 916cc01e7652..5454289d5226 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu_helper.h
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/smu_helper.h
@@ -107,7 +107,7 @@ int smu_get_voltage_dependency_table_ppt_v1(
 		struct phm_ppt_v1_clock_voltage_dependency_table *dep_table);
 
 int smu_set_watermarks_for_clocks_ranges(void *wt_table,
-		struct pp_wm_sets_with_clock_ranges_soc15 *wm_with_clock_ranges);
+		struct dm_pp_wm_sets_with_clock_ranges_soc15 *wm_with_clock_ranges);
 
 #define PHM_FIELD_SHIFT(reg, field) reg##__##field##__SHIFT
 #define PHM_FIELD_MASK(reg, field) reg##__##field##_MASK
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_hwmgr.c
index 3b8d36df52e9..fb86c24394ff 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_hwmgr.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_hwmgr.c
@@ -55,12 +55,6 @@
 
 static const uint32_t channel_number[] = {1, 2, 0, 4, 0, 8, 0, 16, 2};
 
-#define MEM_FREQ_LOW_LATENCY        25000
-#define MEM_FREQ_HIGH_LATENCY       80000
-#define MEM_LATENCY_HIGH            245
-#define MEM_LATENCY_LOW             35
-#define MEM_LATENCY_ERR             0xFFFF
-
 #define mmDF_CS_AON0_DramBaseAddress0                                                                  0x0044
 #define mmDF_CS_AON0_DramBaseAddress0_BASE_IDX                                                         0
 
@@ -295,7 +289,15 @@ static int vega10_odn_initial_default_setting(struct pp_hwmgr *hwmgr)
 	struct phm_ppt_v1_voltage_lookup_table *vddc_lookup_table;
 	struct phm_ppt_v1_clock_voltage_dependency_table *dep_table[3];
 	struct phm_ppt_v1_clock_voltage_dependency_table *od_table[3];
+	struct pp_atomfwctrl_avfs_parameters avfs_params = {0};
 	uint32_t i;
+	int result;
+
+	result = pp_atomfwctrl_get_avfs_information(hwmgr, &avfs_params);
+	if (!result) {
+		data->odn_dpm_table.max_vddc = avfs_params.ulMaxVddc;
+		data->odn_dpm_table.min_vddc = avfs_params.ulMinVddc;
+	}
 
 	od_lookup_table = &odn_table->vddc_lookup_table;
 	vddc_lookup_table = table_info->vddc_lookup_table;
@@ -2078,9 +2080,6 @@ static int vega10_populate_avfs_parameters(struct pp_hwmgr *hwmgr)
 	if (data->smu_features[GNLD_AVFS].supported) {
 		result = pp_atomfwctrl_get_avfs_information(hwmgr, &avfs_params);
 		if (!result) {
-			data->odn_dpm_table.max_vddc = avfs_params.ulMaxVddc;
-			data->odn_dpm_table.min_vddc = avfs_params.ulMinVddc;
-
 			pp_table->MinVoltageVid = (uint8_t)
 					convert_to_vid((uint16_t)(avfs_params.ulMinVddc));
 			pp_table->MaxVoltageVid = (uint8_t)
@@ -2902,11 +2901,6 @@ static int vega10_enable_dpm_tasks(struct pp_hwmgr *hwmgr)
 
 	vega10_enable_disable_PCC_limit_feature(hwmgr, true);
 
-	if ((hwmgr->smu_version == 0x001c2c00) ||
-			(hwmgr->smu_version == 0x001c2d00))
-		smum_send_msg_to_smc_with_parameter(hwmgr,
-				PPSMC_MSG_UpdatePkgPwrPidAlpha, 1);
-
 	smum_send_msg_to_smc_with_parameter(hwmgr,
 		PPSMC_MSG_ConfigureTelemetry, data->config_telemetry);
 
@@ -3223,7 +3217,7 @@ static int vega10_apply_state_adjust_rules(struct pp_hwmgr *hwmgr,
 		/* Find the lowest MCLK frequency that is within
 		 * the tolerable latency defined in DAL
 		 */
-		latency = 0;
+		latency = hwmgr->display_config->dce_tolerable_mclk_in_active_latency;
 		for (i = 0; i < data->mclk_latency_table.count; i++) {
 			if ((data->mclk_latency_table.entries[i].latency <= latency) &&
 				(data->mclk_latency_table.entries[i].frequency >=
@@ -3265,10 +3259,25 @@ static int vega10_populate_and_upload_sclk_mclk_dpm_levels(
 {
 	int result = 0;
 	struct vega10_hwmgr *data = hwmgr->backend;
+	struct vega10_dpm_table *dpm_table = &data->dpm_table;
+	struct vega10_odn_dpm_table *odn_table = &data->odn_dpm_table;
+	struct vega10_odn_clock_voltage_dependency_table *odn_clk_table = &odn_table->vdd_dep_on_sclk;
+	int count;
 
 	if (!data->need_update_dpm_table)
 		return 0;
 
+	if (hwmgr->od_enabled && data->need_update_dpm_table & DPMTABLE_OD_UPDATE_SCLK) {
+		for (count = 0; count < dpm_table->gfx_table.count; count++)
+			dpm_table->gfx_table.dpm_levels[count].value = odn_clk_table->entries[count].clk;
+	}
+
+	odn_clk_table = &odn_table->vdd_dep_on_mclk;
+	if (hwmgr->od_enabled && data->need_update_dpm_table & DPMTABLE_OD_UPDATE_MCLK) {
+		for (count = 0; count < dpm_table->mem_table.count; count++)
+			dpm_table->mem_table.dpm_levels[count].value = odn_clk_table->entries[count].clk;
+	}
+
 	if (data->need_update_dpm_table &
 			(DPMTABLE_OD_UPDATE_SCLK + DPMTABLE_UPDATE_SCLK + DPMTABLE_UPDATE_SOCCLK)) {
 		result = vega10_populate_all_graphic_levels(hwmgr);
@@ -3716,7 +3725,7 @@ static void vega10_notify_smc_display_change(struct pp_hwmgr *hwmgr,
 {
 	smum_send_msg_to_smc_with_parameter(hwmgr,
 			PPSMC_MSG_SetUclkFastSwitch,
-			has_disp ? 0 : 1);
+			has_disp ? 1 : 0);
 }
 
 int vega10_display_clock_voltage_request(struct pp_hwmgr *hwmgr,
@@ -3791,7 +3800,9 @@ static int vega10_notify_smc_display_config_after_ps_adjustment(
 	uint32_t i;
 	struct pp_display_clock_request clock_req;
 
-	if (hwmgr->display_config->num_display > 1)
+	if ((hwmgr->display_config->num_display > 1) &&
+	     !hwmgr->display_config->multi_monitor_in_sync &&
+	     !hwmgr->display_config->nb_pstate_switch_disable)
 		vega10_notify_smc_display_change(hwmgr, false);
 	else
 		vega10_notify_smc_display_change(hwmgr, true);
@@ -3807,7 +3818,7 @@ static int vega10_notify_smc_display_config_after_ps_adjustment(
 
 	if (i < dpm_table->count) {
 		clock_req.clock_type = amd_pp_dcef_clock;
-		clock_req.clock_freq_in_khz = dpm_table->dpm_levels[i].value;
+		clock_req.clock_freq_in_khz = dpm_table->dpm_levels[i].value * 10;
 		if (!vega10_display_clock_voltage_request(hwmgr, &clock_req)) {
 			smum_send_msg_to_smc_with_parameter(
 					hwmgr, PPSMC_MSG_SetMinDeepSleepDcefclk,
@@ -4064,28 +4075,17 @@ static void vega10_get_sclks(struct pp_hwmgr *hwmgr,
 			table_info->vdd_dep_on_sclk;
 	uint32_t i;
 
+	clocks->num_levels = 0;
 	for (i = 0; i < dep_table->count; i++) {
 		if (dep_table->entries[i].clk) {
 			clocks->data[clocks->num_levels].clocks_in_khz =
-					dep_table->entries[i].clk;
+					dep_table->entries[i].clk * 10;
 			clocks->num_levels++;
 		}
 	}
 
 }
 
-static uint32_t vega10_get_mem_latency(struct pp_hwmgr *hwmgr,
-		uint32_t clock)
-{
-	if (clock >= MEM_FREQ_LOW_LATENCY &&
-			clock < MEM_FREQ_HIGH_LATENCY)
-		return MEM_LATENCY_HIGH;
-	else if (clock >= MEM_FREQ_HIGH_LATENCY)
-		return MEM_LATENCY_LOW;
-	else
-		return MEM_LATENCY_ERR;
-}
-
 static void vega10_get_memclocks(struct pp_hwmgr *hwmgr,
 		struct pp_clock_levels_with_latency *clocks)
 {
@@ -4094,26 +4094,22 @@ static void vega10_get_memclocks(struct pp_hwmgr *hwmgr,
 	struct phm_ppt_v1_clock_voltage_dependency_table *dep_table =
 			table_info->vdd_dep_on_mclk;
 	struct vega10_hwmgr *data = hwmgr->backend;
+	uint32_t j = 0;
 	uint32_t i;
 
-	clocks->num_levels = 0;
-	data->mclk_latency_table.count = 0;
-
 	for (i = 0; i < dep_table->count; i++) {
 		if (dep_table->entries[i].clk) {
-			clocks->data[clocks->num_levels].clocks_in_khz =
-			data->mclk_latency_table.entries
-			[data->mclk_latency_table.count].frequency =
-					dep_table->entries[i].clk;
-			clocks->data[clocks->num_levels].latency_in_us =
-			data->mclk_latency_table.entries
-			[data->mclk_latency_table.count].latency =
-					vega10_get_mem_latency(hwmgr,
-						dep_table->entries[i].clk);
-			clocks->num_levels++;
-			data->mclk_latency_table.count++;
+
+			clocks->data[j].clocks_in_khz =
+						dep_table->entries[i].clk * 10;
+			data->mclk_latency_table.entries[j].frequency =
+							dep_table->entries[i].clk;
+			clocks->data[j].latency_in_us =
+				data->mclk_latency_table.entries[j].latency = 25;
+			j++;
 		}
 	}
+	clocks->num_levels = data->mclk_latency_table.count = j;
 }
 
 static void vega10_get_dcefclocks(struct pp_hwmgr *hwmgr,
@@ -4126,7 +4122,7 @@ static void vega10_get_dcefclocks(struct pp_hwmgr *hwmgr,
 	uint32_t i;
 
 	for (i = 0; i < dep_table->count; i++) {
-		clocks->data[i].clocks_in_khz = dep_table->entries[i].clk;
+		clocks->data[i].clocks_in_khz = dep_table->entries[i].clk * 10;
 		clocks->data[i].latency_in_us = 0;
 		clocks->num_levels++;
 	}
@@ -4142,7 +4138,7 @@ static void vega10_get_socclocks(struct pp_hwmgr *hwmgr,
 	uint32_t i;
 
 	for (i = 0; i < dep_table->count; i++) {
-		clocks->data[i].clocks_in_khz = dep_table->entries[i].clk;
+		clocks->data[i].clocks_in_khz = dep_table->entries[i].clk * 10;
 		clocks->data[i].latency_in_us = 0;
 		clocks->num_levels++;
 	}
@@ -4202,7 +4198,7 @@ static int vega10_get_clock_by_type_with_voltage(struct pp_hwmgr *hwmgr,
 	}
 
 	for (i = 0; i < dep_table->count; i++) {
-		clocks->data[i].clocks_in_khz = dep_table->entries[i].clk;
+		clocks->data[i].clocks_in_khz = dep_table->entries[i].clk  * 10;
 		clocks->data[i].voltage_in_mv = (uint32_t)(table_info->vddc_lookup_table->
 				entries[dep_table->entries[i].vddInd].us_vdd);
 		clocks->num_levels++;
@@ -4215,9 +4211,10 @@ static int vega10_get_clock_by_type_with_voltage(struct pp_hwmgr *hwmgr,
 }
 
 static int vega10_set_watermarks_for_clocks_ranges(struct pp_hwmgr *hwmgr,
-		struct pp_wm_sets_with_clock_ranges_soc15 *wm_with_clock_ranges)
+							void *clock_range)
 {
 	struct vega10_hwmgr *data = hwmgr->backend;
+	struct dm_pp_wm_sets_with_clock_ranges_soc15 *wm_with_clock_ranges = clock_range;
 	Watermarks_t *table = &(data->smc_state_table.water_marks_table);
 	int result = 0;
 
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_hwmgr.c
index 782e2098824d..0789d64246ca 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_hwmgr.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_hwmgr.c
@@ -81,6 +81,7 @@ static void vega12_set_default_registry_data(struct pp_hwmgr *hwmgr)
 
 	data->registry_data.disallowed_features = 0x0;
 	data->registry_data.od_state_in_dc_support = 0;
+	data->registry_data.thermal_support = 1;
 	data->registry_data.skip_baco_hardware = 0;
 
 	data->registry_data.log_avfs_param = 0;
@@ -422,6 +423,11 @@ static int vega12_hwmgr_backend_init(struct pp_hwmgr *hwmgr)
 			hwmgr->thermal_controller.advanceFanControlParameters.usFanPWMMinLimit *
 			hwmgr->thermal_controller.fanInfo.ulMaxRPM / 100;
 
+	if (hwmgr->feature_mask & PP_GFXOFF_MASK)
+		data->gfxoff_controlled_by_driver = true;
+	else
+		data->gfxoff_controlled_by_driver = false;
+
 	return result;
 }
 
@@ -453,43 +459,36 @@ static int vega12_setup_asic_task(struct pp_hwmgr *hwmgr)
  */
 static void vega12_init_dpm_state(struct vega12_dpm_state *dpm_state)
 {
-	dpm_state->soft_min_level = 0xff;
-	dpm_state->soft_max_level = 0xff;
-	dpm_state->hard_min_level = 0xff;
-	dpm_state->hard_max_level = 0xff;
+	dpm_state->soft_min_level = 0x0;
+	dpm_state->soft_max_level = 0xffff;
+	dpm_state->hard_min_level = 0x0;
+	dpm_state->hard_max_level = 0xffff;
 }
 
-static int vega12_get_number_dpm_level(struct pp_hwmgr *hwmgr,
-		PPCLK_e clkID, uint32_t *num_dpm_level)
+static int vega12_get_number_of_dpm_level(struct pp_hwmgr *hwmgr,
+		PPCLK_e clk_id, uint32_t *num_of_levels)
 {
-	int result;
-	/*
-	 * SMU expects the Clock ID to be in the top 16 bits.
-	 * Lower 16 bits specify the level however 0xFF is a
-	 * special argument the returns the total number of levels
-	 */
-	PP_ASSERT_WITH_CODE(smum_send_msg_to_smc_with_parameter(hwmgr,
-		PPSMC_MSG_GetDpmFreqByIndex, (clkID << 16 | 0xFF)) == 0,
-		"[GetNumberDpmLevel] Failed to get DPM levels from SMU for CLKID!",
-		return -EINVAL);
-
-	result = vega12_read_arg_from_smc(hwmgr, num_dpm_level);
+	int ret = 0;
 
-	PP_ASSERT_WITH_CODE(*num_dpm_level < MAX_REGULAR_DPM_NUMBER,
-		"[GetNumberDPMLevel] Number of DPM levels is greater than limit",
-		return -EINVAL);
+	ret = smum_send_msg_to_smc_with_parameter(hwmgr,
+			PPSMC_MSG_GetDpmFreqByIndex,
+			(clk_id << 16 | 0xFF));
+	PP_ASSERT_WITH_CODE(!ret,
+			"[GetNumOfDpmLevel] failed to get dpm levels!",
+			return ret);
 
-	PP_ASSERT_WITH_CODE(*num_dpm_level != 0,
-		"[GetNumberDPMLevel] Number of CLK Levels is zero!",
-		return -EINVAL);
+	*num_of_levels = smum_get_argument(hwmgr);
+	PP_ASSERT_WITH_CODE(*num_of_levels > 0,
+			"[GetNumOfDpmLevel] number of clk levels is invalid!",
+			return -EINVAL);
 
-	return result;
+	return ret;
 }
 
 static int vega12_get_dpm_frequency_by_index(struct pp_hwmgr *hwmgr,
 		PPCLK_e clkID, uint32_t index, uint32_t *clock)
 {
-	int result;
+	int result = 0;
 
 	/*
 	 *SMU expects the Clock ID to be in the top 16 bits.
@@ -500,15 +499,36 @@ static int vega12_get_dpm_frequency_by_index(struct pp_hwmgr *hwmgr,
 		"[GetDpmFrequencyByIndex] Failed to get dpm frequency from SMU!",
 		return -EINVAL);
 
-	result = vega12_read_arg_from_smc(hwmgr, clock);
-
-	PP_ASSERT_WITH_CODE(*clock != 0,
-		"[GetDPMFrequencyByIndex] Failed to get dpm frequency by index.!",
-		return -EINVAL);
+	*clock = smum_get_argument(hwmgr);
 
 	return result;
 }
 
+static int vega12_setup_single_dpm_table(struct pp_hwmgr *hwmgr,
+		struct vega12_single_dpm_table *dpm_table, PPCLK_e clk_id)
+{
+	int ret = 0;
+	uint32_t i, num_of_levels, clk;
+
+	ret = vega12_get_number_of_dpm_level(hwmgr, clk_id, &num_of_levels);
+	PP_ASSERT_WITH_CODE(!ret,
+			"[SetupSingleDpmTable] failed to get clk levels!",
+			return ret);
+
+	dpm_table->count = num_of_levels;
+
+	for (i = 0; i < num_of_levels; i++) {
+		ret = vega12_get_dpm_frequency_by_index(hwmgr, clk_id, i, &clk);
+		PP_ASSERT_WITH_CODE(!ret,
+			"[SetupSingleDpmTable] failed to get clk of specific level!",
+			return ret);
+		dpm_table->dpm_levels[i].value = clk;
+		dpm_table->dpm_levels[i].enabled = true;
+	}
+
+	return ret;
+}
+
 /*
  * This function is to initialize all DPM state tables
  * for SMU based on the dependency table.
@@ -519,224 +539,136 @@ static int vega12_get_dpm_frequency_by_index(struct pp_hwmgr *hwmgr,
  */
 static int vega12_setup_default_dpm_tables(struct pp_hwmgr *hwmgr)
 {
-	uint32_t num_levels, i, clock;
 
 	struct vega12_hwmgr *data =
 			(struct vega12_hwmgr *)(hwmgr->backend);
-
 	struct vega12_single_dpm_table *dpm_table;
+	int ret = 0;
 
 	memset(&data->dpm_table, 0, sizeof(data->dpm_table));
 
-	/* Initialize Sclk DPM and SOC DPM table based on allow Sclk values */
+	/* socclk */
 	dpm_table = &(data->dpm_table.soc_table);
-
-	PP_ASSERT_WITH_CODE(vega12_get_number_dpm_level(hwmgr, PPCLK_SOCCLK,
-		&num_levels) == 0,
-		"[SetupDefaultDPMTables] Failed to get DPM levels from SMU for SOCCLK!",
-		return -EINVAL);
-
-	dpm_table->count = num_levels;
-
-	for (i = 0; i < num_levels; i++) {
-		PP_ASSERT_WITH_CODE(vega12_get_dpm_frequency_by_index(hwmgr,
-			PPCLK_SOCCLK, i, &clock) == 0,
-			"[SetupDefaultDPMTables] Failed to get DPM levels from SMU for SOCCLK!",
-			return -EINVAL);
-
-		dpm_table->dpm_levels[i].value = clock;
-		dpm_table->dpm_levels[i].enabled = true;
+	if (data->smu_features[GNLD_DPM_SOCCLK].enabled) {
+		ret = vega12_setup_single_dpm_table(hwmgr, dpm_table, PPCLK_SOCCLK);
+		PP_ASSERT_WITH_CODE(!ret,
+				"[SetupDefaultDpmTable] failed to get socclk dpm levels!",
+				return ret);
+	} else {
+		dpm_table->count = 1;
+		dpm_table->dpm_levels[0].value = data->vbios_boot_state.soc_clock / 100;
 	}
-
 	vega12_init_dpm_state(&(dpm_table->dpm_state));
 
+	/* gfxclk */
 	dpm_table = &(data->dpm_table.gfx_table);
-
-	PP_ASSERT_WITH_CODE(vega12_get_number_dpm_level(hwmgr, PPCLK_GFXCLK,
-		&num_levels) == 0,
-		"[SetupDefaultDPMTables] Failed to get DPM levels from SMU for GFXCLK!",
-		return -EINVAL);
-
-	dpm_table->count = num_levels;
-	for (i = 0; i < num_levels; i++) {
-		PP_ASSERT_WITH_CODE(vega12_get_dpm_frequency_by_index(hwmgr,
-			PPCLK_GFXCLK, i, &clock) == 0,
-			"[SetupDefaultDPMTables] Failed to get DPM levels from SMU for GFXCLK!",
-			return -EINVAL);
-
-		dpm_table->dpm_levels[i].value = clock;
-		dpm_table->dpm_levels[i].enabled = true;
+	if (data->smu_features[GNLD_DPM_GFXCLK].enabled) {
+		ret = vega12_setup_single_dpm_table(hwmgr, dpm_table, PPCLK_GFXCLK);
+		PP_ASSERT_WITH_CODE(!ret,
+				"[SetupDefaultDpmTable] failed to get gfxclk dpm levels!",
+				return ret);
+	} else {
+		dpm_table->count = 1;
+		dpm_table->dpm_levels[0].value = data->vbios_boot_state.gfx_clock / 100;
 	}
-
 	vega12_init_dpm_state(&(dpm_table->dpm_state));
-	/* Initialize Mclk DPM table based on allow Mclk values */
-	dpm_table = &(data->dpm_table.mem_table);
 
-	PP_ASSERT_WITH_CODE(vega12_get_number_dpm_level(hwmgr, PPCLK_UCLK,
-		&num_levels) == 0,
-		"[SetupDefaultDPMTables] Failed to get DPM levels from SMU for UCLK!",
-		return -EINVAL);
-
-	dpm_table->count = num_levels;
-
-	for (i = 0; i < num_levels; i++) {
-		PP_ASSERT_WITH_CODE(vega12_get_dpm_frequency_by_index(hwmgr,
-			PPCLK_UCLK, i, &clock) == 0,
-			"[SetupDefaultDPMTables] Failed to get DPM levels from SMU for UCLK!",
-			return -EINVAL);
-
-		dpm_table->dpm_levels[i].value = clock;
-		dpm_table->dpm_levels[i].enabled = true;
+	/* memclk */
+	dpm_table = &(data->dpm_table.mem_table);
+	if (data->smu_features[GNLD_DPM_UCLK].enabled) {
+		ret = vega12_setup_single_dpm_table(hwmgr, dpm_table, PPCLK_UCLK);
+		PP_ASSERT_WITH_CODE(!ret,
+				"[SetupDefaultDpmTable] failed to get memclk dpm levels!",
+				return ret);
+	} else {
+		dpm_table->count = 1;
+		dpm_table->dpm_levels[0].value = data->vbios_boot_state.mem_clock / 100;
 	}
-
 	vega12_init_dpm_state(&(dpm_table->dpm_state));
 
+	/* eclk */
 	dpm_table = &(data->dpm_table.eclk_table);
-
-	PP_ASSERT_WITH_CODE(vega12_get_number_dpm_level(hwmgr, PPCLK_ECLK,
-		&num_levels) == 0,
-		"[SetupDefaultDPMTables] Failed to get DPM levels from SMU for ECLK!",
-		return -EINVAL);
-
-	dpm_table->count = num_levels;
-
-	for (i = 0; i < num_levels; i++) {
-		PP_ASSERT_WITH_CODE(vega12_get_dpm_frequency_by_index(hwmgr,
-		PPCLK_ECLK, i, &clock) == 0,
-		"[SetupDefaultDPMTables] Failed to get DPM levels from SMU for ECLK!",
-		return -EINVAL);
-
-		dpm_table->dpm_levels[i].value = clock;
-		dpm_table->dpm_levels[i].enabled = true;
+	if (data->smu_features[GNLD_DPM_VCE].enabled) {
+		ret = vega12_setup_single_dpm_table(hwmgr, dpm_table, PPCLK_ECLK);
+		PP_ASSERT_WITH_CODE(!ret,
+				"[SetupDefaultDpmTable] failed to get eclk dpm levels!",
+				return ret);
+	} else {
+		dpm_table->count = 1;
+		dpm_table->dpm_levels[0].value = data->vbios_boot_state.eclock / 100;
 	}
-
 	vega12_init_dpm_state(&(dpm_table->dpm_state));
 
+	/* vclk */
 	dpm_table = &(data->dpm_table.vclk_table);
-
-	PP_ASSERT_WITH_CODE(vega12_get_number_dpm_level(hwmgr, PPCLK_VCLK,
-		&num_levels) == 0,
-		"[SetupDefaultDPMTables] Failed to get DPM levels from SMU for VCLK!",
-		return -EINVAL);
-
-	dpm_table->count = num_levels;
-
-	for (i = 0; i < num_levels; i++) {
-		PP_ASSERT_WITH_CODE(vega12_get_dpm_frequency_by_index(hwmgr,
-			PPCLK_VCLK, i, &clock) == 0,
-			"[SetupDefaultDPMTables] Failed to get DPM levels from SMU for VCLK!",
-			return -EINVAL);
-
-		dpm_table->dpm_levels[i].value = clock;
-		dpm_table->dpm_levels[i].enabled = true;
+	if (data->smu_features[GNLD_DPM_UVD].enabled) {
+		ret = vega12_setup_single_dpm_table(hwmgr, dpm_table, PPCLK_VCLK);
+		PP_ASSERT_WITH_CODE(!ret,
+				"[SetupDefaultDpmTable] failed to get vclk dpm levels!",
+				return ret);
+	} else {
+		dpm_table->count = 1;
+		dpm_table->dpm_levels[0].value = data->vbios_boot_state.vclock / 100;
 	}
-
 	vega12_init_dpm_state(&(dpm_table->dpm_state));
 
+	/* dclk */
 	dpm_table = &(data->dpm_table.dclk_table);
-
-	PP_ASSERT_WITH_CODE(vega12_get_number_dpm_level(hwmgr, PPCLK_DCLK,
-		&num_levels) == 0,
-		"[SetupDefaultDPMTables] Failed to get DPM levels from SMU for DCLK!",
-		return -EINVAL);
-
-	dpm_table->count = num_levels;
-
-	for (i = 0; i < num_levels; i++) {
-		PP_ASSERT_WITH_CODE(vega12_get_dpm_frequency_by_index(hwmgr,
-			PPCLK_DCLK, i, &clock) == 0,
-		"[SetupDefaultDPMTables] Failed to get DPM levels from SMU for DCLK!",
-		return -EINVAL);
-
-		dpm_table->dpm_levels[i].value = clock;
-		dpm_table->dpm_levels[i].enabled = true;
+	if (data->smu_features[GNLD_DPM_UVD].enabled) {
+		ret = vega12_setup_single_dpm_table(hwmgr, dpm_table, PPCLK_DCLK);
+		PP_ASSERT_WITH_CODE(!ret,
+				"[SetupDefaultDpmTable] failed to get dclk dpm levels!",
+				return ret);
+	} else {
+		dpm_table->count = 1;
+		dpm_table->dpm_levels[0].value = data->vbios_boot_state.dclock / 100;
 	}
-
 	vega12_init_dpm_state(&(dpm_table->dpm_state));
 
-	/* Assume there is no headless Vega12 for now */
+	/* dcefclk */
 	dpm_table = &(data->dpm_table.dcef_table);
-
-	PP_ASSERT_WITH_CODE(vega12_get_number_dpm_level(hwmgr,
-		PPCLK_DCEFCLK, &num_levels) == 0,
-		"[SetupDefaultDPMTables] Failed to get DPM levels from SMU for DCEFCLK!",
-		return -EINVAL);
-
-	dpm_table->count = num_levels;
-
-	for (i = 0; i < num_levels; i++) {
-		PP_ASSERT_WITH_CODE(vega12_get_dpm_frequency_by_index(hwmgr,
-			PPCLK_DCEFCLK, i, &clock) == 0,
-			"[SetupDefaultDPMTables] Failed to get DPM levels from SMU for DCEFCLK!",
-			return -EINVAL);
-
-		dpm_table->dpm_levels[i].value = clock;
-		dpm_table->dpm_levels[i].enabled = true;
+	if (data->smu_features[GNLD_DPM_DCEFCLK].enabled) {
+		ret = vega12_setup_single_dpm_table(hwmgr, dpm_table, PPCLK_DCEFCLK);
+		PP_ASSERT_WITH_CODE(!ret,
+				"[SetupDefaultDpmTable] failed to get dcefclk dpm levels!",
+				return ret);
+	} else {
+		dpm_table->count = 1;
+		dpm_table->dpm_levels[0].value = data->vbios_boot_state.dcef_clock / 100;
 	}
-
 	vega12_init_dpm_state(&(dpm_table->dpm_state));
 
+	/* pixclk */
 	dpm_table = &(data->dpm_table.pixel_table);
-
-	PP_ASSERT_WITH_CODE(vega12_get_number_dpm_level(hwmgr,
-		PPCLK_PIXCLK, &num_levels) == 0,
-		"[SetupDefaultDPMTables] Failed to get DPM levels from SMU for PIXCLK!",
-		return -EINVAL);
-
-	dpm_table->count = num_levels;
-
-	for (i = 0; i < num_levels; i++) {
-		PP_ASSERT_WITH_CODE(vega12_get_dpm_frequency_by_index(hwmgr,
-			PPCLK_PIXCLK, i, &clock) == 0,
-			"[SetupDefaultDPMTables] Failed to get DPM levels from SMU for PIXCLK!",
-			return -EINVAL);
-
-		dpm_table->dpm_levels[i].value = clock;
-		dpm_table->dpm_levels[i].enabled = true;
-	}
-
+	if (data->smu_features[GNLD_DPM_DCEFCLK].enabled) {
+		ret = vega12_setup_single_dpm_table(hwmgr, dpm_table, PPCLK_PIXCLK);
+		PP_ASSERT_WITH_CODE(!ret,
+				"[SetupDefaultDpmTable] failed to get pixclk dpm levels!",
+				return ret);
+	} else
+		dpm_table->count = 0;
 	vega12_init_dpm_state(&(dpm_table->dpm_state));
 
+	/* dispclk */
 	dpm_table = &(data->dpm_table.display_table);
-
-	PP_ASSERT_WITH_CODE(vega12_get_number_dpm_level(hwmgr,
-		PPCLK_DISPCLK, &num_levels) == 0,
-		"[SetupDefaultDPMTables] Failed to get DPM levels from SMU for DISPCLK!",
-		return -EINVAL);
-
-	dpm_table->count = num_levels;
-
-	for (i = 0; i < num_levels; i++) {
-		PP_ASSERT_WITH_CODE(vega12_get_dpm_frequency_by_index(hwmgr,
-			PPCLK_DISPCLK, i, &clock) == 0,
-			"[SetupDefaultDPMTables] Failed to get DPM levels from SMU for DISPCLK!",
-			return -EINVAL);
-
-		dpm_table->dpm_levels[i].value = clock;
-		dpm_table->dpm_levels[i].enabled = true;
-	}
-
+	if (data->smu_features[GNLD_DPM_DCEFCLK].enabled) {
+		ret = vega12_setup_single_dpm_table(hwmgr, dpm_table, PPCLK_DISPCLK);
+		PP_ASSERT_WITH_CODE(!ret,
+				"[SetupDefaultDpmTable] failed to get dispclk dpm levels!",
+				return ret);
+	} else
+		dpm_table->count = 0;
 	vega12_init_dpm_state(&(dpm_table->dpm_state));
 
+	/* phyclk */
 	dpm_table = &(data->dpm_table.phy_table);
-
-	PP_ASSERT_WITH_CODE(vega12_get_number_dpm_level(hwmgr,
-		PPCLK_PHYCLK, &num_levels) == 0,
-		"[SetupDefaultDPMTables] Failed to get DPM levels from SMU for PHYCLK!",
-		return -EINVAL);
-
-	dpm_table->count = num_levels;
-
-	for (i = 0; i < num_levels; i++) {
-		PP_ASSERT_WITH_CODE(vega12_get_dpm_frequency_by_index(hwmgr,
-			PPCLK_PHYCLK, i, &clock) == 0,
-			"[SetupDefaultDPMTables] Failed to get DPM levels from SMU for PHYCLK!",
-			return -EINVAL);
-
-		dpm_table->dpm_levels[i].value = clock;
-		dpm_table->dpm_levels[i].enabled = true;
-	}
-
+	if (data->smu_features[GNLD_DPM_DCEFCLK].enabled) {
+		ret = vega12_setup_single_dpm_table(hwmgr, dpm_table, PPCLK_PHYCLK);
+		PP_ASSERT_WITH_CODE(!ret,
+				"[SetupDefaultDpmTable] failed to get phyclk dpm levels!",
+				return ret);
+	} else
+		dpm_table->count = 0;
 	vega12_init_dpm_state(&(dpm_table->dpm_state));
 
 	/* save a copy of the default DPM table */
@@ -803,6 +735,9 @@ static int vega12_init_smc_table(struct pp_hwmgr *hwmgr)
 		data->vbios_boot_state.soc_clock = boot_up_values.ulSocClk;
 		data->vbios_boot_state.dcef_clock = boot_up_values.ulDCEFClk;
 		data->vbios_boot_state.uc_cooling_id = boot_up_values.ucCoolingID;
+		data->vbios_boot_state.eclock = boot_up_values.ulEClk;
+		data->vbios_boot_state.dclock = boot_up_values.ulDClk;
+		data->vbios_boot_state.vclock = boot_up_values.ulVClk;
 		smum_send_msg_to_smc_with_parameter(hwmgr,
 				PPSMC_MSG_SetMinDeepSleepDcefclk,
 			(uint32_t)(data->vbios_boot_state.dcef_clock / 100));
@@ -844,6 +779,21 @@ static int vega12_set_allowed_featuresmask(struct pp_hwmgr *hwmgr)
 	return 0;
 }
 
+static void vega12_init_powergate_state(struct pp_hwmgr *hwmgr)
+{
+	struct vega12_hwmgr *data =
+			(struct vega12_hwmgr *)(hwmgr->backend);
+
+	data->uvd_power_gated = true;
+	data->vce_power_gated = true;
+
+	if (data->smu_features[GNLD_DPM_UVD].enabled)
+		data->uvd_power_gated = false;
+
+	if (data->smu_features[GNLD_DPM_VCE].enabled)
+		data->vce_power_gated = false;
+}
+
 static int vega12_enable_all_smu_features(struct pp_hwmgr *hwmgr)
 {
 	struct vega12_hwmgr *data =
@@ -862,12 +812,11 @@ static int vega12_enable_all_smu_features(struct pp_hwmgr *hwmgr)
 			enabled = (features_enabled & data->smu_features[i].smu_feature_bitmap) ? true : false;
 			data->smu_features[i].enabled = enabled;
 			data->smu_features[i].supported = enabled;
-			PP_ASSERT(
-				!data->smu_features[i].allowed || enabled,
-				"[EnableAllSMUFeatures] Enabled feature is different from allowed, expected disabled!");
 		}
 	}
 
+	vega12_init_powergate_state(hwmgr);
+
 	return 0;
 }
 
@@ -923,6 +872,48 @@ static int vega12_power_control_set_level(struct pp_hwmgr *hwmgr)
 	return result;
 }
 
+static int vega12_get_all_clock_ranges_helper(struct pp_hwmgr *hwmgr,
+		PPCLK_e clkid, struct vega12_clock_range *clock)
+{
+	/* AC Max */
+	PP_ASSERT_WITH_CODE(
+		smum_send_msg_to_smc_with_parameter(hwmgr, PPSMC_MSG_GetMaxDpmFreq, (clkid << 16)) == 0,
+		"[GetClockRanges] Failed to get max ac clock from SMC!",
+		return -EINVAL);
+	clock->ACMax = smum_get_argument(hwmgr);
+
+	/* AC Min */
+	PP_ASSERT_WITH_CODE(
+		smum_send_msg_to_smc_with_parameter(hwmgr, PPSMC_MSG_GetMinDpmFreq, (clkid << 16)) == 0,
+		"[GetClockRanges] Failed to get min ac clock from SMC!",
+		return -EINVAL);
+	clock->ACMin = smum_get_argument(hwmgr);
+
+	/* DC Max */
+	PP_ASSERT_WITH_CODE(
+		smum_send_msg_to_smc_with_parameter(hwmgr, PPSMC_MSG_GetDcModeMaxDpmFreq, (clkid << 16)) == 0,
+		"[GetClockRanges] Failed to get max dc clock from SMC!",
+		return -EINVAL);
+	clock->DCMax = smum_get_argument(hwmgr);
+
+	return 0;
+}
+
+static int vega12_get_all_clock_ranges(struct pp_hwmgr *hwmgr)
+{
+	struct vega12_hwmgr *data =
+			(struct vega12_hwmgr *)(hwmgr->backend);
+	uint32_t i;
+
+	for (i = 0; i < PPCLK_COUNT; i++)
+		PP_ASSERT_WITH_CODE(!vega12_get_all_clock_ranges_helper(hwmgr,
+					i, &(data->clk_range[i])),
+				"Failed to get clk range from SMC!",
+				return -EINVAL);
+
+	return 0;
+}
+
 static int vega12_enable_dpm_tasks(struct pp_hwmgr *hwmgr)
 {
 	int tmp_result, result = 0;
@@ -950,6 +941,11 @@ static int vega12_enable_dpm_tasks(struct pp_hwmgr *hwmgr)
 			"Failed to power control set level!",
 			result = tmp_result);
 
+	result = vega12_get_all_clock_ranges(hwmgr);
+	PP_ASSERT_WITH_CODE(!result,
+			"Failed to get all clock ranges!",
+			return result);
+
 	result = vega12_odn_initialize_default_settings(hwmgr);
 	PP_ASSERT_WITH_CODE(!result,
 			"Failed to power control set level!",
@@ -978,76 +974,172 @@ static uint32_t vega12_find_lowest_dpm_level(
 			break;
 	}
 
+	if (i >= table->count) {
+		i = 0;
+		table->dpm_levels[i].enabled = true;
+	}
+
 	return i;
 }
 
 static uint32_t vega12_find_highest_dpm_level(
 		struct vega12_single_dpm_table *table)
 {
-	uint32_t i = 0;
+	int32_t i = 0;
+	PP_ASSERT_WITH_CODE(table->count <= MAX_REGULAR_DPM_NUMBER,
+			"[FindHighestDPMLevel] DPM Table has too many entries!",
+			return MAX_REGULAR_DPM_NUMBER - 1);
 
-	if (table->count <= MAX_REGULAR_DPM_NUMBER) {
-		for (i = table->count; i > 0; i--) {
-			if (table->dpm_levels[i - 1].enabled)
-				return i - 1;
-		}
-	} else {
-		pr_info("DPM Table Has Too Many Entries!");
-		return MAX_REGULAR_DPM_NUMBER - 1;
+	for (i = table->count - 1; i >= 0; i--) {
+		if (table->dpm_levels[i].enabled)
+			break;
 	}
 
-	return i;
+	if (i < 0) {
+		i = 0;
+		table->dpm_levels[i].enabled = true;
+	}
+
+	return (uint32_t)i;
 }
 
 static int vega12_upload_dpm_min_level(struct pp_hwmgr *hwmgr)
 {
 	struct vega12_hwmgr *data = hwmgr->backend;
-	if (data->smc_state_table.gfx_boot_level !=
-			data->dpm_table.gfx_table.dpm_state.soft_min_level) {
-		smum_send_msg_to_smc_with_parameter(hwmgr,
-			PPSMC_MSG_SetSoftMinByFreq,
-			PPCLK_GFXCLK<<16 | data->dpm_table.gfx_table.dpm_levels[data->smc_state_table.gfx_boot_level].value);
-		data->dpm_table.gfx_table.dpm_state.soft_min_level =
-				data->smc_state_table.gfx_boot_level;
+	uint32_t min_freq;
+	int ret = 0;
+
+	if (data->smu_features[GNLD_DPM_GFXCLK].enabled) {
+		min_freq = data->dpm_table.gfx_table.dpm_state.soft_min_level;
+		PP_ASSERT_WITH_CODE(!(ret = smum_send_msg_to_smc_with_parameter(
+					hwmgr, PPSMC_MSG_SetSoftMinByFreq,
+					(PPCLK_GFXCLK << 16) | (min_freq & 0xffff))),
+					"Failed to set soft min gfxclk !",
+					return ret);
 	}
 
-	if (data->smc_state_table.mem_boot_level !=
-			data->dpm_table.mem_table.dpm_state.soft_min_level) {
-		smum_send_msg_to_smc_with_parameter(hwmgr,
-			PPSMC_MSG_SetSoftMinByFreq,
-			PPCLK_UCLK<<16 | data->dpm_table.mem_table.dpm_levels[data->smc_state_table.mem_boot_level].value);
-		data->dpm_table.mem_table.dpm_state.soft_min_level =
-				data->smc_state_table.mem_boot_level;
+	if (data->smu_features[GNLD_DPM_UCLK].enabled) {
+		min_freq = data->dpm_table.mem_table.dpm_state.soft_min_level;
+		PP_ASSERT_WITH_CODE(!(ret = smum_send_msg_to_smc_with_parameter(
+					hwmgr, PPSMC_MSG_SetSoftMinByFreq,
+					(PPCLK_UCLK << 16) | (min_freq & 0xffff))),
+					"Failed to set soft min memclk !",
+					return ret);
+
+		min_freq = data->dpm_table.mem_table.dpm_state.hard_min_level;
+		PP_ASSERT_WITH_CODE(!(ret = smum_send_msg_to_smc_with_parameter(
+					hwmgr, PPSMC_MSG_SetHardMinByFreq,
+					(PPCLK_UCLK << 16) | (min_freq & 0xffff))),
+					"Failed to set hard min memclk !",
+					return ret);
 	}
 
-	return 0;
+	if (data->smu_features[GNLD_DPM_UVD].enabled) {
+		min_freq = data->dpm_table.vclk_table.dpm_state.soft_min_level;
+
+		PP_ASSERT_WITH_CODE(!(ret = smum_send_msg_to_smc_with_parameter(
+					hwmgr, PPSMC_MSG_SetSoftMinByFreq,
+					(PPCLK_VCLK << 16) | (min_freq & 0xffff))),
+					"Failed to set soft min vclk!",
+					return ret);
+
+		min_freq = data->dpm_table.dclk_table.dpm_state.soft_min_level;
+
+		PP_ASSERT_WITH_CODE(!(ret = smum_send_msg_to_smc_with_parameter(
+					hwmgr, PPSMC_MSG_SetSoftMinByFreq,
+					(PPCLK_DCLK << 16) | (min_freq & 0xffff))),
+					"Failed to set soft min dclk!",
+					return ret);
+	}
+
+	if (data->smu_features[GNLD_DPM_VCE].enabled) {
+		min_freq = data->dpm_table.eclk_table.dpm_state.soft_min_level;
+
+		PP_ASSERT_WITH_CODE(!(ret = smum_send_msg_to_smc_with_parameter(
+					hwmgr, PPSMC_MSG_SetSoftMinByFreq,
+					(PPCLK_ECLK << 16) | (min_freq & 0xffff))),
+					"Failed to set soft min eclk!",
+					return ret);
+	}
+
+	if (data->smu_features[GNLD_DPM_SOCCLK].enabled) {
+		min_freq = data->dpm_table.soc_table.dpm_state.soft_min_level;
+
+		PP_ASSERT_WITH_CODE(!(ret = smum_send_msg_to_smc_with_parameter(
+					hwmgr, PPSMC_MSG_SetSoftMinByFreq,
+					(PPCLK_SOCCLK << 16) | (min_freq & 0xffff))),
+					"Failed to set soft min socclk!",
+					return ret);
+	}
+
+	return ret;
 
 }
 
 static int vega12_upload_dpm_max_level(struct pp_hwmgr *hwmgr)
 {
 	struct vega12_hwmgr *data = hwmgr->backend;
-	if (data->smc_state_table.gfx_max_level !=
-		data->dpm_table.gfx_table.dpm_state.soft_max_level) {
-		smum_send_msg_to_smc_with_parameter(hwmgr,
-			PPSMC_MSG_SetSoftMaxByFreq,
-			/* plus the vale by 1 to align the resolution */
-			PPCLK_GFXCLK<<16 | (data->dpm_table.gfx_table.dpm_levels[data->smc_state_table.gfx_max_level].value + 1));
-		data->dpm_table.gfx_table.dpm_state.soft_max_level =
-				data->smc_state_table.gfx_max_level;
+	uint32_t max_freq;
+	int ret = 0;
+
+	if (data->smu_features[GNLD_DPM_GFXCLK].enabled) {
+		max_freq = data->dpm_table.gfx_table.dpm_state.soft_max_level;
+
+		PP_ASSERT_WITH_CODE(!(ret = smum_send_msg_to_smc_with_parameter(
+					hwmgr, PPSMC_MSG_SetSoftMaxByFreq,
+					(PPCLK_GFXCLK << 16) | (max_freq & 0xffff))),
+					"Failed to set soft max gfxclk!",
+					return ret);
 	}
 
-	if (data->smc_state_table.mem_max_level !=
-		data->dpm_table.mem_table.dpm_state.soft_max_level) {
-		smum_send_msg_to_smc_with_parameter(hwmgr,
-			PPSMC_MSG_SetSoftMaxByFreq,
-			/* plus the vale by 1 to align the resolution */
-			PPCLK_UCLK<<16 | (data->dpm_table.mem_table.dpm_levels[data->smc_state_table.mem_max_level].value + 1));
-		data->dpm_table.mem_table.dpm_state.soft_max_level =
-				data->smc_state_table.mem_max_level;
+	if (data->smu_features[GNLD_DPM_UCLK].enabled) {
+		max_freq = data->dpm_table.mem_table.dpm_state.soft_max_level;
+
+		PP_ASSERT_WITH_CODE(!(ret = smum_send_msg_to_smc_with_parameter(
+					hwmgr, PPSMC_MSG_SetSoftMaxByFreq,
+					(PPCLK_UCLK << 16) | (max_freq & 0xffff))),
+					"Failed to set soft max memclk!",
+					return ret);
 	}
 
-	return 0;
+	if (data->smu_features[GNLD_DPM_UVD].enabled) {
+		max_freq = data->dpm_table.vclk_table.dpm_state.soft_max_level;
+
+		PP_ASSERT_WITH_CODE(!(ret = smum_send_msg_to_smc_with_parameter(
+					hwmgr, PPSMC_MSG_SetSoftMaxByFreq,
+					(PPCLK_VCLK << 16) | (max_freq & 0xffff))),
+					"Failed to set soft max vclk!",
+					return ret);
+
+		max_freq = data->dpm_table.dclk_table.dpm_state.soft_max_level;
+		PP_ASSERT_WITH_CODE(!(ret = smum_send_msg_to_smc_with_parameter(
+					hwmgr, PPSMC_MSG_SetSoftMaxByFreq,
+					(PPCLK_DCLK << 16) | (max_freq & 0xffff))),
+					"Failed to set soft max dclk!",
+					return ret);
+	}
+
+	if (data->smu_features[GNLD_DPM_VCE].enabled) {
+		max_freq = data->dpm_table.eclk_table.dpm_state.soft_max_level;
+
+		PP_ASSERT_WITH_CODE(!(ret = smum_send_msg_to_smc_with_parameter(
+					hwmgr, PPSMC_MSG_SetSoftMaxByFreq,
+					(PPCLK_ECLK << 16) | (max_freq & 0xffff))),
+					"Failed to set soft max eclk!",
+					return ret);
+	}
+
+	if (data->smu_features[GNLD_DPM_SOCCLK].enabled) {
+		max_freq = data->dpm_table.soc_table.dpm_state.soft_max_level;
+
+		PP_ASSERT_WITH_CODE(!(ret = smum_send_msg_to_smc_with_parameter(
+					hwmgr, PPSMC_MSG_SetSoftMaxByFreq,
+					(PPCLK_SOCCLK << 16) | (max_freq & 0xffff))),
+					"Failed to set soft max socclk!",
+					return ret);
+	}
+
+	return ret;
 }
 
 int vega12_enable_disable_vce_dpm(struct pp_hwmgr *hwmgr, bool enable)
@@ -1123,7 +1215,7 @@ static int vega12_get_gpu_power(struct pp_hwmgr *hwmgr, uint32_t *query)
 			"Failed to get current package power!",
 			return -EINVAL);
 
-	vega12_read_arg_from_smc(hwmgr, &value);
+	value = smum_get_argument(hwmgr);
 	/* power value is an integer */
 	*query = value << 8;
 #endif
@@ -1136,14 +1228,11 @@ static int vega12_get_current_gfx_clk_freq(struct pp_hwmgr *hwmgr, uint32_t *gfx
 
 	*gfx_freq = 0;
 
-	PP_ASSERT_WITH_CODE(
-			smum_send_msg_to_smc_with_parameter(hwmgr, PPSMC_MSG_GetDpmClockFreq, (PPCLK_GFXCLK << 16)) == 0,
+	PP_ASSERT_WITH_CODE(smum_send_msg_to_smc_with_parameter(hwmgr,
+			PPSMC_MSG_GetDpmClockFreq, (PPCLK_GFXCLK << 16)) == 0,
 			"[GetCurrentGfxClkFreq] Attempt to get Current GFXCLK Frequency Failed!",
-			return -1);
-	PP_ASSERT_WITH_CODE(
-			vega12_read_arg_from_smc(hwmgr, &gfx_clk) == 0,
-			"[GetCurrentGfxClkFreq] Attempt to read arg from SMC Failed",
-			return -1);
+			return -EINVAL);
+	gfx_clk = smum_get_argument(hwmgr);
 
 	*gfx_freq = gfx_clk * 100;
 
@@ -1159,11 +1248,8 @@ static int vega12_get_current_mclk_freq(struct pp_hwmgr *hwmgr, uint32_t *mclk_f
 	PP_ASSERT_WITH_CODE(
 			smum_send_msg_to_smc_with_parameter(hwmgr, PPSMC_MSG_GetDpmClockFreq, (PPCLK_UCLK << 16)) == 0,
 			"[GetCurrentMClkFreq] Attempt to get Current MCLK Frequency Failed!",
-			return -1);
-	PP_ASSERT_WITH_CODE(
-			vega12_read_arg_from_smc(hwmgr, &mem_clk) == 0,
-			"[GetCurrentMClkFreq] Attempt to read arg from SMC Failed",
-			return -1);
+			return -EINVAL);
+	mem_clk = smum_get_argument(hwmgr);
 
 	*mclk_freq = mem_clk * 100;
 
@@ -1180,16 +1266,12 @@ static int vega12_get_current_activity_percent(
 #if 0
 	ret = smum_send_msg_to_smc_with_parameter(hwmgr, PPSMC_MSG_GetAverageGfxActivity, 0);
 	if (!ret) {
-		ret = vega12_read_arg_from_smc(hwmgr, &current_activity);
-		if (!ret) {
-			if (current_activity > 100) {
-				PP_ASSERT(false,
-					"[GetCurrentActivityPercent] Activity Percentage Exceeds 100!");
-				current_activity = 100;
-			}
-		} else
+		current_activity = smum_get_argument(hwmgr);
+		if (current_activity > 100) {
 			PP_ASSERT(false,
-				"[GetCurrentActivityPercent] Attempt To Read Average Graphics Activity from SMU Failed!");
+				  "[GetCurrentActivityPercent] Activity Percentage Exceeds 100!");
+			current_activity = 100;
+		}
 	} else
 		PP_ASSERT(false,
 			"[GetCurrentActivityPercent] Attempt To Send Get Average Graphics Activity to SMU Failed!");
@@ -1252,7 +1334,7 @@ static int vega12_notify_smc_display_change(struct pp_hwmgr *hwmgr,
 	if (data->smu_features[GNLD_DPM_UCLK].enabled)
 		return smum_send_msg_to_smc_with_parameter(hwmgr,
 			PPSMC_MSG_SetUclkFastSwitch,
-			has_disp ? 0 : 1);
+			has_disp ? 1 : 0);
 
 	return 0;
 }
@@ -1270,7 +1352,6 @@ int vega12_display_clock_voltage_request(struct pp_hwmgr *hwmgr,
 	if (data->smu_features[GNLD_DPM_DCEFCLK].enabled) {
 		switch (clk_type) {
 		case amd_pp_dcef_clock:
-			clk_freq = clock_req->clock_freq_in_khz / 100;
 			clk_select = PPCLK_DCEFCLK;
 			break;
 		case amd_pp_disp_clock:
@@ -1306,9 +1387,10 @@ static int vega12_notify_smc_display_config_after_ps_adjustment(
 			(struct vega12_hwmgr *)(hwmgr->backend);
 	struct PP_Clocks min_clocks = {0};
 	struct pp_display_clock_request clock_req;
-	uint32_t clk_request;
 
-	if (hwmgr->display_config->num_display > 1)
+	if ((hwmgr->display_config->num_display > 1) &&
+	     !hwmgr->display_config->multi_monitor_in_sync &&
+	     !hwmgr->display_config->nb_pstate_switch_disable)
 		vega12_notify_smc_display_change(hwmgr, false);
 	else
 		vega12_notify_smc_display_change(hwmgr, true);
@@ -1319,7 +1401,7 @@ static int vega12_notify_smc_display_config_after_ps_adjustment(
 
 	if (data->smu_features[GNLD_DPM_DCEFCLK].supported) {
 		clock_req.clock_type = amd_pp_dcef_clock;
-		clock_req.clock_freq_in_khz = min_clocks.dcefClock;
+		clock_req.clock_freq_in_khz = min_clocks.dcefClock/10;
 		if (!vega12_display_clock_voltage_request(hwmgr, &clock_req)) {
 			if (data->smu_features[GNLD_DS_DCEFCLK].supported)
 				PP_ASSERT_WITH_CODE(
@@ -1333,15 +1415,6 @@ static int vega12_notify_smc_display_config_after_ps_adjustment(
 		}
 	}
 
-	if (data->smu_features[GNLD_DPM_UCLK].enabled) {
-		clk_request = (PPCLK_UCLK << 16) | (min_clocks.memoryClock) / 100;
-		PP_ASSERT_WITH_CODE(
-			smum_send_msg_to_smc_with_parameter(hwmgr, PPSMC_MSG_SetHardMinByFreq, clk_request) == 0,
-			"[PhwVega12_NotifySMCDisplayConfigAfterPowerStateAdjustment] Attempt to set UCLK HardMin Failed!",
-			return -1);
-		data->dpm_table.mem_table.dpm_state.hard_min_level = min_clocks.memoryClock;
-	}
-
 	return 0;
 }
 
@@ -1350,12 +1423,19 @@ static int vega12_force_dpm_highest(struct pp_hwmgr *hwmgr)
 	struct vega12_hwmgr *data =
 			(struct vega12_hwmgr *)(hwmgr->backend);
 
-	data->smc_state_table.gfx_boot_level =
-	data->smc_state_table.gfx_max_level =
-			vega12_find_highest_dpm_level(&(data->dpm_table.gfx_table));
-	data->smc_state_table.mem_boot_level =
-	data->smc_state_table.mem_max_level =
-			vega12_find_highest_dpm_level(&(data->dpm_table.mem_table));
+	uint32_t soft_level;
+
+	soft_level = vega12_find_highest_dpm_level(&(data->dpm_table.gfx_table));
+
+	data->dpm_table.gfx_table.dpm_state.soft_min_level =
+		data->dpm_table.gfx_table.dpm_state.soft_max_level =
+		data->dpm_table.gfx_table.dpm_levels[soft_level].value;
+
+	soft_level = vega12_find_highest_dpm_level(&(data->dpm_table.mem_table));
+
+	data->dpm_table.mem_table.dpm_state.soft_min_level =
+		data->dpm_table.mem_table.dpm_state.soft_max_level =
+		data->dpm_table.mem_table.dpm_levels[soft_level].value;
 
 	PP_ASSERT_WITH_CODE(!vega12_upload_dpm_min_level(hwmgr),
 			"Failed to upload boot level to highest!",
@@ -1372,13 +1452,19 @@ static int vega12_force_dpm_lowest(struct pp_hwmgr *hwmgr)
 {
 	struct vega12_hwmgr *data =
 			(struct vega12_hwmgr *)(hwmgr->backend);
+	uint32_t soft_level;
+
+	soft_level = vega12_find_lowest_dpm_level(&(data->dpm_table.gfx_table));
+
+	data->dpm_table.gfx_table.dpm_state.soft_min_level =
+		data->dpm_table.gfx_table.dpm_state.soft_max_level =
+		data->dpm_table.gfx_table.dpm_levels[soft_level].value;
 
-	data->smc_state_table.gfx_boot_level =
-	data->smc_state_table.gfx_max_level =
-			vega12_find_lowest_dpm_level(&(data->dpm_table.gfx_table));
-	data->smc_state_table.mem_boot_level =
-	data->smc_state_table.mem_max_level =
-			vega12_find_lowest_dpm_level(&(data->dpm_table.mem_table));
+	soft_level = vega12_find_lowest_dpm_level(&(data->dpm_table.mem_table));
+
+	data->dpm_table.mem_table.dpm_state.soft_min_level =
+		data->dpm_table.mem_table.dpm_state.soft_max_level =
+		data->dpm_table.mem_table.dpm_levels[soft_level].value;
 
 	PP_ASSERT_WITH_CODE(!vega12_upload_dpm_min_level(hwmgr),
 			"Failed to upload boot level to highest!",
@@ -1394,17 +1480,6 @@ static int vega12_force_dpm_lowest(struct pp_hwmgr *hwmgr)
 
 static int vega12_unforce_dpm_levels(struct pp_hwmgr *hwmgr)
 {
-	struct vega12_hwmgr *data = (struct vega12_hwmgr *)(hwmgr->backend);
-
-	data->smc_state_table.gfx_boot_level =
-			vega12_find_lowest_dpm_level(&(data->dpm_table.gfx_table));
-	data->smc_state_table.gfx_max_level =
-			vega12_find_highest_dpm_level(&(data->dpm_table.gfx_table));
-	data->smc_state_table.mem_boot_level =
-			vega12_find_lowest_dpm_level(&(data->dpm_table.mem_table));
-	data->smc_state_table.mem_max_level =
-			vega12_find_highest_dpm_level(&(data->dpm_table.mem_table));
-
 	PP_ASSERT_WITH_CODE(!vega12_upload_dpm_min_level(hwmgr),
 			"Failed to upload DPM Bootup Levels!",
 			return -1);
@@ -1412,22 +1487,28 @@ static int vega12_unforce_dpm_levels(struct pp_hwmgr *hwmgr)
 	PP_ASSERT_WITH_CODE(!vega12_upload_dpm_max_level(hwmgr),
 			"Failed to upload DPM Max Levels!",
 			return -1);
+
 	return 0;
 }
 
-#if 0
 static int vega12_get_profiling_clk_mask(struct pp_hwmgr *hwmgr, enum amd_dpm_forced_level level,
 				uint32_t *sclk_mask, uint32_t *mclk_mask, uint32_t *soc_mask)
 {
-	struct phm_ppt_v2_information *table_info =
-			(struct phm_ppt_v2_information *)(hwmgr->pptable);
+	struct vega12_hwmgr *data = (struct vega12_hwmgr *)(hwmgr->backend);
+	struct vega12_single_dpm_table *gfx_dpm_table = &(data->dpm_table.gfx_table);
+	struct vega12_single_dpm_table *mem_dpm_table = &(data->dpm_table.mem_table);
+	struct vega12_single_dpm_table *soc_dpm_table = &(data->dpm_table.soc_table);
 
-	if (table_info->vdd_dep_on_sclk->count > VEGA12_UMD_PSTATE_GFXCLK_LEVEL &&
-		table_info->vdd_dep_on_socclk->count > VEGA12_UMD_PSTATE_SOCCLK_LEVEL &&
-		table_info->vdd_dep_on_mclk->count > VEGA12_UMD_PSTATE_MCLK_LEVEL) {
+	*sclk_mask = 0;
+	*mclk_mask = 0;
+	*soc_mask  = 0;
+
+	if (gfx_dpm_table->count > VEGA12_UMD_PSTATE_GFXCLK_LEVEL &&
+	    mem_dpm_table->count > VEGA12_UMD_PSTATE_MCLK_LEVEL &&
+	    soc_dpm_table->count > VEGA12_UMD_PSTATE_SOCCLK_LEVEL) {
 		*sclk_mask = VEGA12_UMD_PSTATE_GFXCLK_LEVEL;
-		*soc_mask = VEGA12_UMD_PSTATE_SOCCLK_LEVEL;
 		*mclk_mask = VEGA12_UMD_PSTATE_MCLK_LEVEL;
+		*soc_mask  = VEGA12_UMD_PSTATE_SOCCLK_LEVEL;
 	}
 
 	if (level == AMD_DPM_FORCED_LEVEL_PROFILE_MIN_SCLK) {
@@ -1435,13 +1516,13 @@ static int vega12_get_profiling_clk_mask(struct pp_hwmgr *hwmgr, enum amd_dpm_fo
 	} else if (level == AMD_DPM_FORCED_LEVEL_PROFILE_MIN_MCLK) {
 		*mclk_mask = 0;
 	} else if (level == AMD_DPM_FORCED_LEVEL_PROFILE_PEAK) {
-		*sclk_mask = table_info->vdd_dep_on_sclk->count - 1;
-		*soc_mask = table_info->vdd_dep_on_socclk->count - 1;
-		*mclk_mask = table_info->vdd_dep_on_mclk->count - 1;
+		*sclk_mask = gfx_dpm_table->count - 1;
+		*mclk_mask = mem_dpm_table->count - 1;
+		*soc_mask  = soc_dpm_table->count - 1;
 	}
+
 	return 0;
 }
-#endif
 
 static void vega12_set_fan_control_mode(struct pp_hwmgr *hwmgr, uint32_t mode)
 {
@@ -1465,11 +1546,9 @@ static int vega12_dpm_force_dpm_level(struct pp_hwmgr *hwmgr,
 				enum amd_dpm_forced_level level)
 {
 	int ret = 0;
-#if 0
 	uint32_t sclk_mask = 0;
 	uint32_t mclk_mask = 0;
 	uint32_t soc_mask = 0;
-#endif
 
 	switch (level) {
 	case AMD_DPM_FORCED_LEVEL_HIGH:
@@ -1485,27 +1564,18 @@ static int vega12_dpm_force_dpm_level(struct pp_hwmgr *hwmgr,
 	case AMD_DPM_FORCED_LEVEL_PROFILE_MIN_SCLK:
 	case AMD_DPM_FORCED_LEVEL_PROFILE_MIN_MCLK:
 	case AMD_DPM_FORCED_LEVEL_PROFILE_PEAK:
-#if 0
 		ret = vega12_get_profiling_clk_mask(hwmgr, level, &sclk_mask, &mclk_mask, &soc_mask);
 		if (ret)
 			return ret;
-		vega12_force_clock_level(hwmgr, PP_SCLK, 1<<sclk_mask);
-		vega12_force_clock_level(hwmgr, PP_MCLK, 1<<mclk_mask);
-#endif
+		vega12_force_clock_level(hwmgr, PP_SCLK, 1 << sclk_mask);
+		vega12_force_clock_level(hwmgr, PP_MCLK, 1 << mclk_mask);
 		break;
 	case AMD_DPM_FORCED_LEVEL_MANUAL:
 	case AMD_DPM_FORCED_LEVEL_PROFILE_EXIT:
 	default:
 		break;
 	}
-#if 0
-	if (!ret) {
-		if (level == AMD_DPM_FORCED_LEVEL_PROFILE_PEAK && hwmgr->dpm_level != AMD_DPM_FORCED_LEVEL_PROFILE_PEAK)
-			vega12_set_fan_control_mode(hwmgr, AMD_FAN_CTRL_NONE);
-		else if (level != AMD_DPM_FORCED_LEVEL_PROFILE_PEAK && hwmgr->dpm_level == AMD_DPM_FORCED_LEVEL_PROFILE_PEAK)
-			vega12_set_fan_control_mode(hwmgr, AMD_FAN_CTRL_AUTO);
-	}
-#endif
+
 	return ret;
 }
 
@@ -1539,24 +1609,14 @@ static int vega12_get_clock_ranges(struct pp_hwmgr *hwmgr,
 		PPCLK_e clock_select,
 		bool max)
 {
-	int result;
-	*clock = 0;
+	struct vega12_hwmgr *data = (struct vega12_hwmgr *)(hwmgr->backend);
 
-	if (max) {
-		 PP_ASSERT_WITH_CODE(
-			smum_send_msg_to_smc_with_parameter(hwmgr, PPSMC_MSG_GetMaxDpmFreq, (clock_select << 16)) == 0,
-			"[GetClockRanges] Failed to get max clock from SMC!",
-			return -1);
-		result = vega12_read_arg_from_smc(hwmgr, clock);
-	} else {
-		PP_ASSERT_WITH_CODE(
-			smum_send_msg_to_smc_with_parameter(hwmgr, PPSMC_MSG_GetMinDpmFreq, (clock_select << 16)) == 0,
-			"[GetClockRanges] Failed to get min clock from SMC!",
-			return -1);
-		result = vega12_read_arg_from_smc(hwmgr, clock);
-	}
+	if (max)
+		*clock = data->clk_range[clock_select].ACMax;
+	else
+		*clock = data->clk_range[clock_select].ACMin;
 
-	return result;
+	return 0;
 }
 
 static int vega12_get_sclks(struct pp_hwmgr *hwmgr,
@@ -1571,12 +1631,12 @@ static int vega12_get_sclks(struct pp_hwmgr *hwmgr,
 		return -1;
 
 	dpm_table = &(data->dpm_table.gfx_table);
-	ucount = (dpm_table->count > VG12_PSUEDO_NUM_GFXCLK_DPM_LEVELS) ?
-		VG12_PSUEDO_NUM_GFXCLK_DPM_LEVELS : dpm_table->count;
+	ucount = (dpm_table->count > MAX_NUM_CLOCKS) ?
+		MAX_NUM_CLOCKS : dpm_table->count;
 
 	for (i = 0; i < ucount; i++) {
 		clocks->data[i].clocks_in_khz =
-			dpm_table->dpm_levels[i].value * 100;
+			dpm_table->dpm_levels[i].value * 1000;
 
 		clocks->data[i].latency_in_us = 0;
 	}
@@ -1603,13 +1663,12 @@ static int vega12_get_memclocks(struct pp_hwmgr *hwmgr,
 		return -1;
 
 	dpm_table = &(data->dpm_table.mem_table);
-	ucount = (dpm_table->count > VG12_PSUEDO_NUM_UCLK_DPM_LEVELS) ?
-		VG12_PSUEDO_NUM_UCLK_DPM_LEVELS : dpm_table->count;
+	ucount = (dpm_table->count > MAX_NUM_CLOCKS) ?
+		MAX_NUM_CLOCKS : dpm_table->count;
 
 	for (i = 0; i < ucount; i++) {
-		clocks->data[i].clocks_in_khz =
-			dpm_table->dpm_levels[i].value * 100;
-
+		clocks->data[i].clocks_in_khz = dpm_table->dpm_levels[i].value * 1000;
+		data->mclk_latency_table.entries[i].frequency = dpm_table->dpm_levels[i].value * 100;
 		clocks->data[i].latency_in_us =
 			data->mclk_latency_table.entries[i].latency =
 			vega12_get_mem_latency(hwmgr, dpm_table->dpm_levels[i].value);
@@ -1633,12 +1692,12 @@ static int vega12_get_dcefclocks(struct pp_hwmgr *hwmgr,
 
 
 	dpm_table = &(data->dpm_table.dcef_table);
-	ucount = (dpm_table->count > VG12_PSUEDO_NUM_DCEFCLK_DPM_LEVELS) ?
-		VG12_PSUEDO_NUM_DCEFCLK_DPM_LEVELS : dpm_table->count;
+	ucount = (dpm_table->count > MAX_NUM_CLOCKS) ?
+		MAX_NUM_CLOCKS : dpm_table->count;
 
 	for (i = 0; i < ucount; i++) {
 		clocks->data[i].clocks_in_khz =
-			dpm_table->dpm_levels[i].value * 100;
+			dpm_table->dpm_levels[i].value * 1000;
 
 		clocks->data[i].latency_in_us = 0;
 	}
@@ -1661,12 +1720,12 @@ static int vega12_get_socclocks(struct pp_hwmgr *hwmgr,
 
 
 	dpm_table = &(data->dpm_table.soc_table);
-	ucount = (dpm_table->count > VG12_PSUEDO_NUM_SOCCLK_DPM_LEVELS) ?
-		VG12_PSUEDO_NUM_SOCCLK_DPM_LEVELS : dpm_table->count;
+	ucount = (dpm_table->count > MAX_NUM_CLOCKS) ?
+		MAX_NUM_CLOCKS : dpm_table->count;
 
 	for (i = 0; i < ucount; i++) {
 		clocks->data[i].clocks_in_khz =
-			dpm_table->dpm_levels[i].value * 100;
+			dpm_table->dpm_levels[i].value * 1000;
 
 		clocks->data[i].latency_in_us = 0;
 	}
@@ -1713,99 +1772,69 @@ static int vega12_get_clock_by_type_with_voltage(struct pp_hwmgr *hwmgr,
 }
 
 static int vega12_set_watermarks_for_clocks_ranges(struct pp_hwmgr *hwmgr,
-		struct pp_wm_sets_with_clock_ranges_soc15 *wm_with_clock_ranges)
+							void *clock_ranges)
 {
 	struct vega12_hwmgr *data = (struct vega12_hwmgr *)(hwmgr->backend);
 	Watermarks_t *table = &(data->smc_state_table.water_marks_table);
-	int result = 0;
-	uint32_t i;
+	struct dm_pp_wm_sets_with_clock_ranges_soc15 *wm_with_clock_ranges = clock_ranges;
 
 	if (!data->registry_data.disable_water_mark &&
 			data->smu_features[GNLD_DPM_DCEFCLK].supported &&
 			data->smu_features[GNLD_DPM_SOCCLK].supported) {
-		for (i = 0; i < wm_with_clock_ranges->num_wm_sets_dmif; i++) {
-			table->WatermarkRow[WM_DCEFCLK][i].MinClock =
-				cpu_to_le16((uint16_t)
-				(wm_with_clock_ranges->wm_sets_dmif[i].wm_min_dcefclk_in_khz) /
-				100);
-			table->WatermarkRow[WM_DCEFCLK][i].MaxClock =
-				cpu_to_le16((uint16_t)
-				(wm_with_clock_ranges->wm_sets_dmif[i].wm_max_dcefclk_in_khz) /
-				100);
-			table->WatermarkRow[WM_DCEFCLK][i].MinUclk =
-				cpu_to_le16((uint16_t)
-				(wm_with_clock_ranges->wm_sets_dmif[i].wm_min_memclk_in_khz) /
-				100);
-			table->WatermarkRow[WM_DCEFCLK][i].MaxUclk =
-				cpu_to_le16((uint16_t)
-				(wm_with_clock_ranges->wm_sets_dmif[i].wm_max_memclk_in_khz) /
-				100);
-			table->WatermarkRow[WM_DCEFCLK][i].WmSetting = (uint8_t)
-					wm_with_clock_ranges->wm_sets_dmif[i].wm_set_id;
-		}
-
-		for (i = 0; i < wm_with_clock_ranges->num_wm_sets_mcif; i++) {
-			table->WatermarkRow[WM_SOCCLK][i].MinClock =
-				cpu_to_le16((uint16_t)
-				(wm_with_clock_ranges->wm_sets_mcif[i].wm_min_socclk_in_khz) /
-				100);
-			table->WatermarkRow[WM_SOCCLK][i].MaxClock =
-				cpu_to_le16((uint16_t)
-				(wm_with_clock_ranges->wm_sets_mcif[i].wm_max_socclk_in_khz) /
-				100);
-			table->WatermarkRow[WM_SOCCLK][i].MinUclk =
-				cpu_to_le16((uint16_t)
-				(wm_with_clock_ranges->wm_sets_mcif[i].wm_min_memclk_in_khz) /
-				100);
-			table->WatermarkRow[WM_SOCCLK][i].MaxUclk =
-				cpu_to_le16((uint16_t)
-				(wm_with_clock_ranges->wm_sets_mcif[i].wm_max_memclk_in_khz) /
-				100);
-			table->WatermarkRow[WM_SOCCLK][i].WmSetting = (uint8_t)
-					wm_with_clock_ranges->wm_sets_mcif[i].wm_set_id;
-		}
+		smu_set_watermarks_for_clocks_ranges(table, wm_with_clock_ranges);
 		data->water_marks_bitmap |= WaterMarksExist;
 		data->water_marks_bitmap &= ~WaterMarksLoaded;
 	}
 
-	return result;
+	return 0;
 }
 
 static int vega12_force_clock_level(struct pp_hwmgr *hwmgr,
 		enum pp_clock_type type, uint32_t mask)
 {
 	struct vega12_hwmgr *data = (struct vega12_hwmgr *)(hwmgr->backend);
-
-	if (hwmgr->request_dpm_level & (AMD_DPM_FORCED_LEVEL_AUTO |
-				AMD_DPM_FORCED_LEVEL_LOW |
-				AMD_DPM_FORCED_LEVEL_HIGH))
-		return -EINVAL;
+	uint32_t soft_min_level, soft_max_level;
+	int ret = 0;
 
 	switch (type) {
 	case PP_SCLK:
-		data->smc_state_table.gfx_boot_level = mask ? (ffs(mask) - 1) : 0;
-		data->smc_state_table.gfx_max_level = mask ? (fls(mask) - 1) : 0;
+		soft_min_level = mask ? (ffs(mask) - 1) : 0;
+		soft_max_level = mask ? (fls(mask) - 1) : 0;
+
+		data->dpm_table.gfx_table.dpm_state.soft_min_level =
+			data->dpm_table.gfx_table.dpm_levels[soft_min_level].value;
+		data->dpm_table.gfx_table.dpm_state.soft_max_level =
+			data->dpm_table.gfx_table.dpm_levels[soft_max_level].value;
 
-		PP_ASSERT_WITH_CODE(!vega12_upload_dpm_min_level(hwmgr),
+		ret = vega12_upload_dpm_min_level(hwmgr);
+		PP_ASSERT_WITH_CODE(!ret,
 			"Failed to upload boot level to lowest!",
-			return -EINVAL);
+			return ret);
 
-		PP_ASSERT_WITH_CODE(!vega12_upload_dpm_max_level(hwmgr),
+		ret = vega12_upload_dpm_max_level(hwmgr);
+		PP_ASSERT_WITH_CODE(!ret,
 			"Failed to upload dpm max level to highest!",
-			return -EINVAL);
+			return ret);
 		break;
 
 	case PP_MCLK:
-		data->smc_state_table.mem_boot_level = mask ? (ffs(mask) - 1) : 0;
-		data->smc_state_table.mem_max_level = mask ? (fls(mask) - 1) : 0;
+		soft_min_level = mask ? (ffs(mask) - 1) : 0;
+		soft_max_level = mask ? (fls(mask) - 1) : 0;
 
-		PP_ASSERT_WITH_CODE(!vega12_upload_dpm_min_level(hwmgr),
+		data->dpm_table.mem_table.dpm_state.soft_min_level =
+			data->dpm_table.mem_table.dpm_levels[soft_min_level].value;
+		data->dpm_table.mem_table.dpm_state.soft_max_level =
+			data->dpm_table.mem_table.dpm_levels[soft_max_level].value;
+
+		ret = vega12_upload_dpm_min_level(hwmgr);
+		PP_ASSERT_WITH_CODE(!ret,
 			"Failed to upload boot level to lowest!",
-			return -EINVAL);
+			return ret);
 
-		PP_ASSERT_WITH_CODE(!vega12_upload_dpm_max_level(hwmgr),
+		ret = vega12_upload_dpm_max_level(hwmgr);
+		PP_ASSERT_WITH_CODE(!ret,
 			"Failed to upload dpm max level to highest!",
-			return -EINVAL);
+			return ret);
 
 		break;
 
@@ -1838,8 +1867,8 @@ static int vega12_print_clock_levels(struct pp_hwmgr *hwmgr,
 				return -1);
 		for (i = 0; i < clocks.num_levels; i++)
 			size += sprintf(buf + size, "%d: %uMhz %s\n",
-				i, clocks.data[i].clocks_in_khz / 100,
-				(clocks.data[i].clocks_in_khz == now) ? "*" : "");
+				i, clocks.data[i].clocks_in_khz / 1000,
+				(clocks.data[i].clocks_in_khz / 1000 == now / 100) ? "*" : "");
 		break;
 
 	case PP_MCLK:
@@ -1854,8 +1883,8 @@ static int vega12_print_clock_levels(struct pp_hwmgr *hwmgr,
 				return -1);
 		for (i = 0; i < clocks.num_levels; i++)
 			size += sprintf(buf + size, "%d: %uMhz %s\n",
-				i, clocks.data[i].clocks_in_khz / 100,
-				(clocks.data[i].clocks_in_khz == now) ? "*" : "");
+				i, clocks.data[i].clocks_in_khz / 1000,
+				(clocks.data[i].clocks_in_khz / 1000 == now / 100) ? "*" : "");
 		break;
 
 	case PP_PCIE:
@@ -1867,6 +1896,205 @@ static int vega12_print_clock_levels(struct pp_hwmgr *hwmgr,
 	return size;
 }
 
+static int vega12_apply_clocks_adjust_rules(struct pp_hwmgr *hwmgr)
+{
+	struct vega12_hwmgr *data = (struct vega12_hwmgr *)(hwmgr->backend);
+	struct vega12_single_dpm_table *dpm_table;
+	bool vblank_too_short = false;
+	bool disable_mclk_switching;
+	uint32_t i, latency;
+
+	disable_mclk_switching = ((1 < hwmgr->display_config->num_display) &&
+			          !hwmgr->display_config->multi_monitor_in_sync) ||
+			          vblank_too_short;
+	latency = hwmgr->display_config->dce_tolerable_mclk_in_active_latency;
+
+	/* gfxclk */
+	dpm_table = &(data->dpm_table.gfx_table);
+	dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[0].value;
+	dpm_table->dpm_state.soft_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value;
+	dpm_table->dpm_state.hard_min_level = dpm_table->dpm_levels[0].value;
+	dpm_table->dpm_state.hard_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value;
+
+	if (PP_CAP(PHM_PlatformCaps_UMDPState)) {
+		if (VEGA12_UMD_PSTATE_GFXCLK_LEVEL < dpm_table->count) {
+			dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[VEGA12_UMD_PSTATE_GFXCLK_LEVEL].value;
+			dpm_table->dpm_state.soft_max_level = dpm_table->dpm_levels[VEGA12_UMD_PSTATE_GFXCLK_LEVEL].value;
+		}
+
+		if (hwmgr->dpm_level == AMD_DPM_FORCED_LEVEL_PROFILE_MIN_SCLK) {
+			dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[0].value;
+			dpm_table->dpm_state.soft_max_level = dpm_table->dpm_levels[0].value;
+		}
+
+		if (hwmgr->dpm_level == AMD_DPM_FORCED_LEVEL_PROFILE_PEAK) {
+			dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[dpm_table->count - 1].value;
+			dpm_table->dpm_state.soft_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value;
+		}
+	}
+
+	/* memclk */
+	dpm_table = &(data->dpm_table.mem_table);
+	dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[0].value;
+	dpm_table->dpm_state.soft_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value;
+	dpm_table->dpm_state.hard_min_level = dpm_table->dpm_levels[0].value;
+	dpm_table->dpm_state.hard_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value;
+
+	if (PP_CAP(PHM_PlatformCaps_UMDPState)) {
+		if (VEGA12_UMD_PSTATE_MCLK_LEVEL < dpm_table->count) {
+			dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[VEGA12_UMD_PSTATE_MCLK_LEVEL].value;
+			dpm_table->dpm_state.soft_max_level = dpm_table->dpm_levels[VEGA12_UMD_PSTATE_MCLK_LEVEL].value;
+		}
+
+		if (hwmgr->dpm_level == AMD_DPM_FORCED_LEVEL_PROFILE_MIN_MCLK) {
+			dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[0].value;
+			dpm_table->dpm_state.soft_max_level = dpm_table->dpm_levels[0].value;
+		}
+
+		if (hwmgr->dpm_level == AMD_DPM_FORCED_LEVEL_PROFILE_PEAK) {
+			dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[dpm_table->count - 1].value;
+			dpm_table->dpm_state.soft_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value;
+		}
+	}
+
+	/* honour DAL's UCLK Hardmin */
+	if (dpm_table->dpm_state.hard_min_level < (hwmgr->display_config->min_mem_set_clock / 100))
+		dpm_table->dpm_state.hard_min_level = hwmgr->display_config->min_mem_set_clock / 100;
+
+	/* Hardmin is dependent on displayconfig */
+	if (disable_mclk_switching) {
+		dpm_table->dpm_state.hard_min_level = dpm_table->dpm_levels[dpm_table->count - 1].value;
+		for (i = 0; i < data->mclk_latency_table.count - 1; i++) {
+			if (data->mclk_latency_table.entries[i].latency <= latency) {
+				if (dpm_table->dpm_levels[i].value >= (hwmgr->display_config->min_mem_set_clock / 100)) {
+					dpm_table->dpm_state.hard_min_level = dpm_table->dpm_levels[i].value;
+					break;
+				}
+			}
+		}
+	}
+
+	if (hwmgr->display_config->nb_pstate_switch_disable)
+		dpm_table->dpm_state.hard_min_level = dpm_table->dpm_levels[dpm_table->count - 1].value;
+
+	/* vclk */
+	dpm_table = &(data->dpm_table.vclk_table);
+	dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[0].value;
+	dpm_table->dpm_state.soft_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value;
+	dpm_table->dpm_state.hard_min_level = dpm_table->dpm_levels[0].value;
+	dpm_table->dpm_state.hard_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value;
+
+	if (PP_CAP(PHM_PlatformCaps_UMDPState)) {
+		if (VEGA12_UMD_PSTATE_UVDCLK_LEVEL < dpm_table->count) {
+			dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[VEGA12_UMD_PSTATE_UVDCLK_LEVEL].value;
+			dpm_table->dpm_state.soft_max_level = dpm_table->dpm_levels[VEGA12_UMD_PSTATE_UVDCLK_LEVEL].value;
+		}
+
+		if (hwmgr->dpm_level == AMD_DPM_FORCED_LEVEL_PROFILE_PEAK) {
+			dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[dpm_table->count - 1].value;
+			dpm_table->dpm_state.soft_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value;
+		}
+	}
+
+	/* dclk */
+	dpm_table = &(data->dpm_table.dclk_table);
+	dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[0].value;
+	dpm_table->dpm_state.soft_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value;
+	dpm_table->dpm_state.hard_min_level = dpm_table->dpm_levels[0].value;
+	dpm_table->dpm_state.hard_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value;
+
+	if (PP_CAP(PHM_PlatformCaps_UMDPState)) {
+		if (VEGA12_UMD_PSTATE_UVDCLK_LEVEL < dpm_table->count) {
+			dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[VEGA12_UMD_PSTATE_UVDCLK_LEVEL].value;
+			dpm_table->dpm_state.soft_max_level = dpm_table->dpm_levels[VEGA12_UMD_PSTATE_UVDCLK_LEVEL].value;
+		}
+
+		if (hwmgr->dpm_level == AMD_DPM_FORCED_LEVEL_PROFILE_PEAK) {
+			dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[dpm_table->count - 1].value;
+			dpm_table->dpm_state.soft_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value;
+		}
+	}
+
+	/* socclk */
+	dpm_table = &(data->dpm_table.soc_table);
+	dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[0].value;
+	dpm_table->dpm_state.soft_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value;
+	dpm_table->dpm_state.hard_min_level = dpm_table->dpm_levels[0].value;
+	dpm_table->dpm_state.hard_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value;
+
+	if (PP_CAP(PHM_PlatformCaps_UMDPState)) {
+		if (VEGA12_UMD_PSTATE_SOCCLK_LEVEL < dpm_table->count) {
+			dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[VEGA12_UMD_PSTATE_SOCCLK_LEVEL].value;
+			dpm_table->dpm_state.soft_max_level = dpm_table->dpm_levels[VEGA12_UMD_PSTATE_SOCCLK_LEVEL].value;
+		}
+
+		if (hwmgr->dpm_level == AMD_DPM_FORCED_LEVEL_PROFILE_PEAK) {
+			dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[dpm_table->count - 1].value;
+			dpm_table->dpm_state.soft_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value;
+		}
+	}
+
+	/* eclk */
+	dpm_table = &(data->dpm_table.eclk_table);
+	dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[0].value;
+	dpm_table->dpm_state.soft_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value;
+	dpm_table->dpm_state.hard_min_level = dpm_table->dpm_levels[0].value;
+	dpm_table->dpm_state.hard_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value;
+
+	if (PP_CAP(PHM_PlatformCaps_UMDPState)) {
+		if (VEGA12_UMD_PSTATE_VCEMCLK_LEVEL < dpm_table->count) {
+			dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[VEGA12_UMD_PSTATE_VCEMCLK_LEVEL].value;
+			dpm_table->dpm_state.soft_max_level = dpm_table->dpm_levels[VEGA12_UMD_PSTATE_VCEMCLK_LEVEL].value;
+		}
+
+		if (hwmgr->dpm_level == AMD_DPM_FORCED_LEVEL_PROFILE_PEAK) {
+			dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[dpm_table->count - 1].value;
+			dpm_table->dpm_state.soft_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value;
+		}
+	}
+
+	return 0;
+}
+
+static int vega12_set_uclk_to_highest_dpm_level(struct pp_hwmgr *hwmgr,
+		struct vega12_single_dpm_table *dpm_table)
+{
+	struct vega12_hwmgr *data = (struct vega12_hwmgr *)(hwmgr->backend);
+	int ret = 0;
+
+	if (data->smu_features[GNLD_DPM_UCLK].enabled) {
+		PP_ASSERT_WITH_CODE(dpm_table->count > 0,
+				"[SetUclkToHightestDpmLevel] Dpm table has no entry!",
+				return -EINVAL);
+		PP_ASSERT_WITH_CODE(dpm_table->count <= NUM_UCLK_DPM_LEVELS,
+				"[SetUclkToHightestDpmLevel] Dpm table has too many entries!",
+				return -EINVAL);
+
+		dpm_table->dpm_state.hard_min_level = dpm_table->dpm_levels[dpm_table->count - 1].value;
+		PP_ASSERT_WITH_CODE(!(ret = smum_send_msg_to_smc_with_parameter(hwmgr,
+				PPSMC_MSG_SetHardMinByFreq,
+				(PPCLK_UCLK << 16 ) | dpm_table->dpm_state.hard_min_level)),
+				"[SetUclkToHightestDpmLevel] Set hard min uclk failed!",
+				return ret);
+	}
+
+	return ret;
+}
+
+static int vega12_pre_display_configuration_changed_task(struct pp_hwmgr *hwmgr)
+{
+	struct vega12_hwmgr *data = (struct vega12_hwmgr *)(hwmgr->backend);
+	int ret = 0;
+
+	smum_send_msg_to_smc_with_parameter(hwmgr,
+			PPSMC_MSG_NumOfDisplays, 0);
+
+	ret = vega12_set_uclk_to_highest_dpm_level(hwmgr,
+			&data->dpm_table.mem_table);
+
+	return ret;
+}
+
 static int vega12_display_configuration_changed_task(struct pp_hwmgr *hwmgr)
 {
 	struct vega12_hwmgr *data = (struct vega12_hwmgr *)(hwmgr->backend);
@@ -1911,6 +2139,9 @@ static void vega12_power_gate_vce(struct pp_hwmgr *hwmgr, bool bgate)
 {
 	struct vega12_hwmgr *data = (struct vega12_hwmgr *)(hwmgr->backend);
 
+	if (data->vce_power_gated == bgate)
+		return;
+
 	data->vce_power_gated = bgate;
 	vega12_enable_disable_vce_dpm(hwmgr, !bgate);
 }
@@ -1919,6 +2150,9 @@ static void vega12_power_gate_uvd(struct pp_hwmgr *hwmgr, bool bgate)
 {
 	struct vega12_hwmgr *data = (struct vega12_hwmgr *)(hwmgr->backend);
 
+	if (data->uvd_power_gated == bgate)
+		return;
+
 	data->uvd_power_gated = bgate;
 	vega12_enable_disable_uvd_dpm(hwmgr, !bgate);
 }
@@ -2086,6 +2320,38 @@ static int vega12_get_thermal_temperature_range(struct pp_hwmgr *hwmgr,
 	return 0;
 }
 
+static int vega12_enable_gfx_off(struct pp_hwmgr *hwmgr)
+{
+	struct vega12_hwmgr *data =
+			(struct vega12_hwmgr *)(hwmgr->backend);
+	int ret = 0;
+
+	if (data->gfxoff_controlled_by_driver)
+		ret = smum_send_msg_to_smc(hwmgr, PPSMC_MSG_AllowGfxOff);
+
+	return ret;
+}
+
+static int vega12_disable_gfx_off(struct pp_hwmgr *hwmgr)
+{
+	struct vega12_hwmgr *data =
+			(struct vega12_hwmgr *)(hwmgr->backend);
+	int ret = 0;
+
+	if (data->gfxoff_controlled_by_driver)
+		ret = smum_send_msg_to_smc(hwmgr, PPSMC_MSG_DisallowGfxOff);
+
+	return ret;
+}
+
+static int vega12_gfx_off_control(struct pp_hwmgr *hwmgr, bool enable)
+{
+	if (enable)
+		return vega12_enable_gfx_off(hwmgr);
+	else
+		return vega12_disable_gfx_off(hwmgr);
+}
+
 static const struct pp_hwmgr_func vega12_hwmgr_funcs = {
 	.backend_init = vega12_hwmgr_backend_init,
 	.backend_fini = vega12_hwmgr_backend_fini,
@@ -2113,6 +2379,10 @@ static const struct pp_hwmgr_func vega12_hwmgr_funcs = {
 	.display_clock_voltage_request = vega12_display_clock_voltage_request,
 	.force_clock_level = vega12_force_clock_level,
 	.print_clock_levels = vega12_print_clock_levels,
+	.apply_clocks_adjust_rules =
+		vega12_apply_clocks_adjust_rules,
+	.pre_display_config_changed =
+		vega12_pre_display_configuration_changed_task,
 	.display_config_changed = vega12_display_configuration_changed_task,
 	.powergate_uvd = vega12_power_gate_uvd,
 	.powergate_vce = vega12_power_gate_vce,
@@ -2131,6 +2401,7 @@ static const struct pp_hwmgr_func vega12_hwmgr_funcs = {
 	.get_thermal_temperature_range = vega12_get_thermal_temperature_range,
 	.register_irq_handlers = smu9_register_irq_handlers,
 	.start_thermal_controller = vega12_start_thermal_controller,
+	.powergate_gfx = vega12_gfx_off_control,
 };
 
 int vega12_hwmgr_init(struct pp_hwmgr *hwmgr)
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_hwmgr.h b/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_hwmgr.h
index e81ded1ec198..b3e424d28994 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_hwmgr.h
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_hwmgr.h
@@ -167,6 +167,9 @@ struct vega12_vbios_boot_state {
 	uint32_t    mem_clock;
 	uint32_t    soc_clock;
 	uint32_t    dcef_clock;
+	uint32_t    eclock;
+	uint32_t    dclock;
+	uint32_t    vclock;
 };
 
 #define DPMTABLE_OD_UPDATE_SCLK     0x00000001
@@ -301,6 +304,12 @@ struct vega12_odn_fan_table {
 	bool		force_fan_pwm;
 };
 
+struct vega12_clock_range {
+	uint32_t	ACMax;
+	uint32_t	ACMin;
+	uint32_t	DCMax;
+};
+
 struct vega12_hwmgr {
 	struct vega12_dpm_table          dpm_table;
 	struct vega12_dpm_table          golden_dpm_table;
@@ -382,6 +391,11 @@ struct vega12_hwmgr {
 	uint32_t                       smu_version;
 	struct smu_features            smu_features[GNLD_FEATURES_MAX];
 	struct vega12_smc_state_table  smc_state_table;
+
+	struct vega12_clock_range      clk_range[PPCLK_COUNT];
+
+	/* ---- Gfxoff ---- */
+	bool                           gfxoff_controlled_by_driver;
 };
 
 #define VEGA12_DPM2_NEAR_TDP_DEC                      10
@@ -432,6 +446,8 @@ struct vega12_hwmgr {
 #define VEGA12_UMD_PSTATE_GFXCLK_LEVEL         0x3
 #define VEGA12_UMD_PSTATE_SOCCLK_LEVEL         0x3
 #define VEGA12_UMD_PSTATE_MCLK_LEVEL           0x2
+#define VEGA12_UMD_PSTATE_UVDCLK_LEVEL         0x3
+#define VEGA12_UMD_PSTATE_VCEMCLK_LEVEL        0x3
 
 int vega12_enable_disable_vce_dpm(struct pp_hwmgr *hwmgr, bool enable);
 
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_processpptables.c b/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_processpptables.c
index 888ddca902d8..cb3a5b1737c8 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_processpptables.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_processpptables.c
@@ -224,11 +224,9 @@ static int append_vbios_pptable(struct pp_hwmgr *hwmgr, PPTable_t *ppsmc_pptable
 	ppsmc_pptable->AcgGfxclkSpreadPercent = smc_dpm_table.acggfxclkspreadpercent;
 	ppsmc_pptable->AcgGfxclkSpreadFreq = smc_dpm_table.acggfxclkspreadfreq;
 
-	/* 0xFFFF will disable the ACG feature */
-	if (!(hwmgr->feature_mask & PP_ACG_MASK)) {
-		ppsmc_pptable->AcgThresholdFreqHigh = 0xFFFF;
-		ppsmc_pptable->AcgThresholdFreqLow = 0xFFFF;
-	}
+	ppsmc_pptable->Vr2_I2C_address = smc_dpm_table.Vr2_I2C_address;
+
+	ppsmc_pptable->Vr2_I2C_address = smc_dpm_table.Vr2_I2C_address;
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_thermal.c b/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_thermal.c
index cfd9e6ccb790..904eb2c9155b 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_thermal.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_thermal.c
@@ -34,11 +34,9 @@ static int vega12_get_current_rpm(struct pp_hwmgr *hwmgr, uint32_t *current_rpm)
 	PP_ASSERT_WITH_CODE(!smum_send_msg_to_smc(hwmgr,
 				PPSMC_MSG_GetCurrentRpm),
 			"Attempt to get current RPM from SMC Failed!",
-			return -1);
-	PP_ASSERT_WITH_CODE(!vega12_read_arg_from_smc(hwmgr,
-			current_rpm),
-			"Attempt to read current RPM from SMC Failed!",
-			return -1);
+			return -EINVAL);
+	*current_rpm = smum_get_argument(hwmgr);
+
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/amd/powerplay/inc/hardwaremanager.h b/drivers/gpu/drm/amd/powerplay/inc/hardwaremanager.h
index a202247c9894..429c9c4322da 100644
--- a/drivers/gpu/drm/amd/powerplay/inc/hardwaremanager.h
+++ b/drivers/gpu/drm/amd/powerplay/inc/hardwaremanager.h
@@ -455,7 +455,7 @@ extern int phm_get_clock_by_type_with_voltage(struct pp_hwmgr *hwmgr,
 		enum amd_pp_clock_type type,
 		struct pp_clock_levels_with_voltage *clocks);
 extern int phm_set_watermarks_for_clocks_ranges(struct pp_hwmgr *hwmgr,
-		struct pp_wm_sets_with_clock_ranges_soc15 *wm_with_clock_ranges);
+						void *clock_ranges);
 extern int phm_display_clock_voltage_request(struct pp_hwmgr *hwmgr,
 		struct pp_display_clock_request *clock);
 
diff --git a/drivers/gpu/drm/amd/powerplay/inc/hwmgr.h b/drivers/gpu/drm/amd/powerplay/inc/hwmgr.h
index 40c98ca5feb7..d3d96260f440 100644
--- a/drivers/gpu/drm/amd/powerplay/inc/hwmgr.h
+++ b/drivers/gpu/drm/amd/powerplay/inc/hwmgr.h
@@ -194,7 +194,7 @@ struct pp_smumgr_func {
 	int (*request_smu_load_fw)(struct pp_hwmgr  *hwmgr);
 	int (*request_smu_load_specific_fw)(struct pp_hwmgr  *hwmgr,
 					    uint32_t firmware);
-	int (*get_argument)(struct pp_hwmgr  *hwmgr);
+	uint32_t (*get_argument)(struct pp_hwmgr  *hwmgr);
 	int (*send_msg_to_smc)(struct pp_hwmgr  *hwmgr, uint16_t msg);
 	int (*send_msg_to_smc_with_parameter)(struct pp_hwmgr  *hwmgr,
 					  uint16_t msg, uint32_t parameter);
@@ -293,8 +293,7 @@ struct pp_hwmgr_func {
 	int (*get_clock_by_type_with_voltage)(struct pp_hwmgr *hwmgr,
 			enum amd_pp_clock_type type,
 			struct pp_clock_levels_with_voltage *clocks);
-	int (*set_watermarks_for_clocks_ranges)(struct pp_hwmgr *hwmgr,
-			struct pp_wm_sets_with_clock_ranges_soc15 *wm_with_clock_ranges);
+	int (*set_watermarks_for_clocks_ranges)(struct pp_hwmgr *hwmgr, void *clock_ranges);
 	int (*display_clock_voltage_request)(struct pp_hwmgr *hwmgr,
 			struct pp_display_clock_request *clock);
 	int (*get_max_high_clocks)(struct pp_hwmgr *hwmgr, struct amd_pp_simple_clock_info *clocks);
@@ -302,7 +301,7 @@ struct pp_hwmgr_func {
 	int (*power_off_asic)(struct pp_hwmgr *hwmgr);
 	int (*force_clock_level)(struct pp_hwmgr *hwmgr, enum pp_clock_type type, uint32_t mask);
 	int (*print_clock_levels)(struct pp_hwmgr *hwmgr, enum pp_clock_type type, char *buf);
-	int (*enable_per_cu_power_gating)(struct pp_hwmgr *hwmgr, bool enable);
+	int (*powergate_gfx)(struct pp_hwmgr *hwmgr, bool enable);
 	int (*get_sclk_od)(struct pp_hwmgr *hwmgr);
 	int (*set_sclk_od)(struct pp_hwmgr *hwmgr, uint32_t value);
 	int (*get_mclk_od)(struct pp_hwmgr *hwmgr);
@@ -327,7 +326,7 @@ struct pp_hwmgr_func {
 					enum PP_OD_DPM_TABLE_COMMAND type,
 					long *input, uint32_t size);
 	int (*set_power_limit)(struct pp_hwmgr *hwmgr, uint32_t n);
-	int (*set_mmhub_powergating_by_smu)(struct pp_hwmgr *hwmgr);
+	int (*powergate_mmhub)(struct pp_hwmgr *hwmgr);
 	int (*smus_notify_pwe)(struct pp_hwmgr *hwmgr);
 };
 
diff --git a/drivers/gpu/drm/amd/powerplay/inc/smumgr.h b/drivers/gpu/drm/amd/powerplay/inc/smumgr.h
index 89dfbf53c7e6..82550a8a3a3f 100644
--- a/drivers/gpu/drm/amd/powerplay/inc/smumgr.h
+++ b/drivers/gpu/drm/amd/powerplay/inc/smumgr.h
@@ -80,7 +80,7 @@ enum SMU10_TABLE_ID {
 	SMU10_CLOCKTABLE,
 };
 
-extern int smum_get_argument(struct pp_hwmgr *hwmgr);
+extern uint32_t smum_get_argument(struct pp_hwmgr *hwmgr);
 
 extern int smum_download_powerplay_table(struct pp_hwmgr *hwmgr, void **table);
 
diff --git a/drivers/gpu/drm/amd/powerplay/inc/vega12/smu9_driver_if.h b/drivers/gpu/drm/amd/powerplay/inc/vega12/smu9_driver_if.h
index 2f8a3b983cce..b6ffd08784e7 100644
--- a/drivers/gpu/drm/amd/powerplay/inc/vega12/smu9_driver_if.h
+++ b/drivers/gpu/drm/amd/powerplay/inc/vega12/smu9_driver_if.h
@@ -412,10 +412,10 @@ typedef struct {
   QuadraticInt_t    ReservedEquation2;
   QuadraticInt_t    ReservedEquation3;
 
-	uint16_t     MinVoltageUlvGfx;
-	uint16_t     MinVoltageUlvSoc;
+  uint16_t     MinVoltageUlvGfx;
+  uint16_t     MinVoltageUlvSoc;
 
-	uint32_t     Reserved[14];
+  uint32_t     Reserved[14];
 
 
 
@@ -483,9 +483,9 @@ typedef struct {
   uint8_t      padding8_4;
 
 
-	uint8_t      PllGfxclkSpreadEnabled;
-	uint8_t      PllGfxclkSpreadPercent;
-	uint16_t     PllGfxclkSpreadFreq;
+  uint8_t      PllGfxclkSpreadEnabled;
+  uint8_t      PllGfxclkSpreadPercent;
+  uint16_t     PllGfxclkSpreadFreq;
 
   uint8_t      UclkSpreadEnabled;
   uint8_t      UclkSpreadPercent;
@@ -495,11 +495,14 @@ typedef struct {
   uint8_t      SocclkSpreadPercent;
   uint16_t     SocclkSpreadFreq;
 
-	uint8_t      AcgGfxclkSpreadEnabled;
-	uint8_t      AcgGfxclkSpreadPercent;
-	uint16_t     AcgGfxclkSpreadFreq;
+  uint8_t      AcgGfxclkSpreadEnabled;
+  uint8_t      AcgGfxclkSpreadPercent;
+  uint16_t     AcgGfxclkSpreadFreq;
 
-	uint32_t     BoardReserved[10];
+  uint8_t      Vr2_I2C_address;
+  uint8_t      padding_vr2[3];
+
+  uint32_t     BoardReserved[9];
 
 
   uint32_t     MmHubPadding[7];
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/Makefile b/drivers/gpu/drm/amd/powerplay/smumgr/Makefile
index 0a200406a1ec..8d557accaef2 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/Makefile
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/Makefile
@@ -26,7 +26,7 @@
 SMU_MGR = smumgr.o smu8_smumgr.o tonga_smumgr.o fiji_smumgr.o \
 	  polaris10_smumgr.o iceland_smumgr.o \
 	  smu7_smumgr.o vega10_smumgr.o smu10_smumgr.o ci_smumgr.o \
-	  vega12_smumgr.o vegam_smumgr.o
+	  vega12_smumgr.o vegam_smumgr.o smu9_smumgr.o
 
 AMD_PP_SMUMGR = $(addprefix $(AMD_PP_PATH)/smumgr/,$(SMU_MGR))
 
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smumgr.c
index a4ce199af475..1276f168ff68 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smumgr.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smumgr.c
@@ -1204,7 +1204,6 @@ static int polaris10_populate_smc_acpi_level(struct pp_hwmgr *hwmgr,
 			(struct phm_ppt_v1_information *)(hwmgr->pptable);
 	SMIO_Pattern vol_level;
 	uint32_t mvdd;
-	uint16_t us_mvdd;
 
 	table->ACPILevel.Flags &= ~PPSMC_SWSTATE_FLAG_DC;
 
@@ -1255,16 +1254,11 @@ static int polaris10_populate_smc_acpi_level(struct pp_hwmgr *hwmgr,
 			"in Clock Dependency Table",
 			);
 
-	us_mvdd = 0;
-	if ((SMU7_VOLTAGE_CONTROL_NONE == data->mvdd_control) ||
-			(data->mclk_dpm_key_disabled))
-		us_mvdd = data->vbios_boot_state.mvdd_bootup_value;
-	else {
-		if (!polaris10_populate_mvdd_value(hwmgr,
+	if (!((SMU7_VOLTAGE_CONTROL_NONE == data->mvdd_control) ||
+			(data->mclk_dpm_key_disabled)))
+		polaris10_populate_mvdd_value(hwmgr,
 				data->dpm_table.mclk_table.dpm_levels[0].value,
-				&vol_level))
-			us_mvdd = vol_level.Voltage;
-	}
+				&vol_level);
 
 	if (0 == polaris10_populate_mvdd_value(hwmgr, 0, &vol_level))
 		table->MemoryACPILevel.MinMvdd = PP_HOST_TO_SMC_UL(vol_level.Voltage);
@@ -1517,7 +1511,7 @@ static int polaris10_populate_clock_stretcher_data_table(struct pp_hwmgr *hwmgr)
 	uint32_t ro, efuse, volt_without_cks, volt_with_cks, value, max, min;
 	struct polaris10_smumgr *smu_data = (struct polaris10_smumgr *)(hwmgr->smu_backend);
 
-	uint8_t i, stretch_amount, stretch_amount2, volt_offset = 0;
+	uint8_t i, stretch_amount, volt_offset = 0;
 	struct phm_ppt_v1_information *table_info =
 			(struct phm_ppt_v1_information *)(hwmgr->pptable);
 	struct phm_ppt_v1_clock_voltage_dependency_table *sclk_table =
@@ -1568,11 +1562,7 @@ static int polaris10_populate_clock_stretcher_data_table(struct pp_hwmgr *hwmgr)
 
 	smu_data->smc_state_table.LdoRefSel = (table_info->cac_dtp_table->ucCKS_LDO_REFSEL != 0) ? table_info->cac_dtp_table->ucCKS_LDO_REFSEL : 6;
 	/* Populate CKS Lookup Table */
-	if (stretch_amount == 1 || stretch_amount == 2 || stretch_amount == 5)
-		stretch_amount2 = 0;
-	else if (stretch_amount == 3 || stretch_amount == 4)
-		stretch_amount2 = 1;
-	else {
+	if (stretch_amount == 0 || stretch_amount > 5) {
 		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
 				PHM_PlatformCaps_ClockStretcher);
 		PP_ASSERT_WITH_CODE(false,
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/smu10_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/smu10_smumgr.c
index 0a563f6fe9ea..bb07d43f3874 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/smu10_smumgr.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/smu10_smumgr.c
@@ -68,7 +68,7 @@ static int smu10_send_msg_to_smc_without_waiting(struct pp_hwmgr *hwmgr,
 	return 0;
 }
 
-static int smu10_read_arg_from_smc(struct pp_hwmgr *hwmgr)
+static uint32_t smu10_read_arg_from_smc(struct pp_hwmgr *hwmgr)
 {
 	struct amdgpu_device *adev = hwmgr->adev;
 
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/smu7_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/smu7_smumgr.c
index d644a9bb9078..a029e47c2319 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/smu7_smumgr.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/smu7_smumgr.c
@@ -379,8 +379,7 @@ int smu7_request_smu_load_fw(struct pp_hwmgr *hwmgr)
 {
 	struct smu7_smumgr *smu_data = (struct smu7_smumgr *)(hwmgr->smu_backend);
 	uint32_t fw_to_load;
-	int result = 0;
-	struct SMU_DRAMData_TOC *toc;
+	int r = 0;
 
 	if (!hwmgr->reload_fw) {
 		pr_info("skip reloading...\n");
@@ -421,49 +420,62 @@ int smu7_request_smu_load_fw(struct pp_hwmgr *hwmgr)
 			   + UCODE_ID_CP_MEC_JT2_MASK;
 	}
 
-	toc = (struct SMU_DRAMData_TOC *)smu_data->header;
-	toc->num_entries = 0;
-	toc->structure_version = 1;
+	if (!smu_data->toc) {
+		struct SMU_DRAMData_TOC *toc;
+
+		smu_data->toc = kzalloc(sizeof(struct SMU_DRAMData_TOC), GFP_KERNEL);
+		if (!smu_data->toc)
+			return -ENOMEM;
+		toc = smu_data->toc;
+		toc->num_entries = 0;
+		toc->structure_version = 1;
 
-	PP_ASSERT_WITH_CODE(0 == smu7_populate_single_firmware_entry(hwmgr,
+		PP_ASSERT_WITH_CODE(0 == smu7_populate_single_firmware_entry(hwmgr,
 				UCODE_ID_RLC_G, &toc->entry[toc->num_entries++]),
-				"Failed to Get Firmware Entry.", return -EINVAL);
-	PP_ASSERT_WITH_CODE(0 == smu7_populate_single_firmware_entry(hwmgr,
+				"Failed to Get Firmware Entry.", r = -EINVAL; goto failed);
+		PP_ASSERT_WITH_CODE(0 == smu7_populate_single_firmware_entry(hwmgr,
 				UCODE_ID_CP_CE, &toc->entry[toc->num_entries++]),
-				"Failed to Get Firmware Entry.", return -EINVAL);
-	PP_ASSERT_WITH_CODE(0 == smu7_populate_single_firmware_entry(hwmgr,
+				"Failed to Get Firmware Entry.", r = -EINVAL; goto failed);
+		PP_ASSERT_WITH_CODE(0 == smu7_populate_single_firmware_entry(hwmgr,
 				UCODE_ID_CP_PFP, &toc->entry[toc->num_entries++]),
-				"Failed to Get Firmware Entry.", return -EINVAL);
-	PP_ASSERT_WITH_CODE(0 == smu7_populate_single_firmware_entry(hwmgr,
+				"Failed to Get Firmware Entry.", r = -EINVAL; goto failed);
+		PP_ASSERT_WITH_CODE(0 == smu7_populate_single_firmware_entry(hwmgr,
 				UCODE_ID_CP_ME, &toc->entry[toc->num_entries++]),
-				"Failed to Get Firmware Entry.", return -EINVAL);
-	PP_ASSERT_WITH_CODE(0 == smu7_populate_single_firmware_entry(hwmgr,
+				"Failed to Get Firmware Entry.", r = -EINVAL; goto failed);
+		PP_ASSERT_WITH_CODE(0 == smu7_populate_single_firmware_entry(hwmgr,
 				UCODE_ID_CP_MEC, &toc->entry[toc->num_entries++]),
-				"Failed to Get Firmware Entry.", return -EINVAL);
-	PP_ASSERT_WITH_CODE(0 == smu7_populate_single_firmware_entry(hwmgr,
+				"Failed to Get Firmware Entry.", r = -EINVAL; goto failed);
+		PP_ASSERT_WITH_CODE(0 == smu7_populate_single_firmware_entry(hwmgr,
 				UCODE_ID_CP_MEC_JT1, &toc->entry[toc->num_entries++]),
-				"Failed to Get Firmware Entry.", return -EINVAL);
-	PP_ASSERT_WITH_CODE(0 == smu7_populate_single_firmware_entry(hwmgr,
+				"Failed to Get Firmware Entry.", r = -EINVAL; goto failed);
+		PP_ASSERT_WITH_CODE(0 == smu7_populate_single_firmware_entry(hwmgr,
 				UCODE_ID_CP_MEC_JT2, &toc->entry[toc->num_entries++]),
-				"Failed to Get Firmware Entry.", return -EINVAL);
-	PP_ASSERT_WITH_CODE(0 == smu7_populate_single_firmware_entry(hwmgr,
+				"Failed to Get Firmware Entry.", r = -EINVAL; goto failed);
+		PP_ASSERT_WITH_CODE(0 == smu7_populate_single_firmware_entry(hwmgr,
 				UCODE_ID_SDMA0, &toc->entry[toc->num_entries++]),
-				"Failed to Get Firmware Entry.", return -EINVAL);
-	PP_ASSERT_WITH_CODE(0 == smu7_populate_single_firmware_entry(hwmgr,
-				UCODE_ID_SDMA1, &toc->entry[toc->num_entries++]),
-				"Failed to Get Firmware Entry.", return -EINVAL);
-	if (!hwmgr->not_vf)
+				"Failed to Get Firmware Entry.", r = -EINVAL; goto failed);
 		PP_ASSERT_WITH_CODE(0 == smu7_populate_single_firmware_entry(hwmgr,
+				UCODE_ID_SDMA1, &toc->entry[toc->num_entries++]),
+				"Failed to Get Firmware Entry.", r = -EINVAL; goto failed);
+		if (!hwmgr->not_vf)
+			PP_ASSERT_WITH_CODE(0 == smu7_populate_single_firmware_entry(hwmgr,
 				UCODE_ID_MEC_STORAGE, &toc->entry[toc->num_entries++]),
-				"Failed to Get Firmware Entry.", return -EINVAL);
-
+				"Failed to Get Firmware Entry.", r = -EINVAL; goto failed);
+	}
+	memcpy_toio(smu_data->header_buffer.kaddr, smu_data->toc,
+		    sizeof(struct SMU_DRAMData_TOC));
 	smu7_send_msg_to_smc_with_parameter(hwmgr, PPSMC_MSG_DRV_DRAM_ADDR_HI, upper_32_bits(smu_data->header_buffer.mc_addr));
 	smu7_send_msg_to_smc_with_parameter(hwmgr, PPSMC_MSG_DRV_DRAM_ADDR_LO, lower_32_bits(smu_data->header_buffer.mc_addr));
 
 	if (smu7_send_msg_to_smc_with_parameter(hwmgr, PPSMC_MSG_LoadUcodes, fw_to_load))
 		pr_err("Fail to Request SMU Load uCode");
 
-	return result;
+	return r;
+
+failed:
+	kfree(smu_data->toc);
+	smu_data->toc = NULL;
+	return r;
 }
 
 /* Check if the FW has been loaded, SMU will not return if loading has not finished. */
@@ -570,7 +582,6 @@ int smu7_setup_pwr_virus(struct pp_hwmgr *hwmgr)
 int smu7_init(struct pp_hwmgr *hwmgr)
 {
 	struct smu7_smumgr *smu_data;
-	uint64_t mc_addr = 0;
 	int r;
 	/* Allocate memory for backend private data */
 	smu_data = (struct smu7_smumgr *)(hwmgr->smu_backend);
@@ -584,15 +595,12 @@ int smu7_init(struct pp_hwmgr *hwmgr)
 		PAGE_SIZE,
 		AMDGPU_GEM_DOMAIN_VRAM,
 		&smu_data->header_buffer.handle,
-		&mc_addr,
+		&smu_data->header_buffer.mc_addr,
 		&smu_data->header_buffer.kaddr);
 
 	if (r)
 		return -EINVAL;
 
-	smu_data->header = smu_data->header_buffer.kaddr;
-	smu_data->header_buffer.mc_addr = mc_addr;
-
 	if (!hwmgr->not_vf)
 		return 0;
 
@@ -602,7 +610,7 @@ int smu7_init(struct pp_hwmgr *hwmgr)
 		PAGE_SIZE,
 		AMDGPU_GEM_DOMAIN_VRAM,
 		&smu_data->smu_buffer.handle,
-		&mc_addr,
+		&smu_data->smu_buffer.mc_addr,
 		&smu_data->smu_buffer.kaddr);
 
 	if (r) {
@@ -611,7 +619,6 @@ int smu7_init(struct pp_hwmgr *hwmgr)
 					&smu_data->header_buffer.kaddr);
 		return -EINVAL;
 	}
-	smu_data->smu_buffer.mc_addr = mc_addr;
 
 	if (smum_is_hw_avfs_present(hwmgr))
 		hwmgr->avfs_supported = true;
@@ -633,6 +640,9 @@ int smu7_smu_fini(struct pp_hwmgr *hwmgr)
 					&smu_data->smu_buffer.mc_addr,
 					&smu_data->smu_buffer.kaddr);
 
+
+	kfree(smu_data->toc);
+	smu_data->toc = NULL;
 	kfree(hwmgr->smu_backend);
 	hwmgr->smu_backend = NULL;
 	return 0;
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/smu7_smumgr.h b/drivers/gpu/drm/amd/powerplay/smumgr/smu7_smumgr.h
index 39c9bfda0ab4..01f0538fba6b 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/smu7_smumgr.h
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/smu7_smumgr.h
@@ -37,10 +37,9 @@ struct smu7_buffer_entry {
 };
 
 struct smu7_smumgr {
-	uint8_t *header;
-	uint8_t *mec_image;
 	struct smu7_buffer_entry smu_buffer;
 	struct smu7_buffer_entry header_buffer;
+	struct SMU_DRAMData_TOC *toc;
 
 	uint32_t                             soft_regs_start;
 	uint32_t                             dpm_table_start;
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/smu8_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/smu8_smumgr.c
index c861d3023474..f7e3bc22bb93 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/smu8_smumgr.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/smu8_smumgr.c
@@ -52,10 +52,10 @@ static const enum smu8_scratch_entry firmware_list[] = {
 	SMU8_SCRATCH_ENTRY_UCODE_ID_RLC_G,
 };
 
-static int smu8_get_argument(struct pp_hwmgr *hwmgr)
+static uint32_t smu8_get_argument(struct pp_hwmgr *hwmgr)
 {
 	if (hwmgr == NULL || hwmgr->device == NULL)
-		return -EINVAL;
+		return 0;
 
 	return cgs_read_register(hwmgr->device,
 					mmSMU_MP1_SRBM2P_ARG_0);
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/smu9_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/smu9_smumgr.c
new file mode 100644
index 000000000000..079fc8e8f709
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/smu9_smumgr.c
@@ -0,0 +1,150 @@
+/*
+ * Copyright 2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include "smumgr.h"
+#include "vega10_inc.h"
+#include "soc15_common.h"
+#include "pp_debug.h"
+
+
+/* MP Apertures */
+#define MP0_Public                  0x03800000
+#define MP0_SRAM                    0x03900000
+#define MP1_Public                  0x03b00000
+#define MP1_SRAM                    0x03c00004
+
+#define smnMP1_FIRMWARE_FLAGS                                                                           0x3010028
+
+bool smu9_is_smc_ram_running(struct pp_hwmgr *hwmgr)
+{
+	struct amdgpu_device *adev = hwmgr->adev;
+	uint32_t mp1_fw_flags;
+
+	WREG32_SOC15(NBIF, 0, mmPCIE_INDEX2,
+			(MP1_Public | (smnMP1_FIRMWARE_FLAGS & 0xffffffff)));
+
+	mp1_fw_flags = RREG32_SOC15(NBIF, 0, mmPCIE_DATA2);
+
+	if (mp1_fw_flags & MP1_FIRMWARE_FLAGS__INTERRUPTS_ENABLED_MASK)
+		return true;
+
+	return false;
+}
+
+/*
+ * Check if SMC has responded to previous message.
+ *
+ * @param    smumgr  the address of the powerplay hardware manager.
+ * @return   TRUE    SMC has responded, FALSE otherwise.
+ */
+static uint32_t smu9_wait_for_response(struct pp_hwmgr *hwmgr)
+{
+	struct amdgpu_device *adev = hwmgr->adev;
+	uint32_t reg;
+	uint32_t ret;
+
+	reg = SOC15_REG_OFFSET(MP1, 0, mmMP1_SMN_C2PMSG_90);
+
+	ret = phm_wait_for_register_unequal(hwmgr, reg,
+			0, MP1_C2PMSG_90__CONTENT_MASK);
+
+	if (ret)
+		pr_err("No response from smu\n");
+
+	return RREG32_SOC15(MP1, 0, mmMP1_SMN_C2PMSG_90);
+}
+
+/*
+ * Send a message to the SMC, and do not wait for its response.
+ * @param    smumgr  the address of the powerplay hardware manager.
+ * @param    msg the message to send.
+ * @return   Always return 0.
+ */
+static int smu9_send_msg_to_smc_without_waiting(struct pp_hwmgr *hwmgr,
+						uint16_t msg)
+{
+	struct amdgpu_device *adev = hwmgr->adev;
+
+	WREG32_SOC15(MP1, 0, mmMP1_SMN_C2PMSG_66, msg);
+
+	return 0;
+}
+
+/*
+ * Send a message to the SMC, and wait for its response.
+ * @param    hwmgr  the address of the powerplay hardware manager.
+ * @param    msg the message to send.
+ * @return   Always return 0.
+ */
+int smu9_send_msg_to_smc(struct pp_hwmgr *hwmgr, uint16_t msg)
+{
+	struct amdgpu_device *adev = hwmgr->adev;
+	uint32_t ret;
+
+	smu9_wait_for_response(hwmgr);
+
+	WREG32_SOC15(MP1, 0, mmMP1_SMN_C2PMSG_90, 0);
+
+	smu9_send_msg_to_smc_without_waiting(hwmgr, msg);
+
+	ret = smu9_wait_for_response(hwmgr);
+	if (ret != 1)
+		pr_err("Failed to send message: 0x%x, ret value: 0x%x\n", msg, ret);
+
+	return 0;
+}
+
+/*
+ * Send a message to the SMC with parameter
+ * @param    hwmgr:  the address of the powerplay hardware manager.
+ * @param    msg: the message to send.
+ * @param    parameter: the parameter to send
+ * @return   Always return 0.
+ */
+int smu9_send_msg_to_smc_with_parameter(struct pp_hwmgr *hwmgr,
+					uint16_t msg, uint32_t parameter)
+{
+	struct amdgpu_device *adev = hwmgr->adev;
+	uint32_t ret;
+
+	smu9_wait_for_response(hwmgr);
+
+	WREG32_SOC15(MP1, 0, mmMP1_SMN_C2PMSG_90, 0);
+
+	WREG32_SOC15(MP1, 0, mmMP1_SMN_C2PMSG_82, parameter);
+
+	smu9_send_msg_to_smc_without_waiting(hwmgr, msg);
+
+	ret = smu9_wait_for_response(hwmgr);
+	if (ret != 1)
+		pr_err("Failed message: 0x%x, input parameter: 0x%x, error code: 0x%x\n", msg, parameter, ret);
+
+	return 0;
+}
+
+uint32_t smu9_get_argument(struct pp_hwmgr *hwmgr)
+{
+	struct amdgpu_device *adev = hwmgr->adev;
+
+	return RREG32_SOC15(MP1, 0, mmMP1_SMN_C2PMSG_82);
+}
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/smu9_smumgr.h b/drivers/gpu/drm/amd/powerplay/smumgr/smu9_smumgr.h
new file mode 100644
index 000000000000..1462279ca128
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/smu9_smumgr.h
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef _SMU9_SMUMANAGER_H_
+#define _SMU9_SMUMANAGER_H_
+
+bool smu9_is_smc_ram_running(struct pp_hwmgr *hwmgr);
+int smu9_send_msg_to_smc(struct pp_hwmgr *hwmgr, uint16_t msg);
+int smu9_send_msg_to_smc_with_parameter(struct pp_hwmgr *hwmgr,
+					uint16_t msg, uint32_t parameter);
+uint32_t smu9_get_argument(struct pp_hwmgr *hwmgr);
+
+#endif
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/smumgr.c
index c9837935f0f5..99d5e4f98f49 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/smumgr.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/smumgr.c
@@ -96,7 +96,7 @@ int smum_process_firmware_header(struct pp_hwmgr *hwmgr)
 	return 0;
 }
 
-int smum_get_argument(struct pp_hwmgr *hwmgr)
+uint32_t smum_get_argument(struct pp_hwmgr *hwmgr)
 {
 	if (NULL != hwmgr->smumgr_funcs->get_argument)
 		return hwmgr->smumgr_funcs->get_argument(hwmgr);
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/vega10_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/vega10_smumgr.c
index e84669c448a3..5d19115f410c 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/vega10_smumgr.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/vega10_smumgr.c
@@ -28,142 +28,11 @@
 #include "vega10_hwmgr.h"
 #include "vega10_ppsmc.h"
 #include "smu9_driver_if.h"
+#include "smu9_smumgr.h"
 #include "ppatomctrl.h"
 #include "pp_debug.h"
 
 
-#define AVFS_EN_MSB		1568
-#define AVFS_EN_LSB		1568
-
-/* Microcode file is stored in this buffer */
-#define BUFFER_SIZE                 80000
-#define MAX_STRING_SIZE             15
-#define BUFFER_SIZETWO              131072 /* 128 *1024 */
-
-/* MP Apertures */
-#define MP0_Public                  0x03800000
-#define MP0_SRAM                    0x03900000
-#define MP1_Public                  0x03b00000
-#define MP1_SRAM                    0x03c00004
-
-#define smnMP1_FIRMWARE_FLAGS                                                                           0x3010028
-#define smnMP0_FW_INTF                                                                                  0x3010104
-#define smnMP1_PUB_CTRL                                                                                 0x3010b14
-
-static bool vega10_is_smc_ram_running(struct pp_hwmgr *hwmgr)
-{
-	struct amdgpu_device *adev = hwmgr->adev;
-	uint32_t mp1_fw_flags;
-
-	WREG32_SOC15(NBIF, 0, mmPCIE_INDEX2,
-			(MP1_Public | (smnMP1_FIRMWARE_FLAGS & 0xffffffff)));
-
-	mp1_fw_flags = RREG32_SOC15(NBIF, 0, mmPCIE_DATA2);
-
-	if (mp1_fw_flags & MP1_FIRMWARE_FLAGS__INTERRUPTS_ENABLED_MASK)
-		return true;
-
-	return false;
-}
-
-/*
- * Check if SMC has responded to previous message.
- *
- * @param    smumgr  the address of the powerplay hardware manager.
- * @return   TRUE    SMC has responded, FALSE otherwise.
- */
-static uint32_t vega10_wait_for_response(struct pp_hwmgr *hwmgr)
-{
-	struct amdgpu_device *adev = hwmgr->adev;
-	uint32_t reg;
-	uint32_t ret;
-
-	reg = SOC15_REG_OFFSET(MP1, 0, mmMP1_SMN_C2PMSG_90);
-
-	ret = phm_wait_for_register_unequal(hwmgr, reg,
-			0, MP1_C2PMSG_90__CONTENT_MASK);
-
-	if (ret)
-		pr_err("No response from smu\n");
-
-	return RREG32_SOC15(MP1, 0, mmMP1_SMN_C2PMSG_90);
-}
-
-/*
- * Send a message to the SMC, and do not wait for its response.
- * @param    smumgr  the address of the powerplay hardware manager.
- * @param    msg the message to send.
- * @return   Always return 0.
- */
-static int vega10_send_msg_to_smc_without_waiting(struct pp_hwmgr *hwmgr,
-		uint16_t msg)
-{
-	struct amdgpu_device *adev = hwmgr->adev;
-
-	WREG32_SOC15(MP1, 0, mmMP1_SMN_C2PMSG_66, msg);
-
-	return 0;
-}
-
-/*
- * Send a message to the SMC, and wait for its response.
- * @param    hwmgr  the address of the powerplay hardware manager.
- * @param    msg the message to send.
- * @return   Always return 0.
- */
-static int vega10_send_msg_to_smc(struct pp_hwmgr *hwmgr, uint16_t msg)
-{
-	struct amdgpu_device *adev = hwmgr->adev;
-	uint32_t ret;
-
-	vega10_wait_for_response(hwmgr);
-
-	WREG32_SOC15(MP1, 0, mmMP1_SMN_C2PMSG_90, 0);
-
-	vega10_send_msg_to_smc_without_waiting(hwmgr, msg);
-
-	ret = vega10_wait_for_response(hwmgr);
-	if (ret != 1)
-		pr_err("Failed to send message: 0x%x, ret value: 0x%x\n", msg, ret);
-
-	return 0;
-}
-
-/*
- * Send a message to the SMC with parameter
- * @param    hwmgr:  the address of the powerplay hardware manager.
- * @param    msg: the message to send.
- * @param    parameter: the parameter to send
- * @return   Always return 0.
- */
-static int vega10_send_msg_to_smc_with_parameter(struct pp_hwmgr *hwmgr,
-		uint16_t msg, uint32_t parameter)
-{
-	struct amdgpu_device *adev = hwmgr->adev;
-	uint32_t ret;
-
-	vega10_wait_for_response(hwmgr);
-
-	WREG32_SOC15(MP1, 0, mmMP1_SMN_C2PMSG_90, 0);
-
-	WREG32_SOC15(MP1, 0, mmMP1_SMN_C2PMSG_82, parameter);
-
-	vega10_send_msg_to_smc_without_waiting(hwmgr, msg);
-
-	ret = vega10_wait_for_response(hwmgr);
-	if (ret != 1)
-		pr_err("Failed message: 0x%x, input parameter: 0x%x, error code: 0x%x\n", msg, parameter, ret);
-
-	return 0;
-}
-
-static int vega10_get_argument(struct pp_hwmgr *hwmgr)
-{
-	struct amdgpu_device *adev = hwmgr->adev;
-
-	return RREG32_SOC15(MP1, 0, mmMP1_SMN_C2PMSG_82);
-}
-
 static int vega10_copy_table_from_smc(struct pp_hwmgr *hwmgr,
 		uint8_t *table, int16_t table_id)
 {
@@ -175,13 +44,13 @@ static int vega10_copy_table_from_smc(struct pp_hwmgr *hwmgr,
 			"Invalid SMU Table version!", return -EINVAL);
 	PP_ASSERT_WITH_CODE(priv->smu_tables.entry[table_id].size != 0,
 			"Invalid SMU Table Length!", return -EINVAL);
-	vega10_send_msg_to_smc_with_parameter(hwmgr,
+	smu9_send_msg_to_smc_with_parameter(hwmgr,
 			PPSMC_MSG_SetDriverDramAddrHigh,
 			upper_32_bits(priv->smu_tables.entry[table_id].mc_addr));
-	vega10_send_msg_to_smc_with_parameter(hwmgr,
+	smu9_send_msg_to_smc_with_parameter(hwmgr,
 			PPSMC_MSG_SetDriverDramAddrLow,
 			lower_32_bits(priv->smu_tables.entry[table_id].mc_addr));
-	vega10_send_msg_to_smc_with_parameter(hwmgr,
+	smu9_send_msg_to_smc_with_parameter(hwmgr,
 			PPSMC_MSG_TransferTableSmu2Dram,
 			priv->smu_tables.entry[table_id].table_id);
 
@@ -206,13 +75,13 @@ static int vega10_copy_table_to_smc(struct pp_hwmgr *hwmgr,
 	memcpy(priv->smu_tables.entry[table_id].table, table,
 			priv->smu_tables.entry[table_id].size);
 
-	vega10_send_msg_to_smc_with_parameter(hwmgr,
+	smu9_send_msg_to_smc_with_parameter(hwmgr,
 			PPSMC_MSG_SetDriverDramAddrHigh,
 			upper_32_bits(priv->smu_tables.entry[table_id].mc_addr));
-	vega10_send_msg_to_smc_with_parameter(hwmgr,
+	smu9_send_msg_to_smc_with_parameter(hwmgr,
 			PPSMC_MSG_SetDriverDramAddrLow,
 			lower_32_bits(priv->smu_tables.entry[table_id].mc_addr));
-	vega10_send_msg_to_smc_with_parameter(hwmgr,
+	smu9_send_msg_to_smc_with_parameter(hwmgr,
 			PPSMC_MSG_TransferTableDram2Smu,
 			priv->smu_tables.entry[table_id].table_id);
 
@@ -225,8 +94,8 @@ static int vega10_get_smc_features(struct pp_hwmgr *hwmgr,
 	if (features_enabled == NULL)
 		return -EINVAL;
 
-	vega10_send_msg_to_smc(hwmgr, PPSMC_MSG_GetEnabledSmuFeatures);
-	*features_enabled = vega10_get_argument(hwmgr);
+	smu9_send_msg_to_smc(hwmgr, PPSMC_MSG_GetEnabledSmuFeatures);
+	*features_enabled = smu9_get_argument(hwmgr);
 
 	return 0;
 }
@@ -248,10 +117,10 @@ static int vega10_set_tools_address(struct pp_hwmgr *hwmgr)
 	struct vega10_smumgr *priv = hwmgr->smu_backend;
 
 	if (priv->smu_tables.entry[TOOLSTABLE].mc_addr) {
-		vega10_send_msg_to_smc_with_parameter(hwmgr,
+		smu9_send_msg_to_smc_with_parameter(hwmgr,
 				PPSMC_MSG_SetToolsDramAddrHigh,
 				upper_32_bits(priv->smu_tables.entry[TOOLSTABLE].mc_addr));
-		vega10_send_msg_to_smc_with_parameter(hwmgr,
+		smu9_send_msg_to_smc_with_parameter(hwmgr,
 				PPSMC_MSG_SetToolsDramAddrLow,
 				lower_32_bits(priv->smu_tables.entry[TOOLSTABLE].mc_addr));
 	}
@@ -265,11 +134,11 @@ static int vega10_verify_smc_interface(struct pp_hwmgr *hwmgr)
 	uint32_t dev_id;
 	uint32_t rev_id;
 
-	PP_ASSERT_WITH_CODE(!vega10_send_msg_to_smc(hwmgr,
+	PP_ASSERT_WITH_CODE(!smu9_send_msg_to_smc(hwmgr,
 			PPSMC_MSG_GetDriverIfVersion),
 			"Attempt to get SMC IF Version Number Failed!",
 			return -EINVAL);
-	smc_driver_if_version = vega10_get_argument(hwmgr);
+	smc_driver_if_version = smu9_get_argument(hwmgr);
 
 	dev_id = adev->pdev->device;
 	rev_id = adev->pdev->revision;
@@ -441,7 +310,7 @@ static int vega10_smu_fini(struct pp_hwmgr *hwmgr)
 
 static int vega10_start_smu(struct pp_hwmgr *hwmgr)
 {
-	if (!vega10_is_smc_ram_running(hwmgr))
+	if (!smu9_is_smc_ram_running(hwmgr))
 		return -EINVAL;
 
 	PP_ASSERT_WITH_CODE(!vega10_verify_smc_interface(hwmgr),
@@ -453,7 +322,8 @@ static int vega10_start_smu(struct pp_hwmgr *hwmgr)
 	return 0;
 }
 
-static int vega10_smc_table_manager(struct pp_hwmgr *hwmgr, uint8_t *table, uint16_t table_id, bool rw)
+static int vega10_smc_table_manager(struct pp_hwmgr *hwmgr, uint8_t *table,
+				    uint16_t table_id, bool rw)
 {
 	int ret;
 
@@ -470,11 +340,11 @@ const struct pp_smumgr_func vega10_smu_funcs = {
 	.smu_fini = &vega10_smu_fini,
 	.start_smu = &vega10_start_smu,
 	.request_smu_load_specific_fw = NULL,
-	.send_msg_to_smc = &vega10_send_msg_to_smc,
-	.send_msg_to_smc_with_parameter = &vega10_send_msg_to_smc_with_parameter,
+	.send_msg_to_smc = &smu9_send_msg_to_smc,
+	.send_msg_to_smc_with_parameter = &smu9_send_msg_to_smc_with_parameter,
 	.download_pptable_settings = NULL,
 	.upload_pptable_settings = NULL,
 	.is_dpm_running = vega10_is_dpm_running,
-	.get_argument = vega10_get_argument,
+	.get_argument = smu9_get_argument,
 	.smc_table_manager = vega10_smc_table_manager,
 };
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/vega12_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/vega12_smumgr.c
index 7d9b40e8b1bf..7f0e2109f40d 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/vega12_smumgr.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/vega12_smumgr.c
@@ -24,157 +24,14 @@
 #include "smumgr.h"
 #include "vega12_inc.h"
 #include "soc15_common.h"
+#include "smu9_smumgr.h"
 #include "vega12_smumgr.h"
 #include "vega12_ppsmc.h"
 #include "vega12/smu9_driver_if.h"
-
 #include "ppatomctrl.h"
 #include "pp_debug.h"
 
 
-/* MP Apertures */
-#define MP0_Public                  0x03800000
-#define MP0_SRAM                    0x03900000
-#define MP1_Public                  0x03b00000
-#define MP1_SRAM                    0x03c00004
-
-#define smnMP1_FIRMWARE_FLAGS                                                                           0x3010028
-#define smnMP0_FW_INTF                                                                                  0x3010104
-#define smnMP1_PUB_CTRL                                                                                 0x3010b14
-
-static bool vega12_is_smc_ram_running(struct pp_hwmgr *hwmgr)
-{
-	struct amdgpu_device *adev = hwmgr->adev;
-	uint32_t mp1_fw_flags;
-
-	WREG32_SOC15(NBIF, 0, mmPCIE_INDEX2,
-			(MP1_Public | (smnMP1_FIRMWARE_FLAGS & 0xffffffff)));
-
-	mp1_fw_flags = RREG32_SOC15(NBIF, 0, mmPCIE_DATA2);
-
-	if ((mp1_fw_flags & MP1_FIRMWARE_FLAGS__INTERRUPTS_ENABLED_MASK) >>
-				MP1_FIRMWARE_FLAGS__INTERRUPTS_ENABLED__SHIFT)
-		return true;
-
-	return false;
-}
-
-/*
- * Check if SMC has responded to previous message.
- *
- * @param    smumgr  the address of the powerplay hardware manager.
- * @return   TRUE    SMC has responded, FALSE otherwise.
- */
-static uint32_t vega12_wait_for_response(struct pp_hwmgr *hwmgr)
-{
-	struct amdgpu_device *adev = hwmgr->adev;
-	uint32_t reg;
-
-	reg = SOC15_REG_OFFSET(MP1, 0, mmMP1_SMN_C2PMSG_90);
-
-	phm_wait_for_register_unequal(hwmgr, reg,
-			0, MP1_C2PMSG_90__CONTENT_MASK);
-
-	return RREG32_SOC15(MP1, 0, mmMP1_SMN_C2PMSG_90);
-}
-
-/*
- * Send a message to the SMC, and do not wait for its response.
- * @param    smumgr  the address of the powerplay hardware manager.
- * @param    msg the message to send.
- * @return   Always return 0.
- */
-int vega12_send_msg_to_smc_without_waiting(struct pp_hwmgr *hwmgr,
-		uint16_t msg)
-{
-	struct amdgpu_device *adev = hwmgr->adev;
-
-	WREG32_SOC15(MP1, 0, mmMP1_SMN_C2PMSG_66, msg);
-
-	return 0;
-}
-
-/*
- * Send a message to the SMC, and wait for its response.
- * @param    hwmgr  the address of the powerplay hardware manager.
- * @param    msg the message to send.
- * @return   Always return 0.
- */
-int vega12_send_msg_to_smc(struct pp_hwmgr *hwmgr, uint16_t msg)
-{
-	struct amdgpu_device *adev = hwmgr->adev;
-
-	vega12_wait_for_response(hwmgr);
-
-	WREG32_SOC15(MP1, 0, mmMP1_SMN_C2PMSG_90, 0);
-
-	vega12_send_msg_to_smc_without_waiting(hwmgr, msg);
-
-	if (vega12_wait_for_response(hwmgr) != 1)
-		pr_err("Failed to send message: 0x%x\n", msg);
-
-	return 0;
-}
-
-/*
- * Send a message to the SMC with parameter
- * @param    hwmgr:  the address of the powerplay hardware manager.
- * @param    msg: the message to send.
- * @param    parameter: the parameter to send
- * @return   Always return 0.
- */
-int vega12_send_msg_to_smc_with_parameter(struct pp_hwmgr *hwmgr,
-		uint16_t msg, uint32_t parameter)
-{
-	struct amdgpu_device *adev = hwmgr->adev;
-
-	vega12_wait_for_response(hwmgr);
-
-	WREG32_SOC15(MP1, 0, mmMP1_SMN_C2PMSG_90, 0);
-
-	WREG32_SOC15(MP1, 0, mmMP1_SMN_C2PMSG_82, parameter);
-
-	vega12_send_msg_to_smc_without_waiting(hwmgr, msg);
-
-	if (vega12_wait_for_response(hwmgr) != 1)
-		pr_err("Failed to send message: 0x%x\n", msg);
-
-	return 0;
-}
-
-
-/*
- * Send a message to the SMC with parameter, do not wait for response
- * @param    hwmgr:  the address of the powerplay hardware manager.
- * @param    msg: the message to send.
- * @param    parameter: the parameter to send
- * @return   The response that came from the SMC.
- */
-int vega12_send_msg_to_smc_with_parameter_without_waiting(
-		struct pp_hwmgr *hwmgr, uint16_t msg, uint32_t parameter)
-{
-	struct amdgpu_device *adev = hwmgr->adev;
-
-	WREG32_SOC15(MP1, 0, mmMP1_SMN_C2PMSG_66, parameter);
-
-	return vega12_send_msg_to_smc_without_waiting(hwmgr, msg);
-}
-
-/*
- * Retrieve an argument from SMC.
- * @param    hwmgr  the address of the powerplay hardware manager.
- * @param    arg     pointer to store the argument from SMC.
- * @return   Always return 0.
- */
-int vega12_read_arg_from_smc(struct pp_hwmgr *hwmgr, uint32_t *arg)
-{
-	struct amdgpu_device *adev = hwmgr->adev;
-
-	*arg = RREG32_SOC15(MP1, 0, mmMP1_SMN_C2PMSG_82);
-
-	return 0;
-}
-
 /*
  * Copy table from SMC into driver FB
  * @param   hwmgr    the address of the HW manager
@@ -192,16 +49,16 @@ int vega12_copy_table_from_smc(struct pp_hwmgr *hwmgr,
 			"Invalid SMU Table version!", return -EINVAL);
 	PP_ASSERT_WITH_CODE(priv->smu_tables.entry[table_id].size != 0,
 			"Invalid SMU Table Length!", return -EINVAL);
-	PP_ASSERT_WITH_CODE(vega12_send_msg_to_smc_with_parameter(hwmgr,
+	PP_ASSERT_WITH_CODE(smu9_send_msg_to_smc_with_parameter(hwmgr,
 			PPSMC_MSG_SetDriverDramAddrHigh,
 			upper_32_bits(priv->smu_tables.entry[table_id].mc_addr)) == 0,
 			"[CopyTableFromSMC] Attempt to Set Dram Addr High Failed!", return -EINVAL);
-	PP_ASSERT_WITH_CODE(vega12_send_msg_to_smc_with_parameter(hwmgr,
+	PP_ASSERT_WITH_CODE(smu9_send_msg_to_smc_with_parameter(hwmgr,
 			PPSMC_MSG_SetDriverDramAddrLow,
 			lower_32_bits(priv->smu_tables.entry[table_id].mc_addr)) == 0,
 			"[CopyTableFromSMC] Attempt to Set Dram Addr Low Failed!",
 			return -EINVAL);
-	PP_ASSERT_WITH_CODE(vega12_send_msg_to_smc_with_parameter(hwmgr,
+	PP_ASSERT_WITH_CODE(smu9_send_msg_to_smc_with_parameter(hwmgr,
 			PPSMC_MSG_TransferTableSmu2Dram,
 			table_id) == 0,
 			"[CopyTableFromSMC] Attempt to Transfer Table From SMU Failed!",
@@ -234,17 +91,17 @@ int vega12_copy_table_to_smc(struct pp_hwmgr *hwmgr,
 	memcpy(priv->smu_tables.entry[table_id].table, table,
 			priv->smu_tables.entry[table_id].size);
 
-	PP_ASSERT_WITH_CODE(vega12_send_msg_to_smc_with_parameter(hwmgr,
+	PP_ASSERT_WITH_CODE(smu9_send_msg_to_smc_with_parameter(hwmgr,
 			PPSMC_MSG_SetDriverDramAddrHigh,
 			upper_32_bits(priv->smu_tables.entry[table_id].mc_addr)) == 0,
 			"[CopyTableToSMC] Attempt to Set Dram Addr High Failed!",
 			return -EINVAL;);
-	PP_ASSERT_WITH_CODE(vega12_send_msg_to_smc_with_parameter(hwmgr,
+	PP_ASSERT_WITH_CODE(smu9_send_msg_to_smc_with_parameter(hwmgr,
 			PPSMC_MSG_SetDriverDramAddrLow,
 			lower_32_bits(priv->smu_tables.entry[table_id].mc_addr)) == 0,
 			"[CopyTableToSMC] Attempt to Set Dram Addr Low Failed!",
 			return -EINVAL);
-	PP_ASSERT_WITH_CODE(vega12_send_msg_to_smc_with_parameter(hwmgr,
+	PP_ASSERT_WITH_CODE(smu9_send_msg_to_smc_with_parameter(hwmgr,
 			PPSMC_MSG_TransferTableDram2Smu,
 			table_id) == 0,
 			"[CopyTableToSMC] Attempt to Transfer Table To SMU Failed!",
@@ -262,20 +119,20 @@ int vega12_enable_smc_features(struct pp_hwmgr *hwmgr,
 	smu_features_high = (uint32_t)((feature_mask & SMU_FEATURES_HIGH_MASK) >> SMU_FEATURES_HIGH_SHIFT);
 
 	if (enable) {
-		PP_ASSERT_WITH_CODE(vega12_send_msg_to_smc_with_parameter(hwmgr,
+		PP_ASSERT_WITH_CODE(smu9_send_msg_to_smc_with_parameter(hwmgr,
 				PPSMC_MSG_EnableSmuFeaturesLow, smu_features_low) == 0,
 				"[EnableDisableSMCFeatures] Attemp to enable SMU features Low failed!",
 				return -EINVAL);
-		PP_ASSERT_WITH_CODE(vega12_send_msg_to_smc_with_parameter(hwmgr,
+		PP_ASSERT_WITH_CODE(smu9_send_msg_to_smc_with_parameter(hwmgr,
 				PPSMC_MSG_EnableSmuFeaturesHigh, smu_features_high) == 0,
 				"[EnableDisableSMCFeatures] Attemp to enable SMU features High failed!",
 				return -EINVAL);
 	} else {
-		PP_ASSERT_WITH_CODE(vega12_send_msg_to_smc_with_parameter(hwmgr,
+		PP_ASSERT_WITH_CODE(smu9_send_msg_to_smc_with_parameter(hwmgr,
 				PPSMC_MSG_DisableSmuFeaturesLow, smu_features_low) == 0,
 				"[EnableDisableSMCFeatures] Attemp to disable SMU features Low failed!",
 				return -EINVAL);
-		PP_ASSERT_WITH_CODE(vega12_send_msg_to_smc_with_parameter(hwmgr,
+		PP_ASSERT_WITH_CODE(smu9_send_msg_to_smc_with_parameter(hwmgr,
 				PPSMC_MSG_DisableSmuFeaturesHigh, smu_features_high) == 0,
 				"[EnableDisableSMCFeatures] Attemp to disable SMU features High failed!",
 				return -EINVAL);
@@ -292,22 +149,17 @@ int vega12_get_enabled_smc_features(struct pp_hwmgr *hwmgr,
 	if (features_enabled == NULL)
 		return -EINVAL;
 
-	PP_ASSERT_WITH_CODE(vega12_send_msg_to_smc(hwmgr,
+	PP_ASSERT_WITH_CODE(smu9_send_msg_to_smc(hwmgr,
 			PPSMC_MSG_GetEnabledSmuFeaturesLow) == 0,
 			"[GetEnabledSMCFeatures] Attemp to get SMU features Low failed!",
 			return -EINVAL);
-	PP_ASSERT_WITH_CODE(vega12_read_arg_from_smc(hwmgr,
-			&smc_features_low) == 0,
-			"[GetEnabledSMCFeatures] Attemp to read SMU features Low argument failed!",
-			return -EINVAL);
-	PP_ASSERT_WITH_CODE(vega12_send_msg_to_smc(hwmgr,
+	smc_features_low = smu9_get_argument(hwmgr);
+
+	PP_ASSERT_WITH_CODE(smu9_send_msg_to_smc(hwmgr,
 			PPSMC_MSG_GetEnabledSmuFeaturesHigh) == 0,
 			"[GetEnabledSMCFeatures] Attemp to get SMU features High failed!",
 			return -EINVAL);
-	PP_ASSERT_WITH_CODE(vega12_read_arg_from_smc(hwmgr,
-			&smc_features_high) == 0,
-			"[GetEnabledSMCFeatures] Attemp to read SMU features High argument failed!",
-			return -EINVAL);
+	smc_features_high = smu9_get_argument(hwmgr);
 
 	*features_enabled = ((((uint64_t)smc_features_low << SMU_FEATURES_LOW_SHIFT) & SMU_FEATURES_LOW_MASK) |
 			(((uint64_t)smc_features_high << SMU_FEATURES_HIGH_SHIFT) & SMU_FEATURES_HIGH_MASK));
@@ -333,39 +185,16 @@ static int vega12_set_tools_address(struct pp_hwmgr *hwmgr)
 			(struct vega12_smumgr *)(hwmgr->smu_backend);
 
 	if (priv->smu_tables.entry[TABLE_PMSTATUSLOG].mc_addr) {
-		if (!vega12_send_msg_to_smc_with_parameter(hwmgr,
+		if (!smu9_send_msg_to_smc_with_parameter(hwmgr,
 				PPSMC_MSG_SetToolsDramAddrHigh,
 				upper_32_bits(priv->smu_tables.entry[TABLE_PMSTATUSLOG].mc_addr)))
-			vega12_send_msg_to_smc_with_parameter(hwmgr,
+			smu9_send_msg_to_smc_with_parameter(hwmgr,
 					PPSMC_MSG_SetToolsDramAddrLow,
 					lower_32_bits(priv->smu_tables.entry[TABLE_PMSTATUSLOG].mc_addr));
 	}
 	return 0;
 }
 
-#if 0 /* tentatively remove */
-static int vega12_verify_smc_interface(struct pp_hwmgr *hwmgr)
-{
-	uint32_t smc_driver_if_version;
-
-	PP_ASSERT_WITH_CODE(!vega12_send_msg_to_smc(hwmgr,
-			PPSMC_MSG_GetDriverIfVersion),
-			"Attempt to get SMC IF Version Number Failed!",
-			return -EINVAL);
-	vega12_read_arg_from_smc(hwmgr, &smc_driver_if_version);
-
-	if (smc_driver_if_version != SMU9_DRIVER_IF_VERSION) {
-		pr_err("Your firmware(0x%x) doesn't match \
-			SMU9_DRIVER_IF_VERSION(0x%x). \
-			Please update your firmware!\n",
-			smc_driver_if_version, SMU9_DRIVER_IF_VERSION);
-		return -EINVAL;
-	}
-
-	return 0;
-}
-#endif
-
 static int vega12_smu_init(struct pp_hwmgr *hwmgr)
 {
 	struct vega12_smumgr *priv;
@@ -513,16 +342,10 @@ static int vega12_smu_fini(struct pp_hwmgr *hwmgr)
 
 static int vega12_start_smu(struct pp_hwmgr *hwmgr)
 {
-	PP_ASSERT_WITH_CODE(vega12_is_smc_ram_running(hwmgr),
+	PP_ASSERT_WITH_CODE(smu9_is_smc_ram_running(hwmgr),
 			"SMC is not running!",
 			return -EINVAL);
 
-#if 0 /* tentatively remove */
-	PP_ASSERT_WITH_CODE(!vega12_verify_smc_interface(hwmgr),
-			"Failed to verify SMC interface!",
-			return -EINVAL);
-#endif
-
 	vega12_set_tools_address(hwmgr);
 
 	return 0;
@@ -533,9 +356,10 @@ const struct pp_smumgr_func vega12_smu_funcs = {
 	.smu_fini = &vega12_smu_fini,
 	.start_smu = &vega12_start_smu,
 	.request_smu_load_specific_fw = NULL,
-	.send_msg_to_smc = &vega12_send_msg_to_smc,
-	.send_msg_to_smc_with_parameter = &vega12_send_msg_to_smc_with_parameter,
+	.send_msg_to_smc = &smu9_send_msg_to_smc,
+	.send_msg_to_smc_with_parameter = &smu9_send_msg_to_smc_with_parameter,
 	.download_pptable_settings = NULL,
 	.upload_pptable_settings = NULL,
 	.is_dpm_running = vega12_is_dpm_running,
+	.get_argument = smu9_get_argument,
 };
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/vega12_smumgr.h b/drivers/gpu/drm/amd/powerplay/smumgr/vega12_smumgr.h
index 2810d387b611..b285cbc04019 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/vega12_smumgr.h
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/vega12_smumgr.h
@@ -48,7 +48,6 @@ struct vega12_smumgr {
 #define SMU_FEATURES_HIGH_MASK       0xFFFFFFFF00000000
 #define SMU_FEATURES_HIGH_SHIFT      32
 
-int vega12_read_arg_from_smc(struct pp_hwmgr *hwmgr, uint32_t *arg);
 int vega12_copy_table_from_smc(struct pp_hwmgr *hwmgr,
 		uint8_t *table, int16_t table_id);
 int vega12_copy_table_to_smc(struct pp_hwmgr *hwmgr,
diff --git a/drivers/gpu/drm/arm/hdlcd_crtc.c b/drivers/gpu/drm/arm/hdlcd_crtc.c
index f3f08cd6e9ef..e4d67b70244d 100644
--- a/drivers/gpu/drm/arm/hdlcd_crtc.c
+++ b/drivers/gpu/drm/arm/hdlcd_crtc.c
@@ -229,6 +229,8 @@ static const struct drm_crtc_helper_funcs hdlcd_crtc_helper_funcs = {
 static int hdlcd_plane_atomic_check(struct drm_plane *plane,
 				    struct drm_plane_state *state)
 {
+	int i;
+	struct drm_crtc *crtc;
 	struct drm_crtc_state *crtc_state;
 	u32 src_h = state->src_h >> 16;
 
@@ -238,20 +240,17 @@ static int hdlcd_plane_atomic_check(struct drm_plane *plane,
 		return -EINVAL;
 	}
 
-	if (!state->fb || !state->crtc)
-		return 0;
-
-	crtc_state = drm_atomic_get_existing_crtc_state(state->state,
-							state->crtc);
-	if (!crtc_state) {
-		DRM_DEBUG_KMS("Invalid crtc state\n");
-		return -EINVAL;
+	for_each_new_crtc_in_state(state->state, crtc, crtc_state, i) {
+		/* we cannot disable the plane while the CRTC is active */
+		if (!state->fb && crtc_state->active)
+			return -EINVAL;
+		return drm_atomic_helper_check_plane_state(state, crtc_state,
+						DRM_PLANE_HELPER_NO_SCALING,
+						DRM_PLANE_HELPER_NO_SCALING,
+						false, true);
 	}
 
-	return drm_atomic_helper_check_plane_state(state, crtc_state,
-						   DRM_PLANE_HELPER_NO_SCALING,
-						   DRM_PLANE_HELPER_NO_SCALING,
-						   false, true);
+	return 0;
 }
 
 static void hdlcd_plane_atomic_update(struct drm_plane *plane,
@@ -280,16 +279,10 @@ static const struct drm_plane_helper_funcs hdlcd_plane_helper_funcs = {
 	.atomic_update = hdlcd_plane_atomic_update,
 };
 
-static void hdlcd_plane_destroy(struct drm_plane *plane)
-{
-	drm_plane_helper_disable(plane, NULL);
-	drm_plane_cleanup(plane);
-}
-
 static const struct drm_plane_funcs hdlcd_plane_funcs = {
 	.update_plane		= drm_atomic_helper_update_plane,
 	.disable_plane		= drm_atomic_helper_disable_plane,
-	.destroy		= hdlcd_plane_destroy,
+	.destroy		= drm_plane_cleanup,
 	.reset			= drm_atomic_helper_plane_reset,
 	.atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
 	.atomic_destroy_state	= drm_atomic_helper_plane_destroy_state,
@@ -334,10 +327,8 @@ int hdlcd_setup_crtc(struct drm_device *drm)
 
 	ret = drm_crtc_init_with_planes(drm, &hdlcd->crtc, primary, NULL,
 					&hdlcd_crtc_funcs, NULL);
-	if (ret) {
-		hdlcd_plane_destroy(primary);
+	if (ret)
 		return ret;
-	}
 
 	drm_crtc_helper_add(&hdlcd->crtc, &hdlcd_crtc_helper_funcs);
 	return 0;
diff --git a/drivers/gpu/drm/arm/hdlcd_drv.c b/drivers/gpu/drm/arm/hdlcd_drv.c
index feaa8bc3d7b7..0ed1cde98cf8 100644
--- a/drivers/gpu/drm/arm/hdlcd_drv.c
+++ b/drivers/gpu/drm/arm/hdlcd_drv.c
@@ -27,6 +27,7 @@
 #include <drm/drm_fb_cma_helper.h>
 #include <drm/drm_gem_cma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
+#include <drm/drm_modeset_helper.h>
 #include <drm/drm_of.h>
 
 #include "hdlcd_drv.h"
@@ -100,16 +101,9 @@ setup_fail:
 	return ret;
 }
 
-static void hdlcd_fb_output_poll_changed(struct drm_device *drm)
-{
-	struct hdlcd_drm_private *hdlcd = drm->dev_private;
-
-	drm_fbdev_cma_hotplug_event(hdlcd->fbdev);
-}
-
 static const struct drm_mode_config_funcs hdlcd_mode_config_funcs = {
 	.fb_create = drm_gem_fb_create,
-	.output_poll_changed = hdlcd_fb_output_poll_changed,
+	.output_poll_changed = drm_fb_helper_output_poll_changed,
 	.atomic_check = drm_atomic_helper_check,
 	.atomic_commit = drm_atomic_helper_commit,
 };
@@ -124,13 +118,6 @@ static void hdlcd_setup_mode_config(struct drm_device *drm)
 	drm->mode_config.funcs = &hdlcd_mode_config_funcs;
 }
 
-static void hdlcd_lastclose(struct drm_device *drm)
-{
-	struct hdlcd_drm_private *hdlcd = drm->dev_private;
-
-	drm_fbdev_cma_restore_mode(hdlcd->fbdev);
-}
-
 static irqreturn_t hdlcd_irq(int irq, void *arg)
 {
 	struct drm_device *drm = arg;
@@ -246,7 +233,7 @@ static struct drm_driver hdlcd_driver = {
 	.driver_features = DRIVER_HAVE_IRQ | DRIVER_GEM |
 			   DRIVER_MODESET | DRIVER_PRIME |
 			   DRIVER_ATOMIC,
-	.lastclose = hdlcd_lastclose,
+	.lastclose = drm_fb_helper_lastclose,
 	.irq_handler = hdlcd_irq,
 	.irq_preinstall = hdlcd_irq_preinstall,
 	.irq_postinstall = hdlcd_irq_postinstall,
@@ -321,14 +308,9 @@ static int hdlcd_drm_bind(struct device *dev)
 	drm_mode_config_reset(drm);
 	drm_kms_helper_poll_init(drm);
 
-	hdlcd->fbdev = drm_fbdev_cma_init(drm, 32,
-					  drm->mode_config.num_connector);
-
-	if (IS_ERR(hdlcd->fbdev)) {
-		ret = PTR_ERR(hdlcd->fbdev);
-		hdlcd->fbdev = NULL;
+	ret = drm_fb_cma_fbdev_init(drm, 32, 0);
+	if (ret)
 		goto err_fbdev;
-	}
 
 	ret = drm_dev_register(drm, 0);
 	if (ret)
@@ -337,15 +319,13 @@ static int hdlcd_drm_bind(struct device *dev)
 	return 0;
 
 err_register:
-	if (hdlcd->fbdev) {
-		drm_fbdev_cma_fini(hdlcd->fbdev);
-		hdlcd->fbdev = NULL;
-	}
+	drm_fb_cma_fbdev_fini(drm);
 err_fbdev:
 	drm_kms_helper_poll_fini(drm);
 err_vblank:
 	pm_runtime_disable(drm->dev);
 err_pm_active:
+	drm_atomic_helper_shutdown(drm);
 	component_unbind_all(dev, drm);
 err_unload:
 	of_node_put(hdlcd->crtc.port);
@@ -366,23 +346,23 @@ static void hdlcd_drm_unbind(struct device *dev)
 	struct hdlcd_drm_private *hdlcd = drm->dev_private;
 
 	drm_dev_unregister(drm);
-	if (hdlcd->fbdev) {
-		drm_fbdev_cma_fini(hdlcd->fbdev);
-		hdlcd->fbdev = NULL;
-	}
+	drm_fb_cma_fbdev_fini(drm);
 	drm_kms_helper_poll_fini(drm);
 	component_unbind_all(dev, drm);
 	of_node_put(hdlcd->crtc.port);
 	hdlcd->crtc.port = NULL;
-	pm_runtime_get_sync(drm->dev);
+	pm_runtime_get_sync(dev);
+	drm_crtc_vblank_off(&hdlcd->crtc);
 	drm_irq_uninstall(drm);
-	pm_runtime_put_sync(drm->dev);
-	pm_runtime_disable(drm->dev);
-	of_reserved_mem_device_release(drm->dev);
+	drm_atomic_helper_shutdown(drm);
+	pm_runtime_put(dev);
+	if (pm_runtime_enabled(dev))
+		pm_runtime_disable(dev);
+	of_reserved_mem_device_release(dev);
 	drm_mode_config_cleanup(drm);
-	drm_dev_put(drm);
 	drm->dev_private = NULL;
 	dev_set_drvdata(dev, NULL);
+	drm_dev_put(drm);
 }
 
 static const struct component_master_ops hdlcd_master_ops = {
@@ -427,35 +407,15 @@ MODULE_DEVICE_TABLE(of, hdlcd_of_match);
 static int __maybe_unused hdlcd_pm_suspend(struct device *dev)
 {
 	struct drm_device *drm = dev_get_drvdata(dev);
-	struct hdlcd_drm_private *hdlcd = drm ? drm->dev_private : NULL;
-
-	if (!hdlcd)
-		return 0;
 
-	drm_kms_helper_poll_disable(drm);
-	drm_fbdev_cma_set_suspend_unlocked(hdlcd->fbdev, 1);
-
-	hdlcd->state = drm_atomic_helper_suspend(drm);
-	if (IS_ERR(hdlcd->state)) {
-		drm_fbdev_cma_set_suspend_unlocked(hdlcd->fbdev, 0);
-		drm_kms_helper_poll_enable(drm);
-		return PTR_ERR(hdlcd->state);
-	}
-
-	return 0;
+	return drm_mode_config_helper_suspend(drm);
 }
 
 static int __maybe_unused hdlcd_pm_resume(struct device *dev)
 {
 	struct drm_device *drm = dev_get_drvdata(dev);
-	struct hdlcd_drm_private *hdlcd = drm ? drm->dev_private : NULL;
-
-	if (!hdlcd)
-		return 0;
 
-	drm_atomic_helper_resume(drm, hdlcd->state);
-	drm_fbdev_cma_set_suspend_unlocked(hdlcd->fbdev, 0);
-	drm_kms_helper_poll_enable(drm);
+	drm_mode_config_helper_resume(drm);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/arm/hdlcd_drv.h b/drivers/gpu/drm/arm/hdlcd_drv.h
index 56f34dfff640..fd438d177b64 100644
--- a/drivers/gpu/drm/arm/hdlcd_drv.h
+++ b/drivers/gpu/drm/arm/hdlcd_drv.h
@@ -9,10 +9,8 @@
 struct hdlcd_drm_private {
 	void __iomem			*mmio;
 	struct clk			*clk;
-	struct drm_fbdev_cma		*fbdev;
 	struct drm_crtc			crtc;
 	struct drm_plane		*plane;
-	struct drm_atomic_state		*state;
 #ifdef CONFIG_DEBUG_FS
 	atomic_t buffer_underrun_count;
 	atomic_t bus_error_count;
diff --git a/drivers/gpu/drm/arm/malidp_drv.c b/drivers/gpu/drm/arm/malidp_drv.c
index 5b7260557391..08b5bb219816 100644
--- a/drivers/gpu/drm/arm/malidp_drv.c
+++ b/drivers/gpu/drm/arm/malidp_drv.c
@@ -616,6 +616,7 @@ static int malidp_bind(struct device *dev)
 	struct malidp_hw_device *hwdev;
 	struct platform_device *pdev = to_platform_device(dev);
 	struct of_device_id const *dev_id;
+	struct drm_encoder *encoder;
 	/* number of lines for the R, G and B output */
 	u8 output_width[MAX_OUTPUT_CHANNELS];
 	int ret = 0, i;
@@ -737,6 +738,15 @@ static int malidp_bind(struct device *dev)
 		goto bind_fail;
 	}
 
+	/* We expect to have a maximum of two encoders one for the actual
+	 * display and a virtual one for the writeback connector
+	 */
+	WARN_ON(drm->mode_config.num_encoder > 2);
+	list_for_each_entry(encoder, &drm->mode_config.encoder_list, head) {
+		encoder->possible_clones =
+				(1 << drm->mode_config.num_encoder) -  1;
+	}
+
 	ret = malidp_irq_init(pdev);
 	if (ret < 0)
 		goto irq_init_fail;
diff --git a/drivers/gpu/drm/arm/malidp_mw.c b/drivers/gpu/drm/arm/malidp_mw.c
index cfd718e7e97c..ba6ae66387c9 100644
--- a/drivers/gpu/drm/arm/malidp_mw.c
+++ b/drivers/gpu/drm/arm/malidp_mw.c
@@ -73,7 +73,7 @@ static void malidp_mw_connector_reset(struct drm_connector *connector)
 static enum drm_connector_status
 malidp_mw_connector_detect(struct drm_connector *connector, bool force)
 {
-	return connector_status_disconnected;
+	return connector_status_connected;
 }
 
 static void malidp_mw_connector_destroy(struct drm_connector *connector)
diff --git a/drivers/gpu/drm/armada/Makefile b/drivers/gpu/drm/armada/Makefile
index ecf25cf9f9f5..9bc3c3213724 100644
--- a/drivers/gpu/drm/armada/Makefile
+++ b/drivers/gpu/drm/armada/Makefile
@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0
 armada-y	:= armada_crtc.o armada_drv.o armada_fb.o armada_fbdev.o \
-		   armada_gem.o armada_overlay.o armada_trace.o
+		   armada_gem.o armada_overlay.o armada_plane.o armada_trace.o
 armada-y	+= armada_510.o
 armada-$(CONFIG_DEBUG_FS) += armada_debugfs.o
 
diff --git a/drivers/gpu/drm/armada/armada_510.c b/drivers/gpu/drm/armada/armada_510.c
index 41a784f5a5e6..2f7c048c5361 100644
--- a/drivers/gpu/drm/armada/armada_510.c
+++ b/drivers/gpu/drm/armada/armada_510.c
@@ -27,6 +27,10 @@ static int armada510_crtc_init(struct armada_crtc *dcrtc, struct device *dev)
 	/* Lower the watermark so to eliminate jitter at higher bandwidths */
 	armada_updatel(0x20, (1 << 11) | 0xff, dcrtc->base + LCD_CFG_RDREG4F);
 
+	/* Initialise SPU register */
+	writel_relaxed(ADV_HWC32ENABLE | ADV_HWC32ARGB | ADV_HWC32BLEND,
+		       dcrtc->base + LCD_SPU_ADV_REG);
+
 	return 0;
 }
 
@@ -75,9 +79,27 @@ static int armada510_crtc_compute_clock(struct armada_crtc *dcrtc,
 	return 0;
 }
 
+static void armada510_crtc_disable(struct armada_crtc *dcrtc)
+{
+	if (!IS_ERR(dcrtc->clk)) {
+		clk_disable_unprepare(dcrtc->clk);
+		dcrtc->clk = ERR_PTR(-EINVAL);
+	}
+}
+
+static void armada510_crtc_enable(struct armada_crtc *dcrtc,
+	const struct drm_display_mode *mode)
+{
+	if (IS_ERR(dcrtc->clk)) {
+		dcrtc->clk = dcrtc->extclk[0];
+		WARN_ON(clk_prepare_enable(dcrtc->clk));
+	}
+}
+
 const struct armada_variant armada510_ops = {
 	.has_spu_adv_reg = true,
-	.spu_adv_reg = ADV_HWC32ENABLE | ADV_HWC32ARGB | ADV_HWC32BLEND,
 	.init = armada510_crtc_init,
 	.compute_clock = armada510_crtc_compute_clock,
+	.disable = armada510_crtc_disable,
+	.enable = armada510_crtc_enable,
 };
diff --git a/drivers/gpu/drm/armada/armada_crtc.c b/drivers/gpu/drm/armada/armada_crtc.c
index 03eeee11dd5b..da9360688b55 100644
--- a/drivers/gpu/drm/armada/armada_crtc.c
+++ b/drivers/gpu/drm/armada/armada_crtc.c
@@ -11,6 +11,7 @@
 #include <linux/of_device.h>
 #include <linux/platform_device.h>
 #include <drm/drmP.h>
+#include <drm/drm_atomic.h>
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_plane_helper.h>
 #include <drm/drm_atomic_helper.h>
@@ -19,33 +20,9 @@
 #include "armada_fb.h"
 #include "armada_gem.h"
 #include "armada_hw.h"
+#include "armada_plane.h"
 #include "armada_trace.h"
 
-enum csc_mode {
-	CSC_AUTO = 0,
-	CSC_YUV_CCIR601 = 1,
-	CSC_YUV_CCIR709 = 2,
-	CSC_RGB_COMPUTER = 1,
-	CSC_RGB_STUDIO = 2,
-};
-
-static const uint32_t armada_primary_formats[] = {
-	DRM_FORMAT_UYVY,
-	DRM_FORMAT_YUYV,
-	DRM_FORMAT_VYUY,
-	DRM_FORMAT_YVYU,
-	DRM_FORMAT_ARGB8888,
-	DRM_FORMAT_ABGR8888,
-	DRM_FORMAT_XRGB8888,
-	DRM_FORMAT_XBGR8888,
-	DRM_FORMAT_RGB888,
-	DRM_FORMAT_BGR888,
-	DRM_FORMAT_ARGB1555,
-	DRM_FORMAT_ABGR1555,
-	DRM_FORMAT_RGB565,
-	DRM_FORMAT_BGR565,
-};
-
 /*
  * A note about interlacing.  Let's consider HDMI 1920x1080i.
  * The timing parameters we have from X are:
@@ -115,15 +92,13 @@ armada_drm_crtc_update_regs(struct armada_crtc *dcrtc, struct armada_regs *regs)
 	}
 }
 
-#define dpms_blanked(dpms)	((dpms) != DRM_MODE_DPMS_ON)
-
-static void armada_drm_crtc_update(struct armada_crtc *dcrtc)
+static void armada_drm_crtc_update(struct armada_crtc *dcrtc, bool enable)
 {
 	uint32_t dumb_ctrl;
 
 	dumb_ctrl = dcrtc->cfg_dumb_ctrl;
 
-	if (!dpms_blanked(dcrtc->dpms))
+	if (enable)
 		dumb_ctrl |= CFG_DUMB_ENA;
 
 	/*
@@ -132,295 +107,26 @@ static void armada_drm_crtc_update(struct armada_crtc *dcrtc)
 	 * force LCD_D[23:0] to output blank color, overriding the GPIO or
 	 * SPI usage.  So leave it as-is unless in DUMB24_RGB888_0 mode.
 	 */
-	if (dpms_blanked(dcrtc->dpms) &&
-	    (dumb_ctrl & DUMB_MASK) == DUMB24_RGB888_0) {
+	if (!enable && (dumb_ctrl & DUMB_MASK) == DUMB24_RGB888_0) {
 		dumb_ctrl &= ~DUMB_MASK;
 		dumb_ctrl |= DUMB_BLANK;
 	}
 
-	/*
-	 * The documentation doesn't indicate what the normal state of
-	 * the sync signals are.  Sebastian Hesselbart kindly probed
-	 * these signals on his board to determine their state.
-	 *
-	 * The non-inverted state of the sync signals is active high.
-	 * Setting these bits makes the appropriate signal active low.
-	 */
-	if (dcrtc->crtc.mode.flags & DRM_MODE_FLAG_NCSYNC)
-		dumb_ctrl |= CFG_INV_CSYNC;
-	if (dcrtc->crtc.mode.flags & DRM_MODE_FLAG_NHSYNC)
-		dumb_ctrl |= CFG_INV_HSYNC;
-	if (dcrtc->crtc.mode.flags & DRM_MODE_FLAG_NVSYNC)
-		dumb_ctrl |= CFG_INV_VSYNC;
-
-	if (dcrtc->dumb_ctrl != dumb_ctrl) {
-		dcrtc->dumb_ctrl = dumb_ctrl;
-		writel_relaxed(dumb_ctrl, dcrtc->base + LCD_SPU_DUMB_CTRL);
-	}
-}
-
-void armada_drm_plane_calc_addrs(u32 *addrs, struct drm_framebuffer *fb,
-	int x, int y)
-{
-	const struct drm_format_info *format = fb->format;
-	unsigned int num_planes = format->num_planes;
-	u32 addr = drm_fb_obj(fb)->dev_addr;
-	int i;
-
-	if (num_planes > 3)
-		num_planes = 3;
-
-	addrs[0] = addr + fb->offsets[0] + y * fb->pitches[0] +
-		   x * format->cpp[0];
-
-	y /= format->vsub;
-	x /= format->hsub;
-
-	for (i = 1; i < num_planes; i++)
-		addrs[i] = addr + fb->offsets[i] + y * fb->pitches[i] +
-			     x * format->cpp[i];
-	for (; i < 3; i++)
-		addrs[i] = 0;
-}
-
-static unsigned armada_drm_crtc_calc_fb(struct drm_framebuffer *fb,
-	int x, int y, struct armada_regs *regs, bool interlaced)
-{
-	unsigned pitch = fb->pitches[0];
-	u32 addrs[3], addr_odd, addr_even;
-	unsigned i = 0;
-
-	DRM_DEBUG_DRIVER("pitch %u x %d y %d bpp %d\n",
-		pitch, x, y, fb->format->cpp[0] * 8);
-
-	armada_drm_plane_calc_addrs(addrs, fb, x, y);
-
-	addr_odd = addr_even = addrs[0];
-
-	if (interlaced) {
-		addr_even += pitch;
-		pitch *= 2;
-	}
-
-	/* write offset, base, and pitch */
-	armada_reg_queue_set(regs, i, addr_odd, LCD_CFG_GRA_START_ADDR0);
-	armada_reg_queue_set(regs, i, addr_even, LCD_CFG_GRA_START_ADDR1);
-	armada_reg_queue_mod(regs, i, pitch, 0xffff, LCD_CFG_GRA_PITCH);
-
-	return i;
-}
-
-static void armada_drm_plane_work_call(struct armada_crtc *dcrtc,
-	struct armada_plane_work *work,
-	void (*fn)(struct armada_crtc *, struct armada_plane_work *))
-{
-	struct armada_plane *dplane = drm_to_armada_plane(work->plane);
-	struct drm_pending_vblank_event *event;
-	struct drm_framebuffer *fb;
-
-	if (fn)
-		fn(dcrtc, work);
-	drm_crtc_vblank_put(&dcrtc->crtc);
-
-	event = work->event;
-	fb = work->old_fb;
-	if (event || fb) {
-		struct drm_device *dev = dcrtc->crtc.dev;
-		unsigned long flags;
-
-		spin_lock_irqsave(&dev->event_lock, flags);
-		if (event)
-			drm_crtc_send_vblank_event(&dcrtc->crtc, event);
-		if (fb)
-			__armada_drm_queue_unref_work(dev, fb);
-		spin_unlock_irqrestore(&dev->event_lock, flags);
-	}
-
-	if (work->need_kfree)
-		kfree(work);
-
-	wake_up(&dplane->frame_wait);
+	armada_updatel(dumb_ctrl,
+		       ~(CFG_INV_CSYNC | CFG_INV_HSYNC | CFG_INV_VSYNC),
+		       dcrtc->base + LCD_SPU_DUMB_CTRL);
 }
 
-static void armada_drm_plane_work_run(struct armada_crtc *dcrtc,
-	struct drm_plane *plane)
-{
-	struct armada_plane *dplane = drm_to_armada_plane(plane);
-	struct armada_plane_work *work = xchg(&dplane->work, NULL);
-
-	/* Handle any pending frame work. */
-	if (work)
-		armada_drm_plane_work_call(dcrtc, work, work->fn);
-}
-
-int armada_drm_plane_work_queue(struct armada_crtc *dcrtc,
-	struct armada_plane_work *work)
-{
-	struct armada_plane *plane = drm_to_armada_plane(work->plane);
-	int ret;
-
-	ret = drm_crtc_vblank_get(&dcrtc->crtc);
-	if (ret)
-		return ret;
-
-	ret = cmpxchg(&plane->work, NULL, work) ? -EBUSY : 0;
-	if (ret)
-		drm_crtc_vblank_put(&dcrtc->crtc);
-
-	return ret;
-}
-
-int armada_drm_plane_work_wait(struct armada_plane *plane, long timeout)
-{
-	return wait_event_timeout(plane->frame_wait, !plane->work, timeout);
-}
-
-void armada_drm_plane_work_cancel(struct armada_crtc *dcrtc,
-	struct armada_plane *dplane)
-{
-	struct armada_plane_work *work = xchg(&dplane->work, NULL);
-
-	if (work)
-		armada_drm_plane_work_call(dcrtc, work, work->cancel);
-}
-
-static void armada_drm_crtc_complete_frame_work(struct armada_crtc *dcrtc,
-	struct armada_plane_work *work)
-{
-	unsigned long flags;
-
-	spin_lock_irqsave(&dcrtc->irq_lock, flags);
-	armada_drm_crtc_update_regs(dcrtc, work->regs);
-	spin_unlock_irqrestore(&dcrtc->irq_lock, flags);
-}
-
-static void armada_drm_crtc_complete_disable_work(struct armada_crtc *dcrtc,
-	struct armada_plane_work *work)
-{
-	unsigned long flags;
-
-	if (dcrtc->plane == work->plane)
-		dcrtc->plane = NULL;
-
-	spin_lock_irqsave(&dcrtc->irq_lock, flags);
-	armada_drm_crtc_update_regs(dcrtc, work->regs);
-	spin_unlock_irqrestore(&dcrtc->irq_lock, flags);
-}
-
-static struct armada_plane_work *
-armada_drm_crtc_alloc_plane_work(struct drm_plane *plane)
-{
-	struct armada_plane_work *work;
-	int i = 0;
-
-	work = kzalloc(sizeof(*work), GFP_KERNEL);
-	if (!work)
-		return NULL;
-
-	work->plane = plane;
-	work->fn = armada_drm_crtc_complete_frame_work;
-	work->need_kfree = true;
-	armada_reg_queue_end(work->regs, i);
-
-	return work;
-}
-
-static void armada_drm_crtc_finish_fb(struct armada_crtc *dcrtc,
-	struct drm_framebuffer *fb, bool force)
-{
-	struct armada_plane_work *work;
-
-	if (!fb)
-		return;
-
-	if (force) {
-		/* Display is disabled, so just drop the old fb */
-		drm_framebuffer_put(fb);
-		return;
-	}
-
-	work = armada_drm_crtc_alloc_plane_work(dcrtc->crtc.primary);
-	if (work) {
-		work->old_fb = fb;
-
-		if (armada_drm_plane_work_queue(dcrtc, work) == 0)
-			return;
-
-		kfree(work);
-	}
-
-	/*
-	 * Oops - just drop the reference immediately and hope for
-	 * the best.  The worst that will happen is the buffer gets
-	 * reused before it has finished being displayed.
-	 */
-	drm_framebuffer_put(fb);
-}
-
-static void armada_drm_vblank_off(struct armada_crtc *dcrtc)
-{
-	/*
-	 * Tell the DRM core that vblank IRQs aren't going to happen for
-	 * a while.  This cleans up any pending vblank events for us.
-	 */
-	drm_crtc_vblank_off(&dcrtc->crtc);
-	armada_drm_plane_work_run(dcrtc, dcrtc->crtc.primary);
-}
-
-/* The mode_config.mutex will be held for this call */
-static void armada_drm_crtc_dpms(struct drm_crtc *crtc, int dpms)
-{
-	struct armada_crtc *dcrtc = drm_to_armada_crtc(crtc);
-
-	if (dpms_blanked(dcrtc->dpms) != dpms_blanked(dpms)) {
-		if (dpms_blanked(dpms))
-			armada_drm_vblank_off(dcrtc);
-		else if (!IS_ERR(dcrtc->clk))
-			WARN_ON(clk_prepare_enable(dcrtc->clk));
-		dcrtc->dpms = dpms;
-		armada_drm_crtc_update(dcrtc);
-		if (!dpms_blanked(dpms))
-			drm_crtc_vblank_on(&dcrtc->crtc);
-		else if (!IS_ERR(dcrtc->clk))
-			clk_disable_unprepare(dcrtc->clk);
-	} else if (dcrtc->dpms != dpms) {
-		dcrtc->dpms = dpms;
-	}
-}
-
-/*
- * Prepare for a mode set.  Turn off overlay to ensure that we don't end
- * up with the overlay size being bigger than the active screen size.
- * We rely upon X refreshing this state after the mode set has completed.
- *
- * The mode_config.mutex will be held for this call
- */
-static void armada_drm_crtc_prepare(struct drm_crtc *crtc)
-{
-	struct armada_crtc *dcrtc = drm_to_armada_crtc(crtc);
-	struct drm_plane *plane;
-
-	/*
-	 * If we have an overlay plane associated with this CRTC, disable
-	 * it before the modeset to avoid its coordinates being outside
-	 * the new mode parameters.
-	 */
-	plane = dcrtc->plane;
-	if (plane) {
-		drm_plane_force_disable(plane);
-		WARN_ON(!armada_drm_plane_work_wait(drm_to_armada_plane(plane),
-						    HZ));
-	}
-}
-
-/* The mode_config.mutex will be held for this call */
-static void armada_drm_crtc_commit(struct drm_crtc *crtc)
+static void armada_drm_crtc_queue_state_event(struct drm_crtc *crtc)
 {
 	struct armada_crtc *dcrtc = drm_to_armada_crtc(crtc);
+	struct drm_pending_vblank_event *event;
 
-	if (dcrtc->dpms != DRM_MODE_DPMS_ON) {
-		dcrtc->dpms = DRM_MODE_DPMS_ON;
-		armada_drm_crtc_update(dcrtc);
+	/* If we have an event, we need vblank events enabled */
+	event = xchg(&crtc->state->event, NULL);
+	if (event) {
+		WARN_ON(drm_crtc_vblank_get(crtc) != 0);
+		dcrtc->event = event;
 	}
 }
 
@@ -465,8 +171,8 @@ static void armada_drm_crtc_enable_irq(struct armada_crtc *dcrtc, u32 mask)
 
 static void armada_drm_crtc_irq(struct armada_crtc *dcrtc, u32 stat)
 {
+	struct drm_pending_vblank_event *event;
 	void __iomem *base = dcrtc->base;
-	struct drm_plane *ovl_plane;
 
 	if (stat & DMA_FF_UNDERFLOW)
 		DRM_ERROR("video underflow on crtc %u\n", dcrtc->num);
@@ -476,10 +182,6 @@ static void armada_drm_crtc_irq(struct armada_crtc *dcrtc, u32 stat)
 	if (stat & VSYNC_IRQ)
 		drm_crtc_handle_vblank(&dcrtc->crtc);
 
-	ovl_plane = dcrtc->plane;
-	if (ovl_plane)
-		armada_drm_plane_work_run(dcrtc, ovl_plane);
-
 	spin_lock(&dcrtc->irq_lock);
 	if (stat & GRA_FRAME_IRQ && dcrtc->interlaced) {
 		int i = stat & GRA_FRAME_IRQ0 ? 0 : 1;
@@ -495,22 +197,35 @@ static void armada_drm_crtc_irq(struct armada_crtc *dcrtc, u32 stat)
 		writel_relaxed(val, base + LCD_SPU_ADV_REG);
 	}
 
-	if (stat & DUMB_FRAMEDONE && dcrtc->cursor_update) {
-		writel_relaxed(dcrtc->cursor_hw_pos,
-			       base + LCD_SPU_HWC_OVSA_HPXL_VLN);
-		writel_relaxed(dcrtc->cursor_hw_sz,
-			       base + LCD_SPU_HWC_HPXL_VLN);
-		armada_updatel(CFG_HWC_ENA,
-			       CFG_HWC_ENA | CFG_HWC_1BITMOD | CFG_HWC_1BITENA,
-			       base + LCD_SPU_DMA_CTRL0);
-		dcrtc->cursor_update = false;
+	if (stat & dcrtc->irq_ena & DUMB_FRAMEDONE) {
+		if (dcrtc->update_pending) {
+			armada_drm_crtc_update_regs(dcrtc, dcrtc->regs);
+			dcrtc->update_pending = false;
+		}
+		if (dcrtc->cursor_update) {
+			writel_relaxed(dcrtc->cursor_hw_pos,
+				       base + LCD_SPU_HWC_OVSA_HPXL_VLN);
+			writel_relaxed(dcrtc->cursor_hw_sz,
+				       base + LCD_SPU_HWC_HPXL_VLN);
+			armada_updatel(CFG_HWC_ENA,
+				       CFG_HWC_ENA | CFG_HWC_1BITMOD |
+				       CFG_HWC_1BITENA,
+				       base + LCD_SPU_DMA_CTRL0);
+			dcrtc->cursor_update = false;
+		}
 		armada_drm_crtc_disable_irq(dcrtc, DUMB_FRAMEDONE_ENA);
 	}
-
 	spin_unlock(&dcrtc->irq_lock);
 
-	if (stat & GRA_FRAME_IRQ)
-		armada_drm_plane_work_run(dcrtc, dcrtc->crtc.primary);
+	if (stat & VSYNC_IRQ && !dcrtc->update_pending) {
+		event = xchg(&dcrtc->event, NULL);
+		if (event) {
+			spin_lock(&dcrtc->crtc.dev->event_lock);
+			drm_crtc_send_vblank_event(&dcrtc->crtc, event);
+			spin_unlock(&dcrtc->crtc.dev->event_lock);
+			drm_crtc_vblank_put(&dcrtc->crtc);
+		}
+	}
 }
 
 static irqreturn_t armada_drm_irq(int irq, void *arg)
@@ -519,8 +234,9 @@ static irqreturn_t armada_drm_irq(int irq, void *arg)
 	u32 v, stat = readl_relaxed(dcrtc->base + LCD_SPU_IRQ_ISR);
 
 	/*
-	 * This is rediculous - rather than writing bits to clear, we
-	 * have to set the actual status register value.  This is racy.
+	 * Reading the ISR appears to clear bits provided CLEAN_SPU_IRQ_ISR
+	 * is set.  Writing has some other effect to acknowledge the IRQ -
+	 * without this, we only get a single IRQ.
 	 */
 	writel_relaxed(0, dcrtc->base + LCD_SPU_IRQ_ISR);
 
@@ -536,107 +252,16 @@ static irqreturn_t armada_drm_irq(int irq, void *arg)
 	return IRQ_NONE;
 }
 
-static uint32_t armada_drm_crtc_calculate_csc(struct armada_crtc *dcrtc)
-{
-	struct drm_display_mode *adj = &dcrtc->crtc.mode;
-	uint32_t val = 0;
-
-	if (dcrtc->csc_yuv_mode == CSC_YUV_CCIR709)
-		val |= CFG_CSC_YUV_CCIR709;
-	if (dcrtc->csc_rgb_mode == CSC_RGB_STUDIO)
-		val |= CFG_CSC_RGB_STUDIO;
-
-	/*
-	 * In auto mode, set the colorimetry, based upon the HDMI spec.
-	 * 1280x720p, 1920x1080p and 1920x1080i use ITU709, others use
-	 * ITU601.  It may be more appropriate to set this depending on
-	 * the source - but what if the graphic frame is YUV and the
-	 * video frame is RGB?
-	 */
-	if ((adj->hdisplay == 1280 && adj->vdisplay == 720 &&
-	     !(adj->flags & DRM_MODE_FLAG_INTERLACE)) ||
-	    (adj->hdisplay == 1920 && adj->vdisplay == 1080)) {
-		if (dcrtc->csc_yuv_mode == CSC_AUTO)
-			val |= CFG_CSC_YUV_CCIR709;
-	}
-
-	/*
-	 * We assume we're connected to a TV-like device, so the YUV->RGB
-	 * conversion should produce a limited range.  We should set this
-	 * depending on the connectors attached to this CRTC, and what
-	 * kind of device they report being connected.
-	 */
-	if (dcrtc->csc_rgb_mode == CSC_AUTO)
-		val |= CFG_CSC_RGB_STUDIO;
-
-	return val;
-}
-
-static void armada_drm_gra_plane_regs(struct armada_regs *regs,
-	struct drm_framebuffer *fb, struct armada_plane_state *state,
-	int x, int y, bool interlaced)
-{
-	unsigned int i;
-	u32 ctrl0;
-
-	i = armada_drm_crtc_calc_fb(fb, x, y, regs, interlaced);
-	armada_reg_queue_set(regs, i, state->dst_yx, LCD_SPU_GRA_OVSA_HPXL_VLN);
-	armada_reg_queue_set(regs, i, state->src_hw, LCD_SPU_GRA_HPXL_VLN);
-	armada_reg_queue_set(regs, i, state->dst_hw, LCD_SPU_GZM_HPXL_VLN);
-
-	ctrl0 = state->ctrl0;
-	if (interlaced)
-		ctrl0 |= CFG_GRA_FTOGGLE;
-
-	armada_reg_queue_mod(regs, i, ctrl0, CFG_GRAFORMAT |
-			     CFG_GRA_MOD(CFG_SWAPRB | CFG_SWAPUV |
-					 CFG_SWAPYU | CFG_YUV2RGB) |
-			     CFG_PALETTE_ENA | CFG_GRA_FTOGGLE |
-			     CFG_GRA_HSMOOTH | CFG_GRA_ENA,
-			     LCD_SPU_DMA_CTRL0);
-	armada_reg_queue_end(regs, i);
-}
-
-static void armada_drm_primary_set(struct drm_crtc *crtc,
-	struct drm_plane *plane, int x, int y)
-{
-	struct armada_plane_state *state = &drm_to_armada_plane(plane)->state;
-	struct armada_crtc *dcrtc = drm_to_armada_crtc(crtc);
-	struct armada_regs regs[8];
-	bool interlaced = dcrtc->interlaced;
-
-	armada_drm_gra_plane_regs(regs, plane->fb, state, x, y, interlaced);
-	armada_drm_crtc_update_regs(dcrtc, regs);
-}
-
 /* The mode_config.mutex will be held for this call */
-static int armada_drm_crtc_mode_set(struct drm_crtc *crtc,
-	struct drm_display_mode *mode, struct drm_display_mode *adj,
-	int x, int y, struct drm_framebuffer *old_fb)
+static void armada_drm_crtc_mode_set_nofb(struct drm_crtc *crtc)
 {
+	struct drm_display_mode *adj = &crtc->state->adjusted_mode;
 	struct armada_crtc *dcrtc = drm_to_armada_crtc(crtc);
 	struct armada_regs regs[17];
 	uint32_t lm, rm, tm, bm, val, sclk;
 	unsigned long flags;
 	unsigned i;
-	bool interlaced;
-
-	drm_framebuffer_get(crtc->primary->fb);
-
-	interlaced = !!(adj->flags & DRM_MODE_FLAG_INTERLACE);
-
-	val = CFG_GRA_ENA;
-	val |= CFG_GRA_FMT(drm_fb_to_armada_fb(dcrtc->crtc.primary->fb)->fmt);
-	val |= CFG_GRA_MOD(drm_fb_to_armada_fb(dcrtc->crtc.primary->fb)->mod);
-
-	if (drm_fb_to_armada_fb(dcrtc->crtc.primary->fb)->fmt > CFG_420)
-		val |= CFG_PALETTE_ENA;
-
-	drm_to_armada_plane(crtc->primary)->state.ctrl0 = val;
-	drm_to_armada_plane(crtc->primary)->state.src_hw =
-	drm_to_armada_plane(crtc->primary)->state.dst_hw =
-		adj->crtc_vdisplay << 16 | adj->crtc_hdisplay;
-	drm_to_armada_plane(crtc->primary)->state.dst_yx = 0;
+	bool interlaced = !!(adj->flags & DRM_MODE_FLAG_INTERLACE);
 
 	i = 0;
 	rm = adj->crtc_hsync_start - adj->crtc_hdisplay;
@@ -644,35 +269,15 @@ static int armada_drm_crtc_mode_set(struct drm_crtc *crtc,
 	bm = adj->crtc_vsync_start - adj->crtc_vdisplay;
 	tm = adj->crtc_vtotal - adj->crtc_vsync_end;
 
-	DRM_DEBUG_DRIVER("H: %d %d %d %d lm %d rm %d\n",
-		adj->crtc_hdisplay,
-		adj->crtc_hsync_start,
-		adj->crtc_hsync_end,
-		adj->crtc_htotal, lm, rm);
-	DRM_DEBUG_DRIVER("V: %d %d %d %d tm %d bm %d\n",
-		adj->crtc_vdisplay,
-		adj->crtc_vsync_start,
-		adj->crtc_vsync_end,
-		adj->crtc_vtotal, tm, bm);
-
-	/* Wait for pending flips to complete */
-	armada_drm_plane_work_wait(drm_to_armada_plane(dcrtc->crtc.primary),
-				   MAX_SCHEDULE_TIMEOUT);
-
-	drm_crtc_vblank_off(crtc);
-
-	val = dcrtc->dumb_ctrl & ~CFG_DUMB_ENA;
-	if (val != dcrtc->dumb_ctrl) {
-		dcrtc->dumb_ctrl = val;
-		writel_relaxed(val, dcrtc->base + LCD_SPU_DUMB_CTRL);
-	}
-
-	/*
-	 * If we are blanked, we would have disabled the clock.  Re-enable
-	 * it so that compute_clock() does the right thing.
-	 */
-	if (!IS_ERR(dcrtc->clk) && dpms_blanked(dcrtc->dpms))
-		WARN_ON(clk_prepare_enable(dcrtc->clk));
+	DRM_DEBUG_KMS("[CRTC:%d:%s] mode " DRM_MODE_FMT "\n",
+		      crtc->base.id, crtc->name,
+		      adj->base.id, adj->name, adj->vrefresh, adj->clock,
+		      adj->crtc_hdisplay, adj->crtc_hsync_start,
+		      adj->crtc_hsync_end, adj->crtc_htotal,
+		      adj->crtc_vdisplay, adj->crtc_vsync_start,
+		      adj->crtc_vsync_end, adj->crtc_vtotal,
+		      adj->type, adj->flags);
+	DRM_DEBUG_KMS("lm %d rm %d tm %d bm %d\n", lm, rm, tm, bm);
 
 	/* Now compute the divider for real */
 	dcrtc->variant->compute_clock(dcrtc, adj, &sclk);
@@ -689,25 +294,20 @@ static int armada_drm_crtc_mode_set(struct drm_crtc *crtc,
 
 	spin_lock_irqsave(&dcrtc->irq_lock, flags);
 
-	/* Ensure graphic fifo is enabled */
-	armada_reg_queue_mod(regs, i, 0, CFG_PDWN64x66, LCD_SPU_SRAM_PARA1);
-
 	/* Even interlaced/progressive frame */
 	dcrtc->v[1].spu_v_h_total = adj->crtc_vtotal << 16 |
 				    adj->crtc_htotal;
 	dcrtc->v[1].spu_v_porch = tm << 16 | bm;
 	val = adj->crtc_hsync_start;
-	dcrtc->v[1].spu_adv_reg = val << 20 | val | ADV_VSYNCOFFEN |
-		dcrtc->variant->spu_adv_reg;
+	dcrtc->v[1].spu_adv_reg = val << 20 | val | ADV_VSYNCOFFEN;
 
 	if (interlaced) {
 		/* Odd interlaced frame */
+		val -= adj->crtc_htotal / 2;
+		dcrtc->v[0].spu_adv_reg = val << 20 | val | ADV_VSYNCOFFEN;
 		dcrtc->v[0].spu_v_h_total = dcrtc->v[1].spu_v_h_total +
 						(1 << 16);
 		dcrtc->v[0].spu_v_porch = dcrtc->v[1].spu_v_porch + 1;
-		val = adj->crtc_hsync_start - adj->crtc_htotal / 2;
-		dcrtc->v[0].spu_adv_reg = val << 20 | val | ADV_VSYNCOFFEN |
-			dcrtc->variant->spu_adv_reg;
 	} else {
 		dcrtc->v[0] = dcrtc->v[1];
 	}
@@ -720,77 +320,136 @@ static int armada_drm_crtc_mode_set(struct drm_crtc *crtc,
 	armada_reg_queue_set(regs, i, dcrtc->v[0].spu_v_h_total,
 			   LCD_SPUT_V_H_TOTAL);
 
-	if (dcrtc->variant->has_spu_adv_reg) {
+	if (dcrtc->variant->has_spu_adv_reg)
 		armada_reg_queue_mod(regs, i, dcrtc->v[0].spu_adv_reg,
 				     ADV_VSYNC_L_OFF | ADV_VSYNC_H_OFF |
 				     ADV_VSYNCOFFEN, LCD_SPU_ADV_REG);
-	}
 
 	val = adj->flags & DRM_MODE_FLAG_NVSYNC ? CFG_VSYNC_INV : 0;
 	armada_reg_queue_mod(regs, i, val, CFG_VSYNC_INV, LCD_SPU_DMA_CTRL1);
 
-	val = dcrtc->spu_iopad_ctrl | armada_drm_crtc_calculate_csc(dcrtc);
-	armada_reg_queue_set(regs, i, val, LCD_SPU_IOPAD_CONTROL);
+	/*
+	 * The documentation doesn't indicate what the normal state of
+	 * the sync signals are.  Sebastian Hesselbart kindly probed
+	 * these signals on his board to determine their state.
+	 *
+	 * The non-inverted state of the sync signals is active high.
+	 * Setting these bits makes the appropriate signal active low.
+	 */
+	val = 0;
+	if (adj->flags & DRM_MODE_FLAG_NCSYNC)
+		val |= CFG_INV_CSYNC;
+	if (adj->flags & DRM_MODE_FLAG_NHSYNC)
+		val |= CFG_INV_HSYNC;
+	if (adj->flags & DRM_MODE_FLAG_NVSYNC)
+		val |= CFG_INV_VSYNC;
+	armada_reg_queue_mod(regs, i, val, CFG_INV_CSYNC | CFG_INV_HSYNC |
+			     CFG_INV_VSYNC, LCD_SPU_DUMB_CTRL);
 	armada_reg_queue_end(regs, i);
 
 	armada_drm_crtc_update_regs(dcrtc, regs);
-
-	armada_drm_primary_set(crtc, crtc->primary, x, y);
 	spin_unlock_irqrestore(&dcrtc->irq_lock, flags);
+}
 
-	armada_drm_crtc_update(dcrtc);
+static void armada_drm_crtc_atomic_begin(struct drm_crtc *crtc,
+					 struct drm_crtc_state *old_crtc_state)
+{
+	struct armada_crtc *dcrtc = drm_to_armada_crtc(crtc);
 
-	drm_crtc_vblank_on(crtc);
-	armada_drm_crtc_finish_fb(dcrtc, old_fb, dpms_blanked(dcrtc->dpms));
+	DRM_DEBUG_KMS("[CRTC:%d:%s]\n", crtc->base.id, crtc->name);
 
-	return 0;
+	dcrtc->regs_idx = 0;
+	dcrtc->regs = dcrtc->atomic_regs;
 }
 
-/* The mode_config.mutex will be held for this call */
-static int armada_drm_crtc_mode_set_base(struct drm_crtc *crtc, int x, int y,
-	struct drm_framebuffer *old_fb)
+static void armada_drm_crtc_atomic_flush(struct drm_crtc *crtc,
+					 struct drm_crtc_state *old_crtc_state)
 {
 	struct armada_crtc *dcrtc = drm_to_armada_crtc(crtc);
-	struct armada_regs regs[4];
-	unsigned i;
 
-	i = armada_drm_crtc_calc_fb(crtc->primary->fb, crtc->x, crtc->y, regs,
-				    dcrtc->interlaced);
-	armada_reg_queue_end(regs, i);
+	DRM_DEBUG_KMS("[CRTC:%d:%s]\n", crtc->base.id, crtc->name);
+
+	armada_reg_queue_end(dcrtc->regs, dcrtc->regs_idx);
+
+	/*
+	 * If we aren't doing a full modeset, then we need to queue
+	 * the event here.
+	 */
+	if (!drm_atomic_crtc_needs_modeset(crtc->state)) {
+		dcrtc->update_pending = true;
+		armada_drm_crtc_queue_state_event(crtc);
+		spin_lock_irq(&dcrtc->irq_lock);
+		armada_drm_crtc_enable_irq(dcrtc, DUMB_FRAMEDONE_ENA);
+		spin_unlock_irq(&dcrtc->irq_lock);
+	} else {
+		spin_lock_irq(&dcrtc->irq_lock);
+		armada_drm_crtc_update_regs(dcrtc, dcrtc->regs);
+		spin_unlock_irq(&dcrtc->irq_lock);
+	}
+}
 
-	/* Wait for pending flips to complete */
-	armada_drm_plane_work_wait(drm_to_armada_plane(dcrtc->crtc.primary),
-				   MAX_SCHEDULE_TIMEOUT);
+static void armada_drm_crtc_atomic_disable(struct drm_crtc *crtc,
+					   struct drm_crtc_state *old_state)
+{
+	struct armada_crtc *dcrtc = drm_to_armada_crtc(crtc);
+	struct drm_pending_vblank_event *event;
 
-	/* Take a reference to the new fb as we're using it */
-	drm_framebuffer_get(crtc->primary->fb);
+	DRM_DEBUG_KMS("[CRTC:%d:%s]\n", crtc->base.id, crtc->name);
 
-	/* Update the base in the CRTC */
-	armada_drm_crtc_update_regs(dcrtc, regs);
+	drm_crtc_vblank_off(crtc);
+	armada_drm_crtc_update(dcrtc, false);
 
-	/* Drop our previously held reference */
-	armada_drm_crtc_finish_fb(dcrtc, old_fb, dpms_blanked(dcrtc->dpms));
+	if (!crtc->state->active) {
+		/*
+		 * This modeset will be leaving the CRTC disabled, so
+		 * call the backend to disable upstream clocks etc.
+		 */
+		if (dcrtc->variant->disable)
+			dcrtc->variant->disable(dcrtc);
 
-	return 0;
+		/*
+		 * We will not receive any further vblank events.
+		 * Send the flip_done event manually.
+		 */
+		event = crtc->state->event;
+		crtc->state->event = NULL;
+		if (event) {
+			spin_lock_irq(&crtc->dev->event_lock);
+			drm_crtc_send_vblank_event(crtc, event);
+			spin_unlock_irq(&crtc->dev->event_lock);
+		}
+	}
 }
 
-/* The mode_config.mutex will be held for this call */
-static void armada_drm_crtc_disable(struct drm_crtc *crtc)
+static void armada_drm_crtc_atomic_enable(struct drm_crtc *crtc,
+					  struct drm_crtc_state *old_state)
 {
-	armada_drm_crtc_dpms(crtc, DRM_MODE_DPMS_OFF);
+	struct armada_crtc *dcrtc = drm_to_armada_crtc(crtc);
+
+	DRM_DEBUG_KMS("[CRTC:%d:%s]\n", crtc->base.id, crtc->name);
+
+	if (!old_state->active) {
+		/*
+		 * This modeset is enabling the CRTC after it having
+		 * been disabled.  Reverse the call to ->disable in
+		 * the atomic_disable().
+		 */
+		if (dcrtc->variant->enable)
+			dcrtc->variant->enable(dcrtc, &crtc->state->adjusted_mode);
+	}
+	armada_drm_crtc_update(dcrtc, true);
+	drm_crtc_vblank_on(crtc);
 
-	/* Disable our primary plane when we disable the CRTC. */
-	crtc->primary->funcs->disable_plane(crtc->primary, NULL);
+	armada_drm_crtc_queue_state_event(crtc);
 }
 
 static const struct drm_crtc_helper_funcs armada_crtc_helper_funcs = {
-	.dpms		= armada_drm_crtc_dpms,
-	.prepare	= armada_drm_crtc_prepare,
-	.commit		= armada_drm_crtc_commit,
 	.mode_fixup	= armada_drm_crtc_mode_fixup,
-	.mode_set	= armada_drm_crtc_mode_set,
-	.mode_set_base	= armada_drm_crtc_mode_set_base,
-	.disable	= armada_drm_crtc_disable,
+	.mode_set_nofb	= armada_drm_crtc_mode_set_nofb,
+	.atomic_begin	= armada_drm_crtc_atomic_begin,
+	.atomic_flush	= armada_drm_crtc_atomic_flush,
+	.atomic_disable	= armada_drm_crtc_atomic_disable,
+	.atomic_enable	= armada_drm_crtc_atomic_enable,
 };
 
 static void armada_load_cursor_argb(void __iomem *base, uint32_t *pix,
@@ -883,7 +542,6 @@ static int armada_drm_crtc_cursor_update(struct armada_crtc *dcrtc, bool reload)
 
 	if (!dcrtc->cursor_obj || !h || !w) {
 		spin_lock_irq(&dcrtc->irq_lock);
-		armada_drm_crtc_disable_irq(dcrtc, DUMB_FRAMEDONE_ENA);
 		dcrtc->cursor_update = false;
 		armada_updatel(0, CFG_HWC_ENA, dcrtc->base + LCD_SPU_DMA_CTRL0);
 		spin_unlock_irq(&dcrtc->irq_lock);
@@ -907,7 +565,6 @@ static int armada_drm_crtc_cursor_update(struct armada_crtc *dcrtc, bool reload)
 
 	if (dcrtc->cursor_hw_sz != (h << 16 | w)) {
 		spin_lock_irq(&dcrtc->irq_lock);
-		armada_drm_crtc_disable_irq(dcrtc, DUMB_FRAMEDONE_ENA);
 		dcrtc->cursor_update = false;
 		armada_updatel(0, CFG_HWC_ENA, dcrtc->base + LCD_SPU_DMA_CTRL0);
 		spin_unlock_irq(&dcrtc->irq_lock);
@@ -1015,8 +672,8 @@ static void armada_drm_crtc_destroy(struct drm_crtc *crtc)
 	priv->dcrtc[dcrtc->num] = NULL;
 	drm_crtc_cleanup(&dcrtc->crtc);
 
-	if (!IS_ERR(dcrtc->clk))
-		clk_disable_unprepare(dcrtc->clk);
+	if (dcrtc->variant->disable)
+		dcrtc->variant->disable(dcrtc);
 
 	writel_relaxed(0, dcrtc->base + LCD_SPU_IRQ_ENA);
 
@@ -1025,361 +682,51 @@ static void armada_drm_crtc_destroy(struct drm_crtc *crtc)
 	kfree(dcrtc);
 }
 
-/*
- * The mode_config lock is held here, to prevent races between this
- * and a mode_set.
- */
-static int armada_drm_crtc_page_flip(struct drm_crtc *crtc,
-	struct drm_framebuffer *fb, struct drm_pending_vblank_event *event, uint32_t page_flip_flags,
-	struct drm_modeset_acquire_ctx *ctx)
-{
-	struct armada_crtc *dcrtc = drm_to_armada_crtc(crtc);
-	struct armada_plane_work *work;
-	unsigned i;
-	int ret;
-
-	/* We don't support changing the pixel format */
-	if (fb->format != crtc->primary->fb->format)
-		return -EINVAL;
-
-	work = armada_drm_crtc_alloc_plane_work(dcrtc->crtc.primary);
-	if (!work)
-		return -ENOMEM;
-
-	work->event = event;
-	work->old_fb = dcrtc->crtc.primary->fb;
-
-	i = armada_drm_crtc_calc_fb(fb, crtc->x, crtc->y, work->regs,
-				    dcrtc->interlaced);
-	armada_reg_queue_end(work->regs, i);
-
-	/*
-	 * Ensure that we hold a reference on the new framebuffer.
-	 * This has to match the behaviour in mode_set.
-	 */
-	drm_framebuffer_get(fb);
-
-	ret = armada_drm_plane_work_queue(dcrtc, work);
-	if (ret) {
-		/* Undo our reference above */
-		drm_framebuffer_put(fb);
-		kfree(work);
-		return ret;
-	}
-
-	/*
-	 * Don't take a reference on the new framebuffer;
-	 * drm_mode_page_flip_ioctl() has already grabbed a reference and
-	 * will _not_ drop that reference on successful return from this
-	 * function.  Simply mark this new framebuffer as the current one.
-	 */
-	dcrtc->crtc.primary->fb = fb;
-
-	/*
-	 * Finally, if the display is blanked, we won't receive an
-	 * interrupt, so complete it now.
-	 */
-	if (dpms_blanked(dcrtc->dpms))
-		armada_drm_plane_work_run(dcrtc, dcrtc->crtc.primary);
-
-	return 0;
-}
-
-static int
-armada_drm_crtc_set_property(struct drm_crtc *crtc,
-	struct drm_property *property, uint64_t val)
-{
-	struct armada_private *priv = crtc->dev->dev_private;
-	struct armada_crtc *dcrtc = drm_to_armada_crtc(crtc);
-	bool update_csc = false;
-
-	if (property == priv->csc_yuv_prop) {
-		dcrtc->csc_yuv_mode = val;
-		update_csc = true;
-	} else if (property == priv->csc_rgb_prop) {
-		dcrtc->csc_rgb_mode = val;
-		update_csc = true;
-	}
-
-	if (update_csc) {
-		uint32_t val;
-
-		val = dcrtc->spu_iopad_ctrl |
-		      armada_drm_crtc_calculate_csc(dcrtc);
-		writel_relaxed(val, dcrtc->base + LCD_SPU_IOPAD_CONTROL);
-	}
-
-	return 0;
-}
-
 /* These are called under the vbl_lock. */
 static int armada_drm_crtc_enable_vblank(struct drm_crtc *crtc)
 {
 	struct armada_crtc *dcrtc = drm_to_armada_crtc(crtc);
+	unsigned long flags;
 
+	spin_lock_irqsave(&dcrtc->irq_lock, flags);
 	armada_drm_crtc_enable_irq(dcrtc, VSYNC_IRQ_ENA);
+	spin_unlock_irqrestore(&dcrtc->irq_lock, flags);
 	return 0;
 }
 
 static void armada_drm_crtc_disable_vblank(struct drm_crtc *crtc)
 {
 	struct armada_crtc *dcrtc = drm_to_armada_crtc(crtc);
+	unsigned long flags;
 
+	spin_lock_irqsave(&dcrtc->irq_lock, flags);
 	armada_drm_crtc_disable_irq(dcrtc, VSYNC_IRQ_ENA);
+	spin_unlock_irqrestore(&dcrtc->irq_lock, flags);
 }
 
 static const struct drm_crtc_funcs armada_crtc_funcs = {
+	.reset		= drm_atomic_helper_crtc_reset,
 	.cursor_set	= armada_drm_crtc_cursor_set,
 	.cursor_move	= armada_drm_crtc_cursor_move,
 	.destroy	= armada_drm_crtc_destroy,
-	.set_config	= drm_crtc_helper_set_config,
-	.page_flip	= armada_drm_crtc_page_flip,
-	.set_property	= armada_drm_crtc_set_property,
+	.set_config	= drm_atomic_helper_set_config,
+	.page_flip	= drm_atomic_helper_page_flip,
+	.atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_crtc_destroy_state,
 	.enable_vblank	= armada_drm_crtc_enable_vblank,
 	.disable_vblank	= armada_drm_crtc_disable_vblank,
 };
 
-static void armada_drm_primary_update_state(struct drm_plane_state *state,
-	struct armada_regs *regs)
-{
-	struct armada_plane *dplane = drm_to_armada_plane(state->plane);
-	struct armada_crtc *dcrtc = drm_to_armada_crtc(state->crtc);
-	struct armada_framebuffer *dfb = drm_fb_to_armada_fb(state->fb);
-	bool was_disabled;
-	unsigned int idx = 0;
-	u32 val;
-
-	val = CFG_GRA_FMT(dfb->fmt) | CFG_GRA_MOD(dfb->mod);
-	if (dfb->fmt > CFG_420)
-		val |= CFG_PALETTE_ENA;
-	if (state->visible)
-		val |= CFG_GRA_ENA;
-	if (drm_rect_width(&state->src) >> 16 != drm_rect_width(&state->dst))
-		val |= CFG_GRA_HSMOOTH;
-
-	was_disabled = !(dplane->state.ctrl0 & CFG_GRA_ENA);
-	if (was_disabled)
-		armada_reg_queue_mod(regs, idx,
-				     0, CFG_PDWN64x66, LCD_SPU_SRAM_PARA1);
-
-	dplane->state.ctrl0 = val;
-	dplane->state.src_hw = (drm_rect_height(&state->src) & 0xffff0000) |
-				drm_rect_width(&state->src) >> 16;
-	dplane->state.dst_hw = drm_rect_height(&state->dst) << 16 |
-			       drm_rect_width(&state->dst);
-	dplane->state.dst_yx = state->dst.y1 << 16 | state->dst.x1;
-
-	armada_drm_gra_plane_regs(regs + idx, &dfb->fb, &dplane->state,
-				  state->src.x1 >> 16, state->src.y1 >> 16,
-				  dcrtc->interlaced);
-
-	dplane->state.vsync_update = !was_disabled;
-	dplane->state.changed = true;
-}
-
-static int armada_drm_primary_update(struct drm_plane *plane,
-	struct drm_crtc *crtc, struct drm_framebuffer *fb,
-	int crtc_x, int crtc_y, unsigned int crtc_w, unsigned int crtc_h,
-	uint32_t src_x, uint32_t src_y, uint32_t src_w, uint32_t src_h,
-	struct drm_modeset_acquire_ctx *ctx)
-{
-	struct armada_plane *dplane = drm_to_armada_plane(plane);
-	struct armada_crtc *dcrtc = drm_to_armada_crtc(crtc);
-	struct armada_plane_work *work;
-	struct drm_plane_state state = {
-		.plane = plane,
-		.crtc = crtc,
-		.fb = fb,
-		.src_x = src_x,
-		.src_y = src_y,
-		.src_w = src_w,
-		.src_h = src_h,
-		.crtc_x = crtc_x,
-		.crtc_y = crtc_y,
-		.crtc_w = crtc_w,
-		.crtc_h = crtc_h,
-		.rotation = DRM_MODE_ROTATE_0,
-	};
-	struct drm_crtc_state crtc_state = {
-		.crtc = crtc,
-		.enable = crtc->enabled,
-		.mode = crtc->mode,
-	};
-	int ret;
-
-	ret = drm_atomic_helper_check_plane_state(&state, &crtc_state, 0,
-						  INT_MAX, true, false);
-	if (ret)
-		return ret;
-
-	work = &dplane->works[dplane->next_work];
-	work->fn = armada_drm_crtc_complete_frame_work;
-
-	if (plane->fb != fb) {
-		/*
-		 * Take a reference on the new framebuffer - we want to
-		 * hold on to it while the hardware is displaying it.
-		 */
-		drm_framebuffer_reference(fb);
-
-		work->old_fb = plane->fb;
-	} else {
-		work->old_fb = NULL;
-	}
-
-	armada_drm_primary_update_state(&state, work->regs);
-
-	if (!dplane->state.changed)
-		return 0;
-
-	/* Wait for pending work to complete */
-	if (armada_drm_plane_work_wait(dplane, HZ / 10) == 0)
-		armada_drm_plane_work_cancel(dcrtc, dplane);
-
-	if (!dplane->state.vsync_update) {
-		work->fn(dcrtc, work);
-		if (work->old_fb)
-			drm_framebuffer_unreference(work->old_fb);
-		return 0;
-	}
-
-	/* Queue it for update on the next interrupt if we are enabled */
-	ret = armada_drm_plane_work_queue(dcrtc, work);
-	if (ret) {
-		work->fn(dcrtc, work);
-		if (work->old_fb)
-			drm_framebuffer_unreference(work->old_fb);
-	}
-
-	dplane->next_work = !dplane->next_work;
-
-	return 0;
-}
-
-int armada_drm_plane_disable(struct drm_plane *plane,
-			     struct drm_modeset_acquire_ctx *ctx)
-{
-	struct armada_plane *dplane = drm_to_armada_plane(plane);
-	struct armada_crtc *dcrtc;
-	struct armada_plane_work *work;
-	unsigned int idx = 0;
-	u32 sram_para1, enable_mask;
-
-	if (!plane->crtc)
-		return 0;
-
-	/*
-	 * Arrange to power down most RAMs and FIFOs if this is the primary
-	 * plane, otherwise just the YUV FIFOs for the overlay plane.
-	 */
-	if (plane->type == DRM_PLANE_TYPE_PRIMARY) {
-		sram_para1 = CFG_PDWN256x32 | CFG_PDWN256x24 | CFG_PDWN256x8 |
-			     CFG_PDWN32x32 | CFG_PDWN64x66;
-		enable_mask = CFG_GRA_ENA;
-	} else {
-		sram_para1 = CFG_PDWN16x66 | CFG_PDWN32x66;
-		enable_mask = CFG_DMA_ENA;
-	}
-
-	dplane->state.ctrl0 &= ~enable_mask;
-
-	dcrtc = drm_to_armada_crtc(plane->crtc);
-
-	/*
-	 * Try to disable the plane and drop our ref on the framebuffer
-	 * at the next frame update. If we fail for any reason, disable
-	 * the plane immediately.
-	 */
-	work = &dplane->works[dplane->next_work];
-	work->fn = armada_drm_crtc_complete_disable_work;
-	work->cancel = armada_drm_crtc_complete_disable_work;
-	work->old_fb = plane->fb;
-
-	armada_reg_queue_mod(work->regs, idx,
-			     0, enable_mask, LCD_SPU_DMA_CTRL0);
-	armada_reg_queue_mod(work->regs, idx,
-			     sram_para1, 0, LCD_SPU_SRAM_PARA1);
-	armada_reg_queue_end(work->regs, idx);
-
-	/* Wait for any preceding work to complete, but don't wedge */
-	if (WARN_ON(!armada_drm_plane_work_wait(dplane, HZ)))
-		armada_drm_plane_work_cancel(dcrtc, dplane);
-
-	if (armada_drm_plane_work_queue(dcrtc, work)) {
-		work->fn(dcrtc, work);
-		if (work->old_fb)
-			drm_framebuffer_unreference(work->old_fb);
-	}
-
-	dplane->next_work = !dplane->next_work;
-
-	return 0;
-}
-
-static const struct drm_plane_funcs armada_primary_plane_funcs = {
-	.update_plane	= armada_drm_primary_update,
-	.disable_plane	= armada_drm_plane_disable,
-	.destroy	= drm_primary_helper_destroy,
-};
-
-int armada_drm_plane_init(struct armada_plane *plane)
-{
-	unsigned int i;
-
-	for (i = 0; i < ARRAY_SIZE(plane->works); i++)
-		plane->works[i].plane = &plane->base;
-
-	init_waitqueue_head(&plane->frame_wait);
-
-	return 0;
-}
-
-static const struct drm_prop_enum_list armada_drm_csc_yuv_enum_list[] = {
-	{ CSC_AUTO,        "Auto" },
-	{ CSC_YUV_CCIR601, "CCIR601" },
-	{ CSC_YUV_CCIR709, "CCIR709" },
-};
-
-static const struct drm_prop_enum_list armada_drm_csc_rgb_enum_list[] = {
-	{ CSC_AUTO,         "Auto" },
-	{ CSC_RGB_COMPUTER, "Computer system" },
-	{ CSC_RGB_STUDIO,   "Studio" },
-};
-
-static int armada_drm_crtc_create_properties(struct drm_device *dev)
-{
-	struct armada_private *priv = dev->dev_private;
-
-	if (priv->csc_yuv_prop)
-		return 0;
-
-	priv->csc_yuv_prop = drm_property_create_enum(dev, 0,
-				"CSC_YUV", armada_drm_csc_yuv_enum_list,
-				ARRAY_SIZE(armada_drm_csc_yuv_enum_list));
-	priv->csc_rgb_prop = drm_property_create_enum(dev, 0,
-				"CSC_RGB", armada_drm_csc_rgb_enum_list,
-				ARRAY_SIZE(armada_drm_csc_rgb_enum_list));
-
-	if (!priv->csc_yuv_prop || !priv->csc_rgb_prop)
-		return -ENOMEM;
-
-	return 0;
-}
-
 static int armada_drm_crtc_create(struct drm_device *drm, struct device *dev,
 	struct resource *res, int irq, const struct armada_variant *variant,
 	struct device_node *port)
 {
 	struct armada_private *priv = drm->dev_private;
 	struct armada_crtc *dcrtc;
-	struct armada_plane *primary;
+	struct drm_plane *primary;
 	void __iomem *base;
 	int ret;
 
-	ret = armada_drm_crtc_create_properties(drm);
-	if (ret)
-		return ret;
-
 	base = devm_ioremap_resource(dev, res);
 	if (IS_ERR(base))
 		return PTR_ERR(base);
@@ -1397,8 +744,6 @@ static int armada_drm_crtc_create(struct drm_device *drm, struct device *dev,
 	dcrtc->base = base;
 	dcrtc->num = drm->mode_config.num_crtc;
 	dcrtc->clk = ERR_PTR(-EINVAL);
-	dcrtc->csc_yuv_mode = CSC_AUTO;
-	dcrtc->csc_rgb_mode = CSC_AUTO;
 	dcrtc->cfg_dumb_ctrl = DUMB24_RGB888_0;
 	dcrtc->spu_iopad_ctrl = CFG_VSCALE_LN_EN | CFG_IOPAD_DUMB24;
 	spin_lock_init(&dcrtc->irq_lock);
@@ -1415,6 +760,7 @@ static int armada_drm_crtc_create(struct drm_device *drm, struct device *dev,
 		       CFG_PDWN64x66, dcrtc->base + LCD_SPU_SRAM_PARA1);
 	writel_relaxed(0x2032ff81, dcrtc->base + LCD_SPU_DMA_CTRL1);
 	writel_relaxed(dcrtc->irq_ena, dcrtc->base + LCD_SPU_IRQ_ENA);
+	readl_relaxed(dcrtc->base + LCD_SPU_IRQ_ISR);
 	writel_relaxed(0, dcrtc->base + LCD_SPU_IRQ_ISR);
 
 	ret = devm_request_irq(dev, irq, armada_drm_irq, 0, "armada_drm_crtc",
@@ -1441,39 +787,23 @@ static int armada_drm_crtc_create(struct drm_device *drm, struct device *dev,
 		goto err_crtc;
 	}
 
-	ret = armada_drm_plane_init(primary);
-	if (ret) {
-		kfree(primary);
-		goto err_crtc;
-	}
-
-	ret = drm_universal_plane_init(drm, &primary->base, 0,
-				       &armada_primary_plane_funcs,
-				       armada_primary_formats,
-				       ARRAY_SIZE(armada_primary_formats),
-				       NULL,
-				       DRM_PLANE_TYPE_PRIMARY, NULL);
+	ret = armada_drm_primary_plane_init(drm, primary);
 	if (ret) {
 		kfree(primary);
 		goto err_crtc;
 	}
 
-	ret = drm_crtc_init_with_planes(drm, &dcrtc->crtc, &primary->base, NULL,
+	ret = drm_crtc_init_with_planes(drm, &dcrtc->crtc, primary, NULL,
 					&armada_crtc_funcs, NULL);
 	if (ret)
 		goto err_crtc_init;
 
 	drm_crtc_helper_add(&dcrtc->crtc, &armada_crtc_helper_funcs);
 
-	drm_object_attach_property(&dcrtc->crtc.base, priv->csc_yuv_prop,
-				   dcrtc->csc_yuv_mode);
-	drm_object_attach_property(&dcrtc->crtc.base, priv->csc_rgb_prop,
-				   dcrtc->csc_rgb_mode);
-
 	return armada_overlay_plane_create(drm, 1 << dcrtc->num);
 
 err_crtc_init:
-	primary->base.funcs->destroy(&primary->base);
+	primary->funcs->destroy(primary);
 err_crtc:
 	kfree(dcrtc);
 
diff --git a/drivers/gpu/drm/armada/armada_crtc.h b/drivers/gpu/drm/armada/armada_crtc.h
index 445829b8877a..7ebd337b60af 100644
--- a/drivers/gpu/drm/armada/armada_crtc.h
+++ b/drivers/gpu/drm/armada/armada_crtc.h
@@ -32,49 +32,8 @@ struct armada_regs {
 	armada_reg_queue_mod(_r, _i, 0, 0, ~0)
 
 struct armada_crtc;
-struct armada_plane;
 struct armada_variant;
 
-struct armada_plane_work {
-	void (*fn)(struct armada_crtc *, struct armada_plane_work *);
-	void (*cancel)(struct armada_crtc *, struct armada_plane_work *);
-	bool need_kfree;
-	struct drm_plane *plane;
-	struct drm_framebuffer *old_fb;
-	struct drm_pending_vblank_event *event;
-	struct armada_regs regs[14];
-};
-
-struct armada_plane_state {
-	u16 src_x;
-	u16 src_y;
-	u32 src_hw;
-	u32 dst_hw;
-	u32 dst_yx;
-	u32 ctrl0;
-	bool changed;
-	bool vsync_update;
-};
-
-struct armada_plane {
-	struct drm_plane	base;
-	wait_queue_head_t	frame_wait;
-	bool			next_work;
-	struct armada_plane_work works[2];
-	struct armada_plane_work *work;
-	struct armada_plane_state state;
-};
-#define drm_to_armada_plane(p) container_of(p, struct armada_plane, base)
-
-int armada_drm_plane_init(struct armada_plane *plane);
-int armada_drm_plane_work_queue(struct armada_crtc *dcrtc,
-	struct armada_plane_work *work);
-int armada_drm_plane_work_wait(struct armada_plane *plane, long timeout);
-void armada_drm_plane_work_cancel(struct armada_crtc *dcrtc,
-	struct armada_plane *plane);
-void armada_drm_plane_calc_addrs(u32 *addrs, struct drm_framebuffer *fb,
-	int x, int y);
-
 struct armada_crtc {
 	struct drm_crtc		crtc;
 	const struct armada_variant *variant;
@@ -89,10 +48,6 @@ struct armada_crtc {
 	} v[2];
 	bool			interlaced;
 	bool			cursor_update;
-	uint8_t			csc_yuv_mode;
-	uint8_t			csc_rgb_mode;
-
-	struct drm_plane	*plane;
 
 	struct armada_gem_object	*cursor_obj;
 	int			cursor_x;
@@ -102,21 +57,22 @@ struct armada_crtc {
 	uint32_t		cursor_w;
 	uint32_t		cursor_h;
 
-	int			dpms;
 	uint32_t		cfg_dumb_ctrl;
-	uint32_t		dumb_ctrl;
 	uint32_t		spu_iopad_ctrl;
 
 	spinlock_t		irq_lock;
 	uint32_t		irq_ena;
+
+	bool			update_pending;
+	struct drm_pending_vblank_event *event;
+	struct armada_regs	atomic_regs[32];
+	struct armada_regs	*regs;
+	unsigned int		regs_idx;
 };
 #define drm_to_armada_crtc(c) container_of(c, struct armada_crtc, crtc)
 
 void armada_drm_crtc_update_regs(struct armada_crtc *, struct armada_regs *);
 
-int armada_drm_plane_disable(struct drm_plane *plane,
-			     struct drm_modeset_acquire_ctx *ctx);
-
 extern struct platform_driver armada_lcd_platform_driver;
 
 #endif
diff --git a/drivers/gpu/drm/armada/armada_drm.h b/drivers/gpu/drm/armada/armada_drm.h
index cc4c557c9f66..f09083ff15d3 100644
--- a/drivers/gpu/drm/armada/armada_drm.h
+++ b/drivers/gpu/drm/armada/armada_drm.h
@@ -42,11 +42,12 @@ struct armada_private;
 
 struct armada_variant {
 	bool has_spu_adv_reg;
-	uint32_t spu_adv_reg;
 	int (*init)(struct armada_crtc *, struct device *);
 	int (*compute_clock)(struct armada_crtc *,
 			     const struct drm_display_mode *,
 			     uint32_t *);
+	void (*disable)(struct armada_crtc *);
+	void (*enable)(struct armada_crtc *, const struct drm_display_mode *);
 };
 
 /* Variant ops */
@@ -54,14 +55,10 @@ extern const struct armada_variant armada510_ops;
 
 struct armada_private {
 	struct drm_device	drm;
-	struct work_struct	fb_unref_work;
-	DECLARE_KFIFO(fb_unref, struct drm_framebuffer *, 8);
 	struct drm_fb_helper	*fbdev;
 	struct armada_crtc	*dcrtc[2];
 	struct drm_mm		linear; /* protected by linear_lock */
 	struct mutex		linear_lock;
-	struct drm_property	*csc_yuv_prop;
-	struct drm_property	*csc_rgb_prop;
 	struct drm_property	*colorkey_prop;
 	struct drm_property	*colorkey_min_prop;
 	struct drm_property	*colorkey_max_prop;
@@ -76,13 +73,6 @@ struct armada_private {
 #endif
 };
 
-void __armada_drm_queue_unref_work(struct drm_device *,
-	struct drm_framebuffer *);
-void armada_drm_queue_unref_work(struct drm_device *,
-	struct drm_framebuffer *);
-
-extern const struct drm_mode_config_funcs armada_drm_mode_config_funcs;
-
 int armada_fbdev_init(struct drm_device *);
 void armada_fbdev_fini(struct drm_device *);
 
diff --git a/drivers/gpu/drm/armada/armada_drv.c b/drivers/gpu/drm/armada/armada_drv.c
index 4b11b6b52f1d..fa31589b4fc0 100644
--- a/drivers/gpu/drm/armada/armada_drv.c
+++ b/drivers/gpu/drm/armada/armada_drv.c
@@ -9,46 +9,18 @@
 #include <linux/component.h>
 #include <linux/module.h>
 #include <linux/of_graph.h>
+#include <drm/drm_atomic_helper.h>
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_fb_helper.h>
 #include <drm/drm_of.h>
 #include "armada_crtc.h"
 #include "armada_drm.h"
 #include "armada_gem.h"
+#include "armada_fb.h"
 #include "armada_hw.h"
 #include <drm/armada_drm.h>
 #include "armada_ioctlP.h"
 
-static void armada_drm_unref_work(struct work_struct *work)
-{
-	struct armada_private *priv =
-		container_of(work, struct armada_private, fb_unref_work);
-	struct drm_framebuffer *fb;
-
-	while (kfifo_get(&priv->fb_unref, &fb))
-		drm_framebuffer_put(fb);
-}
-
-/* Must be called with dev->event_lock held */
-void __armada_drm_queue_unref_work(struct drm_device *dev,
-	struct drm_framebuffer *fb)
-{
-	struct armada_private *priv = dev->dev_private;
-
-	WARN_ON(!kfifo_put(&priv->fb_unref, fb));
-	schedule_work(&priv->fb_unref_work);
-}
-
-void armada_drm_queue_unref_work(struct drm_device *dev,
-	struct drm_framebuffer *fb)
-{
-	unsigned long flags;
-
-	spin_lock_irqsave(&dev->event_lock, flags);
-	__armada_drm_queue_unref_work(dev, fb);
-	spin_unlock_irqrestore(&dev->event_lock, flags);
-}
-
 static struct drm_ioctl_desc armada_ioctls[] = {
 	DRM_IOCTL_DEF_DRV(ARMADA_GEM_CREATE, armada_gem_create_ioctl,0),
 	DRM_IOCTL_DEF_DRV(ARMADA_GEM_MMAP, armada_gem_mmap_ioctl, 0),
@@ -72,11 +44,18 @@ static struct drm_driver armada_drm_driver = {
 	.desc			= "Armada SoC DRM",
 	.date			= "20120730",
 	.driver_features	= DRIVER_GEM | DRIVER_MODESET |
-				  DRIVER_PRIME,
+				  DRIVER_PRIME | DRIVER_ATOMIC,
 	.ioctls			= armada_ioctls,
 	.fops			= &armada_drm_fops,
 };
 
+static const struct drm_mode_config_funcs armada_drm_mode_config_funcs = {
+	.fb_create		= armada_fb_create,
+	.output_poll_changed	= drm_fb_helper_output_poll_changed,
+	.atomic_check		= drm_atomic_helper_check,
+	.atomic_commit		= drm_atomic_helper_commit,
+};
+
 static int armada_drm_bind(struct device *dev)
 {
 	struct armada_private *priv;
@@ -109,7 +88,7 @@ static int armada_drm_bind(struct device *dev)
 
 	/*
 	 * The drm_device structure must be at the start of
-	 * armada_private for drm_dev_unref() to work correctly.
+	 * armada_private for drm_dev_put() to work correctly.
 	 */
 	BUILD_BUG_ON(offsetof(struct armada_private, drm) != 0);
 
@@ -125,9 +104,6 @@ static int armada_drm_bind(struct device *dev)
 
 	dev_set_drvdata(dev, &priv->drm);
 
-	INIT_WORK(&priv->fb_unref_work, armada_drm_unref_work);
-	INIT_KFIFO(priv->fb_unref);
-
 	/* Mode setting support */
 	drm_mode_config_init(&priv->drm);
 	priv->drm.mode_config.min_width = 320;
@@ -155,6 +131,8 @@ static int armada_drm_bind(struct device *dev)
 
 	priv->drm.irq_enabled = true;
 
+	drm_mode_config_reset(&priv->drm);
+
 	ret = armada_fbdev_init(&priv->drm);
 	if (ret)
 		goto err_comp;
@@ -179,8 +157,7 @@ static int armada_drm_bind(struct device *dev)
  err_kms:
 	drm_mode_config_cleanup(&priv->drm);
 	drm_mm_takedown(&priv->linear);
-	flush_work(&priv->fb_unref_work);
-	drm_dev_unref(&priv->drm);
+	drm_dev_put(&priv->drm);
 	return ret;
 }
 
@@ -198,9 +175,8 @@ static void armada_drm_unbind(struct device *dev)
 
 	drm_mode_config_cleanup(&priv->drm);
 	drm_mm_takedown(&priv->linear);
-	flush_work(&priv->fb_unref_work);
 
-	drm_dev_unref(&priv->drm);
+	drm_dev_put(&priv->drm);
 }
 
 static int compare_of(struct device *dev, void *data)
diff --git a/drivers/gpu/drm/armada/armada_fb.c b/drivers/gpu/drm/armada/armada_fb.c
index edd15126bde9..6bd638a54579 100644
--- a/drivers/gpu/drm/armada/armada_fb.c
+++ b/drivers/gpu/drm/armada/armada_fb.c
@@ -84,7 +84,7 @@ struct armada_framebuffer *armada_framebuffer_create(struct drm_device *dev,
 	return dfb;
 }
 
-static struct drm_framebuffer *armada_fb_create(struct drm_device *dev,
+struct drm_framebuffer *armada_fb_create(struct drm_device *dev,
 	struct drm_file *dfile, const struct drm_mode_fb_cmd2 *mode)
 {
 	struct armada_gem_object *obj;
@@ -138,8 +138,3 @@ static struct drm_framebuffer *armada_fb_create(struct drm_device *dev,
 	DRM_ERROR("failed to initialize framebuffer: %d\n", ret);
 	return ERR_PTR(ret);
 }
-
-const struct drm_mode_config_funcs armada_drm_mode_config_funcs = {
-	.fb_create		= armada_fb_create,
-	.output_poll_changed	= drm_fb_helper_output_poll_changed,
-};
diff --git a/drivers/gpu/drm/armada/armada_fb.h b/drivers/gpu/drm/armada/armada_fb.h
index 5c130ff5da77..476daad0a36a 100644
--- a/drivers/gpu/drm/armada/armada_fb.h
+++ b/drivers/gpu/drm/armada/armada_fb.h
@@ -19,5 +19,6 @@ struct armada_framebuffer {
 
 struct armada_framebuffer *armada_framebuffer_create(struct drm_device *,
 	const struct drm_mode_fb_cmd2 *, struct armada_gem_object *);
-
+struct drm_framebuffer *armada_fb_create(struct drm_device *dev,
+	struct drm_file *dfile, const struct drm_mode_fb_cmd2 *mode);
 #endif
diff --git a/drivers/gpu/drm/armada/armada_fbdev.c b/drivers/gpu/drm/armada/armada_fbdev.c
index 2a59db0994b2..8d23700848df 100644
--- a/drivers/gpu/drm/armada/armada_fbdev.c
+++ b/drivers/gpu/drm/armada/armada_fbdev.c
@@ -24,7 +24,7 @@ static /*const*/ struct fb_ops armada_fb_ops = {
 	.fb_imageblit	= drm_fb_helper_cfb_imageblit,
 };
 
-static int armada_fb_create(struct drm_fb_helper *fbh,
+static int armada_fbdev_create(struct drm_fb_helper *fbh,
 	struct drm_fb_helper_surface_size *sizes)
 {
 	struct drm_device *dev = fbh->dev;
@@ -108,7 +108,7 @@ static int armada_fb_probe(struct drm_fb_helper *fbh,
 	int ret = 0;
 
 	if (!fbh->fb) {
-		ret = armada_fb_create(fbh, sizes);
+		ret = armada_fbdev_create(fbh, sizes);
 		if (ret == 0)
 			ret = 1;
 	}
diff --git a/drivers/gpu/drm/armada/armada_gem.c b/drivers/gpu/drm/armada/armada_gem.c
index 3fb37c75c065..892c1d9304bb 100644
--- a/drivers/gpu/drm/armada/armada_gem.c
+++ b/drivers/gpu/drm/armada/armada_gem.c
@@ -13,25 +13,14 @@
 #include <drm/armada_drm.h>
 #include "armada_ioctlP.h"
 
-static int armada_gem_vm_fault(struct vm_fault *vmf)
+static vm_fault_t armada_gem_vm_fault(struct vm_fault *vmf)
 {
 	struct drm_gem_object *gobj = vmf->vma->vm_private_data;
 	struct armada_gem_object *obj = drm_to_armada_gem(gobj);
 	unsigned long pfn = obj->phys_addr >> PAGE_SHIFT;
-	int ret;
 
 	pfn += (vmf->address - vmf->vma->vm_start) >> PAGE_SHIFT;
-	ret = vm_insert_pfn(vmf->vma, vmf->address, pfn);
-
-	switch (ret) {
-	case 0:
-	case -EBUSY:
-		return VM_FAULT_NOPAGE;
-	case -ENOMEM:
-		return VM_FAULT_OOM;
-	default:
-		return VM_FAULT_SIGBUS;
-	}
+	return vmf_insert_pfn(vmf->vma, vmf->address, pfn);
 }
 
 const struct vm_operations_struct armada_gem_vm_ops = {
diff --git a/drivers/gpu/drm/armada/armada_hw.h b/drivers/gpu/drm/armada/armada_hw.h
index 27319a8335e2..277580b36758 100644
--- a/drivers/gpu/drm/armada/armada_hw.h
+++ b/drivers/gpu/drm/armada/armada_hw.h
@@ -160,6 +160,7 @@ enum {
 	CFG_ALPHAM_GRA		= 0x1 << 16,
 	CFG_ALPHAM_CFG		= 0x2 << 16,
 	CFG_ALPHA_MASK		= 0xff << 8,
+#define CFG_ALPHA(x)		((x) << 8)
 	CFG_PIXCMD_MASK		= 0xff,
 };
 
@@ -315,4 +316,19 @@ enum {
 	PWRDN_IRQ_LEVEL		= 1 << 0,
 };
 
+static inline u32 armada_rect_hw_fp(struct drm_rect *r)
+{
+	return (drm_rect_height(r) & 0xffff0000) | drm_rect_width(r) >> 16;
+}
+
+static inline u32 armada_rect_hw(struct drm_rect *r)
+{
+	return drm_rect_height(r) << 16 | (drm_rect_width(r) & 0x0000ffff);
+}
+
+static inline u32 armada_rect_yx(struct drm_rect *r)
+{
+	return (r)->y1 << 16 | ((r)->x1 & 0x0000ffff);
+}
+
 #endif
diff --git a/drivers/gpu/drm/armada/armada_overlay.c b/drivers/gpu/drm/armada/armada_overlay.c
index c391955009d6..eb7dfb65ef47 100644
--- a/drivers/gpu/drm/armada/armada_overlay.c
+++ b/drivers/gpu/drm/armada/armada_overlay.c
@@ -7,346 +7,468 @@
  * published by the Free Software Foundation.
  */
 #include <drm/drmP.h>
+#include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
+#include <drm/drm_plane_helper.h>
+#include <drm/armada_drm.h>
 #include "armada_crtc.h"
 #include "armada_drm.h"
 #include "armada_fb.h"
 #include "armada_gem.h"
 #include "armada_hw.h"
-#include <drm/armada_drm.h>
 #include "armada_ioctlP.h"
+#include "armada_plane.h"
 #include "armada_trace.h"
 
-struct armada_ovl_plane_properties {
-	uint32_t colorkey_yr;
-	uint32_t colorkey_ug;
-	uint32_t colorkey_vb;
-#define K2R(val) (((val) >> 0) & 0xff)
-#define K2G(val) (((val) >> 8) & 0xff)
-#define K2B(val) (((val) >> 16) & 0xff)
-	int16_t  brightness;
-	uint16_t contrast;
-	uint16_t saturation;
-	uint32_t colorkey_mode;
-};
-
-struct armada_ovl_plane {
-	struct armada_plane base;
-	struct armada_ovl_plane_properties prop;
+#define DEFAULT_BRIGHTNESS	0
+#define DEFAULT_CONTRAST	0x4000
+#define DEFAULT_SATURATION	0x4000
+#define DEFAULT_ENCODING	DRM_COLOR_YCBCR_BT601
+
+struct armada_overlay_state {
+	struct drm_plane_state base;
+	u32 colorkey_yr;
+	u32 colorkey_ug;
+	u32 colorkey_vb;
+	u32 colorkey_mode;
+	u32 colorkey_enable;
+	s16 brightness;
+	u16 contrast;
+	u16 saturation;
 };
-#define drm_to_armada_ovl_plane(p) \
-	container_of(p, struct armada_ovl_plane, base.base)
-
+#define drm_to_overlay_state(s) \
+	container_of(s, struct armada_overlay_state, base)
 
-static void
-armada_ovl_update_attr(struct armada_ovl_plane_properties *prop,
-	struct armada_crtc *dcrtc)
+static inline u32 armada_spu_contrast(struct drm_plane_state *state)
 {
-	writel_relaxed(prop->colorkey_yr, dcrtc->base + LCD_SPU_COLORKEY_Y);
-	writel_relaxed(prop->colorkey_ug, dcrtc->base + LCD_SPU_COLORKEY_U);
-	writel_relaxed(prop->colorkey_vb, dcrtc->base + LCD_SPU_COLORKEY_V);
+	return drm_to_overlay_state(state)->brightness << 16 |
+	       drm_to_overlay_state(state)->contrast;
+}
 
-	writel_relaxed(prop->brightness << 16 | prop->contrast,
-		       dcrtc->base + LCD_SPU_CONTRAST);
+static inline u32 armada_spu_saturation(struct drm_plane_state *state)
+{
 	/* Docs say 15:0, but it seems to actually be 31:16 on Armada 510 */
-	writel_relaxed(prop->saturation << 16,
-		       dcrtc->base + LCD_SPU_SATURATION);
-	writel_relaxed(0x00002000, dcrtc->base + LCD_SPU_CBSH_HUE);
-
-	spin_lock_irq(&dcrtc->irq_lock);
-	armada_updatel(prop->colorkey_mode | CFG_ALPHAM_GRA,
-		     CFG_CKMODE_MASK | CFG_ALPHAM_MASK | CFG_ALPHA_MASK,
-		     dcrtc->base + LCD_SPU_DMA_CTRL1);
+	return drm_to_overlay_state(state)->saturation << 16;
+}
 
-	armada_updatel(ADV_GRACOLORKEY, 0, dcrtc->base + LCD_SPU_ADV_REG);
-	spin_unlock_irq(&dcrtc->irq_lock);
+static inline u32 armada_csc(struct drm_plane_state *state)
+{
+	/*
+	 * The CFG_CSC_RGB_* settings control the output of the colour space
+	 * converter, setting the range of output values it produces.  Since
+	 * we will be blending with the full-range graphics, we need to
+	 * produce full-range RGB output from the conversion.
+	 */
+	return CFG_CSC_RGB_COMPUTER |
+	       (state->color_encoding == DRM_COLOR_YCBCR_BT709 ?
+			CFG_CSC_YUV_CCIR709 : CFG_CSC_YUV_CCIR601);
 }
 
 /* === Plane support === */
-static void armada_ovl_plane_work(struct armada_crtc *dcrtc,
-	struct armada_plane_work *work)
+static void armada_drm_overlay_plane_atomic_update(struct drm_plane *plane,
+	struct drm_plane_state *old_state)
 {
-	unsigned long flags;
+	struct drm_plane_state *state = plane->state;
+	struct armada_crtc *dcrtc;
+	struct armada_regs *regs;
+	unsigned int idx;
+	u32 cfg, cfg_mask, val;
 
-	trace_armada_ovl_plane_work(&dcrtc->crtc, work->plane);
+	DRM_DEBUG_KMS("[PLANE:%d:%s]\n", plane->base.id, plane->name);
 
-	spin_lock_irqsave(&dcrtc->irq_lock, flags);
-	armada_drm_crtc_update_regs(dcrtc, work->regs);
-	spin_unlock_irqrestore(&dcrtc->irq_lock, flags);
-}
-
-static void armada_ovl_plane_update_state(struct drm_plane_state *state,
-	struct armada_regs *regs)
-{
-	struct armada_ovl_plane *dplane = drm_to_armada_ovl_plane(state->plane);
-	struct armada_framebuffer *dfb = drm_fb_to_armada_fb(state->fb);
-	const struct drm_format_info *format;
-	unsigned int idx = 0;
-	bool fb_changed;
-	u32 val, ctrl0;
-	u16 src_x, src_y;
+	if (!state->fb || WARN_ON(!state->crtc))
+		return;
 
-	ctrl0 = CFG_DMA_FMT(dfb->fmt) | CFG_DMA_MOD(dfb->mod) | CFG_CBSH_ENA;
-	if (state->visible)
-		ctrl0 |= CFG_DMA_ENA;
-	if (drm_rect_width(&state->src) >> 16 != drm_rect_width(&state->dst))
-		ctrl0 |= CFG_DMA_HSMOOTH;
+	DRM_DEBUG_KMS("[PLANE:%d:%s] is on [CRTC:%d:%s] with [FB:%d] visible %u->%u\n",
+		plane->base.id, plane->name,
+		state->crtc->base.id, state->crtc->name,
+		state->fb->base.id,
+		old_state->visible, state->visible);
 
-	/*
-	 * Shifting a YUV packed format image by one pixel causes the U/V
-	 * planes to swap.  Compensate for it by also toggling the UV swap.
-	 */
-	format = dfb->fb.format;
-	if (format->num_planes == 1 && state->src.x1 >> 16 & (format->hsub - 1))
-		ctrl0 ^= CFG_DMA_MOD(CFG_SWAPUV);
+	dcrtc = drm_to_armada_crtc(state->crtc);
+	regs = dcrtc->regs + dcrtc->regs_idx;
 
-	if (~dplane->base.state.ctrl0 & ctrl0 & CFG_DMA_ENA) {
-		/* Power up the Y/U/V FIFOs on ENA 0->1 transitions */
+	idx = 0;
+	if (!old_state->visible && state->visible)
 		armada_reg_queue_mod(regs, idx,
 				     0, CFG_PDWN16x66 | CFG_PDWN32x66,
 				     LCD_SPU_SRAM_PARA1);
-	}
-
-	fb_changed = dplane->base.base.fb != &dfb->fb ||
-		     dplane->base.state.src_x != state->src.x1 >> 16 ||
-	             dplane->base.state.src_y != state->src.y1 >> 16;
-
-	dplane->base.state.vsync_update = fb_changed;
-
+	val = armada_rect_hw_fp(&state->src);
+	if (armada_rect_hw_fp(&old_state->src) != val)
+		armada_reg_queue_set(regs, idx, val, LCD_SPU_DMA_HPXL_VLN);
+	val = armada_rect_yx(&state->dst);
+	if (armada_rect_yx(&old_state->dst) != val)
+		armada_reg_queue_set(regs, idx, val, LCD_SPU_DMA_OVSA_HPXL_VLN);
+	val = armada_rect_hw(&state->dst);
+	if (armada_rect_hw(&old_state->dst) != val)
+		armada_reg_queue_set(regs, idx, val, LCD_SPU_DZM_HPXL_VLN);
 	/* FIXME: overlay on an interlaced display */
-	if (fb_changed) {
-		u32 addrs[3];
-
-		dplane->base.state.src_y = src_y = state->src.y1 >> 16;
-		dplane->base.state.src_x = src_x = state->src.x1 >> 16;
+	if (old_state->src.x1 != state->src.x1 ||
+	    old_state->src.y1 != state->src.y1 ||
+	    old_state->fb != state->fb) {
+		const struct drm_format_info *format;
+		u16 src_x, pitches[3];
+		u32 addrs[2][3];
 
-		armada_drm_plane_calc_addrs(addrs, &dfb->fb, src_x, src_y);
+		armada_drm_plane_calc(state, addrs, pitches, false);
 
-		armada_reg_queue_set(regs, idx, addrs[0],
+		armada_reg_queue_set(regs, idx, addrs[0][0],
 				     LCD_SPU_DMA_START_ADDR_Y0);
-		armada_reg_queue_set(regs, idx, addrs[1],
+		armada_reg_queue_set(regs, idx, addrs[0][1],
 				     LCD_SPU_DMA_START_ADDR_U0);
-		armada_reg_queue_set(regs, idx, addrs[2],
+		armada_reg_queue_set(regs, idx, addrs[0][2],
 				     LCD_SPU_DMA_START_ADDR_V0);
-		armada_reg_queue_set(regs, idx, addrs[0],
+		armada_reg_queue_set(regs, idx, addrs[1][0],
 				     LCD_SPU_DMA_START_ADDR_Y1);
-		armada_reg_queue_set(regs, idx, addrs[1],
+		armada_reg_queue_set(regs, idx, addrs[1][1],
 				     LCD_SPU_DMA_START_ADDR_U1);
-		armada_reg_queue_set(regs, idx, addrs[2],
+		armada_reg_queue_set(regs, idx, addrs[1][2],
 				     LCD_SPU_DMA_START_ADDR_V1);
 
-		val = dfb->fb.pitches[0] << 16 | dfb->fb.pitches[0];
-		armada_reg_queue_set(regs, idx, val,
-				     LCD_SPU_DMA_PITCH_YC);
-		val = dfb->fb.pitches[1] << 16 | dfb->fb.pitches[2];
-		armada_reg_queue_set(regs, idx, val,
-				     LCD_SPU_DMA_PITCH_UV);
-	}
+		val = pitches[0] << 16 | pitches[0];
+		armada_reg_queue_set(regs, idx, val, LCD_SPU_DMA_PITCH_YC);
+		val = pitches[1] << 16 | pitches[2];
+		armada_reg_queue_set(regs, idx, val, LCD_SPU_DMA_PITCH_UV);
 
-	val = (drm_rect_height(&state->src) & 0xffff0000) |
-	       drm_rect_width(&state->src) >> 16;
-	if (dplane->base.state.src_hw != val) {
-		dplane->base.state.src_hw = val;
-		armada_reg_queue_set(regs, idx, val,
-				     LCD_SPU_DMA_HPXL_VLN);
-	}
+		cfg = CFG_DMA_FMT(drm_fb_to_armada_fb(state->fb)->fmt) |
+		      CFG_DMA_MOD(drm_fb_to_armada_fb(state->fb)->mod) |
+		      CFG_CBSH_ENA;
+		if (state->visible)
+			cfg |= CFG_DMA_ENA;
 
-	val = drm_rect_height(&state->dst) << 16 | drm_rect_width(&state->dst);
-	if (dplane->base.state.dst_hw != val) {
-		dplane->base.state.dst_hw = val;
-		armada_reg_queue_set(regs, idx, val,
-				     LCD_SPU_DZM_HPXL_VLN);
+		/*
+		 * Shifting a YUV packed format image by one pixel causes the
+		 * U/V planes to swap.  Compensate for it by also toggling
+		 * the UV swap.
+		 */
+		format = state->fb->format;
+		src_x = state->src.x1 >> 16;
+		if (format->num_planes == 1 && src_x & (format->hsub - 1))
+			cfg ^= CFG_DMA_MOD(CFG_SWAPUV);
+		cfg_mask = CFG_CBSH_ENA | CFG_DMAFORMAT |
+			   CFG_DMA_MOD(CFG_SWAPRB | CFG_SWAPUV |
+				       CFG_SWAPYU | CFG_YUV2RGB) |
+			   CFG_DMA_FTOGGLE | CFG_DMA_TSTMODE |
+			   CFG_DMA_ENA;
+	} else if (old_state->visible != state->visible) {
+		cfg = state->visible ? CFG_DMA_ENA : 0;
+		cfg_mask = CFG_DMA_ENA;
+	} else {
+		cfg = cfg_mask = 0;
 	}
-
-	val = state->dst.y1 << 16 | state->dst.x1;
-	if (dplane->base.state.dst_yx != val) {
-		dplane->base.state.dst_yx = val;
-		armada_reg_queue_set(regs, idx, val,
-				     LCD_SPU_DMA_OVSA_HPXL_VLN);
+	if (drm_rect_width(&old_state->src) != drm_rect_width(&state->src) ||
+	    drm_rect_width(&old_state->dst) != drm_rect_width(&state->dst)) {
+		cfg_mask |= CFG_DMA_HSMOOTH;
+		if (drm_rect_width(&state->src) >> 16 !=
+		    drm_rect_width(&state->dst))
+			cfg |= CFG_DMA_HSMOOTH;
 	}
 
-	if (dplane->base.state.ctrl0 != ctrl0) {
-		dplane->base.state.ctrl0 = ctrl0;
-		armada_reg_queue_mod(regs, idx, ctrl0,
-			CFG_CBSH_ENA | CFG_DMAFORMAT | CFG_DMA_FTOGGLE |
-			CFG_DMA_HSMOOTH | CFG_DMA_TSTMODE |
-			CFG_DMA_MOD(CFG_SWAPRB | CFG_SWAPUV | CFG_SWAPYU |
-			CFG_YUV2RGB) | CFG_DMA_ENA,
-			LCD_SPU_DMA_CTRL0);
-		dplane->base.state.vsync_update = true;
-	}
+	if (cfg_mask)
+		armada_reg_queue_mod(regs, idx, cfg, cfg_mask,
+				     LCD_SPU_DMA_CTRL0);
+
+	val = armada_spu_contrast(state);
+	if ((!old_state->visible && state->visible) ||
+	    armada_spu_contrast(old_state) != val)
+		armada_reg_queue_set(regs, idx, val, LCD_SPU_CONTRAST);
+	val = armada_spu_saturation(state);
+	if ((!old_state->visible && state->visible) ||
+	    armada_spu_saturation(old_state) != val)
+		armada_reg_queue_set(regs, idx, val, LCD_SPU_SATURATION);
+	if (!old_state->visible && state->visible)
+		armada_reg_queue_set(regs, idx, 0x00002000, LCD_SPU_CBSH_HUE);
+	val = armada_csc(state);
+	if ((!old_state->visible && state->visible) ||
+	    armada_csc(old_state) != val)
+		armada_reg_queue_mod(regs, idx, val, CFG_CSC_MASK,
+				     LCD_SPU_IOPAD_CONTROL);
+	val = drm_to_overlay_state(state)->colorkey_yr;
+	if ((!old_state->visible && state->visible) ||
+	    drm_to_overlay_state(old_state)->colorkey_yr != val)
+		armada_reg_queue_set(regs, idx, val, LCD_SPU_COLORKEY_Y);
+	val = drm_to_overlay_state(state)->colorkey_ug;
+	if ((!old_state->visible && state->visible) ||
+	    drm_to_overlay_state(old_state)->colorkey_ug != val)
+		armada_reg_queue_set(regs, idx, val, LCD_SPU_COLORKEY_U);
+	val = drm_to_overlay_state(state)->colorkey_vb;
+	if ((!old_state->visible && state->visible) ||
+	    drm_to_overlay_state(old_state)->colorkey_vb != val)
+		armada_reg_queue_set(regs, idx, val, LCD_SPU_COLORKEY_V);
+	val = drm_to_overlay_state(state)->colorkey_mode;
+	if ((!old_state->visible && state->visible) ||
+	    drm_to_overlay_state(old_state)->colorkey_mode != val)
+		armada_reg_queue_mod(regs, idx, val, CFG_CKMODE_MASK |
+				     CFG_ALPHAM_MASK | CFG_ALPHA_MASK,
+				     LCD_SPU_DMA_CTRL1);
+	val = drm_to_overlay_state(state)->colorkey_enable;
+	if (((!old_state->visible && state->visible) ||
+	     drm_to_overlay_state(old_state)->colorkey_enable != val) &&
+	    dcrtc->variant->has_spu_adv_reg)
+		armada_reg_queue_mod(regs, idx, val, ADV_GRACOLORKEY |
+				     ADV_VIDCOLORKEY, LCD_SPU_ADV_REG);
+
+	dcrtc->regs_idx += idx;
+}
+
+static void armada_drm_overlay_plane_atomic_disable(struct drm_plane *plane,
+	struct drm_plane_state *old_state)
+{
+	struct armada_crtc *dcrtc;
+	struct armada_regs *regs;
+	unsigned int idx = 0;
+
+	DRM_DEBUG_KMS("[PLANE:%d:%s]\n", plane->base.id, plane->name);
+
+	if (!old_state->crtc)
+		return;
 
-	dplane->base.state.changed = idx != 0;
+	DRM_DEBUG_KMS("[PLANE:%d:%s] was on [CRTC:%d:%s] with [FB:%d]\n",
+		plane->base.id, plane->name,
+		old_state->crtc->base.id, old_state->crtc->name,
+		old_state->fb->base.id);
 
-	armada_reg_queue_end(regs, idx);
+	dcrtc = drm_to_armada_crtc(old_state->crtc);
+	regs = dcrtc->regs + dcrtc->regs_idx;
+
+	/* Disable plane and power down the YUV FIFOs */
+	armada_reg_queue_mod(regs, idx, 0, CFG_DMA_ENA, LCD_SPU_DMA_CTRL0);
+	armada_reg_queue_mod(regs, idx, CFG_PDWN16x66 | CFG_PDWN32x66, 0,
+			     LCD_SPU_SRAM_PARA1);
+
+	dcrtc->regs_idx += idx;
 }
 
+static const struct drm_plane_helper_funcs armada_overlay_plane_helper_funcs = {
+	.prepare_fb	= armada_drm_plane_prepare_fb,
+	.cleanup_fb	= armada_drm_plane_cleanup_fb,
+	.atomic_check	= armada_drm_plane_atomic_check,
+	.atomic_update	= armada_drm_overlay_plane_atomic_update,
+	.atomic_disable	= armada_drm_overlay_plane_atomic_disable,
+};
+
 static int
-armada_ovl_plane_update(struct drm_plane *plane, struct drm_crtc *crtc,
+armada_overlay_plane_update(struct drm_plane *plane, struct drm_crtc *crtc,
 	struct drm_framebuffer *fb,
 	int crtc_x, int crtc_y, unsigned crtc_w, unsigned crtc_h,
 	uint32_t src_x, uint32_t src_y, uint32_t src_w, uint32_t src_h,
 	struct drm_modeset_acquire_ctx *ctx)
 {
-	struct armada_ovl_plane *dplane = drm_to_armada_ovl_plane(plane);
-	struct armada_crtc *dcrtc = drm_to_armada_crtc(crtc);
-	struct armada_plane_work *work;
-	struct drm_plane_state state = {
-		.plane = plane,
-		.crtc = crtc,
-		.fb = fb,
-		.src_x = src_x,
-		.src_y = src_y,
-		.src_w = src_w,
-		.src_h = src_h,
-		.crtc_x = crtc_x,
-		.crtc_y = crtc_y,
-		.crtc_w = crtc_w,
-		.crtc_h = crtc_h,
-		.rotation = DRM_MODE_ROTATE_0,
-	};
-	struct drm_crtc_state crtc_state = {
-		.crtc = crtc,
-		.enable = crtc->enabled,
-		.mode = crtc->mode,
-	};
-	int ret;
+	struct drm_atomic_state *state;
+	struct drm_plane_state *plane_state;
+	int ret = 0;
 
 	trace_armada_ovl_plane_update(plane, crtc, fb,
 				 crtc_x, crtc_y, crtc_w, crtc_h,
 				 src_x, src_y, src_w, src_h);
 
-	ret = drm_atomic_helper_check_plane_state(&state, &crtc_state, 0,
-						  INT_MAX, true, false);
-	if (ret)
-		return ret;
-
-	work = &dplane->base.works[dplane->base.next_work];
-
-	if (plane->fb != fb) {
-		/*
-		 * Take a reference on the new framebuffer - we want to
-		 * hold on to it while the hardware is displaying it.
-		 */
-		drm_framebuffer_reference(fb);
+	state = drm_atomic_state_alloc(plane->dev);
+	if (!state)
+		return -ENOMEM;
 
-		work->old_fb = plane->fb;
-	} else {
-		work->old_fb = NULL;
+	state->acquire_ctx = ctx;
+	plane_state = drm_atomic_get_plane_state(state, plane);
+	if (IS_ERR(plane_state)) {
+		ret = PTR_ERR(plane_state);
+		goto fail;
 	}
 
-	armada_ovl_plane_update_state(&state, work->regs);
-
-	if (!dplane->base.state.changed)
-		return 0;
-
-	/* Wait for pending work to complete */
-	if (armada_drm_plane_work_wait(&dplane->base, HZ / 25) == 0)
-		armada_drm_plane_work_cancel(dcrtc, &dplane->base);
+	ret = drm_atomic_set_crtc_for_plane(plane_state, crtc);
+	if (ret != 0)
+		goto fail;
+
+	drm_atomic_set_fb_for_plane(plane_state, fb);
+	plane_state->crtc_x = crtc_x;
+	plane_state->crtc_y = crtc_y;
+	plane_state->crtc_h = crtc_h;
+	plane_state->crtc_w = crtc_w;
+	plane_state->src_x = src_x;
+	plane_state->src_y = src_y;
+	plane_state->src_h = src_h;
+	plane_state->src_w = src_w;
+
+	ret = drm_atomic_nonblocking_commit(state);
+fail:
+	drm_atomic_state_put(state);
+	return ret;
+}
 
-	/* Just updating the position/size? */
-	if (!dplane->base.state.vsync_update) {
-		armada_ovl_plane_work(dcrtc, work);
-		return 0;
-	}
+static void armada_ovl_plane_destroy(struct drm_plane *plane)
+{
+	drm_plane_cleanup(plane);
+	kfree(plane);
+}
 
-	if (!dcrtc->plane) {
-		dcrtc->plane = plane;
-		armada_ovl_update_attr(&dplane->prop, dcrtc);
+static void armada_overlay_reset(struct drm_plane *plane)
+{
+	struct armada_overlay_state *state;
+
+	if (plane->state)
+		__drm_atomic_helper_plane_destroy_state(plane->state);
+	kfree(plane->state);
+
+	state = kzalloc(sizeof(*state), GFP_KERNEL);
+	if (state) {
+		state->base.plane = plane;
+		state->base.color_encoding = DEFAULT_ENCODING;
+		state->base.color_range = DRM_COLOR_YCBCR_LIMITED_RANGE;
+		state->base.rotation = DRM_MODE_ROTATE_0;
+		state->colorkey_yr = 0xfefefe00;
+		state->colorkey_ug = 0x01010100;
+		state->colorkey_vb = 0x01010100;
+		state->colorkey_mode = CFG_CKMODE(CKMODE_RGB) |
+				       CFG_ALPHAM_GRA | CFG_ALPHA(0);
+		state->colorkey_enable = ADV_GRACOLORKEY;
+		state->brightness = DEFAULT_BRIGHTNESS;
+		state->contrast = DEFAULT_CONTRAST;
+		state->saturation = DEFAULT_SATURATION;
 	}
-
-	/* Queue it for update on the next interrupt if we are enabled */
-	ret = armada_drm_plane_work_queue(dcrtc, work);
-	if (ret)
-		DRM_ERROR("failed to queue plane work: %d\n", ret);
-
-	dplane->base.next_work = !dplane->base.next_work;
-
-	return 0;
+	plane->state = &state->base;
 }
 
-static void armada_ovl_plane_destroy(struct drm_plane *plane)
+struct drm_plane_state *
+armada_overlay_duplicate_state(struct drm_plane *plane)
 {
-	struct armada_ovl_plane *dplane = drm_to_armada_ovl_plane(plane);
+	struct armada_overlay_state *state;
 
-	drm_plane_cleanup(plane);
+	if (WARN_ON(!plane->state))
+		return NULL;
 
-	kfree(dplane);
+	state = kmemdup(plane->state, sizeof(*state), GFP_KERNEL);
+	if (state)
+		__drm_atomic_helper_plane_duplicate_state(plane, &state->base);
+	return &state->base;
 }
 
-static int armada_ovl_plane_set_property(struct drm_plane *plane,
-	struct drm_property *property, uint64_t val)
+static int armada_overlay_set_property(struct drm_plane *plane,
+	struct drm_plane_state *state, struct drm_property *property,
+	uint64_t val)
 {
 	struct armada_private *priv = plane->dev->dev_private;
-	struct armada_ovl_plane *dplane = drm_to_armada_ovl_plane(plane);
-	bool update_attr = false;
 
+#define K2R(val) (((val) >> 0) & 0xff)
+#define K2G(val) (((val) >> 8) & 0xff)
+#define K2B(val) (((val) >> 16) & 0xff)
 	if (property == priv->colorkey_prop) {
 #define CCC(v) ((v) << 24 | (v) << 16 | (v) << 8)
-		dplane->prop.colorkey_yr = CCC(K2R(val));
-		dplane->prop.colorkey_ug = CCC(K2G(val));
-		dplane->prop.colorkey_vb = CCC(K2B(val));
+		drm_to_overlay_state(state)->colorkey_yr = CCC(K2R(val));
+		drm_to_overlay_state(state)->colorkey_ug = CCC(K2G(val));
+		drm_to_overlay_state(state)->colorkey_vb = CCC(K2B(val));
 #undef CCC
-		update_attr = true;
 	} else if (property == priv->colorkey_min_prop) {
-		dplane->prop.colorkey_yr &= ~0x00ff0000;
-		dplane->prop.colorkey_yr |= K2R(val) << 16;
-		dplane->prop.colorkey_ug &= ~0x00ff0000;
-		dplane->prop.colorkey_ug |= K2G(val) << 16;
-		dplane->prop.colorkey_vb &= ~0x00ff0000;
-		dplane->prop.colorkey_vb |= K2B(val) << 16;
-		update_attr = true;
+		drm_to_overlay_state(state)->colorkey_yr &= ~0x00ff0000;
+		drm_to_overlay_state(state)->colorkey_yr |= K2R(val) << 16;
+		drm_to_overlay_state(state)->colorkey_ug &= ~0x00ff0000;
+		drm_to_overlay_state(state)->colorkey_ug |= K2G(val) << 16;
+		drm_to_overlay_state(state)->colorkey_vb &= ~0x00ff0000;
+		drm_to_overlay_state(state)->colorkey_vb |= K2B(val) << 16;
 	} else if (property == priv->colorkey_max_prop) {
-		dplane->prop.colorkey_yr &= ~0xff000000;
-		dplane->prop.colorkey_yr |= K2R(val) << 24;
-		dplane->prop.colorkey_ug &= ~0xff000000;
-		dplane->prop.colorkey_ug |= K2G(val) << 24;
-		dplane->prop.colorkey_vb &= ~0xff000000;
-		dplane->prop.colorkey_vb |= K2B(val) << 24;
-		update_attr = true;
+		drm_to_overlay_state(state)->colorkey_yr &= ~0xff000000;
+		drm_to_overlay_state(state)->colorkey_yr |= K2R(val) << 24;
+		drm_to_overlay_state(state)->colorkey_ug &= ~0xff000000;
+		drm_to_overlay_state(state)->colorkey_ug |= K2G(val) << 24;
+		drm_to_overlay_state(state)->colorkey_vb &= ~0xff000000;
+		drm_to_overlay_state(state)->colorkey_vb |= K2B(val) << 24;
 	} else if (property == priv->colorkey_val_prop) {
-		dplane->prop.colorkey_yr &= ~0x0000ff00;
-		dplane->prop.colorkey_yr |= K2R(val) << 8;
-		dplane->prop.colorkey_ug &= ~0x0000ff00;
-		dplane->prop.colorkey_ug |= K2G(val) << 8;
-		dplane->prop.colorkey_vb &= ~0x0000ff00;
-		dplane->prop.colorkey_vb |= K2B(val) << 8;
-		update_attr = true;
+		drm_to_overlay_state(state)->colorkey_yr &= ~0x0000ff00;
+		drm_to_overlay_state(state)->colorkey_yr |= K2R(val) << 8;
+		drm_to_overlay_state(state)->colorkey_ug &= ~0x0000ff00;
+		drm_to_overlay_state(state)->colorkey_ug |= K2G(val) << 8;
+		drm_to_overlay_state(state)->colorkey_vb &= ~0x0000ff00;
+		drm_to_overlay_state(state)->colorkey_vb |= K2B(val) << 8;
 	} else if (property == priv->colorkey_alpha_prop) {
-		dplane->prop.colorkey_yr &= ~0x000000ff;
-		dplane->prop.colorkey_yr |= K2R(val);
-		dplane->prop.colorkey_ug &= ~0x000000ff;
-		dplane->prop.colorkey_ug |= K2G(val);
-		dplane->prop.colorkey_vb &= ~0x000000ff;
-		dplane->prop.colorkey_vb |= K2B(val);
-		update_attr = true;
+		drm_to_overlay_state(state)->colorkey_yr &= ~0x000000ff;
+		drm_to_overlay_state(state)->colorkey_yr |= K2R(val);
+		drm_to_overlay_state(state)->colorkey_ug &= ~0x000000ff;
+		drm_to_overlay_state(state)->colorkey_ug |= K2G(val);
+		drm_to_overlay_state(state)->colorkey_vb &= ~0x000000ff;
+		drm_to_overlay_state(state)->colorkey_vb |= K2B(val);
 	} else if (property == priv->colorkey_mode_prop) {
-		dplane->prop.colorkey_mode &= ~CFG_CKMODE_MASK;
-		dplane->prop.colorkey_mode |= CFG_CKMODE(val);
-		update_attr = true;
+		if (val == CKMODE_DISABLE) {
+			drm_to_overlay_state(state)->colorkey_mode =
+				CFG_CKMODE(CKMODE_DISABLE) |
+				CFG_ALPHAM_CFG | CFG_ALPHA(255);
+			drm_to_overlay_state(state)->colorkey_enable = 0;
+		} else {
+			drm_to_overlay_state(state)->colorkey_mode =
+				CFG_CKMODE(val) |
+				CFG_ALPHAM_GRA | CFG_ALPHA(0);
+			drm_to_overlay_state(state)->colorkey_enable =
+				ADV_GRACOLORKEY;
+		}
 	} else if (property == priv->brightness_prop) {
-		dplane->prop.brightness = val - 256;
-		update_attr = true;
+		drm_to_overlay_state(state)->brightness = val - 256;
 	} else if (property == priv->contrast_prop) {
-		dplane->prop.contrast = val;
-		update_attr = true;
+		drm_to_overlay_state(state)->contrast = val;
 	} else if (property == priv->saturation_prop) {
-		dplane->prop.saturation = val;
-		update_attr = true;
+		drm_to_overlay_state(state)->saturation = val;
+	} else {
+		return -EINVAL;
 	}
+	return 0;
+}
 
-	if (update_attr && dplane->base.base.crtc)
-		armada_ovl_update_attr(&dplane->prop,
-				       drm_to_armada_crtc(dplane->base.base.crtc));
+static int armada_overlay_get_property(struct drm_plane *plane,
+	const struct drm_plane_state *state, struct drm_property *property,
+	uint64_t *val)
+{
+	struct armada_private *priv = plane->dev->dev_private;
 
+#define C2K(c,s)	(((c) >> (s)) & 0xff)
+#define R2BGR(r,g,b,s)	(C2K(r,s) << 0 | C2K(g,s) << 8 | C2K(b,s) << 16)
+	if (property == priv->colorkey_prop) {
+		/* Do best-efforts here for this property */
+		*val = R2BGR(drm_to_overlay_state(state)->colorkey_yr,
+			     drm_to_overlay_state(state)->colorkey_ug,
+			     drm_to_overlay_state(state)->colorkey_vb, 16);
+		/* If min != max, or min != val, error out */
+		if (*val != R2BGR(drm_to_overlay_state(state)->colorkey_yr,
+				  drm_to_overlay_state(state)->colorkey_ug,
+				  drm_to_overlay_state(state)->colorkey_vb, 24) ||
+		    *val != R2BGR(drm_to_overlay_state(state)->colorkey_yr,
+				  drm_to_overlay_state(state)->colorkey_ug,
+				  drm_to_overlay_state(state)->colorkey_vb, 8))
+			return -EINVAL;
+	} else if (property == priv->colorkey_min_prop) {
+		*val = R2BGR(drm_to_overlay_state(state)->colorkey_yr,
+			     drm_to_overlay_state(state)->colorkey_ug,
+			     drm_to_overlay_state(state)->colorkey_vb, 16);
+	} else if (property == priv->colorkey_max_prop) {
+		*val = R2BGR(drm_to_overlay_state(state)->colorkey_yr,
+			     drm_to_overlay_state(state)->colorkey_ug,
+			     drm_to_overlay_state(state)->colorkey_vb, 24);
+	} else if (property == priv->colorkey_val_prop) {
+		*val = R2BGR(drm_to_overlay_state(state)->colorkey_yr,
+			     drm_to_overlay_state(state)->colorkey_ug,
+			     drm_to_overlay_state(state)->colorkey_vb, 8);
+	} else if (property == priv->colorkey_alpha_prop) {
+		*val = R2BGR(drm_to_overlay_state(state)->colorkey_yr,
+			     drm_to_overlay_state(state)->colorkey_ug,
+			     drm_to_overlay_state(state)->colorkey_vb, 0);
+	} else if (property == priv->colorkey_mode_prop) {
+		*val = (drm_to_overlay_state(state)->colorkey_mode &
+			CFG_CKMODE_MASK) >> ffs(CFG_CKMODE_MASK);
+	} else if (property == priv->brightness_prop) {
+		*val = drm_to_overlay_state(state)->brightness + 256;
+	} else if (property == priv->contrast_prop) {
+		*val = drm_to_overlay_state(state)->contrast;
+	} else if (property == priv->saturation_prop) {
+		*val = drm_to_overlay_state(state)->saturation;
+	} else {
+		return -EINVAL;
+	}
 	return 0;
 }
 
 static const struct drm_plane_funcs armada_ovl_plane_funcs = {
-	.update_plane	= armada_ovl_plane_update,
-	.disable_plane	= armada_drm_plane_disable,
+	.update_plane	= armada_overlay_plane_update,
+	.disable_plane	= drm_atomic_helper_disable_plane,
 	.destroy	= armada_ovl_plane_destroy,
-	.set_property	= armada_ovl_plane_set_property,
+	.reset		= armada_overlay_reset,
+	.atomic_duplicate_state = armada_overlay_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_plane_destroy_state,
+	.atomic_set_property = armada_overlay_set_property,
+	.atomic_get_property = armada_overlay_get_property,
 };
 
 static const uint32_t armada_ovl_formats[] = {
@@ -419,46 +541,31 @@ int armada_overlay_plane_create(struct drm_device *dev, unsigned long crtcs)
 {
 	struct armada_private *priv = dev->dev_private;
 	struct drm_mode_object *mobj;
-	struct armada_ovl_plane *dplane;
+	struct drm_plane *overlay;
 	int ret;
 
 	ret = armada_overlay_create_properties(dev);
 	if (ret)
 		return ret;
 
-	dplane = kzalloc(sizeof(*dplane), GFP_KERNEL);
-	if (!dplane)
+	overlay = kzalloc(sizeof(*overlay), GFP_KERNEL);
+	if (!overlay)
 		return -ENOMEM;
 
-	ret = armada_drm_plane_init(&dplane->base);
-	if (ret) {
-		kfree(dplane);
-		return ret;
-	}
-
-	dplane->base.works[0].fn = armada_ovl_plane_work;
-	dplane->base.works[1].fn = armada_ovl_plane_work;
+	drm_plane_helper_add(overlay, &armada_overlay_plane_helper_funcs);
 
-	ret = drm_universal_plane_init(dev, &dplane->base.base, crtcs,
+	ret = drm_universal_plane_init(dev, overlay, crtcs,
 				       &armada_ovl_plane_funcs,
 				       armada_ovl_formats,
 				       ARRAY_SIZE(armada_ovl_formats),
 				       NULL,
 				       DRM_PLANE_TYPE_OVERLAY, NULL);
 	if (ret) {
-		kfree(dplane);
+		kfree(overlay);
 		return ret;
 	}
 
-	dplane->prop.colorkey_yr = 0xfefefe00;
-	dplane->prop.colorkey_ug = 0x01010100;
-	dplane->prop.colorkey_vb = 0x01010100;
-	dplane->prop.colorkey_mode = CFG_CKMODE(CKMODE_RGB);
-	dplane->prop.brightness = 0;
-	dplane->prop.contrast = 0x4000;
-	dplane->prop.saturation = 0x4000;
-
-	mobj = &dplane->base.base.base;
+	mobj = &overlay->base;
 	drm_object_attach_property(mobj, priv->colorkey_prop,
 				   0x0101fe);
 	drm_object_attach_property(mobj, priv->colorkey_min_prop,
@@ -471,11 +578,19 @@ int armada_overlay_plane_create(struct drm_device *dev, unsigned long crtcs)
 				   0x000000);
 	drm_object_attach_property(mobj, priv->colorkey_mode_prop,
 				   CKMODE_RGB);
-	drm_object_attach_property(mobj, priv->brightness_prop, 256);
+	drm_object_attach_property(mobj, priv->brightness_prop,
+				   256 + DEFAULT_BRIGHTNESS);
 	drm_object_attach_property(mobj, priv->contrast_prop,
-				   dplane->prop.contrast);
+				   DEFAULT_CONTRAST);
 	drm_object_attach_property(mobj, priv->saturation_prop,
-				   dplane->prop.saturation);
+				   DEFAULT_SATURATION);
 
-	return 0;
+	ret = drm_plane_create_color_properties(overlay,
+						BIT(DRM_COLOR_YCBCR_BT601) |
+						BIT(DRM_COLOR_YCBCR_BT709),
+						BIT(DRM_COLOR_YCBCR_LIMITED_RANGE),
+						DEFAULT_ENCODING,
+						DRM_COLOR_YCBCR_LIMITED_RANGE);
+
+	return ret;
 }
diff --git a/drivers/gpu/drm/armada/armada_plane.c b/drivers/gpu/drm/armada/armada_plane.c
new file mode 100644
index 000000000000..9f36423dd394
--- /dev/null
+++ b/drivers/gpu/drm/armada/armada_plane.c
@@ -0,0 +1,289 @@
+/*
+ * Copyright (C) 2012 Russell King
+ *  Rewritten from the dovefb driver, and Armada510 manuals.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+#include <drm/drmP.h>
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_plane_helper.h>
+#include "armada_crtc.h"
+#include "armada_drm.h"
+#include "armada_fb.h"
+#include "armada_gem.h"
+#include "armada_hw.h"
+#include "armada_plane.h"
+#include "armada_trace.h"
+
+static const uint32_t armada_primary_formats[] = {
+	DRM_FORMAT_UYVY,
+	DRM_FORMAT_YUYV,
+	DRM_FORMAT_VYUY,
+	DRM_FORMAT_YVYU,
+	DRM_FORMAT_ARGB8888,
+	DRM_FORMAT_ABGR8888,
+	DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_XBGR8888,
+	DRM_FORMAT_RGB888,
+	DRM_FORMAT_BGR888,
+	DRM_FORMAT_ARGB1555,
+	DRM_FORMAT_ABGR1555,
+	DRM_FORMAT_RGB565,
+	DRM_FORMAT_BGR565,
+};
+
+void armada_drm_plane_calc(struct drm_plane_state *state, u32 addrs[2][3],
+	u16 pitches[3], bool interlaced)
+{
+	struct drm_framebuffer *fb = state->fb;
+	const struct drm_format_info *format = fb->format;
+	unsigned int num_planes = format->num_planes;
+	unsigned int x = state->src.x1 >> 16;
+	unsigned int y = state->src.y1 >> 16;
+	u32 addr = drm_fb_obj(fb)->dev_addr;
+	int i;
+
+	DRM_DEBUG_KMS("pitch %u x %d y %d bpp %d\n",
+		      fb->pitches[0], x, y, format->cpp[0] * 8);
+
+	if (num_planes > 3)
+		num_planes = 3;
+
+	addrs[0][0] = addr + fb->offsets[0] + y * fb->pitches[0] +
+		      x * format->cpp[0];
+	pitches[0] = fb->pitches[0];
+
+	y /= format->vsub;
+	x /= format->hsub;
+
+	for (i = 1; i < num_planes; i++) {
+		addrs[0][i] = addr + fb->offsets[i] + y * fb->pitches[i] +
+			      x * format->cpp[i];
+		pitches[i] = fb->pitches[i];
+	}
+	for (; i < 3; i++) {
+		addrs[0][i] = 0;
+		pitches[i] = 0;
+	}
+	if (interlaced) {
+		for (i = 0; i < 3; i++) {
+			addrs[1][i] = addrs[0][i] + pitches[i];
+			pitches[i] *= 2;
+		}
+	} else {
+		for (i = 0; i < 3; i++)
+			addrs[1][i] = addrs[0][i];
+	}
+}
+
+static unsigned armada_drm_crtc_calc_fb(struct drm_plane_state *state,
+	struct armada_regs *regs, bool interlaced)
+{
+	u16 pitches[3];
+	u32 addrs[2][3];
+	unsigned i = 0;
+
+	armada_drm_plane_calc(state, addrs, pitches, interlaced);
+
+	/* write offset, base, and pitch */
+	armada_reg_queue_set(regs, i, addrs[0][0], LCD_CFG_GRA_START_ADDR0);
+	armada_reg_queue_set(regs, i, addrs[1][0], LCD_CFG_GRA_START_ADDR1);
+	armada_reg_queue_mod(regs, i, pitches[0], 0xffff, LCD_CFG_GRA_PITCH);
+
+	return i;
+}
+
+int armada_drm_plane_prepare_fb(struct drm_plane *plane,
+	struct drm_plane_state *state)
+{
+	DRM_DEBUG_KMS("[PLANE:%d:%s] [FB:%d]\n",
+		plane->base.id, plane->name,
+		state->fb ? state->fb->base.id : 0);
+
+	/*
+	 * Take a reference on the new framebuffer - we want to
+	 * hold on to it while the hardware is displaying it.
+	 */
+	if (state->fb)
+		drm_framebuffer_get(state->fb);
+	return 0;
+}
+
+void armada_drm_plane_cleanup_fb(struct drm_plane *plane,
+	struct drm_plane_state *old_state)
+{
+	DRM_DEBUG_KMS("[PLANE:%d:%s] [FB:%d]\n",
+		plane->base.id, plane->name,
+		old_state->fb ? old_state->fb->base.id : 0);
+
+	if (old_state->fb)
+		drm_framebuffer_put(old_state->fb);
+}
+
+int armada_drm_plane_atomic_check(struct drm_plane *plane,
+	struct drm_plane_state *state)
+{
+	if (state->fb && !WARN_ON(!state->crtc)) {
+		struct drm_crtc *crtc = state->crtc;
+		struct drm_crtc_state *crtc_state;
+
+		if (state->state)
+			crtc_state = drm_atomic_get_existing_crtc_state(state->state, crtc);
+		else
+			crtc_state = crtc->state;
+		return drm_atomic_helper_check_plane_state(state, crtc_state,
+							   0, INT_MAX,
+							   true, false);
+	} else {
+		state->visible = false;
+	}
+	return 0;
+}
+
+static void armada_drm_primary_plane_atomic_update(struct drm_plane *plane,
+	struct drm_plane_state *old_state)
+{
+	struct drm_plane_state *state = plane->state;
+	struct armada_crtc *dcrtc;
+	struct armada_regs *regs;
+	u32 cfg, cfg_mask, val;
+	unsigned int idx;
+
+	DRM_DEBUG_KMS("[PLANE:%d:%s]\n", plane->base.id, plane->name);
+
+	if (!state->fb || WARN_ON(!state->crtc))
+		return;
+
+	DRM_DEBUG_KMS("[PLANE:%d:%s] is on [CRTC:%d:%s] with [FB:%d] visible %u->%u\n",
+		plane->base.id, plane->name,
+		state->crtc->base.id, state->crtc->name,
+		state->fb->base.id,
+		old_state->visible, state->visible);
+
+	dcrtc = drm_to_armada_crtc(state->crtc);
+	regs = dcrtc->regs + dcrtc->regs_idx;
+
+	idx = 0;
+	if (!old_state->visible && state->visible) {
+		val = CFG_PDWN64x66;
+		if (drm_fb_to_armada_fb(state->fb)->fmt > CFG_420)
+			val |= CFG_PDWN256x24;
+		armada_reg_queue_mod(regs, idx, 0, val, LCD_SPU_SRAM_PARA1);
+	}
+	val = armada_rect_hw_fp(&state->src);
+	if (armada_rect_hw_fp(&old_state->src) != val)
+		armada_reg_queue_set(regs, idx, val, LCD_SPU_GRA_HPXL_VLN);
+	val = armada_rect_yx(&state->dst);
+	if (armada_rect_yx(&old_state->dst) != val)
+		armada_reg_queue_set(regs, idx, val, LCD_SPU_GRA_OVSA_HPXL_VLN);
+	val = armada_rect_hw(&state->dst);
+	if (armada_rect_hw(&old_state->dst) != val)
+		armada_reg_queue_set(regs, idx, val, LCD_SPU_GZM_HPXL_VLN);
+	if (old_state->src.x1 != state->src.x1 ||
+	    old_state->src.y1 != state->src.y1 ||
+	    old_state->fb != state->fb ||
+	    state->crtc->state->mode_changed) {
+		idx += armada_drm_crtc_calc_fb(state, regs + idx,
+					       dcrtc->interlaced);
+	}
+	if (old_state->fb != state->fb ||
+	    state->crtc->state->mode_changed) {
+		cfg = CFG_GRA_FMT(drm_fb_to_armada_fb(state->fb)->fmt) |
+		      CFG_GRA_MOD(drm_fb_to_armada_fb(state->fb)->mod);
+		if (drm_fb_to_armada_fb(state->fb)->fmt > CFG_420)
+			cfg |= CFG_PALETTE_ENA;
+		if (state->visible)
+			cfg |= CFG_GRA_ENA;
+		if (dcrtc->interlaced)
+			cfg |= CFG_GRA_FTOGGLE;
+		cfg_mask = CFG_GRAFORMAT |
+			   CFG_GRA_MOD(CFG_SWAPRB | CFG_SWAPUV |
+				       CFG_SWAPYU | CFG_YUV2RGB) |
+			   CFG_PALETTE_ENA | CFG_GRA_FTOGGLE |
+			   CFG_GRA_ENA;
+	} else if (old_state->visible != state->visible) {
+		cfg = state->visible ? CFG_GRA_ENA : 0;
+		cfg_mask = CFG_GRA_ENA;
+	} else {
+		cfg = cfg_mask = 0;
+	}
+	if (drm_rect_width(&old_state->src) != drm_rect_width(&state->src) ||
+	    drm_rect_width(&old_state->dst) != drm_rect_width(&state->dst)) {
+		cfg_mask |= CFG_GRA_HSMOOTH;
+		if (drm_rect_width(&state->src) >> 16 !=
+		    drm_rect_width(&state->dst))
+			cfg |= CFG_GRA_HSMOOTH;
+	}
+
+	if (cfg_mask)
+		armada_reg_queue_mod(regs, idx, cfg, cfg_mask,
+				     LCD_SPU_DMA_CTRL0);
+
+	dcrtc->regs_idx += idx;
+}
+
+static void armada_drm_primary_plane_atomic_disable(struct drm_plane *plane,
+	struct drm_plane_state *old_state)
+{
+	struct armada_crtc *dcrtc;
+	struct armada_regs *regs;
+	unsigned int idx = 0;
+
+	DRM_DEBUG_KMS("[PLANE:%d:%s]\n", plane->base.id, plane->name);
+
+	if (!old_state->crtc)
+		return;
+
+	DRM_DEBUG_KMS("[PLANE:%d:%s] was on [CRTC:%d:%s] with [FB:%d]\n",
+		plane->base.id, plane->name,
+		old_state->crtc->base.id, old_state->crtc->name,
+		old_state->fb->base.id);
+
+	dcrtc = drm_to_armada_crtc(old_state->crtc);
+	regs = dcrtc->regs + dcrtc->regs_idx;
+
+	/* Disable plane and power down most RAMs and FIFOs */
+	armada_reg_queue_mod(regs, idx, 0, CFG_GRA_ENA, LCD_SPU_DMA_CTRL0);
+	armada_reg_queue_mod(regs, idx, CFG_PDWN256x32 | CFG_PDWN256x24 |
+			     CFG_PDWN256x8 | CFG_PDWN32x32 | CFG_PDWN64x66,
+			     0, LCD_SPU_SRAM_PARA1);
+
+	dcrtc->regs_idx += idx;
+}
+
+static const struct drm_plane_helper_funcs armada_primary_plane_helper_funcs = {
+	.prepare_fb	= armada_drm_plane_prepare_fb,
+	.cleanup_fb	= armada_drm_plane_cleanup_fb,
+	.atomic_check	= armada_drm_plane_atomic_check,
+	.atomic_update	= armada_drm_primary_plane_atomic_update,
+	.atomic_disable	= armada_drm_primary_plane_atomic_disable,
+};
+
+static const struct drm_plane_funcs armada_primary_plane_funcs = {
+	.update_plane	= drm_atomic_helper_update_plane,
+	.disable_plane	= drm_atomic_helper_disable_plane,
+	.destroy	= drm_primary_helper_destroy,
+	.reset		= drm_atomic_helper_plane_reset,
+	.atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_plane_destroy_state,
+};
+
+int armada_drm_primary_plane_init(struct drm_device *drm,
+	struct drm_plane *primary)
+{
+	int ret;
+
+	drm_plane_helper_add(primary, &armada_primary_plane_helper_funcs);
+
+	ret = drm_universal_plane_init(drm, primary, 0,
+				       &armada_primary_plane_funcs,
+				       armada_primary_formats,
+				       ARRAY_SIZE(armada_primary_formats),
+				       NULL,
+				       DRM_PLANE_TYPE_PRIMARY, NULL);
+
+	return ret;
+}
diff --git a/drivers/gpu/drm/armada/armada_plane.h b/drivers/gpu/drm/armada/armada_plane.h
new file mode 100644
index 000000000000..ff4281ba7fad
--- /dev/null
+++ b/drivers/gpu/drm/armada/armada_plane.h
@@ -0,0 +1,15 @@
+#ifndef ARMADA_PLANE_H
+#define ARMADA_PLANE_H
+
+void armada_drm_plane_calc(struct drm_plane_state *state, u32 addrs[2][3],
+	u16 pitches[3], bool interlaced);
+int armada_drm_plane_prepare_fb(struct drm_plane *plane,
+	struct drm_plane_state *state);
+void armada_drm_plane_cleanup_fb(struct drm_plane *plane,
+	struct drm_plane_state *old_state);
+int armada_drm_plane_atomic_check(struct drm_plane *plane,
+	struct drm_plane_state *state);
+int armada_drm_primary_plane_init(struct drm_device *drm,
+	struct drm_plane *primary);
+
+#endif
diff --git a/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c b/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c
index 6437b878724a..85c2d407a52e 100644
--- a/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c
+++ b/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c
@@ -429,6 +429,18 @@ static void adv7511_hpd_work(struct work_struct *work)
 	else
 		status = connector_status_disconnected;
 
+	/*
+	 * The bridge resets its registers on unplug. So when we get a plug
+	 * event and we're already supposed to be powered, cycle the bridge to
+	 * restore its state.
+	 */
+	if (status == connector_status_connected &&
+	    adv7511->connector.status == connector_status_disconnected &&
+	    adv7511->powered) {
+		regcache_mark_dirty(adv7511->regmap);
+		adv7511_power_on(adv7511);
+	}
+
 	if (adv7511->connector.status != status) {
 		adv7511->connector.status = status;
 		if (status == connector_status_disconnected)
diff --git a/drivers/gpu/drm/bridge/sil-sii8620.c b/drivers/gpu/drm/bridge/sil-sii8620.c
index 250effa0e6b8..a6e8f4591e63 100644
--- a/drivers/gpu/drm/bridge/sil-sii8620.c
+++ b/drivers/gpu/drm/bridge/sil-sii8620.c
@@ -14,6 +14,7 @@
 #include <drm/bridge/mhl.h>
 #include <drm/drm_crtc.h>
 #include <drm/drm_edid.h>
+#include <drm/drm_encoder.h>
 
 #include <linux/clk.h>
 #include <linux/delay.h>
@@ -72,9 +73,7 @@ struct sii8620 {
 	struct regulator_bulk_data supplies[2];
 	struct mutex lock; /* context lock, protects fields below */
 	int error;
-	int pixel_clock;
 	unsigned int use_packed_pixel:1;
-	int video_code;
 	enum sii8620_mode mode;
 	enum sii8620_sink_type sink_type;
 	u8 cbus_status;
@@ -82,7 +81,6 @@ struct sii8620 {
 	u8 xstat[MHL_XDS_SIZE];
 	u8 devcap[MHL_DCAP_SIZE];
 	u8 xdevcap[MHL_XDC_SIZE];
-	u8 avif[HDMI_INFOFRAME_SIZE(AVI)];
 	bool feature_complete;
 	bool devcap_read;
 	bool sink_detected;
@@ -1017,21 +1015,36 @@ static void sii8620_stop_video(struct sii8620 *ctx)
 
 static void sii8620_set_format(struct sii8620 *ctx)
 {
+	u8 out_fmt;
+
 	if (sii8620_is_mhl3(ctx)) {
 		sii8620_setbits(ctx, REG_M3_P0CTRL,
 				BIT_M3_P0CTRL_MHL3_P0_PIXEL_MODE_PACKED,
 				ctx->use_packed_pixel ? ~0 : 0);
 	} else {
+		if (ctx->use_packed_pixel) {
+			sii8620_write_seq_static(ctx,
+				REG_VID_MODE, BIT_VID_MODE_M1080P,
+				REG_MHL_TOP_CTL, BIT_MHL_TOP_CTL_MHL_PP_SEL | 1,
+				REG_MHLTX_CTL6, 0x60
+			);
+		} else {
 			sii8620_write_seq_static(ctx,
 				REG_VID_MODE, 0,
 				REG_MHL_TOP_CTL, 1,
 				REG_MHLTX_CTL6, 0xa0
 			);
+		}
 	}
 
+	if (ctx->use_packed_pixel)
+		out_fmt = VAL_TPI_FORMAT(YCBCR422, FULL);
+	else
+		out_fmt = VAL_TPI_FORMAT(RGB, FULL);
+
 	sii8620_write_seq(ctx,
 		REG_TPI_INPUT, VAL_TPI_FORMAT(RGB, FULL),
-		REG_TPI_OUTPUT, VAL_TPI_FORMAT(RGB, FULL),
+		REG_TPI_OUTPUT, out_fmt,
 	);
 }
 
@@ -1082,18 +1095,28 @@ static ssize_t mhl3_infoframe_pack(struct mhl3_infoframe *frame,
 	return frm_len;
 }
 
-static void sii8620_set_infoframes(struct sii8620 *ctx)
+static void sii8620_set_infoframes(struct sii8620 *ctx,
+				   struct drm_display_mode *mode)
 {
 	struct mhl3_infoframe mhl_frm;
 	union hdmi_infoframe frm;
 	u8 buf[31];
 	int ret;
 
+	ret = drm_hdmi_avi_infoframe_from_display_mode(&frm.avi,
+						       mode,
+						       true);
+	if (ctx->use_packed_pixel)
+		frm.avi.colorspace = HDMI_COLORSPACE_YUV422;
+
+	if (!ret)
+		ret = hdmi_avi_infoframe_pack(&frm.avi, buf, ARRAY_SIZE(buf));
+	if (ret > 0)
+		sii8620_write_buf(ctx, REG_TPI_AVI_CHSUM, buf + 3, ret - 3);
+
 	if (!sii8620_is_mhl3(ctx) || !ctx->use_packed_pixel) {
 		sii8620_write(ctx, REG_TPI_SC,
 			BIT_TPI_SC_TPI_OUTPUT_MODE_0_HDMI);
-		sii8620_write_buf(ctx, REG_TPI_AVI_CHSUM, ctx->avif + 3,
-			ARRAY_SIZE(ctx->avif) - 3);
 		sii8620_write(ctx, REG_PKT_FILTER_0,
 			BIT_PKT_FILTER_0_DROP_CEA_GAMUT_PKT |
 			BIT_PKT_FILTER_0_DROP_MPEG_PKT |
@@ -1102,16 +1125,6 @@ static void sii8620_set_infoframes(struct sii8620 *ctx)
 		return;
 	}
 
-	ret = hdmi_avi_infoframe_init(&frm.avi);
-	frm.avi.colorspace = HDMI_COLORSPACE_YUV422;
-	frm.avi.active_aspect = HDMI_ACTIVE_ASPECT_PICTURE;
-	frm.avi.picture_aspect = HDMI_PICTURE_ASPECT_16_9;
-	frm.avi.colorimetry = HDMI_COLORIMETRY_ITU_709;
-	frm.avi.video_code = ctx->video_code;
-	if (!ret)
-		ret = hdmi_avi_infoframe_pack(&frm.avi, buf, ARRAY_SIZE(buf));
-	if (ret > 0)
-		sii8620_write_buf(ctx, REG_TPI_AVI_CHSUM, buf + 3, ret - 3);
 	sii8620_write(ctx, REG_PKT_FILTER_0,
 		BIT_PKT_FILTER_0_DROP_CEA_GAMUT_PKT |
 		BIT_PKT_FILTER_0_DROP_MPEG_PKT |
@@ -1131,6 +1144,9 @@ static void sii8620_set_infoframes(struct sii8620 *ctx)
 
 static void sii8620_start_video(struct sii8620 *ctx)
 {
+	struct drm_display_mode *mode =
+		&ctx->bridge.encoder->crtc->state->adjusted_mode;
+
 	if (!sii8620_is_mhl3(ctx))
 		sii8620_stop_video(ctx);
 
@@ -1149,8 +1165,14 @@ static void sii8620_start_video(struct sii8620 *ctx)
 	sii8620_set_format(ctx);
 
 	if (!sii8620_is_mhl3(ctx)) {
-		sii8620_mt_write_stat(ctx, MHL_DST_REG(LINK_MODE),
-			MHL_DST_LM_CLK_MODE_NORMAL | MHL_DST_LM_PATH_ENABLED);
+		u8 link_mode = MHL_DST_LM_PATH_ENABLED;
+
+		if (ctx->use_packed_pixel)
+			link_mode |= MHL_DST_LM_CLK_MODE_PACKED_PIXEL;
+		else
+			link_mode |= MHL_DST_LM_CLK_MODE_NORMAL;
+
+		sii8620_mt_write_stat(ctx, MHL_DST_REG(LINK_MODE), link_mode);
 		sii8620_set_auto_zone(ctx);
 	} else {
 		static const struct {
@@ -1167,7 +1189,7 @@ static void sii8620_start_video(struct sii8620 *ctx)
 			  MHL_XDS_LINK_RATE_6_0_GBPS, 0x40 },
 		};
 		u8 p0_ctrl = BIT_M3_P0CTRL_MHL3_P0_PORT_EN;
-		int clk = ctx->pixel_clock * (ctx->use_packed_pixel ? 2 : 3);
+		int clk = mode->clock * (ctx->use_packed_pixel ? 2 : 3);
 		int i;
 
 		for (i = 0; i < ARRAY_SIZE(clk_spec) - 1; ++i)
@@ -1196,7 +1218,7 @@ static void sii8620_start_video(struct sii8620 *ctx)
 			clk_spec[i].link_rate);
 	}
 
-	sii8620_set_infoframes(ctx);
+	sii8620_set_infoframes(ctx, mode);
 }
 
 static void sii8620_disable_hpd(struct sii8620 *ctx)
@@ -1661,14 +1683,18 @@ static void sii8620_status_dcap_ready(struct sii8620 *ctx)
 
 static void sii8620_status_changed_path(struct sii8620 *ctx)
 {
-	if (ctx->stat[MHL_DST_LINK_MODE] & MHL_DST_LM_PATH_ENABLED) {
-		sii8620_mt_write_stat(ctx, MHL_DST_REG(LINK_MODE),
-				      MHL_DST_LM_CLK_MODE_NORMAL
-				      | MHL_DST_LM_PATH_ENABLED);
-	} else {
-		sii8620_mt_write_stat(ctx, MHL_DST_REG(LINK_MODE),
-				      MHL_DST_LM_CLK_MODE_NORMAL);
-	}
+	u8 link_mode;
+
+	if (ctx->use_packed_pixel)
+		link_mode = MHL_DST_LM_CLK_MODE_PACKED_PIXEL;
+	else
+		link_mode = MHL_DST_LM_CLK_MODE_NORMAL;
+
+	if (ctx->stat[MHL_DST_LINK_MODE] & MHL_DST_LM_PATH_ENABLED)
+		link_mode |= MHL_DST_LM_PATH_ENABLED;
+
+	sii8620_mt_write_stat(ctx, MHL_DST_REG(LINK_MODE),
+			      link_mode);
 }
 
 static void sii8620_msc_mr_write_stat(struct sii8620 *ctx)
@@ -2242,8 +2268,6 @@ static bool sii8620_mode_fixup(struct drm_bridge *bridge,
 	mutex_lock(&ctx->lock);
 
 	ctx->use_packed_pixel = sii8620_is_packing_required(ctx, adjusted_mode);
-	ctx->video_code = drm_match_cea_mode(adjusted_mode);
-	ctx->pixel_clock = adjusted_mode->clock;
 
 	mutex_unlock(&ctx->lock);
 
diff --git a/drivers/gpu/drm/drm_atomic_helper.c b/drivers/gpu/drm/drm_atomic_helper.c
index 284a5d2bc11d..2c23a48482da 100644
--- a/drivers/gpu/drm/drm_atomic_helper.c
+++ b/drivers/gpu/drm/drm_atomic_helper.c
@@ -1538,8 +1538,9 @@ int drm_atomic_helper_async_check(struct drm_device *dev,
 {
 	struct drm_crtc *crtc;
 	struct drm_crtc_state *crtc_state;
-	struct drm_plane *plane;
-	struct drm_plane_state *old_plane_state, *new_plane_state;
+	struct drm_plane *plane = NULL;
+	struct drm_plane_state *old_plane_state = NULL;
+	struct drm_plane_state *new_plane_state = NULL;
 	const struct drm_plane_helper_funcs *funcs;
 	int i, n_planes = 0;
 
@@ -1555,7 +1556,8 @@ int drm_atomic_helper_async_check(struct drm_device *dev,
 	if (n_planes != 1)
 		return -EINVAL;
 
-	if (!new_plane_state->crtc)
+	if (!new_plane_state->crtc ||
+	    old_plane_state->crtc != new_plane_state->crtc)
 		return -EINVAL;
 
 	funcs = plane->helper_private;
diff --git a/drivers/gpu/drm/drm_context.c b/drivers/gpu/drm/drm_context.c
index 3c4000facb36..f973d287696a 100644
--- a/drivers/gpu/drm/drm_context.c
+++ b/drivers/gpu/drm/drm_context.c
@@ -372,7 +372,7 @@ int drm_legacy_addctx(struct drm_device *dev, void *data,
 		ctx->handle = drm_legacy_ctxbitmap_next(dev);
 	}
 	DRM_DEBUG("%d\n", ctx->handle);
-	if (ctx->handle == -1) {
+	if (ctx->handle < 0) {
 		DRM_DEBUG("Not enough free contexts.\n");
 		/* Should this return -EBUSY instead? */
 		return -ENOMEM;
diff --git a/drivers/gpu/drm/drm_dp_cec.c b/drivers/gpu/drm/drm_dp_cec.c
index ddb1c5adebb9..988513346e9c 100644
--- a/drivers/gpu/drm/drm_dp_cec.c
+++ b/drivers/gpu/drm/drm_dp_cec.c
@@ -157,7 +157,7 @@ static void drm_dp_cec_adap_status(struct cec_adapter *adap,
 
 	if (drm_dp_read_desc(aux, &desc, true))
 		return;
-	seq_printf(file, "OUI: %*pdH\n",
+	seq_printf(file, "OUI: %*phD\n",
 		   (int)sizeof(id->oui), id->oui);
 	seq_printf(file, "ID: %*pE\n",
 		   (int)strnlen(id->device_id, sizeof(id->device_id)),
diff --git a/drivers/gpu/drm/drm_edid.c b/drivers/gpu/drm/drm_edid.c
index 5dc742b27ca0..3c9fc99648b7 100644
--- a/drivers/gpu/drm/drm_edid.c
+++ b/drivers/gpu/drm/drm_edid.c
@@ -116,6 +116,9 @@ static const struct edid_quirk {
 	/* CPT panel of Asus UX303LA reports 8 bpc, but is a 6 bpc panel */
 	{ "CPT", 0x17df, EDID_QUIRK_FORCE_6BPC },
 
+	/* SDC panel of Lenovo B50-80 reports 8 bpc, but is a 6 bpc panel */
+	{ "SDC", 0x3652, EDID_QUIRK_FORCE_6BPC },
+
 	/* Belinea 10 15 55 */
 	{ "MAX", 1516, EDID_QUIRK_PREFER_LARGE_60 },
 	{ "MAX", 0x77e, EDID_QUIRK_PREFER_LARGE_60 },
diff --git a/drivers/gpu/drm/drm_gem.c b/drivers/gpu/drm/drm_gem.c
index 4a16d7b26c89..bf90625df3c5 100644
--- a/drivers/gpu/drm/drm_gem.c
+++ b/drivers/gpu/drm/drm_gem.c
@@ -1036,6 +1036,15 @@ int drm_gem_mmap(struct file *filp, struct vm_area_struct *vma)
 		return -EACCES;
 	}
 
+	if (node->readonly) {
+		if (vma->vm_flags & VM_WRITE) {
+			drm_gem_object_put_unlocked(obj);
+			return -EINVAL;
+		}
+
+		vma->vm_flags &= ~VM_MAYWRITE;
+	}
+
 	ret = drm_gem_mmap_obj(obj, drm_vma_node_size(node) << PAGE_SHIFT,
 			       vma);
 
diff --git a/drivers/gpu/drm/drm_lease.c b/drivers/gpu/drm/drm_lease.c
index 50c73c0a20b9..b54fb78a283c 100644
--- a/drivers/gpu/drm/drm_lease.c
+++ b/drivers/gpu/drm/drm_lease.c
@@ -553,24 +553,13 @@ int drm_mode_create_lease_ioctl(struct drm_device *dev,
 
 	/* Clone the lessor file to create a new file for us */
 	DRM_DEBUG_LEASE("Allocating lease file\n");
-	path_get(&lessor_file->f_path);
-	lessee_file = alloc_file(&lessor_file->f_path,
-				 lessor_file->f_mode,
-				 fops_get(lessor_file->f_inode->i_fop));
-
+	lessee_file = file_clone_open(lessor_file);
 	if (IS_ERR(lessee_file)) {
 		ret = PTR_ERR(lessee_file);
 		goto out_lessee;
 	}
 
-	/* Initialize the new file for DRM */
-	DRM_DEBUG_LEASE("Initializing the file with %p\n", lessee_file->f_op->open);
-	ret = lessee_file->f_op->open(lessee_file->f_inode, lessee_file);
-	if (ret)
-		goto out_lessee_file;
-
 	lessee_priv = lessee_file->private_data;
-
 	/* Change the file to a master one */
 	drm_master_put(&lessee_priv->master);
 	lessee_priv->master = lessee;
@@ -588,9 +577,6 @@ int drm_mode_create_lease_ioctl(struct drm_device *dev,
 	DRM_DEBUG_LEASE("drm_mode_create_lease_ioctl succeeded\n");
 	return 0;
 
-out_lessee_file:
-	fput(lessee_file);
-
 out_lessee:
 	drm_master_put(&lessee);
 
diff --git a/drivers/gpu/drm/drm_mipi_dsi.c b/drivers/gpu/drm/drm_mipi_dsi.c
index bc73b7f5b9fc..80b75501f5c6 100644
--- a/drivers/gpu/drm/drm_mipi_dsi.c
+++ b/drivers/gpu/drm/drm_mipi_dsi.c
@@ -392,6 +392,7 @@ bool mipi_dsi_packet_format_is_short(u8 type)
 	case MIPI_DSI_DCS_SHORT_WRITE:
 	case MIPI_DSI_DCS_SHORT_WRITE_PARAM:
 	case MIPI_DSI_DCS_READ:
+	case MIPI_DSI_DCS_COMPRESSION_MODE:
 	case MIPI_DSI_SET_MAXIMUM_RETURN_PACKET_SIZE:
 		return true;
 	}
@@ -410,6 +411,7 @@ EXPORT_SYMBOL(mipi_dsi_packet_format_is_short);
 bool mipi_dsi_packet_format_is_long(u8 type)
 {
 	switch (type) {
+	case MIPI_DSI_PPS_LONG_WRITE:
 	case MIPI_DSI_NULL_PACKET:
 	case MIPI_DSI_BLANKING_PACKET:
 	case MIPI_DSI_GENERIC_LONG_WRITE:
diff --git a/drivers/gpu/drm/drm_pci.c b/drivers/gpu/drm/drm_pci.c
index 4db9c515b74f..896e42a34895 100644
--- a/drivers/gpu/drm/drm_pci.c
+++ b/drivers/gpu/drm/drm_pci.c
@@ -326,64 +326,6 @@ int drm_legacy_pci_init(struct drm_driver *driver, struct pci_driver *pdriver)
 }
 EXPORT_SYMBOL(drm_legacy_pci_init);
 
-int drm_pcie_get_speed_cap_mask(struct drm_device *dev, u32 *mask)
-{
-	struct pci_dev *root;
-	u32 lnkcap, lnkcap2;
-
-	*mask = 0;
-	if (!dev->pdev)
-		return -EINVAL;
-
-	root = dev->pdev->bus->self;
-
-	/* we've been informed via and serverworks don't make the cut */
-	if (root->vendor == PCI_VENDOR_ID_VIA ||
-	    root->vendor == PCI_VENDOR_ID_SERVERWORKS)
-		return -EINVAL;
-
-	pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
-	pcie_capability_read_dword(root, PCI_EXP_LNKCAP2, &lnkcap2);
-
-	if (lnkcap2) {	/* PCIe r3.0-compliant */
-		if (lnkcap2 & PCI_EXP_LNKCAP2_SLS_2_5GB)
-			*mask |= DRM_PCIE_SPEED_25;
-		if (lnkcap2 & PCI_EXP_LNKCAP2_SLS_5_0GB)
-			*mask |= DRM_PCIE_SPEED_50;
-		if (lnkcap2 & PCI_EXP_LNKCAP2_SLS_8_0GB)
-			*mask |= DRM_PCIE_SPEED_80;
-	} else {	/* pre-r3.0 */
-		if (lnkcap & PCI_EXP_LNKCAP_SLS_2_5GB)
-			*mask |= DRM_PCIE_SPEED_25;
-		if (lnkcap & PCI_EXP_LNKCAP_SLS_5_0GB)
-			*mask |= (DRM_PCIE_SPEED_25 | DRM_PCIE_SPEED_50);
-	}
-
-	DRM_INFO("probing gen 2 caps for device %x:%x = %x/%x\n", root->vendor, root->device, lnkcap, lnkcap2);
-	return 0;
-}
-EXPORT_SYMBOL(drm_pcie_get_speed_cap_mask);
-
-int drm_pcie_get_max_link_width(struct drm_device *dev, u32 *mlw)
-{
-	struct pci_dev *root;
-	u32 lnkcap;
-
-	*mlw = 0;
-	if (!dev->pdev)
-		return -EINVAL;
-
-	root = dev->pdev->bus->self;
-
-	pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
-
-	*mlw = (lnkcap & PCI_EXP_LNKCAP_MLW) >> 4;
-
-	DRM_INFO("probing mlw for device %x:%x = %x\n", root->vendor, root->device, lnkcap);
-	return 0;
-}
-EXPORT_SYMBOL(drm_pcie_get_max_link_width);
-
 #else
 
 void drm_pci_agp_destroy(struct drm_device *dev) {}
diff --git a/drivers/gpu/drm/drm_print.c b/drivers/gpu/drm/drm_print.c
index b25f98f33f6c..0e7fc3e7dfb4 100644
--- a/drivers/gpu/drm/drm_print.c
+++ b/drivers/gpu/drm/drm_print.c
@@ -30,6 +30,100 @@
 #include <drm/drmP.h>
 #include <drm/drm_print.h>
 
+void __drm_puts_coredump(struct drm_printer *p, const char *str)
+{
+	struct drm_print_iterator *iterator = p->arg;
+	ssize_t len;
+
+	if (!iterator->remain)
+		return;
+
+	if (iterator->offset < iterator->start) {
+		ssize_t copy;
+
+		len = strlen(str);
+
+		if (iterator->offset + len <= iterator->start) {
+			iterator->offset += len;
+			return;
+		}
+
+		copy = len - (iterator->start - iterator->offset);
+
+		if (copy > iterator->remain)
+			copy = iterator->remain;
+
+		/* Copy out the bit of the string that we need */
+		memcpy(iterator->data,
+			str + (iterator->start - iterator->offset), copy);
+
+		iterator->offset = iterator->start + copy;
+		iterator->remain -= copy;
+	} else {
+		ssize_t pos = iterator->offset - iterator->start;
+
+		len = min_t(ssize_t, strlen(str), iterator->remain);
+
+		memcpy(iterator->data + pos, str, len);
+
+		iterator->offset += len;
+		iterator->remain -= len;
+	}
+}
+EXPORT_SYMBOL(__drm_puts_coredump);
+
+void __drm_printfn_coredump(struct drm_printer *p, struct va_format *vaf)
+{
+	struct drm_print_iterator *iterator = p->arg;
+	size_t len;
+	char *buf;
+
+	if (!iterator->remain)
+		return;
+
+	/* Figure out how big the string will be */
+	len = snprintf(NULL, 0, "%pV", vaf);
+
+	/* This is the easiest path, we've already advanced beyond the offset */
+	if (iterator->offset + len <= iterator->start) {
+		iterator->offset += len;
+		return;
+	}
+
+	/* Then check if we can directly copy into the target buffer */
+	if ((iterator->offset >= iterator->start) && (len < iterator->remain)) {
+		ssize_t pos = iterator->offset - iterator->start;
+
+		snprintf(((char *) iterator->data) + pos,
+			iterator->remain, "%pV", vaf);
+
+		iterator->offset += len;
+		iterator->remain -= len;
+
+		return;
+	}
+
+	/*
+	 * Finally, hit the slow path and make a temporary string to copy over
+	 * using _drm_puts_coredump
+	 */
+	buf = kmalloc(len + 1, GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY);
+	if (!buf)
+		return;
+
+	snprintf(buf, len + 1, "%pV", vaf);
+	__drm_puts_coredump(p, (const char *) buf);
+
+	kfree(buf);
+}
+EXPORT_SYMBOL(__drm_printfn_coredump);
+
+void __drm_puts_seq_file(struct drm_printer *p, const char *str)
+{
+	seq_puts(p->arg, str);
+}
+EXPORT_SYMBOL(__drm_puts_seq_file);
+
 void __drm_printfn_seq_file(struct drm_printer *p, struct va_format *vaf)
 {
 	seq_printf(p->arg, "%pV", vaf);
@@ -49,6 +143,23 @@ void __drm_printfn_debug(struct drm_printer *p, struct va_format *vaf)
 EXPORT_SYMBOL(__drm_printfn_debug);
 
 /**
+ * drm_puts - print a const string to a &drm_printer stream
+ * @p: the &drm printer
+ * @str: const string
+ *
+ * Allow &drm_printer types that have a constant string
+ * option to use it.
+ */
+void drm_puts(struct drm_printer *p, const char *str)
+{
+	if (p->puts)
+		p->puts(p, str);
+	else
+		drm_printf(p, "%s", str);
+}
+EXPORT_SYMBOL(drm_puts);
+
+/**
  * drm_printf - print to a &drm_printer stream
  * @p: the &drm_printer
  * @f: format string
diff --git a/drivers/gpu/drm/drm_property.c b/drivers/gpu/drm/drm_property.c
index 1f8031e30f53..cdb10f885a4f 100644
--- a/drivers/gpu/drm/drm_property.c
+++ b/drivers/gpu/drm/drm_property.c
@@ -532,7 +532,7 @@ static void drm_property_free_blob(struct kref *kref)
 
 	drm_mode_object_unregister(blob->dev, &blob->base);
 
-	kfree(blob);
+	kvfree(blob);
 }
 
 /**
@@ -559,7 +559,7 @@ drm_property_create_blob(struct drm_device *dev, size_t length,
 	if (!length || length > ULONG_MAX - sizeof(struct drm_property_blob))
 		return ERR_PTR(-EINVAL);
 
-	blob = kzalloc(sizeof(struct drm_property_blob)+length, GFP_KERNEL);
+	blob = kvzalloc(sizeof(struct drm_property_blob)+length, GFP_KERNEL);
 	if (!blob)
 		return ERR_PTR(-ENOMEM);
 
@@ -576,7 +576,7 @@ drm_property_create_blob(struct drm_device *dev, size_t length,
 	ret = __drm_mode_object_add(dev, &blob->base, DRM_MODE_OBJECT_BLOB,
 				    true, drm_property_free_blob);
 	if (ret) {
-		kfree(blob);
+		kvfree(blob);
 		return ERR_PTR(-EINVAL);
 	}
 
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.c b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
index e5013a999147..9b2720b41571 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_drv.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
@@ -49,12 +49,12 @@ static int etnaviv_open(struct drm_device *dev, struct drm_file *file)
 
 	for (i = 0; i < ETNA_MAX_PIPES; i++) {
 		struct etnaviv_gpu *gpu = priv->gpu[i];
+		struct drm_sched_rq *rq;
 
 		if (gpu) {
-			drm_sched_entity_init(&gpu->sched,
-				&ctx->sched_entity[i],
-				&gpu->sched.sched_rq[DRM_SCHED_PRIORITY_NORMAL],
-				NULL);
+			rq = &gpu->sched.sched_rq[DRM_SCHED_PRIORITY_NORMAL];
+			drm_sched_entity_init(&ctx->sched_entity[i],
+					      &rq, 1, NULL);
 			}
 	}
 
@@ -78,8 +78,7 @@ static void etnaviv_postclose(struct drm_device *dev, struct drm_file *file)
 				gpu->lastctx = NULL;
 			mutex_unlock(&gpu->lock);
 
-			drm_sched_entity_fini(&gpu->sched,
-					      &ctx->sched_entity[i]);
+			drm_sched_entity_destroy(&ctx->sched_entity[i]);
 		}
 	}
 
@@ -631,8 +630,11 @@ static struct platform_driver etnaviv_platform_driver = {
 	},
 };
 
+static struct platform_device *etnaviv_drm;
+
 static int __init etnaviv_init(void)
 {
+	struct platform_device *pdev;
 	int ret;
 	struct device_node *np;
 
@@ -644,7 +646,7 @@ static int __init etnaviv_init(void)
 
 	ret = platform_driver_register(&etnaviv_platform_driver);
 	if (ret != 0)
-		platform_driver_unregister(&etnaviv_gpu_driver);
+		goto unregister_gpu_driver;
 
 	/*
 	 * If the DT contains at least one available GPU device, instantiate
@@ -653,20 +655,33 @@ static int __init etnaviv_init(void)
 	for_each_compatible_node(np, NULL, "vivante,gc") {
 		if (!of_device_is_available(np))
 			continue;
-
-		platform_device_register_simple("etnaviv", -1, NULL, 0);
+		pdev = platform_device_register_simple("etnaviv", -1,
+						       NULL, 0);
+		if (IS_ERR(pdev)) {
+			ret = PTR_ERR(pdev);
+			of_node_put(np);
+			goto unregister_platform_driver;
+		}
+		etnaviv_drm = pdev;
 		of_node_put(np);
 		break;
 	}
 
+	return 0;
+
+unregister_platform_driver:
+	platform_driver_unregister(&etnaviv_platform_driver);
+unregister_gpu_driver:
+	platform_driver_unregister(&etnaviv_gpu_driver);
 	return ret;
 }
 module_init(etnaviv_init);
 
 static void __exit etnaviv_exit(void)
 {
-	platform_driver_unregister(&etnaviv_gpu_driver);
+	platform_device_unregister(etnaviv_drm);
 	platform_driver_unregister(&etnaviv_platform_driver);
+	platform_driver_unregister(&etnaviv_gpu_driver);
 }
 module_exit(etnaviv_exit);
 
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.h b/drivers/gpu/drm/etnaviv/etnaviv_drv.h
index d36c7bbe66db..8d02d1b7dcf5 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_drv.h
+++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.h
@@ -18,6 +18,7 @@
 #include <linux/time64.h>
 #include <linux/types.h>
 #include <linux/sizes.h>
+#include <linux/mm_types.h>
 
 #include <drm/drmP.h>
 #include <drm/drm_crtc_helper.h>
@@ -53,7 +54,7 @@ int etnaviv_ioctl_gem_submit(struct drm_device *dev, void *data,
 		struct drm_file *file);
 
 int etnaviv_gem_mmap(struct file *filp, struct vm_area_struct *vma);
-int etnaviv_gem_fault(struct vm_fault *vmf);
+vm_fault_t etnaviv_gem_fault(struct vm_fault *vmf);
 int etnaviv_gem_mmap_offset(struct drm_gem_object *obj, u64 *offset);
 struct sg_table *etnaviv_gem_prime_get_sg_table(struct drm_gem_object *obj);
 void *etnaviv_gem_prime_vmap(struct drm_gem_object *obj);
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem.c b/drivers/gpu/drm/etnaviv/etnaviv_gem.c
index 209ef1274b80..1fa74226db91 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gem.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gem.c
@@ -169,31 +169,30 @@ int etnaviv_gem_mmap(struct file *filp, struct vm_area_struct *vma)
 	return obj->ops->mmap(obj, vma);
 }
 
-int etnaviv_gem_fault(struct vm_fault *vmf)
+vm_fault_t etnaviv_gem_fault(struct vm_fault *vmf)
 {
 	struct vm_area_struct *vma = vmf->vma;
 	struct drm_gem_object *obj = vma->vm_private_data;
 	struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj);
 	struct page **pages, *page;
 	pgoff_t pgoff;
-	int ret;
+	int err;
 
 	/*
 	 * Make sure we don't parallel update on a fault, nor move or remove
-	 * something from beneath our feet.  Note that vm_insert_page() is
+	 * something from beneath our feet.  Note that vmf_insert_page() is
 	 * specifically coded to take care of this, so we don't have to.
 	 */
-	ret = mutex_lock_interruptible(&etnaviv_obj->lock);
-	if (ret)
-		goto out;
-
+	err = mutex_lock_interruptible(&etnaviv_obj->lock);
+	if (err)
+		return VM_FAULT_NOPAGE;
 	/* make sure we have pages attached now */
 	pages = etnaviv_gem_get_pages(etnaviv_obj);
 	mutex_unlock(&etnaviv_obj->lock);
 
 	if (IS_ERR(pages)) {
-		ret = PTR_ERR(pages);
-		goto out;
+		err = PTR_ERR(pages);
+		return vmf_error(err);
 	}
 
 	/* We don't use vmf->pgoff since that has the fake offset: */
@@ -204,25 +203,7 @@ int etnaviv_gem_fault(struct vm_fault *vmf)
 	VERB("Inserting %p pfn %lx, pa %lx", (void *)vmf->address,
 	     page_to_pfn(page), page_to_pfn(page) << PAGE_SHIFT);
 
-	ret = vm_insert_page(vma, vmf->address, page);
-
-out:
-	switch (ret) {
-	case -EAGAIN:
-	case 0:
-	case -ERESTARTSYS:
-	case -EINTR:
-	case -EBUSY:
-		/*
-		 * EBUSY is ok: this just means that another thread
-		 * already did the job.
-		 */
-		return VM_FAULT_NOPAGE;
-	case -ENOMEM:
-		return VM_FAULT_OOM;
-	default:
-		return VM_FAULT_SIGBUS;
-	}
+	return vmf_insert_page(vma, vmf->address, page);
 }
 
 int etnaviv_gem_mmap_offset(struct drm_gem_object *obj, u64 *offset)
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c b/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c
index 46ecd3e66ac9..983e67f19e45 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c
@@ -388,9 +388,9 @@ static void submit_cleanup(struct kref *kref)
 		dma_fence_put(submit->in_fence);
 	if (submit->out_fence) {
 		/* first remove from IDR, so fence can not be found anymore */
-		mutex_lock(&submit->gpu->fence_idr_lock);
+		mutex_lock(&submit->gpu->fence_lock);
 		idr_remove(&submit->gpu->fence_idr, submit->out_fence_id);
-		mutex_unlock(&submit->gpu->fence_idr_lock);
+		mutex_unlock(&submit->gpu->fence_lock);
 		dma_fence_put(submit->out_fence);
 	}
 	kfree(submit->pmrs);
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
index 19b09a59e30e..f225fbc6edd2 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
@@ -799,6 +799,7 @@ int etnaviv_gpu_init(struct etnaviv_gpu *gpu)
 
 free_buffer:
 	etnaviv_cmdbuf_free(&gpu->buffer);
+	gpu->buffer.suballoc = NULL;
 destroy_iommu:
 	etnaviv_iommu_destroy(gpu->mmu);
 	gpu->mmu = NULL;
@@ -1726,7 +1727,7 @@ static int etnaviv_gpu_platform_probe(struct platform_device *pdev)
 
 	gpu->dev = &pdev->dev;
 	mutex_init(&gpu->lock);
-	mutex_init(&gpu->fence_idr_lock);
+	mutex_init(&gpu->fence_lock);
 
 	/* Map registers: */
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.h b/drivers/gpu/drm/etnaviv/etnaviv_gpu.h
index dd430f0f8ff5..9a75a6937268 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.h
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.h
@@ -118,7 +118,7 @@ struct etnaviv_gpu {
 	u32 idle_mask;
 
 	/* Fencing support */
-	struct mutex fence_idr_lock;
+	struct mutex fence_lock;
 	struct idr fence_idr;
 	u32 next_fence;
 	u32 active_fence;
@@ -131,6 +131,9 @@ struct etnaviv_gpu {
 	struct work_struct sync_point_work;
 	int sync_point_event;
 
+	/* hang detection */
+	u32 hangcheck_dma_addr;
+
 	void __iomem *mmio;
 	int irq;
 
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.c b/drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.c
index 71fbc1f96cb6..f1c88d8ad5ba 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.c
@@ -119,8 +119,7 @@ static size_t etnaviv_iommuv2_unmap(struct etnaviv_iommu_domain *domain,
 
 static int etnaviv_iommuv2_init(struct etnaviv_iommuv2_domain *etnaviv_domain)
 {
-	u32 *p;
-	int ret, i;
+	int ret;
 
 	/* allocate scratch page */
 	etnaviv_domain->base.bad_page_cpu =
@@ -131,9 +130,9 @@ static int etnaviv_iommuv2_init(struct etnaviv_iommuv2_domain *etnaviv_domain)
 		ret = -ENOMEM;
 		goto fail_mem;
 	}
-	p = etnaviv_domain->base.bad_page_cpu;
-	for (i = 0; i < SZ_4K / 4; i++)
-		*p++ = 0xdead55aa;
+
+	memset32(etnaviv_domain->base.bad_page_cpu, 0xdead55aa,
+		 SZ_4K / sizeof(u32));
 
 	etnaviv_domain->pta_cpu = dma_alloc_wc(etnaviv_domain->base.dev,
 					       SZ_4K, &etnaviv_domain->pta_dma,
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_sched.c b/drivers/gpu/drm/etnaviv/etnaviv_sched.c
index a74eb57af15b..69e9b431bf1f 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_sched.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_sched.c
@@ -10,6 +10,7 @@
 #include "etnaviv_gem.h"
 #include "etnaviv_gpu.h"
 #include "etnaviv_sched.h"
+#include "state.xml.h"
 
 static int etnaviv_job_hang_limit = 0;
 module_param_named(job_hang_limit, etnaviv_job_hang_limit, int , 0444);
@@ -85,6 +86,29 @@ static void etnaviv_sched_timedout_job(struct drm_sched_job *sched_job)
 {
 	struct etnaviv_gem_submit *submit = to_etnaviv_submit(sched_job);
 	struct etnaviv_gpu *gpu = submit->gpu;
+	u32 dma_addr;
+	int change;
+
+	/*
+	 * If the GPU managed to complete this jobs fence, the timout is
+	 * spurious. Bail out.
+	 */
+	if (fence_completed(gpu, submit->out_fence->seqno))
+		return;
+
+	/*
+	 * If the GPU is still making forward progress on the front-end (which
+	 * should never loop) we shift out the timeout to give it a chance to
+	 * finish the job.
+	 */
+	dma_addr = gpu_read(gpu, VIVS_FE_DMA_ADDRESS);
+	change = dma_addr - gpu->hangcheck_dma_addr;
+	if (change < 0 || change > 16) {
+		gpu->hangcheck_dma_addr = dma_addr;
+		schedule_delayed_work(&sched_job->work_tdr,
+				      sched_job->sched->timeout);
+		return;
+	}
 
 	/* block scheduler */
 	kthread_park(gpu->sched.thread);
@@ -116,28 +140,38 @@ static const struct drm_sched_backend_ops etnaviv_sched_ops = {
 int etnaviv_sched_push_job(struct drm_sched_entity *sched_entity,
 			   struct etnaviv_gem_submit *submit)
 {
-	int ret;
+	int ret = 0;
 
-	ret = drm_sched_job_init(&submit->sched_job, &submit->gpu->sched,
-				 sched_entity, submit->cmdbuf.ctx);
+	/*
+	 * Hold the fence lock across the whole operation to avoid jobs being
+	 * pushed out of order with regard to their sched fence seqnos as
+	 * allocated in drm_sched_job_init.
+	 */
+	mutex_lock(&submit->gpu->fence_lock);
+
+	ret = drm_sched_job_init(&submit->sched_job, sched_entity,
+				 submit->cmdbuf.ctx);
 	if (ret)
-		return ret;
+		goto out_unlock;
 
 	submit->out_fence = dma_fence_get(&submit->sched_job.s_fence->finished);
-	mutex_lock(&submit->gpu->fence_idr_lock);
 	submit->out_fence_id = idr_alloc_cyclic(&submit->gpu->fence_idr,
 						submit->out_fence, 0,
 						INT_MAX, GFP_KERNEL);
-	mutex_unlock(&submit->gpu->fence_idr_lock);
-	if (submit->out_fence_id < 0)
-		return -ENOMEM;
+	if (submit->out_fence_id < 0) {
+		ret = -ENOMEM;
+		goto out_unlock;
+	}
 
 	/* the scheduler holds on to the job now */
 	kref_get(&submit->refcount);
 
 	drm_sched_entity_push_job(&submit->sched_job, sched_entity);
 
-	return 0;
+out_unlock:
+	mutex_unlock(&submit->gpu->fence_lock);
+
+	return ret;
 }
 
 int etnaviv_sched_init(struct etnaviv_gpu *gpu)
diff --git a/drivers/gpu/drm/exynos/Makefile b/drivers/gpu/drm/exynos/Makefile
index 3b323f1e0475..2ad146bbf4f5 100644
--- a/drivers/gpu/drm/exynos/Makefile
+++ b/drivers/gpu/drm/exynos/Makefile
@@ -4,7 +4,7 @@
 # Direct Rendering Infrastructure (DRI) in XFree86 4.1.0 and higher.
 
 exynosdrm-y := exynos_drm_drv.o exynos_drm_crtc.o exynos_drm_fb.o \
-		exynos_drm_gem.o exynos_drm_core.o exynos_drm_plane.o
+		exynos_drm_gem.o exynos_drm_plane.o
 
 exynosdrm-$(CONFIG_DRM_FBDEV_EMULATION) += exynos_drm_fbdev.o
 exynosdrm-$(CONFIG_DRM_EXYNOS_IOMMU) += exynos_drm_iommu.o
diff --git a/drivers/gpu/drm/exynos/exynos5433_drm_decon.c b/drivers/gpu/drm/exynos/exynos5433_drm_decon.c
index 82c95c34447f..94529aa82339 100644
--- a/drivers/gpu/drm/exynos/exynos5433_drm_decon.c
+++ b/drivers/gpu/drm/exynos/exynos5433_drm_decon.c
@@ -265,7 +265,7 @@ static void decon_win_set_pixfmt(struct decon_context *ctx, unsigned int win,
 	unsigned long val;
 
 	val = readl(ctx->addr + DECON_WINCONx(win));
-	val &= ~WINCONx_BPPMODE_MASK;
+	val &= WINCONx_ENWIN_F;
 
 	switch (fb->format->format) {
 	case DRM_FORMAT_XRGB1555:
@@ -356,8 +356,8 @@ static void decon_update_plane(struct exynos_drm_crtc *crtc,
 		writel(val, ctx->addr + DECON_VIDOSDxB(win));
 	}
 
-	val = VIDOSD_Wx_ALPHA_R_F(0x0) | VIDOSD_Wx_ALPHA_G_F(0x0) |
-		VIDOSD_Wx_ALPHA_B_F(0x0);
+	val = VIDOSD_Wx_ALPHA_R_F(0xff) | VIDOSD_Wx_ALPHA_G_F(0xff) |
+		VIDOSD_Wx_ALPHA_B_F(0xff);
 	writel(val, ctx->addr + DECON_VIDOSDxC(win));
 
 	val = VIDOSD_Wx_ALPHA_R_F(0x0) | VIDOSD_Wx_ALPHA_G_F(0x0) |
@@ -673,6 +673,8 @@ err:
 static const struct dev_pm_ops exynos5433_decon_pm_ops = {
 	SET_RUNTIME_PM_OPS(exynos5433_decon_suspend, exynos5433_decon_resume,
 			   NULL)
+	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
+				     pm_runtime_force_resume)
 };
 
 static const struct of_device_id exynos5433_decon_driver_dt_match[] = {
diff --git a/drivers/gpu/drm/exynos/exynos7_drm_decon.c b/drivers/gpu/drm/exynos/exynos7_drm_decon.c
index 3931d5e33fe0..88cbd000eb09 100644
--- a/drivers/gpu/drm/exynos/exynos7_drm_decon.c
+++ b/drivers/gpu/drm/exynos/exynos7_drm_decon.c
@@ -832,6 +832,8 @@ static int exynos7_decon_resume(struct device *dev)
 static const struct dev_pm_ops exynos7_decon_pm_ops = {
 	SET_RUNTIME_PM_OPS(exynos7_decon_suspend, exynos7_decon_resume,
 			   NULL)
+	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
+				pm_runtime_force_resume)
 };
 
 struct platform_driver decon_driver = {
diff --git a/drivers/gpu/drm/exynos/exynos_dp.c b/drivers/gpu/drm/exynos/exynos_dp.c
index af7ab1ceb50f..c8449ae4f4fe 100644
--- a/drivers/gpu/drm/exynos/exynos_dp.c
+++ b/drivers/gpu/drm/exynos/exynos_dp.c
@@ -16,6 +16,7 @@
 #include <linux/clk.h>
 #include <linux/of_graph.h>
 #include <linux/component.h>
+#include <linux/pm_runtime.h>
 #include <video/of_display_timing.h>
 #include <video/of_videomode.h>
 #include <video/videomode.h>
@@ -278,6 +279,8 @@ static int exynos_dp_resume(struct device *dev)
 
 static const struct dev_pm_ops exynos_dp_pm_ops = {
 	SET_RUNTIME_PM_OPS(exynos_dp_suspend, exynos_dp_resume, NULL)
+	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
+				pm_runtime_force_resume)
 };
 
 static const struct of_device_id exynos_dp_match[] = {
diff --git a/drivers/gpu/drm/exynos/exynos_drm_core.c b/drivers/gpu/drm/exynos/exynos_drm_core.c
deleted file mode 100644
index b0c0621fcdf7..000000000000
--- a/drivers/gpu/drm/exynos/exynos_drm_core.c
+++ /dev/null
@@ -1,119 +0,0 @@
-/* exynos_drm_core.c
- *
- * Copyright (c) 2011 Samsung Electronics Co., Ltd.
- * Author:
- *	Inki Dae <inki.dae@samsung.com>
- *	Joonyoung Shim <jy0922.shim@samsung.com>
- *	Seung-Woo Kim <sw0312.kim@samsung.com>
- *
- * This program is free software; you can redistribute  it and/or modify it
- * under  the terms of  the GNU General  Public License as published by the
- * Free Software Foundation;  either version 2 of the  License, or (at your
- * option) any later version.
- */
-
-#include <drm/drmP.h>
-
-#include "exynos_drm_drv.h"
-#include "exynos_drm_crtc.h"
-
-static LIST_HEAD(exynos_drm_subdrv_list);
-
-int exynos_drm_subdrv_register(struct exynos_drm_subdrv *subdrv)
-{
-	if (!subdrv)
-		return -EINVAL;
-
-	list_add_tail(&subdrv->list, &exynos_drm_subdrv_list);
-
-	return 0;
-}
-
-int exynos_drm_subdrv_unregister(struct exynos_drm_subdrv *subdrv)
-{
-	if (!subdrv)
-		return -EINVAL;
-
-	list_del(&subdrv->list);
-
-	return 0;
-}
-
-int exynos_drm_device_subdrv_probe(struct drm_device *dev)
-{
-	struct exynos_drm_subdrv *subdrv, *n;
-	int err;
-
-	if (!dev)
-		return -EINVAL;
-
-	list_for_each_entry_safe(subdrv, n, &exynos_drm_subdrv_list, list) {
-		if (subdrv->probe) {
-			subdrv->drm_dev = dev;
-
-			/*
-			 * this probe callback would be called by sub driver
-			 * after setting of all resources to this sub driver,
-			 * such as clock, irq and register map are done.
-			 */
-			err = subdrv->probe(dev, subdrv->dev);
-			if (err) {
-				DRM_DEBUG("exynos drm subdrv probe failed.\n");
-				list_del(&subdrv->list);
-				continue;
-			}
-		}
-	}
-
-	return 0;
-}
-
-int exynos_drm_device_subdrv_remove(struct drm_device *dev)
-{
-	struct exynos_drm_subdrv *subdrv;
-
-	if (!dev) {
-		WARN(1, "Unexpected drm device unregister!\n");
-		return -EINVAL;
-	}
-
-	list_for_each_entry(subdrv, &exynos_drm_subdrv_list, list) {
-		if (subdrv->remove)
-			subdrv->remove(dev, subdrv->dev);
-	}
-
-	return 0;
-}
-
-int exynos_drm_subdrv_open(struct drm_device *dev, struct drm_file *file)
-{
-	struct exynos_drm_subdrv *subdrv;
-	int ret;
-
-	list_for_each_entry(subdrv, &exynos_drm_subdrv_list, list) {
-		if (subdrv->open) {
-			ret = subdrv->open(dev, subdrv->dev, file);
-			if (ret)
-				goto err;
-		}
-	}
-
-	return 0;
-
-err:
-	list_for_each_entry_continue_reverse(subdrv, &exynos_drm_subdrv_list, list) {
-		if (subdrv->close)
-			subdrv->close(dev, subdrv->dev, file);
-	}
-	return ret;
-}
-
-void exynos_drm_subdrv_close(struct drm_device *dev, struct drm_file *file)
-{
-	struct exynos_drm_subdrv *subdrv;
-
-	list_for_each_entry(subdrv, &exynos_drm_subdrv_list, list) {
-		if (subdrv->close)
-			subdrv->close(dev, subdrv->dev, file);
-	}
-}
diff --git a/drivers/gpu/drm/exynos/exynos_drm_drv.c b/drivers/gpu/drm/exynos/exynos_drm_drv.c
index a81b4a5e24a7..b599f74692e5 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_drv.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_drv.c
@@ -55,8 +55,7 @@ static int exynos_drm_open(struct drm_device *dev, struct drm_file *file)
 		return -ENOMEM;
 
 	file->driver_priv = file_priv;
-
-	ret = exynos_drm_subdrv_open(dev, file);
+	ret = g2d_open(dev, file);
 	if (ret)
 		goto err_file_priv_free;
 
@@ -70,7 +69,7 @@ err_file_priv_free:
 
 static void exynos_drm_postclose(struct drm_device *dev, struct drm_file *file)
 {
-	exynos_drm_subdrv_close(dev, file);
+	g2d_close(dev, file);
 	kfree(file->driver_priv);
 	file->driver_priv = NULL;
 }
@@ -147,13 +146,12 @@ static struct drm_driver exynos_drm_driver = {
 	.minor	= DRIVER_MINOR,
 };
 
-#ifdef CONFIG_PM_SLEEP
 static int exynos_drm_suspend(struct device *dev)
 {
 	struct drm_device *drm_dev = dev_get_drvdata(dev);
 	struct exynos_drm_private *private;
 
-	if (pm_runtime_suspended(dev) || !drm_dev)
+	if (!drm_dev)
 		return 0;
 
 	private = drm_dev->dev_private;
@@ -170,25 +168,23 @@ static int exynos_drm_suspend(struct device *dev)
 	return 0;
 }
 
-static int exynos_drm_resume(struct device *dev)
+static void exynos_drm_resume(struct device *dev)
 {
 	struct drm_device *drm_dev = dev_get_drvdata(dev);
 	struct exynos_drm_private *private;
 
-	if (pm_runtime_suspended(dev) || !drm_dev)
-		return 0;
+	if (!drm_dev)
+		return;
 
 	private = drm_dev->dev_private;
 	drm_atomic_helper_resume(drm_dev, private->suspend_state);
 	exynos_drm_fbdev_resume(drm_dev);
 	drm_kms_helper_poll_enable(drm_dev);
-
-	return 0;
 }
-#endif
 
 static const struct dev_pm_ops exynos_drm_pm_ops = {
-	SET_SYSTEM_SLEEP_PM_OPS(exynos_drm_suspend, exynos_drm_resume)
+	.prepare = exynos_drm_suspend,
+	.complete = exynos_drm_resume,
 };
 
 /* forward declaration */
@@ -240,6 +236,7 @@ static struct exynos_drm_driver_info exynos_drm_drivers[] = {
 		DRM_COMPONENT_DRIVER | DRM_VIRTUAL_DEVICE
 	}, {
 		DRV_PTR(g2d_driver, CONFIG_DRM_EXYNOS_G2D),
+		DRM_COMPONENT_DRIVER
 	}, {
 		DRV_PTR(fimc_driver, CONFIG_DRM_EXYNOS_FIMC),
 		DRM_COMPONENT_DRIVER | DRM_FIMC_DEVICE,
@@ -376,11 +373,6 @@ static int exynos_drm_bind(struct device *dev)
 	if (ret)
 		goto err_unbind_all;
 
-	/* Probe non kms sub drivers and virtual display driver. */
-	ret = exynos_drm_device_subdrv_probe(drm);
-	if (ret)
-		goto err_unbind_all;
-
 	drm_mode_config_reset(drm);
 
 	/*
@@ -411,7 +403,6 @@ err_cleanup_fbdev:
 	exynos_drm_fbdev_fini(drm);
 err_cleanup_poll:
 	drm_kms_helper_poll_fini(drm);
-	exynos_drm_device_subdrv_remove(drm);
 err_unbind_all:
 	component_unbind_all(drm->dev, drm);
 err_mode_config_cleanup:
@@ -420,7 +411,7 @@ err_mode_config_cleanup:
 err_free_private:
 	kfree(private);
 err_free_drm:
-	drm_dev_unref(drm);
+	drm_dev_put(drm);
 
 	return ret;
 }
@@ -431,8 +422,6 @@ static void exynos_drm_unbind(struct device *dev)
 
 	drm_dev_unregister(drm);
 
-	exynos_drm_device_subdrv_remove(drm);
-
 	exynos_drm_fbdev_fini(drm);
 	drm_kms_helper_poll_fini(drm);
 
@@ -444,7 +433,7 @@ static void exynos_drm_unbind(struct device *dev)
 	drm->dev_private = NULL;
 	dev_set_drvdata(dev, NULL);
 
-	drm_dev_unref(drm);
+	drm_dev_put(drm);
 }
 
 static const struct component_master_ops exynos_drm_ops = {
diff --git a/drivers/gpu/drm/exynos/exynos_drm_drv.h b/drivers/gpu/drm/exynos/exynos_drm_drv.h
index 0f6d079a55c9..c737c4bd2c19 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_drv.h
+++ b/drivers/gpu/drm/exynos/exynos_drm_drv.h
@@ -179,17 +179,13 @@ static inline void exynos_drm_pipe_clk_enable(struct exynos_drm_crtc *crtc,
 		crtc->pipe_clk->enable(crtc->pipe_clk, enable);
 }
 
-struct exynos_drm_g2d_private {
-	struct device		*dev;
+struct drm_exynos_file_private {
+	/* for g2d api */
 	struct list_head	inuse_cmdlist;
 	struct list_head	event_list;
 	struct list_head	userptr_list;
 };
 
-struct drm_exynos_file_private {
-	struct exynos_drm_g2d_private	*g2d_priv;
-};
-
 /*
  * Exynos drm private structure.
  *
@@ -201,6 +197,7 @@ struct exynos_drm_private {
 	struct drm_fb_helper *fb_helper;
 	struct drm_atomic_state *suspend_state;
 
+	struct device *g2d_dev;
 	struct device *dma_dev;
 	void *mapping;
 
@@ -217,44 +214,6 @@ static inline struct device *to_dma_dev(struct drm_device *dev)
 	return priv->dma_dev;
 }
 
-/*
- * Exynos drm sub driver structure.
- *
- * @list: sub driver has its own list object to register to exynos drm driver.
- * @dev: pointer to device object for subdrv device driver.
- * @drm_dev: pointer to drm_device and this pointer would be set
- *	when sub driver calls exynos_drm_subdrv_register().
- * @probe: this callback would be called by exynos drm driver after
- *     subdrv is registered to it.
- * @remove: this callback is used to release resources created
- *     by probe callback.
- * @open: this would be called with drm device file open.
- * @close: this would be called with drm device file close.
- */
-struct exynos_drm_subdrv {
-	struct list_head list;
-	struct device *dev;
-	struct drm_device *drm_dev;
-
-	int (*probe)(struct drm_device *drm_dev, struct device *dev);
-	void (*remove)(struct drm_device *drm_dev, struct device *dev);
-	int (*open)(struct drm_device *drm_dev, struct device *dev,
-			struct drm_file *file);
-	void (*close)(struct drm_device *drm_dev, struct device *dev,
-			struct drm_file *file);
-};
-
- /* This function would be called by non kms drivers such as g2d and ipp. */
-int exynos_drm_subdrv_register(struct exynos_drm_subdrv *drm_subdrv);
-
-/* this function removes subdrv list from exynos drm driver */
-int exynos_drm_subdrv_unregister(struct exynos_drm_subdrv *drm_subdrv);
-
-int exynos_drm_device_subdrv_probe(struct drm_device *dev);
-int exynos_drm_device_subdrv_remove(struct drm_device *dev);
-int exynos_drm_subdrv_open(struct drm_device *dev, struct drm_file *file);
-void exynos_drm_subdrv_close(struct drm_device *dev, struct drm_file *file);
-
 #ifdef CONFIG_DRM_EXYNOS_DPI
 struct drm_encoder *exynos_dpi_probe(struct device *dev);
 int exynos_dpi_remove(struct drm_encoder *encoder);
diff --git a/drivers/gpu/drm/exynos/exynos_drm_dsi.c b/drivers/gpu/drm/exynos/exynos_drm_dsi.c
index a1ed6146a3b5..781b82c2c579 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_dsi.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_dsi.c
@@ -1863,6 +1863,8 @@ err_clk:
 
 static const struct dev_pm_ops exynos_dsi_pm_ops = {
 	SET_RUNTIME_PM_OPS(exynos_dsi_suspend, exynos_dsi_resume, NULL)
+	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
+				pm_runtime_force_resume)
 };
 
 struct platform_driver dsi_driver = {
diff --git a/drivers/gpu/drm/exynos/exynos_drm_fb.c b/drivers/gpu/drm/exynos/exynos_drm_fb.c
index 7fcc1a7ab1a0..9f52382e19ee 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_fb.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_fb.c
@@ -101,7 +101,6 @@ exynos_user_fb_create(struct drm_device *dev, struct drm_file *file_priv,
 {
 	const struct drm_format_info *info = drm_get_format_info(dev, mode_cmd);
 	struct exynos_drm_gem *exynos_gem[MAX_FB_BUFFER];
-	struct drm_gem_object *obj;
 	struct drm_framebuffer *fb;
 	int i;
 	int ret;
@@ -112,15 +111,14 @@ exynos_user_fb_create(struct drm_device *dev, struct drm_file *file_priv,
 		unsigned long size = height * mode_cmd->pitches[i] +
 				     mode_cmd->offsets[i];
 
-		obj = drm_gem_object_lookup(file_priv, mode_cmd->handles[i]);
-		if (!obj) {
+		exynos_gem[i] = exynos_drm_gem_get(file_priv,
+						   mode_cmd->handles[i]);
+		if (!exynos_gem[i]) {
 			DRM_ERROR("failed to lookup gem object\n");
 			ret = -ENOENT;
 			goto err;
 		}
 
-		exynos_gem[i] = to_exynos_gem(obj);
-
 		if (size > exynos_gem[i]->size) {
 			i++;
 			ret = -EINVAL;
@@ -138,7 +136,7 @@ exynos_user_fb_create(struct drm_device *dev, struct drm_file *file_priv,
 
 err:
 	while (i--)
-		drm_gem_object_unreference_unlocked(&exynos_gem[i]->base);
+		exynos_drm_gem_put(exynos_gem[i]);
 
 	return ERR_PTR(ret);
 }
diff --git a/drivers/gpu/drm/exynos/exynos_drm_fimc.c b/drivers/gpu/drm/exynos/exynos_drm_fimc.c
index 6127ef25acd6..e8d0670bb5f8 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_fimc.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_fimc.c
@@ -470,17 +470,18 @@ static void fimc_src_set_transf(struct fimc_context *ctx, unsigned int rotation)
 static void fimc_set_window(struct fimc_context *ctx,
 			    struct exynos_drm_ipp_buffer *buf)
 {
+	unsigned int real_width = buf->buf.pitch[0] / buf->format->cpp[0];
 	u32 cfg, h1, h2, v1, v2;
 
 	/* cropped image */
 	h1 = buf->rect.x;
-	h2 = buf->buf.width - buf->rect.w - buf->rect.x;
+	h2 = real_width - buf->rect.w - buf->rect.x;
 	v1 = buf->rect.y;
 	v2 = buf->buf.height - buf->rect.h - buf->rect.y;
 
 	DRM_DEBUG_KMS("x[%d]y[%d]w[%d]h[%d]hsize[%d]vsize[%d]\n",
 		buf->rect.x, buf->rect.y, buf->rect.w, buf->rect.h,
-		buf->buf.width, buf->buf.height);
+		real_width, buf->buf.height);
 	DRM_DEBUG_KMS("h1[%d]h2[%d]v1[%d]v2[%d]\n", h1, h2, v1, v2);
 
 	/*
@@ -503,12 +504,13 @@ static void fimc_set_window(struct fimc_context *ctx,
 static void fimc_src_set_size(struct fimc_context *ctx,
 			      struct exynos_drm_ipp_buffer *buf)
 {
+	unsigned int real_width = buf->buf.pitch[0] / buf->format->cpp[0];
 	u32 cfg;
 
-	DRM_DEBUG_KMS("hsize[%d]vsize[%d]\n", buf->buf.width, buf->buf.height);
+	DRM_DEBUG_KMS("hsize[%d]vsize[%d]\n", real_width, buf->buf.height);
 
 	/* original size */
-	cfg = (EXYNOS_ORGISIZE_HORIZONTAL(buf->buf.width) |
+	cfg = (EXYNOS_ORGISIZE_HORIZONTAL(real_width) |
 		EXYNOS_ORGISIZE_VERTICAL(buf->buf.height));
 
 	fimc_write(ctx, cfg, EXYNOS_ORGISIZE);
@@ -529,7 +531,7 @@ static void fimc_src_set_size(struct fimc_context *ctx,
 	 * for now, we support only ITU601 8 bit mode
 	 */
 	cfg = (EXYNOS_CISRCFMT_ITU601_8BIT |
-		EXYNOS_CISRCFMT_SOURCEHSIZE(buf->buf.width) |
+		EXYNOS_CISRCFMT_SOURCEHSIZE(real_width) |
 		EXYNOS_CISRCFMT_SOURCEVSIZE(buf->buf.height));
 	fimc_write(ctx, cfg, EXYNOS_CISRCFMT);
 
@@ -842,12 +844,13 @@ static void fimc_set_scaler(struct fimc_context *ctx, struct fimc_scaler *sc)
 static void fimc_dst_set_size(struct fimc_context *ctx,
 			     struct exynos_drm_ipp_buffer *buf)
 {
+	unsigned int real_width = buf->buf.pitch[0] / buf->format->cpp[0];
 	u32 cfg, cfg_ext;
 
-	DRM_DEBUG_KMS("hsize[%d]vsize[%d]\n", buf->buf.width, buf->buf.height);
+	DRM_DEBUG_KMS("hsize[%d]vsize[%d]\n", real_width, buf->buf.height);
 
 	/* original size */
-	cfg = (EXYNOS_ORGOSIZE_HORIZONTAL(buf->buf.width) |
+	cfg = (EXYNOS_ORGOSIZE_HORIZONTAL(real_width) |
 		EXYNOS_ORGOSIZE_VERTICAL(buf->buf.height));
 
 	fimc_write(ctx, cfg, EXYNOS_ORGOSIZE);
diff --git a/drivers/gpu/drm/exynos/exynos_drm_fimd.c b/drivers/gpu/drm/exynos/exynos_drm_fimd.c
index 01b1570d0c3a..b7f56935a46b 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_fimd.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_fimd.c
@@ -1192,6 +1192,8 @@ static int exynos_fimd_resume(struct device *dev)
 
 static const struct dev_pm_ops exynos_fimd_pm_ops = {
 	SET_RUNTIME_PM_OPS(exynos_fimd_suspend, exynos_fimd_resume, NULL)
+	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
+				pm_runtime_force_resume)
 };
 
 struct platform_driver fimd_driver = {
diff --git a/drivers/gpu/drm/exynos/exynos_drm_g2d.c b/drivers/gpu/drm/exynos/exynos_drm_g2d.c
index f68ef1b3a28c..f2481a2014bb 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_g2d.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_g2d.c
@@ -9,6 +9,7 @@
 
 #include <linux/kernel.h>
 #include <linux/clk.h>
+#include <linux/component.h>
 #include <linux/err.h>
 #include <linux/interrupt.h>
 #include <linux/io.h>
@@ -190,7 +191,7 @@ struct g2d_buf_desc {
 struct g2d_buf_info {
 	unsigned int		map_nr;
 	enum g2d_reg_type	reg_types[MAX_REG_TYPE_NR];
-	unsigned long		handles[MAX_REG_TYPE_NR];
+	void			*obj[MAX_REG_TYPE_NR];
 	unsigned int		types[MAX_REG_TYPE_NR];
 	struct g2d_buf_desc	descs[MAX_REG_TYPE_NR];
 };
@@ -237,7 +238,7 @@ struct g2d_data {
 	int				irq;
 	struct workqueue_struct		*g2d_workq;
 	struct work_struct		runqueue_work;
-	struct exynos_drm_subdrv	subdrv;
+	struct drm_device		*drm_dev;
 	unsigned long			flags;
 
 	/* cmdlist */
@@ -268,14 +269,13 @@ static int g2d_init_cmdlist(struct g2d_data *g2d)
 {
 	struct device *dev = g2d->dev;
 	struct g2d_cmdlist_node *node = g2d->cmdlist_node;
-	struct exynos_drm_subdrv *subdrv = &g2d->subdrv;
 	int nr;
 	int ret;
 	struct g2d_buf_info *buf_info;
 
 	g2d->cmdlist_dma_attrs = DMA_ATTR_WRITE_COMBINE;
 
-	g2d->cmdlist_pool_virt = dma_alloc_attrs(to_dma_dev(subdrv->drm_dev),
+	g2d->cmdlist_pool_virt = dma_alloc_attrs(to_dma_dev(g2d->drm_dev),
 						G2D_CMDLIST_POOL_SIZE,
 						&g2d->cmdlist_pool, GFP_KERNEL,
 						g2d->cmdlist_dma_attrs);
@@ -308,7 +308,7 @@ static int g2d_init_cmdlist(struct g2d_data *g2d)
 	return 0;
 
 err:
-	dma_free_attrs(to_dma_dev(subdrv->drm_dev), G2D_CMDLIST_POOL_SIZE,
+	dma_free_attrs(to_dma_dev(g2d->drm_dev), G2D_CMDLIST_POOL_SIZE,
 			g2d->cmdlist_pool_virt,
 			g2d->cmdlist_pool, g2d->cmdlist_dma_attrs);
 	return ret;
@@ -316,12 +316,10 @@ err:
 
 static void g2d_fini_cmdlist(struct g2d_data *g2d)
 {
-	struct exynos_drm_subdrv *subdrv = &g2d->subdrv;
-
 	kfree(g2d->cmdlist_node);
 
 	if (g2d->cmdlist_pool_virt && g2d->cmdlist_pool) {
-		dma_free_attrs(to_dma_dev(subdrv->drm_dev),
+		dma_free_attrs(to_dma_dev(g2d->drm_dev),
 				G2D_CMDLIST_POOL_SIZE,
 				g2d->cmdlist_pool_virt,
 				g2d->cmdlist_pool, g2d->cmdlist_dma_attrs);
@@ -355,32 +353,31 @@ static void g2d_put_cmdlist(struct g2d_data *g2d, struct g2d_cmdlist_node *node)
 	mutex_unlock(&g2d->cmdlist_mutex);
 }
 
-static void g2d_add_cmdlist_to_inuse(struct exynos_drm_g2d_private *g2d_priv,
+static void g2d_add_cmdlist_to_inuse(struct drm_exynos_file_private *file_priv,
 				     struct g2d_cmdlist_node *node)
 {
 	struct g2d_cmdlist_node *lnode;
 
-	if (list_empty(&g2d_priv->inuse_cmdlist))
+	if (list_empty(&file_priv->inuse_cmdlist))
 		goto add_to_list;
 
 	/* this links to base address of new cmdlist */
-	lnode = list_entry(g2d_priv->inuse_cmdlist.prev,
+	lnode = list_entry(file_priv->inuse_cmdlist.prev,
 				struct g2d_cmdlist_node, list);
 	lnode->cmdlist->data[lnode->cmdlist->last] = node->dma_addr;
 
 add_to_list:
-	list_add_tail(&node->list, &g2d_priv->inuse_cmdlist);
+	list_add_tail(&node->list, &file_priv->inuse_cmdlist);
 
 	if (node->event)
-		list_add_tail(&node->event->base.link, &g2d_priv->event_list);
+		list_add_tail(&node->event->base.link, &file_priv->event_list);
 }
 
-static void g2d_userptr_put_dma_addr(struct drm_device *drm_dev,
-					unsigned long obj,
+static void g2d_userptr_put_dma_addr(struct g2d_data *g2d,
+					void *obj,
 					bool force)
 {
-	struct g2d_cmdlist_userptr *g2d_userptr =
-					(struct g2d_cmdlist_userptr *)obj;
+	struct g2d_cmdlist_userptr *g2d_userptr = obj;
 	struct page **pages;
 
 	if (!obj)
@@ -398,7 +395,7 @@ static void g2d_userptr_put_dma_addr(struct drm_device *drm_dev,
 		return;
 
 out:
-	dma_unmap_sg(to_dma_dev(drm_dev), g2d_userptr->sgt->sgl,
+	dma_unmap_sg(to_dma_dev(g2d->drm_dev), g2d_userptr->sgt->sgl,
 			g2d_userptr->sgt->nents, DMA_BIDIRECTIONAL);
 
 	pages = frame_vector_pages(g2d_userptr->vec);
@@ -419,16 +416,14 @@ out:
 	kfree(g2d_userptr);
 }
 
-static dma_addr_t *g2d_userptr_get_dma_addr(struct drm_device *drm_dev,
+static dma_addr_t *g2d_userptr_get_dma_addr(struct g2d_data *g2d,
 					unsigned long userptr,
 					unsigned long size,
 					struct drm_file *filp,
-					unsigned long *obj)
+					void **obj)
 {
 	struct drm_exynos_file_private *file_priv = filp->driver_priv;
-	struct exynos_drm_g2d_private *g2d_priv = file_priv->g2d_priv;
 	struct g2d_cmdlist_userptr *g2d_userptr;
-	struct g2d_data *g2d;
 	struct sg_table	*sgt;
 	unsigned long start, end;
 	unsigned int npages, offset;
@@ -439,10 +434,8 @@ static dma_addr_t *g2d_userptr_get_dma_addr(struct drm_device *drm_dev,
 		return ERR_PTR(-EINVAL);
 	}
 
-	g2d = dev_get_drvdata(g2d_priv->dev);
-
 	/* check if userptr already exists in userptr_list. */
-	list_for_each_entry(g2d_userptr, &g2d_priv->userptr_list, list) {
+	list_for_each_entry(g2d_userptr, &file_priv->userptr_list, list) {
 		if (g2d_userptr->userptr == userptr) {
 			/*
 			 * also check size because there could be same address
@@ -450,7 +443,7 @@ static dma_addr_t *g2d_userptr_get_dma_addr(struct drm_device *drm_dev,
 			 */
 			if (g2d_userptr->size == size) {
 				atomic_inc(&g2d_userptr->refcount);
-				*obj = (unsigned long)g2d_userptr;
+				*obj = g2d_userptr;
 
 				return &g2d_userptr->dma_addr;
 			}
@@ -517,7 +510,7 @@ static dma_addr_t *g2d_userptr_get_dma_addr(struct drm_device *drm_dev,
 
 	g2d_userptr->sgt = sgt;
 
-	if (!dma_map_sg(to_dma_dev(drm_dev), sgt->sgl, sgt->nents,
+	if (!dma_map_sg(to_dma_dev(g2d->drm_dev), sgt->sgl, sgt->nents,
 				DMA_BIDIRECTIONAL)) {
 		DRM_ERROR("failed to map sgt with dma region.\n");
 		ret = -ENOMEM;
@@ -527,14 +520,14 @@ static dma_addr_t *g2d_userptr_get_dma_addr(struct drm_device *drm_dev,
 	g2d_userptr->dma_addr = sgt->sgl[0].dma_address;
 	g2d_userptr->userptr = userptr;
 
-	list_add_tail(&g2d_userptr->list, &g2d_priv->userptr_list);
+	list_add_tail(&g2d_userptr->list, &file_priv->userptr_list);
 
 	if (g2d->current_pool + (npages << PAGE_SHIFT) < g2d->max_pool) {
 		g2d->current_pool += npages << PAGE_SHIFT;
 		g2d_userptr->in_pool = true;
 	}
 
-	*obj = (unsigned long)g2d_userptr;
+	*obj = g2d_userptr;
 
 	return &g2d_userptr->dma_addr;
 
@@ -556,19 +549,14 @@ err_free:
 	return ERR_PTR(ret);
 }
 
-static void g2d_userptr_free_all(struct drm_device *drm_dev,
-					struct g2d_data *g2d,
-					struct drm_file *filp)
+static void g2d_userptr_free_all(struct g2d_data *g2d, struct drm_file *filp)
 {
 	struct drm_exynos_file_private *file_priv = filp->driver_priv;
-	struct exynos_drm_g2d_private *g2d_priv = file_priv->g2d_priv;
 	struct g2d_cmdlist_userptr *g2d_userptr, *n;
 
-	list_for_each_entry_safe(g2d_userptr, n, &g2d_priv->userptr_list, list)
+	list_for_each_entry_safe(g2d_userptr, n, &file_priv->userptr_list, list)
 		if (g2d_userptr->in_pool)
-			g2d_userptr_put_dma_addr(drm_dev,
-						(unsigned long)g2d_userptr,
-						true);
+			g2d_userptr_put_dma_addr(g2d, g2d_userptr, true);
 
 	g2d->current_pool = 0;
 }
@@ -723,26 +711,23 @@ static int g2d_map_cmdlist_gem(struct g2d_data *g2d,
 		buf_desc = &buf_info->descs[reg_type];
 
 		if (buf_info->types[reg_type] == BUF_TYPE_GEM) {
-			unsigned long size;
+			struct exynos_drm_gem *exynos_gem;
 
-			size = exynos_drm_gem_get_size(drm_dev, handle, file);
-			if (!size) {
+			exynos_gem = exynos_drm_gem_get(file, handle);
+			if (!exynos_gem) {
 				ret = -EFAULT;
 				goto err;
 			}
 
-			if (!g2d_check_buf_desc_is_valid(buf_desc, reg_type,
-									size)) {
+			if (!g2d_check_buf_desc_is_valid(buf_desc,
+							 reg_type, exynos_gem->size)) {
+				exynos_drm_gem_put(exynos_gem);
 				ret = -EFAULT;
 				goto err;
 			}
 
-			addr = exynos_drm_gem_get_dma_addr(drm_dev, handle,
-								file);
-			if (IS_ERR(addr)) {
-				ret = -EFAULT;
-				goto err;
-			}
+			addr = &exynos_gem->dma_addr;
+			buf_info->obj[reg_type] = exynos_gem;
 		} else {
 			struct drm_exynos_g2d_userptr g2d_userptr;
 
@@ -758,11 +743,11 @@ static int g2d_map_cmdlist_gem(struct g2d_data *g2d,
 				goto err;
 			}
 
-			addr = g2d_userptr_get_dma_addr(drm_dev,
+			addr = g2d_userptr_get_dma_addr(g2d,
 							g2d_userptr.userptr,
 							g2d_userptr.size,
 							file,
-							&handle);
+							&buf_info->obj[reg_type]);
 			if (IS_ERR(addr)) {
 				ret = -EFAULT;
 				goto err;
@@ -771,7 +756,6 @@ static int g2d_map_cmdlist_gem(struct g2d_data *g2d,
 
 		cmdlist->data[reg_pos + 1] = *addr;
 		buf_info->reg_types[i] = reg_type;
-		buf_info->handles[reg_type] = handle;
 	}
 
 	return 0;
@@ -785,29 +769,26 @@ static void g2d_unmap_cmdlist_gem(struct g2d_data *g2d,
 				  struct g2d_cmdlist_node *node,
 				  struct drm_file *filp)
 {
-	struct exynos_drm_subdrv *subdrv = &g2d->subdrv;
 	struct g2d_buf_info *buf_info = &node->buf_info;
 	int i;
 
 	for (i = 0; i < buf_info->map_nr; i++) {
 		struct g2d_buf_desc *buf_desc;
 		enum g2d_reg_type reg_type;
-		unsigned long handle;
+		void *obj;
 
 		reg_type = buf_info->reg_types[i];
 
 		buf_desc = &buf_info->descs[reg_type];
-		handle = buf_info->handles[reg_type];
+		obj = buf_info->obj[reg_type];
 
 		if (buf_info->types[reg_type] == BUF_TYPE_GEM)
-			exynos_drm_gem_put_dma_addr(subdrv->drm_dev, handle,
-							filp);
+			exynos_drm_gem_put(obj);
 		else
-			g2d_userptr_put_dma_addr(subdrv->drm_dev, handle,
-							false);
+			g2d_userptr_put_dma_addr(g2d, obj, false);
 
 		buf_info->reg_types[i] = REG_TYPE_NONE;
-		buf_info->handles[reg_type] = 0;
+		buf_info->obj[reg_type] = NULL;
 		buf_info->types[reg_type] = 0;
 		memset(buf_desc, 0x00, sizeof(*buf_desc));
 	}
@@ -922,7 +903,7 @@ static void g2d_runqueue_worker(struct work_struct *work)
 
 static void g2d_finish_event(struct g2d_data *g2d, u32 cmdlist_no)
 {
-	struct drm_device *drm_dev = g2d->subdrv.drm_dev;
+	struct drm_device *drm_dev = g2d->drm_dev;
 	struct g2d_runqueue_node *runqueue_node = g2d->runqueue_node;
 	struct drm_exynos_pending_g2d_event *e;
 	struct timespec64 now;
@@ -1031,7 +1012,7 @@ out:
 	mutex_unlock(&g2d->runqueue_mutex);
 }
 
-static int g2d_check_reg_offset(struct device *dev,
+static int g2d_check_reg_offset(struct g2d_data *g2d,
 				struct g2d_cmdlist_node *node,
 				int nr, bool for_addr)
 {
@@ -1131,7 +1112,7 @@ static int g2d_check_reg_offset(struct device *dev,
 	return 0;
 
 err:
-	dev_err(dev, "Bad register offset: 0x%lx\n", cmdlist->data[index]);
+	dev_err(g2d->dev, "Bad register offset: 0x%lx\n", cmdlist->data[index]);
 	return -EINVAL;
 }
 
@@ -1139,23 +1120,8 @@ err:
 int exynos_g2d_get_ver_ioctl(struct drm_device *drm_dev, void *data,
 			     struct drm_file *file)
 {
-	struct drm_exynos_file_private *file_priv = file->driver_priv;
-	struct exynos_drm_g2d_private *g2d_priv = file_priv->g2d_priv;
-	struct device *dev;
-	struct g2d_data *g2d;
 	struct drm_exynos_g2d_get_ver *ver = data;
 
-	if (!g2d_priv)
-		return -ENODEV;
-
-	dev = g2d_priv->dev;
-	if (!dev)
-		return -ENODEV;
-
-	g2d = dev_get_drvdata(dev);
-	if (!g2d)
-		return -EFAULT;
-
 	ver->major = G2D_HW_MAJOR_VER;
 	ver->minor = G2D_HW_MINOR_VER;
 
@@ -1166,9 +1132,8 @@ int exynos_g2d_set_cmdlist_ioctl(struct drm_device *drm_dev, void *data,
 				 struct drm_file *file)
 {
 	struct drm_exynos_file_private *file_priv = file->driver_priv;
-	struct exynos_drm_g2d_private *g2d_priv = file_priv->g2d_priv;
-	struct device *dev;
-	struct g2d_data *g2d;
+	struct exynos_drm_private *priv = drm_dev->dev_private;
+	struct g2d_data *g2d = dev_get_drvdata(priv->g2d_dev);
 	struct drm_exynos_g2d_set_cmdlist *req = data;
 	struct drm_exynos_g2d_cmd *cmd;
 	struct drm_exynos_pending_g2d_event *e;
@@ -1177,17 +1142,6 @@ int exynos_g2d_set_cmdlist_ioctl(struct drm_device *drm_dev, void *data,
 	int size;
 	int ret;
 
-	if (!g2d_priv)
-		return -ENODEV;
-
-	dev = g2d_priv->dev;
-	if (!dev)
-		return -ENODEV;
-
-	g2d = dev_get_drvdata(dev);
-	if (!g2d)
-		return -EFAULT;
-
 	node = g2d_get_cmdlist(g2d);
 	if (!node)
 		return -ENOMEM;
@@ -1199,7 +1153,7 @@ int exynos_g2d_set_cmdlist_ioctl(struct drm_device *drm_dev, void *data,
 	 */
 	if (req->cmd_nr > G2D_CMDLIST_DATA_NUM ||
 	    req->cmd_buf_nr > G2D_CMDLIST_DATA_NUM) {
-		dev_err(dev, "number of submitted G2D commands exceeds limit\n");
+		dev_err(g2d->dev, "number of submitted G2D commands exceeds limit\n");
 		return -EINVAL;
 	}
 
@@ -1267,7 +1221,7 @@ int exynos_g2d_set_cmdlist_ioctl(struct drm_device *drm_dev, void *data,
 	 */
 	size = cmdlist->last + req->cmd_nr * 2 + req->cmd_buf_nr * 2 + 2;
 	if (size > G2D_CMDLIST_DATA_NUM) {
-		dev_err(dev, "cmdlist size is too big\n");
+		dev_err(g2d->dev, "cmdlist size is too big\n");
 		ret = -EINVAL;
 		goto err_free_event;
 	}
@@ -1282,7 +1236,7 @@ int exynos_g2d_set_cmdlist_ioctl(struct drm_device *drm_dev, void *data,
 	}
 	cmdlist->last += req->cmd_nr * 2;
 
-	ret = g2d_check_reg_offset(dev, node, req->cmd_nr, false);
+	ret = g2d_check_reg_offset(g2d, node, req->cmd_nr, false);
 	if (ret < 0)
 		goto err_free_event;
 
@@ -1301,7 +1255,7 @@ int exynos_g2d_set_cmdlist_ioctl(struct drm_device *drm_dev, void *data,
 		}
 		cmdlist->last += req->cmd_buf_nr * 2;
 
-		ret = g2d_check_reg_offset(dev, node, req->cmd_buf_nr, true);
+		ret = g2d_check_reg_offset(g2d, node, req->cmd_buf_nr, true);
 		if (ret < 0)
 			goto err_free_event;
 
@@ -1319,7 +1273,7 @@ int exynos_g2d_set_cmdlist_ioctl(struct drm_device *drm_dev, void *data,
 	/* tail */
 	cmdlist->data[cmdlist->last] = 0;
 
-	g2d_add_cmdlist_to_inuse(g2d_priv, node);
+	g2d_add_cmdlist_to_inuse(file_priv, node);
 
 	return 0;
 
@@ -1337,25 +1291,13 @@ int exynos_g2d_exec_ioctl(struct drm_device *drm_dev, void *data,
 			  struct drm_file *file)
 {
 	struct drm_exynos_file_private *file_priv = file->driver_priv;
-	struct exynos_drm_g2d_private *g2d_priv = file_priv->g2d_priv;
-	struct device *dev;
-	struct g2d_data *g2d;
+	struct exynos_drm_private *priv = drm_dev->dev_private;
+	struct g2d_data *g2d = dev_get_drvdata(priv->g2d_dev);
 	struct drm_exynos_g2d_exec *req = data;
 	struct g2d_runqueue_node *runqueue_node;
 	struct list_head *run_cmdlist;
 	struct list_head *event_list;
 
-	if (!g2d_priv)
-		return -ENODEV;
-
-	dev = g2d_priv->dev;
-	if (!dev)
-		return -ENODEV;
-
-	g2d = dev_get_drvdata(dev);
-	if (!g2d)
-		return -EFAULT;
-
 	runqueue_node = kmem_cache_alloc(g2d->runqueue_slab, GFP_KERNEL);
 	if (!runqueue_node)
 		return -ENOMEM;
@@ -1367,11 +1309,11 @@ int exynos_g2d_exec_ioctl(struct drm_device *drm_dev, void *data,
 	init_completion(&runqueue_node->complete);
 	runqueue_node->async = req->async;
 
-	list_splice_init(&g2d_priv->inuse_cmdlist, run_cmdlist);
-	list_splice_init(&g2d_priv->event_list, event_list);
+	list_splice_init(&file_priv->inuse_cmdlist, run_cmdlist);
+	list_splice_init(&file_priv->event_list, event_list);
 
 	if (list_empty(run_cmdlist)) {
-		dev_err(dev, "there is no inuse cmdlist\n");
+		dev_err(g2d->dev, "there is no inuse cmdlist\n");
 		kmem_cache_free(g2d->runqueue_slab, runqueue_node);
 		return -EPERM;
 	}
@@ -1395,71 +1337,28 @@ out:
 	return 0;
 }
 
-static int g2d_subdrv_probe(struct drm_device *drm_dev, struct device *dev)
-{
-	struct g2d_data *g2d;
-	int ret;
-
-	g2d = dev_get_drvdata(dev);
-	if (!g2d)
-		return -EFAULT;
-
-	/* allocate dma-aware cmdlist buffer. */
-	ret = g2d_init_cmdlist(g2d);
-	if (ret < 0) {
-		dev_err(dev, "cmdlist init failed\n");
-		return ret;
-	}
-
-	ret = drm_iommu_attach_device(drm_dev, dev);
-	if (ret < 0) {
-		dev_err(dev, "failed to enable iommu.\n");
-		g2d_fini_cmdlist(g2d);
-	}
-
-	return ret;
-
-}
-
-static void g2d_subdrv_remove(struct drm_device *drm_dev, struct device *dev)
-{
-	drm_iommu_detach_device(drm_dev, dev);
-}
-
-static int g2d_open(struct drm_device *drm_dev, struct device *dev,
-			struct drm_file *file)
+int g2d_open(struct drm_device *drm_dev, struct drm_file *file)
 {
 	struct drm_exynos_file_private *file_priv = file->driver_priv;
-	struct exynos_drm_g2d_private *g2d_priv;
-
-	g2d_priv = kzalloc(sizeof(*g2d_priv), GFP_KERNEL);
-	if (!g2d_priv)
-		return -ENOMEM;
 
-	g2d_priv->dev = dev;
-	file_priv->g2d_priv = g2d_priv;
-
-	INIT_LIST_HEAD(&g2d_priv->inuse_cmdlist);
-	INIT_LIST_HEAD(&g2d_priv->event_list);
-	INIT_LIST_HEAD(&g2d_priv->userptr_list);
+	INIT_LIST_HEAD(&file_priv->inuse_cmdlist);
+	INIT_LIST_HEAD(&file_priv->event_list);
+	INIT_LIST_HEAD(&file_priv->userptr_list);
 
 	return 0;
 }
 
-static void g2d_close(struct drm_device *drm_dev, struct device *dev,
-			struct drm_file *file)
+void g2d_close(struct drm_device *drm_dev, struct drm_file *file)
 {
 	struct drm_exynos_file_private *file_priv = file->driver_priv;
-	struct exynos_drm_g2d_private *g2d_priv = file_priv->g2d_priv;
+	struct exynos_drm_private *priv = drm_dev->dev_private;
 	struct g2d_data *g2d;
 	struct g2d_cmdlist_node *node, *n;
 
-	if (!dev)
+	if (!priv->g2d_dev)
 		return;
 
-	g2d = dev_get_drvdata(dev);
-	if (!g2d)
-		return;
+	g2d = dev_get_drvdata(priv->g2d_dev);
 
 	/* Remove the runqueue nodes that belong to us. */
 	mutex_lock(&g2d->runqueue_mutex);
@@ -1480,24 +1379,70 @@ static void g2d_close(struct drm_device *drm_dev, struct device *dev,
 	 * Properly unmap these buffers here.
 	 */
 	mutex_lock(&g2d->cmdlist_mutex);
-	list_for_each_entry_safe(node, n, &g2d_priv->inuse_cmdlist, list) {
+	list_for_each_entry_safe(node, n, &file_priv->inuse_cmdlist, list) {
 		g2d_unmap_cmdlist_gem(g2d, node, file);
 		list_move_tail(&node->list, &g2d->free_cmdlist);
 	}
 	mutex_unlock(&g2d->cmdlist_mutex);
 
 	/* release all g2d_userptr in pool. */
-	g2d_userptr_free_all(drm_dev, g2d, file);
+	g2d_userptr_free_all(g2d, file);
+}
+
+static int g2d_bind(struct device *dev, struct device *master, void *data)
+{
+	struct g2d_data *g2d = dev_get_drvdata(dev);
+	struct drm_device *drm_dev = data;
+	struct exynos_drm_private *priv = drm_dev->dev_private;
+	int ret;
+
+	g2d->drm_dev = drm_dev;
 
-	kfree(file_priv->g2d_priv);
+	/* allocate dma-aware cmdlist buffer. */
+	ret = g2d_init_cmdlist(g2d);
+	if (ret < 0) {
+		dev_err(dev, "cmdlist init failed\n");
+		return ret;
+	}
+
+	ret = drm_iommu_attach_device(drm_dev, dev);
+	if (ret < 0) {
+		dev_err(dev, "failed to enable iommu.\n");
+		g2d_fini_cmdlist(g2d);
+		return ret;
+	}
+	priv->g2d_dev = dev;
+
+	dev_info(dev, "The Exynos G2D (ver %d.%d) successfully registered.\n",
+			G2D_HW_MAJOR_VER, G2D_HW_MINOR_VER);
+	return 0;
+}
+
+static void g2d_unbind(struct device *dev, struct device *master, void *data)
+{
+	struct g2d_data *g2d = dev_get_drvdata(dev);
+	struct drm_device *drm_dev = data;
+	struct exynos_drm_private *priv = drm_dev->dev_private;
+
+	/* Suspend operation and wait for engine idle. */
+	set_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags);
+	g2d_wait_finish(g2d, NULL);
+	priv->g2d_dev = NULL;
+
+	cancel_work_sync(&g2d->runqueue_work);
+	drm_iommu_detach_device(g2d->drm_dev, dev);
 }
 
+static const struct component_ops g2d_component_ops = {
+	.bind	= g2d_bind,
+	.unbind = g2d_unbind,
+};
+
 static int g2d_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct resource *res;
 	struct g2d_data *g2d;
-	struct exynos_drm_subdrv *subdrv;
 	int ret;
 
 	g2d = devm_kzalloc(dev, sizeof(*g2d), GFP_KERNEL);
@@ -1564,22 +1509,12 @@ static int g2d_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, g2d);
 
-	subdrv = &g2d->subdrv;
-	subdrv->dev = dev;
-	subdrv->probe = g2d_subdrv_probe;
-	subdrv->remove = g2d_subdrv_remove;
-	subdrv->open = g2d_open;
-	subdrv->close = g2d_close;
-
-	ret = exynos_drm_subdrv_register(subdrv);
+	ret = component_add(dev, &g2d_component_ops);
 	if (ret < 0) {
 		dev_err(dev, "failed to register drm g2d device\n");
 		goto err_put_clk;
 	}
 
-	dev_info(dev, "The Exynos G2D (ver %d.%d) successfully probed.\n",
-			G2D_HW_MAJOR_VER, G2D_HW_MINOR_VER);
-
 	return 0;
 
 err_put_clk:
@@ -1595,12 +1530,7 @@ static int g2d_remove(struct platform_device *pdev)
 {
 	struct g2d_data *g2d = platform_get_drvdata(pdev);
 
-	/* Suspend operation and wait for engine idle. */
-	set_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags);
-	g2d_wait_finish(g2d, NULL);
-
-	cancel_work_sync(&g2d->runqueue_work);
-	exynos_drm_subdrv_unregister(&g2d->subdrv);
+	component_del(&pdev->dev, &g2d_component_ops);
 
 	/* There should be no locking needed here. */
 	g2d_remove_runqueue_nodes(g2d, NULL);
diff --git a/drivers/gpu/drm/exynos/exynos_drm_g2d.h b/drivers/gpu/drm/exynos/exynos_drm_g2d.h
index 1a9c7ca8c15b..287b2ed8f178 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_g2d.h
+++ b/drivers/gpu/drm/exynos/exynos_drm_g2d.h
@@ -14,6 +14,9 @@ extern int exynos_g2d_set_cmdlist_ioctl(struct drm_device *dev, void *data,
 					struct drm_file *file_priv);
 extern int exynos_g2d_exec_ioctl(struct drm_device *dev, void *data,
 				 struct drm_file *file_priv);
+
+extern int g2d_open(struct drm_device *drm_dev, struct drm_file *file);
+extern void g2d_close(struct drm_device *drm_dev, struct drm_file *file);
 #else
 static inline int exynos_g2d_get_ver_ioctl(struct drm_device *dev, void *data,
 					   struct drm_file *file_priv)
@@ -33,4 +36,12 @@ static inline int exynos_g2d_exec_ioctl(struct drm_device *dev, void *data,
 {
 	return -ENODEV;
 }
+
+int g2d_open(struct drm_device *drm_dev, struct drm_file *file)
+{
+	return 0;
+}
+
+void g2d_close(struct drm_device *drm_dev, struct drm_file *file)
+{ }
 #endif
diff --git a/drivers/gpu/drm/exynos/exynos_drm_gem.c b/drivers/gpu/drm/exynos/exynos_drm_gem.c
index 6e1494fa71b4..34ace85feb68 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_gem.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_gem.c
@@ -143,7 +143,7 @@ static int exynos_drm_gem_handle_create(struct drm_gem_object *obj,
 	DRM_DEBUG_KMS("gem handle = 0x%x\n", *handle);
 
 	/* drop reference from allocate - handle holds it now. */
-	drm_gem_object_unreference_unlocked(obj);
+	drm_gem_object_put_unlocked(obj);
 
 	return 0;
 }
@@ -171,26 +171,6 @@ void exynos_drm_gem_destroy(struct exynos_drm_gem *exynos_gem)
 	kfree(exynos_gem);
 }
 
-unsigned long exynos_drm_gem_get_size(struct drm_device *dev,
-						unsigned int gem_handle,
-						struct drm_file *file_priv)
-{
-	struct exynos_drm_gem *exynos_gem;
-	struct drm_gem_object *obj;
-
-	obj = drm_gem_object_lookup(file_priv, gem_handle);
-	if (!obj) {
-		DRM_ERROR("failed to lookup gem object.\n");
-		return 0;
-	}
-
-	exynos_gem = to_exynos_gem(obj);
-
-	drm_gem_object_unreference_unlocked(obj);
-
-	return exynos_gem->size;
-}
-
 static struct exynos_drm_gem *exynos_drm_gem_init(struct drm_device *dev,
 						  unsigned long size)
 {
@@ -299,43 +279,15 @@ int exynos_drm_gem_map_ioctl(struct drm_device *dev, void *data,
 				       &args->offset);
 }
 
-dma_addr_t *exynos_drm_gem_get_dma_addr(struct drm_device *dev,
-					unsigned int gem_handle,
-					struct drm_file *filp)
-{
-	struct exynos_drm_gem *exynos_gem;
-	struct drm_gem_object *obj;
-
-	obj = drm_gem_object_lookup(filp, gem_handle);
-	if (!obj) {
-		DRM_ERROR("failed to lookup gem object.\n");
-		return ERR_PTR(-EINVAL);
-	}
-
-	exynos_gem = to_exynos_gem(obj);
-
-	return &exynos_gem->dma_addr;
-}
-
-void exynos_drm_gem_put_dma_addr(struct drm_device *dev,
-					unsigned int gem_handle,
-					struct drm_file *filp)
+struct exynos_drm_gem *exynos_drm_gem_get(struct drm_file *filp,
+					  unsigned int gem_handle)
 {
 	struct drm_gem_object *obj;
 
 	obj = drm_gem_object_lookup(filp, gem_handle);
-	if (!obj) {
-		DRM_ERROR("failed to lookup gem object.\n");
-		return;
-	}
-
-	drm_gem_object_unreference_unlocked(obj);
-
-	/*
-	 * decrease obj->refcount one more time because we has already
-	 * increased it at exynos_drm_gem_get_dma_addr().
-	 */
-	drm_gem_object_unreference_unlocked(obj);
+	if (!obj)
+		return NULL;
+	return to_exynos_gem(obj);
 }
 
 static int exynos_drm_gem_mmap_buffer(struct exynos_drm_gem *exynos_gem,
@@ -383,7 +335,7 @@ int exynos_drm_gem_get_ioctl(struct drm_device *dev, void *data,
 	args->flags = exynos_gem->flags;
 	args->size = exynos_gem->size;
 
-	drm_gem_object_unreference_unlocked(obj);
+	drm_gem_object_put_unlocked(obj);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/exynos/exynos_drm_gem.h b/drivers/gpu/drm/exynos/exynos_drm_gem.h
index 9057d7f1d6ed..d46a62c30812 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_gem.h
+++ b/drivers/gpu/drm/exynos/exynos_drm_gem.h
@@ -77,32 +77,26 @@ int exynos_drm_gem_map_ioctl(struct drm_device *dev, void *data,
 			     struct drm_file *file_priv);
 
 /*
- * get dma address from gem handle and this function could be used for
+ * get exynos drm object from gem handle, this function could be used for
  * other drivers such as 2d/3d acceleration drivers.
  * with this function call, gem object reference count would be increased.
  */
-dma_addr_t *exynos_drm_gem_get_dma_addr(struct drm_device *dev,
-					unsigned int gem_handle,
-					struct drm_file *filp);
+struct exynos_drm_gem *exynos_drm_gem_get(struct drm_file *filp,
+					  unsigned int gem_handle);
 
 /*
- * put dma address from gem handle and this function could be used for
- * other drivers such as 2d/3d acceleration drivers.
- * with this function call, gem object reference count would be decreased.
+ * put exynos drm object acquired from exynos_drm_gem_get(),
+ * gem object reference count would be decreased.
  */
-void exynos_drm_gem_put_dma_addr(struct drm_device *dev,
-					unsigned int gem_handle,
-					struct drm_file *filp);
+static inline void exynos_drm_gem_put(struct exynos_drm_gem *exynos_gem)
+{
+	drm_gem_object_put_unlocked(&exynos_gem->base);
+}
 
 /* get buffer information to memory region allocated by gem. */
 int exynos_drm_gem_get_ioctl(struct drm_device *dev, void *data,
 				      struct drm_file *file_priv);
 
-/* get buffer size to gem handle. */
-unsigned long exynos_drm_gem_get_size(struct drm_device *dev,
-						unsigned int gem_handle,
-						struct drm_file *file_priv);
-
 /* free gem object. */
 void exynos_drm_gem_free_object(struct drm_gem_object *obj);
 
diff --git a/drivers/gpu/drm/exynos/exynos_drm_gsc.c b/drivers/gpu/drm/exynos/exynos_drm_gsc.c
index 35ac66730563..7ba414b52faa 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_gsc.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_gsc.c
@@ -492,21 +492,25 @@ static void gsc_src_set_fmt(struct gsc_context *ctx, u32 fmt)
 			GSC_IN_CHROMA_ORDER_CRCB);
 		break;
 	case DRM_FORMAT_NV21:
+		cfg |= (GSC_IN_CHROMA_ORDER_CRCB | GSC_IN_YUV420_2P);
+		break;
 	case DRM_FORMAT_NV61:
-		cfg |= (GSC_IN_CHROMA_ORDER_CRCB |
-			GSC_IN_YUV420_2P);
+		cfg |= (GSC_IN_CHROMA_ORDER_CRCB | GSC_IN_YUV422_2P);
 		break;
 	case DRM_FORMAT_YUV422:
 		cfg |= GSC_IN_YUV422_3P;
 		break;
 	case DRM_FORMAT_YUV420:
+		cfg |= (GSC_IN_CHROMA_ORDER_CBCR | GSC_IN_YUV420_3P);
+		break;
 	case DRM_FORMAT_YVU420:
-		cfg |= GSC_IN_YUV420_3P;
+		cfg |= (GSC_IN_CHROMA_ORDER_CRCB | GSC_IN_YUV420_3P);
 		break;
 	case DRM_FORMAT_NV12:
+		cfg |= (GSC_IN_CHROMA_ORDER_CBCR | GSC_IN_YUV420_2P);
+		break;
 	case DRM_FORMAT_NV16:
-		cfg |= (GSC_IN_CHROMA_ORDER_CBCR |
-			GSC_IN_YUV420_2P);
+		cfg |= (GSC_IN_CHROMA_ORDER_CBCR | GSC_IN_YUV422_2P);
 		break;
 	}
 
@@ -523,30 +527,30 @@ static void gsc_src_set_transf(struct gsc_context *ctx, unsigned int rotation)
 
 	switch (degree) {
 	case DRM_MODE_ROTATE_0:
-		if (rotation & DRM_MODE_REFLECT_Y)
-			cfg |= GSC_IN_ROT_XFLIP;
 		if (rotation & DRM_MODE_REFLECT_X)
+			cfg |= GSC_IN_ROT_XFLIP;
+		if (rotation & DRM_MODE_REFLECT_Y)
 			cfg |= GSC_IN_ROT_YFLIP;
 		break;
 	case DRM_MODE_ROTATE_90:
 		cfg |= GSC_IN_ROT_90;
-		if (rotation & DRM_MODE_REFLECT_Y)
-			cfg |= GSC_IN_ROT_XFLIP;
 		if (rotation & DRM_MODE_REFLECT_X)
+			cfg |= GSC_IN_ROT_XFLIP;
+		if (rotation & DRM_MODE_REFLECT_Y)
 			cfg |= GSC_IN_ROT_YFLIP;
 		break;
 	case DRM_MODE_ROTATE_180:
 		cfg |= GSC_IN_ROT_180;
-		if (rotation & DRM_MODE_REFLECT_Y)
-			cfg &= ~GSC_IN_ROT_XFLIP;
 		if (rotation & DRM_MODE_REFLECT_X)
+			cfg &= ~GSC_IN_ROT_XFLIP;
+		if (rotation & DRM_MODE_REFLECT_Y)
 			cfg &= ~GSC_IN_ROT_YFLIP;
 		break;
 	case DRM_MODE_ROTATE_270:
 		cfg |= GSC_IN_ROT_270;
-		if (rotation & DRM_MODE_REFLECT_Y)
-			cfg &= ~GSC_IN_ROT_XFLIP;
 		if (rotation & DRM_MODE_REFLECT_X)
+			cfg &= ~GSC_IN_ROT_XFLIP;
+		if (rotation & DRM_MODE_REFLECT_Y)
 			cfg &= ~GSC_IN_ROT_YFLIP;
 		break;
 	}
@@ -577,7 +581,7 @@ static void gsc_src_set_size(struct gsc_context *ctx,
 	cfg &= ~(GSC_SRCIMG_HEIGHT_MASK |
 		GSC_SRCIMG_WIDTH_MASK);
 
-	cfg |= (GSC_SRCIMG_WIDTH(buf->buf.width) |
+	cfg |= (GSC_SRCIMG_WIDTH(buf->buf.pitch[0] / buf->format->cpp[0]) |
 		GSC_SRCIMG_HEIGHT(buf->buf.height));
 
 	gsc_write(cfg, GSC_SRCIMG_SIZE);
@@ -672,18 +676,25 @@ static void gsc_dst_set_fmt(struct gsc_context *ctx, u32 fmt)
 			GSC_OUT_CHROMA_ORDER_CRCB);
 		break;
 	case DRM_FORMAT_NV21:
-	case DRM_FORMAT_NV61:
 		cfg |= (GSC_OUT_CHROMA_ORDER_CRCB | GSC_OUT_YUV420_2P);
 		break;
+	case DRM_FORMAT_NV61:
+		cfg |= (GSC_OUT_CHROMA_ORDER_CRCB | GSC_OUT_YUV422_2P);
+		break;
 	case DRM_FORMAT_YUV422:
+		cfg |= GSC_OUT_YUV422_3P;
+		break;
 	case DRM_FORMAT_YUV420:
+		cfg |= (GSC_OUT_CHROMA_ORDER_CBCR | GSC_OUT_YUV420_3P);
+		break;
 	case DRM_FORMAT_YVU420:
-		cfg |= GSC_OUT_YUV420_3P;
+		cfg |= (GSC_OUT_CHROMA_ORDER_CRCB | GSC_OUT_YUV420_3P);
 		break;
 	case DRM_FORMAT_NV12:
+		cfg |= (GSC_OUT_CHROMA_ORDER_CBCR | GSC_OUT_YUV420_2P);
+		break;
 	case DRM_FORMAT_NV16:
-		cfg |= (GSC_OUT_CHROMA_ORDER_CBCR |
-			GSC_OUT_YUV420_2P);
+		cfg |= (GSC_OUT_CHROMA_ORDER_CBCR | GSC_OUT_YUV422_2P);
 		break;
 	}
 
@@ -868,7 +879,7 @@ static void gsc_dst_set_size(struct gsc_context *ctx,
 	/* original size */
 	cfg = gsc_read(GSC_DSTIMG_SIZE);
 	cfg &= ~(GSC_DSTIMG_HEIGHT_MASK | GSC_DSTIMG_WIDTH_MASK);
-	cfg |= GSC_DSTIMG_WIDTH(buf->buf.width) |
+	cfg |= GSC_DSTIMG_WIDTH(buf->buf.pitch[0] / buf->format->cpp[0]) |
 	       GSC_DSTIMG_HEIGHT(buf->buf.height);
 	gsc_write(cfg, GSC_DSTIMG_SIZE);
 
@@ -1341,7 +1352,7 @@ static const struct drm_exynos_ipp_limit gsc_5420_limits[] = {
 };
 
 static const struct drm_exynos_ipp_limit gsc_5433_limits[] = {
-	{ IPP_SIZE_LIMIT(BUFFER, .h = { 32, 8191, 2 }, .v = { 16, 8191, 2 }) },
+	{ IPP_SIZE_LIMIT(BUFFER, .h = { 32, 8191, 16 }, .v = { 16, 8191, 2 }) },
 	{ IPP_SIZE_LIMIT(AREA, .h = { 16, 4800, 1 }, .v = { 8, 3344, 1 }) },
 	{ IPP_SIZE_LIMIT(ROTATED, .h = { 32, 2047 }, .v = { 8, 8191 }) },
 	{ IPP_SCALE_LIMIT(.h = { (1 << 16) / 16, (1 << 16) * 8 },
diff --git a/drivers/gpu/drm/exynos/exynos_drm_ipp.c b/drivers/gpu/drm/exynos/exynos_drm_ipp.c
index 26374e58c557..23226a0212e8 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_ipp.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_ipp.c
@@ -345,39 +345,18 @@ static int exynos_drm_ipp_task_setup_buffer(struct exynos_drm_ipp_buffer *buf,
 	int ret = 0;
 	int i;
 
-	/* basic checks */
-	if (buf->buf.width == 0 || buf->buf.height == 0)
-		return -EINVAL;
-	buf->format = drm_format_info(buf->buf.fourcc);
-	for (i = 0; i < buf->format->num_planes; i++) {
-		unsigned int width = (i == 0) ? buf->buf.width :
-			     DIV_ROUND_UP(buf->buf.width, buf->format->hsub);
-
-		if (buf->buf.pitch[i] == 0)
-			buf->buf.pitch[i] = width * buf->format->cpp[i];
-		if (buf->buf.pitch[i] < width * buf->format->cpp[i])
-			return -EINVAL;
-		if (!buf->buf.gem_id[i])
-			return -ENOENT;
-	}
-
-	/* pitch for additional planes must match */
-	if (buf->format->num_planes > 2 &&
-	    buf->buf.pitch[1] != buf->buf.pitch[2])
-		return -EINVAL;
-
 	/* get GEM buffers and check their size */
 	for (i = 0; i < buf->format->num_planes; i++) {
 		unsigned int height = (i == 0) ? buf->buf.height :
 			     DIV_ROUND_UP(buf->buf.height, buf->format->vsub);
 		unsigned long size = height * buf->buf.pitch[i];
-		struct drm_gem_object *obj = drm_gem_object_lookup(filp,
+		struct exynos_drm_gem *gem = exynos_drm_gem_get(filp,
 							    buf->buf.gem_id[i]);
-		if (!obj) {
+		if (!gem) {
 			ret = -ENOENT;
 			goto gem_free;
 		}
-		buf->exynos_gem[i] = to_exynos_gem(obj);
+		buf->exynos_gem[i] = gem;
 
 		if (size + buf->buf.offset[i] > buf->exynos_gem[i]->size) {
 			i++;
@@ -391,7 +370,7 @@ static int exynos_drm_ipp_task_setup_buffer(struct exynos_drm_ipp_buffer *buf,
 	return 0;
 gem_free:
 	while (i--) {
-		drm_gem_object_put_unlocked(&buf->exynos_gem[i]->base);
+		exynos_drm_gem_put(buf->exynos_gem[i]);
 		buf->exynos_gem[i] = NULL;
 	}
 	return ret;
@@ -404,7 +383,7 @@ static void exynos_drm_ipp_task_release_buf(struct exynos_drm_ipp_buffer *buf)
 	if (!buf->exynos_gem[0])
 		return;
 	for (i = 0; i < buf->format->num_planes; i++)
-		drm_gem_object_put_unlocked(&buf->exynos_gem[i]->base);
+		exynos_drm_gem_put(buf->exynos_gem[i]);
 }
 
 static void exynos_drm_ipp_task_free(struct exynos_drm_ipp *ipp,
@@ -428,7 +407,7 @@ enum drm_ipp_size_id {
 	IPP_LIMIT_BUFFER, IPP_LIMIT_AREA, IPP_LIMIT_ROTATED, IPP_LIMIT_MAX
 };
 
-static const enum drm_ipp_size_id limit_id_fallback[IPP_LIMIT_MAX][4] = {
+static const enum drm_exynos_ipp_limit_type limit_id_fallback[IPP_LIMIT_MAX][4] = {
 	[IPP_LIMIT_BUFFER]  = { DRM_EXYNOS_IPP_LIMIT_SIZE_BUFFER },
 	[IPP_LIMIT_AREA]    = { DRM_EXYNOS_IPP_LIMIT_SIZE_AREA,
 				DRM_EXYNOS_IPP_LIMIT_SIZE_BUFFER },
@@ -495,12 +474,13 @@ static int exynos_drm_ipp_check_size_limits(struct exynos_drm_ipp_buffer *buf,
 	enum drm_ipp_size_id id = rotate ? IPP_LIMIT_ROTATED : IPP_LIMIT_AREA;
 	struct drm_ipp_limit l;
 	struct drm_exynos_ipp_limit_val *lh = &l.h, *lv = &l.v;
+	int real_width = buf->buf.pitch[0] / buf->format->cpp[0];
 
 	if (!limits)
 		return 0;
 
 	__get_size_limit(limits, num_limits, IPP_LIMIT_BUFFER, &l);
-	if (!__size_limit_check(buf->buf.width, &l.h) ||
+	if (!__size_limit_check(real_width, &l.h) ||
 	    !__size_limit_check(buf->buf.height, &l.v))
 		return -EINVAL;
 
@@ -560,10 +540,62 @@ static int exynos_drm_ipp_check_scale_limits(
 	return 0;
 }
 
+static int exynos_drm_ipp_check_format(struct exynos_drm_ipp_task *task,
+				       struct exynos_drm_ipp_buffer *buf,
+				       struct exynos_drm_ipp_buffer *src,
+				       struct exynos_drm_ipp_buffer *dst,
+				       bool rotate, bool swap)
+{
+	const struct exynos_drm_ipp_formats *fmt;
+	int ret, i;
+
+	fmt = __ipp_format_get(task->ipp, buf->buf.fourcc, buf->buf.modifier,
+			       buf == src ? DRM_EXYNOS_IPP_FORMAT_SOURCE :
+					    DRM_EXYNOS_IPP_FORMAT_DESTINATION);
+	if (!fmt) {
+		DRM_DEBUG_DRIVER("Task %pK: %s format not supported\n", task,
+				 buf == src ? "src" : "dst");
+		return -EINVAL;
+	}
+
+	/* basic checks */
+	if (buf->buf.width == 0 || buf->buf.height == 0)
+		return -EINVAL;
+
+	buf->format = drm_format_info(buf->buf.fourcc);
+	for (i = 0; i < buf->format->num_planes; i++) {
+		unsigned int width = (i == 0) ? buf->buf.width :
+			     DIV_ROUND_UP(buf->buf.width, buf->format->hsub);
+
+		if (buf->buf.pitch[i] == 0)
+			buf->buf.pitch[i] = width * buf->format->cpp[i];
+		if (buf->buf.pitch[i] < width * buf->format->cpp[i])
+			return -EINVAL;
+		if (!buf->buf.gem_id[i])
+			return -ENOENT;
+	}
+
+	/* pitch for additional planes must match */
+	if (buf->format->num_planes > 2 &&
+	    buf->buf.pitch[1] != buf->buf.pitch[2])
+		return -EINVAL;
+
+	/* check driver limits */
+	ret = exynos_drm_ipp_check_size_limits(buf, fmt->limits,
+					       fmt->num_limits,
+					       rotate,
+					       buf == dst ? swap : false);
+	if (ret)
+		return ret;
+	ret = exynos_drm_ipp_check_scale_limits(&src->rect, &dst->rect,
+						fmt->limits,
+						fmt->num_limits, swap);
+	return ret;
+}
+
 static int exynos_drm_ipp_task_check(struct exynos_drm_ipp_task *task)
 {
 	struct exynos_drm_ipp *ipp = task->ipp;
-	const struct exynos_drm_ipp_formats *src_fmt, *dst_fmt;
 	struct exynos_drm_ipp_buffer *src = &task->src, *dst = &task->dst;
 	unsigned int rotation = task->transform.rotation;
 	int ret = 0;
@@ -607,37 +639,11 @@ static int exynos_drm_ipp_task_check(struct exynos_drm_ipp_task *task)
 		return -EINVAL;
 	}
 
-	src_fmt = __ipp_format_get(ipp, src->buf.fourcc, src->buf.modifier,
-				   DRM_EXYNOS_IPP_FORMAT_SOURCE);
-	if (!src_fmt) {
-		DRM_DEBUG_DRIVER("Task %pK: src format not supported\n", task);
-		return -EINVAL;
-	}
-	ret = exynos_drm_ipp_check_size_limits(src, src_fmt->limits,
-					       src_fmt->num_limits,
-					       rotate, false);
-	if (ret)
-		return ret;
-	ret = exynos_drm_ipp_check_scale_limits(&src->rect, &dst->rect,
-						src_fmt->limits,
-						src_fmt->num_limits, swap);
+	ret = exynos_drm_ipp_check_format(task, src, src, dst, rotate, swap);
 	if (ret)
 		return ret;
 
-	dst_fmt = __ipp_format_get(ipp, dst->buf.fourcc, dst->buf.modifier,
-				   DRM_EXYNOS_IPP_FORMAT_DESTINATION);
-	if (!dst_fmt) {
-		DRM_DEBUG_DRIVER("Task %pK: dst format not supported\n", task);
-		return -EINVAL;
-	}
-	ret = exynos_drm_ipp_check_size_limits(dst, dst_fmt->limits,
-					       dst_fmt->num_limits,
-					       false, swap);
-	if (ret)
-		return ret;
-	ret = exynos_drm_ipp_check_scale_limits(&src->rect, &dst->rect,
-						dst_fmt->limits,
-						dst_fmt->num_limits, swap);
+	ret = exynos_drm_ipp_check_format(task, dst, src, dst, false, swap);
 	if (ret)
 		return ret;
 
diff --git a/drivers/gpu/drm/exynos/exynos_drm_mic.c b/drivers/gpu/drm/exynos/exynos_drm_mic.c
index 2174814273e2..2fd299a58297 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_mic.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_mic.c
@@ -367,6 +367,8 @@ static int exynos_mic_resume(struct device *dev)
 
 static const struct dev_pm_ops exynos_mic_pm_ops = {
 	SET_RUNTIME_PM_OPS(exynos_mic_suspend, exynos_mic_resume, NULL)
+	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
+				pm_runtime_force_resume)
 };
 
 static int exynos_mic_probe(struct platform_device *pdev)
diff --git a/drivers/gpu/drm/exynos/exynos_drm_plane.c b/drivers/gpu/drm/exynos/exynos_drm_plane.c
index eb9915da7dec..dba29aec59b4 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_plane.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_plane.c
@@ -132,7 +132,7 @@ static void exynos_drm_plane_reset(struct drm_plane *plane)
 	if (plane->state) {
 		exynos_state = to_exynos_plane_state(plane->state);
 		if (exynos_state->base.fb)
-			drm_framebuffer_unreference(exynos_state->base.fb);
+			drm_framebuffer_put(exynos_state->base.fb);
 		kfree(exynos_state);
 		plane->state = NULL;
 	}
diff --git a/drivers/gpu/drm/exynos/exynos_drm_rotator.c b/drivers/gpu/drm/exynos/exynos_drm_rotator.c
index 1a76dd3d52e1..a820a68429b9 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_rotator.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_rotator.c
@@ -168,9 +168,9 @@ static void rotator_dst_set_transf(struct rot_context *rot,
 	val &= ~ROT_CONTROL_FLIP_MASK;
 
 	if (rotation & DRM_MODE_REFLECT_X)
-		val |= ROT_CONTROL_FLIP_HORIZONTAL;
-	if (rotation & DRM_MODE_REFLECT_Y)
 		val |= ROT_CONTROL_FLIP_VERTICAL;
+	if (rotation & DRM_MODE_REFLECT_Y)
+		val |= ROT_CONTROL_FLIP_HORIZONTAL;
 
 	val &= ~ROT_CONTROL_ROT_MASK;
 
diff --git a/drivers/gpu/drm/exynos/exynos_drm_scaler.c b/drivers/gpu/drm/exynos/exynos_drm_scaler.c
index 91d4382343d0..0ddb6eec7b11 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_scaler.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_scaler.c
@@ -30,6 +30,7 @@
 #define scaler_write(cfg, offset)	writel(cfg, scaler->regs + (offset))
 #define SCALER_MAX_CLK			4
 #define SCALER_AUTOSUSPEND_DELAY	2000
+#define SCALER_RESET_WAIT_RETRIES	100
 
 struct scaler_data {
 	const char	*clk_name[SCALER_MAX_CLK];
@@ -51,9 +52,9 @@ struct scaler_context {
 static u32 scaler_get_format(u32 drm_fmt)
 {
 	switch (drm_fmt) {
-	case DRM_FORMAT_NV21:
-		return SCALER_YUV420_2P_UV;
 	case DRM_FORMAT_NV12:
+		return SCALER_YUV420_2P_UV;
+	case DRM_FORMAT_NV21:
 		return SCALER_YUV420_2P_VU;
 	case DRM_FORMAT_YUV420:
 		return SCALER_YUV420_3P;
@@ -63,15 +64,15 @@ static u32 scaler_get_format(u32 drm_fmt)
 		return SCALER_YUV422_1P_UYVY;
 	case DRM_FORMAT_YVYU:
 		return SCALER_YUV422_1P_YVYU;
-	case DRM_FORMAT_NV61:
-		return SCALER_YUV422_2P_UV;
 	case DRM_FORMAT_NV16:
+		return SCALER_YUV422_2P_UV;
+	case DRM_FORMAT_NV61:
 		return SCALER_YUV422_2P_VU;
 	case DRM_FORMAT_YUV422:
 		return SCALER_YUV422_3P;
-	case DRM_FORMAT_NV42:
-		return SCALER_YUV444_2P_UV;
 	case DRM_FORMAT_NV24:
+		return SCALER_YUV444_2P_UV;
+	case DRM_FORMAT_NV42:
 		return SCALER_YUV444_2P_VU;
 	case DRM_FORMAT_YUV444:
 		return SCALER_YUV444_3P;
@@ -100,6 +101,23 @@ static u32 scaler_get_format(u32 drm_fmt)
 	return 0;
 }
 
+static inline int scaler_reset(struct scaler_context *scaler)
+{
+	int retry = SCALER_RESET_WAIT_RETRIES;
+
+	scaler_write(SCALER_CFG_SOFT_RESET, SCALER_CFG);
+	do {
+		cpu_relax();
+	} while (retry > 1 &&
+		 scaler_read(SCALER_CFG) & SCALER_CFG_SOFT_RESET);
+	do {
+		cpu_relax();
+		scaler_write(1, SCALER_INT_EN);
+	} while (retry > 0 && scaler_read(SCALER_INT_EN) != 1);
+
+	return retry ? 0 : -EIO;
+}
+
 static inline void scaler_enable_int(struct scaler_context *scaler)
 {
 	u32 val;
@@ -354,9 +372,13 @@ static int scaler_commit(struct exynos_drm_ipp *ipp,
 	u32 dst_fmt = scaler_get_format(task->dst.buf.fourcc);
 	struct drm_exynos_ipp_task_rect *dst_pos = &task->dst.rect;
 
-	scaler->task = task;
-
 	pm_runtime_get_sync(scaler->dev);
+	if (scaler_reset(scaler)) {
+		pm_runtime_put(scaler->dev);
+		return -EIO;
+	}
+
+	scaler->task = task;
 
 	scaler_set_src_fmt(scaler, src_fmt);
 	scaler_set_src_base(scaler, &task->src);
@@ -394,7 +416,11 @@ static inline void scaler_disable_int(struct scaler_context *scaler)
 
 static inline u32 scaler_get_int_status(struct scaler_context *scaler)
 {
-	return scaler_read(SCALER_INT_STATUS);
+	u32 val = scaler_read(SCALER_INT_STATUS);
+
+	scaler_write(val, SCALER_INT_STATUS);
+
+	return val;
 }
 
 static inline int scaler_task_done(u32 val)
diff --git a/drivers/gpu/drm/exynos/exynos_hdmi.c b/drivers/gpu/drm/exynos/exynos_hdmi.c
index 3a11c719a580..2092a650df7d 100644
--- a/drivers/gpu/drm/exynos/exynos_hdmi.c
+++ b/drivers/gpu/drm/exynos/exynos_hdmi.c
@@ -2093,6 +2093,8 @@ static int __maybe_unused exynos_hdmi_resume(struct device *dev)
 
 static const struct dev_pm_ops exynos_hdmi_pm_ops = {
 	SET_RUNTIME_PM_OPS(exynos_hdmi_suspend, exynos_hdmi_resume, NULL)
+	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
+				pm_runtime_force_resume)
 };
 
 struct platform_driver hdmi_driver = {
diff --git a/drivers/gpu/drm/exynos/exynos_mixer.c b/drivers/gpu/drm/exynos/exynos_mixer.c
index 272c79f5f5bf..ffbf4a950f69 100644
--- a/drivers/gpu/drm/exynos/exynos_mixer.c
+++ b/drivers/gpu/drm/exynos/exynos_mixer.c
@@ -837,8 +837,6 @@ static int mixer_initialize(struct mixer_context *mixer_ctx,
 			struct drm_device *drm_dev)
 {
 	int ret;
-	struct exynos_drm_private *priv;
-	priv = drm_dev->dev_private;
 
 	mixer_ctx->drm_dev = drm_dev;
 
@@ -1271,6 +1269,8 @@ static int __maybe_unused exynos_mixer_resume(struct device *dev)
 
 static const struct dev_pm_ops exynos_mixer_pm_ops = {
 	SET_RUNTIME_PM_OPS(exynos_mixer_suspend, exynos_mixer_resume, NULL)
+	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
+				pm_runtime_force_resume)
 };
 
 struct platform_driver mixer_driver = {
diff --git a/drivers/gpu/drm/exynos/regs-gsc.h b/drivers/gpu/drm/exynos/regs-gsc.h
index 4704a993cbb7..16b39734115c 100644
--- a/drivers/gpu/drm/exynos/regs-gsc.h
+++ b/drivers/gpu/drm/exynos/regs-gsc.h
@@ -138,6 +138,7 @@
 #define GSC_OUT_YUV420_3P		(3 << 4)
 #define GSC_OUT_YUV422_1P		(4 << 4)
 #define GSC_OUT_YUV422_2P		(5 << 4)
+#define GSC_OUT_YUV422_3P		(6 << 4)
 #define GSC_OUT_YUV444			(7 << 4)
 #define GSC_OUT_TILE_TYPE_MASK		(1 << 2)
 #define GSC_OUT_TILE_C_16x8		(0 << 2)
diff --git a/drivers/gpu/drm/gma500/framebuffer.c b/drivers/gpu/drm/gma500/framebuffer.c
index 2f00a37684a2..adefae58b5fc 100644
--- a/drivers/gpu/drm/gma500/framebuffer.c
+++ b/drivers/gpu/drm/gma500/framebuffer.c
@@ -108,7 +108,7 @@ static int psbfb_pan(struct fb_var_screeninfo *var, struct fb_info *info)
         return 0;
 }
 
-static int psbfb_vm_fault(struct vm_fault *vmf)
+static vm_fault_t psbfb_vm_fault(struct vm_fault *vmf)
 {
 	struct vm_area_struct *vma = vmf->vma;
 	struct psb_framebuffer *psbfb = vma->vm_private_data;
@@ -118,7 +118,7 @@ static int psbfb_vm_fault(struct vm_fault *vmf)
 	int page_num;
 	int i;
 	unsigned long address;
-	int ret;
+	vm_fault_t ret = VM_FAULT_SIGBUS;
 	unsigned long pfn;
 	unsigned long phys_addr = (unsigned long)dev_priv->stolen_base +
 				  gtt->offset;
@@ -131,18 +131,14 @@ static int psbfb_vm_fault(struct vm_fault *vmf)
 	for (i = 0; i < page_num; i++) {
 		pfn = (phys_addr >> PAGE_SHIFT);
 
-		ret = vm_insert_mixed(vma, address,
+		ret = vmf_insert_mixed(vma, address,
 				__pfn_to_pfn_t(pfn, PFN_DEV));
-		if (unlikely((ret == -EBUSY) || (ret != 0 && i > 0)))
+		if (unlikely(ret & VM_FAULT_ERROR))
 			break;
-		else if (unlikely(ret != 0)) {
-			ret = (ret == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS;
-			return ret;
-		}
 		address += PAGE_SIZE;
 		phys_addr += PAGE_SIZE;
 	}
-	return VM_FAULT_NOPAGE;
+	return ret;
 }
 
 static void psbfb_vm_open(struct vm_area_struct *vma)
diff --git a/drivers/gpu/drm/gma500/gem.c b/drivers/gpu/drm/gma500/gem.c
index 913bf4c256fa..576f1b272f23 100644
--- a/drivers/gpu/drm/gma500/gem.c
+++ b/drivers/gpu/drm/gma500/gem.c
@@ -134,12 +134,13 @@ int psb_gem_dumb_create(struct drm_file *file, struct drm_device *dev,
  *	vma->vm_private_data points to the GEM object that is backing this
  *	mapping.
  */
-int psb_gem_fault(struct vm_fault *vmf)
+vm_fault_t psb_gem_fault(struct vm_fault *vmf)
 {
 	struct vm_area_struct *vma = vmf->vma;
 	struct drm_gem_object *obj;
 	struct gtt_range *r;
-	int ret;
+	int err;
+	vm_fault_t ret;
 	unsigned long pfn;
 	pgoff_t page_offset;
 	struct drm_device *dev;
@@ -158,9 +159,10 @@ int psb_gem_fault(struct vm_fault *vmf)
 	/* For now the mmap pins the object and it stays pinned. As things
 	   stand that will do us no harm */
 	if (r->mmapping == 0) {
-		ret = psb_gtt_pin(r);
-		if (ret < 0) {
-			dev_err(dev->dev, "gma500: pin failed: %d\n", ret);
+		err = psb_gtt_pin(r);
+		if (err < 0) {
+			dev_err(dev->dev, "gma500: pin failed: %d\n", err);
+			ret = vmf_error(err);
 			goto fail;
 		}
 		r->mmapping = 1;
@@ -175,18 +177,9 @@ int psb_gem_fault(struct vm_fault *vmf)
 		pfn = (dev_priv->stolen_base + r->offset) >> PAGE_SHIFT;
 	else
 		pfn = page_to_pfn(r->pages[page_offset]);
-	ret = vm_insert_pfn(vma, vmf->address, pfn);
-
+	ret = vmf_insert_pfn(vma, vmf->address, pfn);
 fail:
 	mutex_unlock(&dev_priv->mmap_mutex);
-	switch (ret) {
-	case 0:
-	case -ERESTARTSYS:
-	case -EINTR:
-		return VM_FAULT_NOPAGE;
-	case -ENOMEM:
-		return VM_FAULT_OOM;
-	default:
-		return VM_FAULT_SIGBUS;
-	}
+
+	return ret;
 }
diff --git a/drivers/gpu/drm/gma500/psb_drv.h b/drivers/gpu/drm/gma500/psb_drv.h
index d876ed5e9265..941b238bdcc9 100644
--- a/drivers/gpu/drm/gma500/psb_drv.h
+++ b/drivers/gpu/drm/gma500/psb_drv.h
@@ -21,6 +21,7 @@
 #define _PSB_DRV_H_
 
 #include <linux/kref.h>
+#include <linux/mm_types.h>
 
 #include <drm/drmP.h>
 #include <drm/gma_drm.h>
@@ -748,7 +749,7 @@ extern int psb_gem_get_aperture(struct drm_device *dev, void *data,
 			struct drm_file *file);
 extern int psb_gem_dumb_create(struct drm_file *file, struct drm_device *dev,
 			struct drm_mode_create_dumb *args);
-extern int psb_gem_fault(struct vm_fault *vmf);
+extern vm_fault_t psb_gem_fault(struct vm_fault *vmf);
 
 /* psb_device.c */
 extern const struct psb_ops psb_chip_ops;
diff --git a/drivers/gpu/drm/i2c/tda9950.c b/drivers/gpu/drm/i2c/tda9950.c
index 3f7396caad48..5d2f0d548469 100644
--- a/drivers/gpu/drm/i2c/tda9950.c
+++ b/drivers/gpu/drm/i2c/tda9950.c
@@ -76,9 +76,12 @@ struct tda9950_priv {
 static int tda9950_write_range(struct i2c_client *client, u8 addr, u8 *p, int cnt)
 {
 	struct i2c_msg msg;
-	u8 buf[cnt + 1];
+	u8 buf[CEC_MAX_MSG_SIZE + 3];
 	int ret;
 
+	if (WARN_ON(cnt > sizeof(buf) - 1))
+		return -EINVAL;
+
 	buf[0] = addr;
 	memcpy(buf + 1, p, cnt);
 
diff --git a/drivers/gpu/drm/i2c/tda998x_drv.c b/drivers/gpu/drm/i2c/tda998x_drv.c
index eecdc327b9f8..a7c39f39793f 100644
--- a/drivers/gpu/drm/i2c/tda998x_drv.c
+++ b/drivers/gpu/drm/i2c/tda998x_drv.c
@@ -69,6 +69,7 @@ struct tda998x_priv {
 	bool edid_delay_active;
 
 	struct drm_encoder encoder;
+	struct drm_bridge bridge;
 	struct drm_connector connector;
 
 	struct tda998x_audio_port audio_port[2];
@@ -79,9 +80,10 @@ struct tda998x_priv {
 
 #define conn_to_tda998x_priv(x) \
 	container_of(x, struct tda998x_priv, connector)
-
 #define enc_to_tda998x_priv(x) \
 	container_of(x, struct tda998x_priv, encoder)
+#define bridge_to_tda998x_priv(x) \
+	container_of(x, struct tda998x_priv, bridge)
 
 /* The TDA9988 series of devices use a paged register scheme.. to simplify
  * things we encode the page # in upper bits of the register #.  To read/
@@ -762,7 +764,7 @@ static void tda998x_detect_work(struct work_struct *work)
 {
 	struct tda998x_priv *priv =
 		container_of(work, struct tda998x_priv, detect_work);
-	struct drm_device *dev = priv->encoder.dev;
+	struct drm_device *dev = priv->connector.dev;
 
 	if (dev)
 		drm_kms_helper_hotplug_event(dev);
@@ -1104,29 +1106,6 @@ static int tda998x_audio_codec_init(struct tda998x_priv *priv,
 
 /* DRM connector functions */
 
-static int tda998x_connector_fill_modes(struct drm_connector *connector,
-					uint32_t maxX, uint32_t maxY)
-{
-	struct tda998x_priv *priv = conn_to_tda998x_priv(connector);
-	int ret;
-
-	mutex_lock(&priv->audio_mutex);
-	ret = drm_helper_probe_single_connector_modes(connector, maxX, maxY);
-
-	if (connector->edid_blob_ptr) {
-		struct edid *edid = (void *)connector->edid_blob_ptr->data;
-
-		cec_notifier_set_phys_addr_from_edid(priv->cec_notify, edid);
-
-		priv->sink_has_audio = drm_detect_monitor_audio(edid);
-	} else {
-		priv->sink_has_audio = false;
-	}
-	mutex_unlock(&priv->audio_mutex);
-
-	return ret;
-}
-
 static enum drm_connector_status
 tda998x_connector_detect(struct drm_connector *connector, bool force)
 {
@@ -1145,7 +1124,7 @@ static void tda998x_connector_destroy(struct drm_connector *connector)
 static const struct drm_connector_funcs tda998x_connector_funcs = {
 	.dpms = drm_helper_connector_dpms,
 	.reset = drm_atomic_helper_connector_reset,
-	.fill_modes = tda998x_connector_fill_modes,
+	.fill_modes = drm_helper_probe_single_connector_modes,
 	.detect = tda998x_connector_detect,
 	.destroy = tda998x_connector_destroy,
 	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
@@ -1244,40 +1223,29 @@ static int tda998x_connector_get_modes(struct drm_connector *connector)
 	}
 
 	drm_connector_update_edid_property(connector, edid);
+	cec_notifier_set_phys_addr_from_edid(priv->cec_notify, edid);
+
+	mutex_lock(&priv->audio_mutex);
 	n = drm_add_edid_modes(connector, edid);
+	priv->sink_has_audio = drm_detect_monitor_audio(edid);
+	mutex_unlock(&priv->audio_mutex);
 
 	kfree(edid);
 
 	return n;
 }
 
-static enum drm_mode_status tda998x_connector_mode_valid(struct drm_connector *connector,
-					struct drm_display_mode *mode)
-{
-	/* TDA19988 dotclock can go up to 165MHz */
-	struct tda998x_priv *priv = conn_to_tda998x_priv(connector);
-
-	if (mode->clock > ((priv->rev == TDA19988) ? 165000 : 150000))
-		return MODE_CLOCK_HIGH;
-	if (mode->htotal >= BIT(13))
-		return MODE_BAD_HVALUE;
-	if (mode->vtotal >= BIT(11))
-		return MODE_BAD_VVALUE;
-	return MODE_OK;
-}
-
 static struct drm_encoder *
 tda998x_connector_best_encoder(struct drm_connector *connector)
 {
 	struct tda998x_priv *priv = conn_to_tda998x_priv(connector);
 
-	return &priv->encoder;
+	return priv->bridge.encoder;
 }
 
 static
 const struct drm_connector_helper_funcs tda998x_connector_helper_funcs = {
 	.get_modes = tda998x_connector_get_modes,
-	.mode_valid = tda998x_connector_mode_valid,
 	.best_encoder = tda998x_connector_best_encoder,
 };
 
@@ -1301,25 +1269,48 @@ static int tda998x_connector_init(struct tda998x_priv *priv,
 	if (ret)
 		return ret;
 
-	drm_connector_attach_encoder(&priv->connector, &priv->encoder);
+	drm_connector_attach_encoder(&priv->connector,
+				     priv->bridge.encoder);
 
 	return 0;
 }
 
-/* DRM encoder functions */
+/* DRM bridge functions */
 
-static void tda998x_encoder_dpms(struct drm_encoder *encoder, int mode)
+static int tda998x_bridge_attach(struct drm_bridge *bridge)
 {
-	struct tda998x_priv *priv = enc_to_tda998x_priv(encoder);
-	bool on;
+	struct tda998x_priv *priv = bridge_to_tda998x_priv(bridge);
 
-	/* we only care about on or off: */
-	on = mode == DRM_MODE_DPMS_ON;
+	return tda998x_connector_init(priv, bridge->dev);
+}
 
-	if (on == priv->is_on)
-		return;
+static void tda998x_bridge_detach(struct drm_bridge *bridge)
+{
+	struct tda998x_priv *priv = bridge_to_tda998x_priv(bridge);
 
-	if (on) {
+	drm_connector_cleanup(&priv->connector);
+}
+
+static enum drm_mode_status tda998x_bridge_mode_valid(struct drm_bridge *bridge,
+				     const struct drm_display_mode *mode)
+{
+	/* TDA19988 dotclock can go up to 165MHz */
+	struct tda998x_priv *priv = bridge_to_tda998x_priv(bridge);
+
+	if (mode->clock > ((priv->rev == TDA19988) ? 165000 : 150000))
+		return MODE_CLOCK_HIGH;
+	if (mode->htotal >= BIT(13))
+		return MODE_BAD_HVALUE;
+	if (mode->vtotal >= BIT(11))
+		return MODE_BAD_VVALUE;
+	return MODE_OK;
+}
+
+static void tda998x_bridge_enable(struct drm_bridge *bridge)
+{
+	struct tda998x_priv *priv = bridge_to_tda998x_priv(bridge);
+
+	if (!priv->is_on) {
 		/* enable video ports, audio will be enabled later */
 		reg_write(priv, REG_ENA_VP_0, 0xff);
 		reg_write(priv, REG_ENA_VP_1, 0xff);
@@ -1330,7 +1321,14 @@ static void tda998x_encoder_dpms(struct drm_encoder *encoder, int mode)
 		reg_write(priv, REG_VIP_CNTRL_2, priv->vip_cntrl_2);
 
 		priv->is_on = true;
-	} else {
+	}
+}
+
+static void tda998x_bridge_disable(struct drm_bridge *bridge)
+{
+	struct tda998x_priv *priv = bridge_to_tda998x_priv(bridge);
+
+	if (priv->is_on) {
 		/* disable video ports */
 		reg_write(priv, REG_ENA_VP_0, 0x00);
 		reg_write(priv, REG_ENA_VP_1, 0x00);
@@ -1340,12 +1338,12 @@ static void tda998x_encoder_dpms(struct drm_encoder *encoder, int mode)
 	}
 }
 
-static void
-tda998x_encoder_mode_set(struct drm_encoder *encoder,
-			 struct drm_display_mode *mode,
-			 struct drm_display_mode *adjusted_mode)
+static void tda998x_bridge_mode_set(struct drm_bridge *bridge,
+				    struct drm_display_mode *mode,
+				    struct drm_display_mode *adjusted_mode)
 {
-	struct tda998x_priv *priv = enc_to_tda998x_priv(encoder);
+	struct tda998x_priv *priv = bridge_to_tda998x_priv(bridge);
+	unsigned long tmds_clock;
 	u16 ref_pix, ref_line, n_pix, n_line;
 	u16 hs_pix_s, hs_pix_e;
 	u16 vs1_pix_s, vs1_pix_e, vs1_line_s, vs1_line_e;
@@ -1416,12 +1414,19 @@ tda998x_encoder_mode_set(struct drm_encoder *encoder,
 			       (mode->vsync_end - mode->vsync_start)/2;
 	}
 
-	div = 148500 / mode->clock;
-	if (div != 0) {
-		div--;
-		if (div > 3)
-			div = 3;
-	}
+	tmds_clock = mode->clock;
+
+	/*
+	 * The divisor is power-of-2. The TDA9983B datasheet gives
+	 * this as ranges of Msample/s, which is 10x the TMDS clock:
+	 *   0 - 800 to 1500 Msample/s
+	 *   1 - 400 to 800 Msample/s
+	 *   2 - 200 to 400 Msample/s
+	 *   3 - as 2 above
+	 */
+	for (div = 0; div < 3; div++)
+		if (80000 >> div <= tmds_clock)
+			break;
 
 	mutex_lock(&priv->audio_mutex);
 
@@ -1552,26 +1557,14 @@ tda998x_encoder_mode_set(struct drm_encoder *encoder,
 	mutex_unlock(&priv->audio_mutex);
 }
 
-static void tda998x_destroy(struct tda998x_priv *priv)
-{
-	/* disable all IRQs and free the IRQ handler */
-	cec_write(priv, REG_CEC_RXSHPDINTENA, 0);
-	reg_clear(priv, REG_INT_FLAGS_2, INT_FLAGS_2_EDID_BLK_RD);
-
-	if (priv->audio_pdev)
-		platform_device_unregister(priv->audio_pdev);
-
-	if (priv->hdmi->irq)
-		free_irq(priv->hdmi->irq, priv);
-
-	del_timer_sync(&priv->edid_delay_timer);
-	cancel_work_sync(&priv->detect_work);
-
-	i2c_unregister_device(priv->cec);
-
-	if (priv->cec_notify)
-		cec_notifier_put(priv->cec_notify);
-}
+static const struct drm_bridge_funcs tda998x_bridge_funcs = {
+	.attach = tda998x_bridge_attach,
+	.detach = tda998x_bridge_detach,
+	.mode_valid = tda998x_bridge_mode_valid,
+	.disable = tda998x_bridge_disable,
+	.mode_set = tda998x_bridge_mode_set,
+	.enable = tda998x_bridge_enable,
+};
 
 /* I2C driver functions */
 
@@ -1617,16 +1610,69 @@ static int tda998x_get_audio_ports(struct tda998x_priv *priv,
 	return 0;
 }
 
-static int tda998x_create(struct i2c_client *client, struct tda998x_priv *priv)
+static void tda998x_set_config(struct tda998x_priv *priv,
+			       const struct tda998x_encoder_params *p)
 {
+	priv->vip_cntrl_0 = VIP_CNTRL_0_SWAP_A(p->swap_a) |
+			    (p->mirr_a ? VIP_CNTRL_0_MIRR_A : 0) |
+			    VIP_CNTRL_0_SWAP_B(p->swap_b) |
+			    (p->mirr_b ? VIP_CNTRL_0_MIRR_B : 0);
+	priv->vip_cntrl_1 = VIP_CNTRL_1_SWAP_C(p->swap_c) |
+			    (p->mirr_c ? VIP_CNTRL_1_MIRR_C : 0) |
+			    VIP_CNTRL_1_SWAP_D(p->swap_d) |
+			    (p->mirr_d ? VIP_CNTRL_1_MIRR_D : 0);
+	priv->vip_cntrl_2 = VIP_CNTRL_2_SWAP_E(p->swap_e) |
+			    (p->mirr_e ? VIP_CNTRL_2_MIRR_E : 0) |
+			    VIP_CNTRL_2_SWAP_F(p->swap_f) |
+			    (p->mirr_f ? VIP_CNTRL_2_MIRR_F : 0);
+
+	priv->audio_params = p->audio_params;
+}
+
+static void tda998x_destroy(struct device *dev)
+{
+	struct tda998x_priv *priv = dev_get_drvdata(dev);
+
+	drm_bridge_remove(&priv->bridge);
+
+	/* disable all IRQs and free the IRQ handler */
+	cec_write(priv, REG_CEC_RXSHPDINTENA, 0);
+	reg_clear(priv, REG_INT_FLAGS_2, INT_FLAGS_2_EDID_BLK_RD);
+
+	if (priv->audio_pdev)
+		platform_device_unregister(priv->audio_pdev);
+
+	if (priv->hdmi->irq)
+		free_irq(priv->hdmi->irq, priv);
+
+	del_timer_sync(&priv->edid_delay_timer);
+	cancel_work_sync(&priv->detect_work);
+
+	i2c_unregister_device(priv->cec);
+
+	if (priv->cec_notify)
+		cec_notifier_put(priv->cec_notify);
+}
+
+static int tda998x_create(struct device *dev)
+{
+	struct i2c_client *client = to_i2c_client(dev);
 	struct device_node *np = client->dev.of_node;
 	struct i2c_board_info cec_info;
+	struct tda998x_priv *priv;
 	u32 video;
 	int rev_lo, rev_hi, ret;
 
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	dev_set_drvdata(dev, priv);
+
 	mutex_init(&priv->mutex);	/* protect the page access */
 	mutex_init(&priv->audio_mutex); /* protect access from audio thread */
 	mutex_init(&priv->edid_mutex);
+	INIT_LIST_HEAD(&priv->bridge.list);
 	init_waitqueue_head(&priv->edid_delay_waitq);
 	timer_setup(&priv->edid_delay_timer, tda998x_edid_delay_done, 0);
 	INIT_WORK(&priv->detect_work, tda998x_detect_work);
@@ -1649,13 +1695,13 @@ static int tda998x_create(struct i2c_client *client, struct tda998x_priv *priv)
 	/* read version: */
 	rev_lo = reg_read(priv, REG_VERSION_LSB);
 	if (rev_lo < 0) {
-		dev_err(&client->dev, "failed to read version: %d\n", rev_lo);
+		dev_err(dev, "failed to read version: %d\n", rev_lo);
 		return rev_lo;
 	}
 
 	rev_hi = reg_read(priv, REG_VERSION_MSB);
 	if (rev_hi < 0) {
-		dev_err(&client->dev, "failed to read version: %d\n", rev_hi);
+		dev_err(dev, "failed to read version: %d\n", rev_hi);
 		return rev_hi;
 	}
 
@@ -1666,20 +1712,19 @@ static int tda998x_create(struct i2c_client *client, struct tda998x_priv *priv)
 
 	switch (priv->rev) {
 	case TDA9989N2:
-		dev_info(&client->dev, "found TDA9989 n2");
+		dev_info(dev, "found TDA9989 n2");
 		break;
 	case TDA19989:
-		dev_info(&client->dev, "found TDA19989");
+		dev_info(dev, "found TDA19989");
 		break;
 	case TDA19989N2:
-		dev_info(&client->dev, "found TDA19989 n2");
+		dev_info(dev, "found TDA19989 n2");
 		break;
 	case TDA19988:
-		dev_info(&client->dev, "found TDA19988");
+		dev_info(dev, "found TDA19988");
 		break;
 	default:
-		dev_err(&client->dev, "found unsupported device: %04x\n",
-			priv->rev);
+		dev_err(dev, "found unsupported device: %04x\n", priv->rev);
 		return -ENXIO;
 	}
 
@@ -1722,8 +1767,7 @@ static int tda998x_create(struct i2c_client *client, struct tda998x_priv *priv)
 					   tda998x_irq_thread, irq_flags,
 					   "tda998x", priv);
 		if (ret) {
-			dev_err(&client->dev,
-				"failed to request IRQ#%u: %d\n",
+			dev_err(dev, "failed to request IRQ#%u: %d\n",
 				client->irq, ret);
 			goto err_irq;
 		}
@@ -1732,13 +1776,13 @@ static int tda998x_create(struct i2c_client *client, struct tda998x_priv *priv)
 		cec_write(priv, REG_CEC_RXSHPDINTENA, CEC_RXSHPDLEV_HPD);
 	}
 
-	priv->cec_notify = cec_notifier_get(&client->dev);
+	priv->cec_notify = cec_notifier_get(dev);
 	if (!priv->cec_notify) {
 		ret = -ENOMEM;
 		goto fail;
 	}
 
-	priv->cec_glue.parent = &client->dev;
+	priv->cec_glue.parent = dev;
 	priv->cec_glue.data = priv;
 	priv->cec_glue.init = tda998x_cec_hook_init;
 	priv->cec_glue.exit = tda998x_cec_hook_exit;
@@ -1768,61 +1812,44 @@ static int tda998x_create(struct i2c_client *client, struct tda998x_priv *priv)
 	/* enable EDID read irq: */
 	reg_set(priv, REG_INT_FLAGS_2, INT_FLAGS_2_EDID_BLK_RD);
 
-	if (!np)
-		return 0;		/* non-DT */
+	if (np) {
+		/* get the device tree parameters */
+		ret = of_property_read_u32(np, "video-ports", &video);
+		if (ret == 0) {
+			priv->vip_cntrl_0 = video >> 16;
+			priv->vip_cntrl_1 = video >> 8;
+			priv->vip_cntrl_2 = video;
+		}
+
+		ret = tda998x_get_audio_ports(priv, np);
+		if (ret)
+			goto fail;
 
-	/* get the device tree parameters */
-	ret = of_property_read_u32(np, "video-ports", &video);
-	if (ret == 0) {
-		priv->vip_cntrl_0 = video >> 16;
-		priv->vip_cntrl_1 = video >> 8;
-		priv->vip_cntrl_2 = video;
+		if (priv->audio_port[0].format != AFMT_UNUSED)
+			tda998x_audio_codec_init(priv, &client->dev);
+	} else if (dev->platform_data) {
+		tda998x_set_config(priv, dev->platform_data);
 	}
 
-	ret = tda998x_get_audio_ports(priv, np);
-	if (ret)
-		goto fail;
+	priv->bridge.funcs = &tda998x_bridge_funcs;
+#ifdef CONFIG_OF
+	priv->bridge.of_node = dev->of_node;
+#endif
 
-	if (priv->audio_port[0].format != AFMT_UNUSED)
-		tda998x_audio_codec_init(priv, &client->dev);
+	drm_bridge_add(&priv->bridge);
 
 	return 0;
 
 fail:
-	/* if encoder_init fails, the encoder slave is never registered,
-	 * so cleanup here:
-	 */
-	i2c_unregister_device(priv->cec);
-	if (priv->cec_notify)
-		cec_notifier_put(priv->cec_notify);
-	if (client->irq)
-		free_irq(client->irq, priv);
+	tda998x_destroy(dev);
 err_irq:
 	return ret;
 }
 
-static void tda998x_encoder_prepare(struct drm_encoder *encoder)
-{
-	tda998x_encoder_dpms(encoder, DRM_MODE_DPMS_OFF);
-}
-
-static void tda998x_encoder_commit(struct drm_encoder *encoder)
-{
-	tda998x_encoder_dpms(encoder, DRM_MODE_DPMS_ON);
-}
-
-static const struct drm_encoder_helper_funcs tda998x_encoder_helper_funcs = {
-	.dpms = tda998x_encoder_dpms,
-	.prepare = tda998x_encoder_prepare,
-	.commit = tda998x_encoder_commit,
-	.mode_set = tda998x_encoder_mode_set,
-};
+/* DRM encoder functions */
 
 static void tda998x_encoder_destroy(struct drm_encoder *encoder)
 {
-	struct tda998x_priv *priv = enc_to_tda998x_priv(encoder);
-
-	tda998x_destroy(priv);
 	drm_encoder_cleanup(encoder);
 }
 
@@ -1830,40 +1857,12 @@ static const struct drm_encoder_funcs tda998x_encoder_funcs = {
 	.destroy = tda998x_encoder_destroy,
 };
 
-static void tda998x_set_config(struct tda998x_priv *priv,
-			       const struct tda998x_encoder_params *p)
-{
-	priv->vip_cntrl_0 = VIP_CNTRL_0_SWAP_A(p->swap_a) |
-			    (p->mirr_a ? VIP_CNTRL_0_MIRR_A : 0) |
-			    VIP_CNTRL_0_SWAP_B(p->swap_b) |
-			    (p->mirr_b ? VIP_CNTRL_0_MIRR_B : 0);
-	priv->vip_cntrl_1 = VIP_CNTRL_1_SWAP_C(p->swap_c) |
-			    (p->mirr_c ? VIP_CNTRL_1_MIRR_C : 0) |
-			    VIP_CNTRL_1_SWAP_D(p->swap_d) |
-			    (p->mirr_d ? VIP_CNTRL_1_MIRR_D : 0);
-	priv->vip_cntrl_2 = VIP_CNTRL_2_SWAP_E(p->swap_e) |
-			    (p->mirr_e ? VIP_CNTRL_2_MIRR_E : 0) |
-			    VIP_CNTRL_2_SWAP_F(p->swap_f) |
-			    (p->mirr_f ? VIP_CNTRL_2_MIRR_F : 0);
-
-	priv->audio_params = p->audio_params;
-}
-
-static int tda998x_bind(struct device *dev, struct device *master, void *data)
+static int tda998x_encoder_init(struct device *dev, struct drm_device *drm)
 {
-	struct tda998x_encoder_params *params = dev->platform_data;
-	struct i2c_client *client = to_i2c_client(dev);
-	struct drm_device *drm = data;
-	struct tda998x_priv *priv;
+	struct tda998x_priv *priv = dev_get_drvdata(dev);
 	u32 crtcs = 0;
 	int ret;
 
-	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
-	if (!priv)
-		return -ENOMEM;
-
-	dev_set_drvdata(dev, priv);
-
 	if (dev->of_node)
 		crtcs = drm_of_find_possible_crtcs(drm, dev->of_node);
 
@@ -1875,40 +1874,36 @@ static int tda998x_bind(struct device *dev, struct device *master, void *data)
 
 	priv->encoder.possible_crtcs = crtcs;
 
-	ret = tda998x_create(client, priv);
-	if (ret)
-		return ret;
-
-	if (!dev->of_node && params)
-		tda998x_set_config(priv, params);
-
-	drm_encoder_helper_add(&priv->encoder, &tda998x_encoder_helper_funcs);
 	ret = drm_encoder_init(drm, &priv->encoder, &tda998x_encoder_funcs,
 			       DRM_MODE_ENCODER_TMDS, NULL);
 	if (ret)
 		goto err_encoder;
 
-	ret = tda998x_connector_init(priv, drm);
+	ret = drm_bridge_attach(&priv->encoder, &priv->bridge, NULL);
 	if (ret)
-		goto err_connector;
+		goto err_bridge;
 
 	return 0;
 
-err_connector:
+err_bridge:
 	drm_encoder_cleanup(&priv->encoder);
 err_encoder:
-	tda998x_destroy(priv);
 	return ret;
 }
 
+static int tda998x_bind(struct device *dev, struct device *master, void *data)
+{
+	struct drm_device *drm = data;
+
+	return tda998x_encoder_init(dev, drm);
+}
+
 static void tda998x_unbind(struct device *dev, struct device *master,
 			   void *data)
 {
 	struct tda998x_priv *priv = dev_get_drvdata(dev);
 
-	drm_connector_cleanup(&priv->connector);
 	drm_encoder_cleanup(&priv->encoder);
-	tda998x_destroy(priv);
 }
 
 static const struct component_ops tda998x_ops = {
@@ -1919,16 +1914,27 @@ static const struct component_ops tda998x_ops = {
 static int
 tda998x_probe(struct i2c_client *client, const struct i2c_device_id *id)
 {
+	int ret;
+
 	if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C)) {
 		dev_warn(&client->dev, "adapter does not support I2C\n");
 		return -EIO;
 	}
-	return component_add(&client->dev, &tda998x_ops);
+
+	ret = tda998x_create(&client->dev);
+	if (ret)
+		return ret;
+
+	ret = component_add(&client->dev, &tda998x_ops);
+	if (ret)
+		tda998x_destroy(&client->dev);
+	return ret;
 }
 
 static int tda998x_remove(struct i2c_client *client)
 {
 	component_del(&client->dev, &tda998x_ops);
+	tda998x_destroy(&client->dev);
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/i915/Kconfig b/drivers/gpu/drm/i915/Kconfig
index dfd95889f4b7..33a458b7f1fc 100644
--- a/drivers/gpu/drm/i915/Kconfig
+++ b/drivers/gpu/drm/i915/Kconfig
@@ -23,6 +23,8 @@ config DRM_I915
 	select SYNC_FILE
 	select IOSF_MBI
 	select CRC32
+	select SND_HDA_I915 if SND_HDA_CORE
+	select CEC_CORE if CEC_NOTIFIER
 	help
 	  Choose this option if you have a system that has "Intel Graphics
 	  Media Accelerator" or "HD Graphics" integrated graphics,
diff --git a/drivers/gpu/drm/i915/Kconfig.debug b/drivers/gpu/drm/i915/Kconfig.debug
index 9de8b1c51a5c..459f8f88a34c 100644
--- a/drivers/gpu/drm/i915/Kconfig.debug
+++ b/drivers/gpu/drm/i915/Kconfig.debug
@@ -51,6 +51,18 @@ config DRM_I915_DEBUG_GEM
 
           If in doubt, say "N".
 
+config DRM_I915_ERRLOG_GEM
+	bool "Insert extra logging (very verbose) for common GEM errors"
+	default n
+	depends on DRM_I915_DEBUG_GEM
+	help
+	  Enable additional logging that may help track down the cause of
+	  principally userspace errors.
+
+	  Recommended for driver developers only.
+
+	  If in doubt, say "N".
+
 config DRM_I915_TRACE_GEM
 	bool "Insert extra ftrace output from the GEM internals"
 	depends on DRM_I915_DEBUG_GEM
diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile
index 4c6adae23e18..5794f102f9b8 100644
--- a/drivers/gpu/drm/i915/Makefile
+++ b/drivers/gpu/drm/i915/Makefile
@@ -135,15 +135,14 @@ i915-y += dvo_ch7017.o \
 	  dvo_ns2501.o \
 	  dvo_sil164.o \
 	  dvo_tfp410.o \
+	  icl_dsi.o \
 	  intel_crt.o \
 	  intel_ddi.o \
 	  intel_dp_aux_backlight.o \
 	  intel_dp_link_training.o \
 	  intel_dp_mst.o \
 	  intel_dp.o \
-	  intel_dsi.o \
 	  intel_dsi_dcs_backlight.o \
-	  intel_dsi_pll.o \
 	  intel_dsi_vbt.o \
 	  intel_dvo.o \
 	  intel_hdmi.o \
@@ -152,7 +151,9 @@ i915-y += dvo_ch7017.o \
 	  intel_lvds.o \
 	  intel_panel.o \
 	  intel_sdvo.o \
-	  intel_tv.o
+	  intel_tv.o \
+	  vlv_dsi.o \
+	  vlv_dsi_pll.o
 
 # Post-mortem debug and GPU hang state capture
 i915-$(CONFIG_DRM_I915_CAPTURE_ERROR) += i915_gpu_error.o
diff --git a/drivers/gpu/drm/i915/gvt/aperture_gm.c b/drivers/gpu/drm/i915/gvt/aperture_gm.c
index 380eeb2a0e83..fe754022e356 100644
--- a/drivers/gpu/drm/i915/gvt/aperture_gm.c
+++ b/drivers/gpu/drm/i915/gvt/aperture_gm.c
@@ -131,7 +131,7 @@ void intel_vgpu_write_fence(struct intel_vgpu *vgpu,
 
 	assert_rpm_wakelock_held(dev_priv);
 
-	if (WARN_ON(fence > vgpu_fence_sz(vgpu)))
+	if (WARN_ON(fence >= vgpu_fence_sz(vgpu)))
 		return;
 
 	reg = vgpu->fence.regs[fence];
diff --git a/drivers/gpu/drm/i915/gvt/cmd_parser.c b/drivers/gpu/drm/i915/gvt/cmd_parser.c
index 0651e63b25fb..a614db310ea2 100644
--- a/drivers/gpu/drm/i915/gvt/cmd_parser.c
+++ b/drivers/gpu/drm/i915/gvt/cmd_parser.c
@@ -863,6 +863,7 @@ static int cmd_reg_handler(struct parser_exec_state *s,
 {
 	struct intel_vgpu *vgpu = s->vgpu;
 	struct intel_gvt *gvt = vgpu->gvt;
+	u32 ctx_sr_ctl;
 
 	if (offset + 4 > gvt->device_info.mmio_size) {
 		gvt_vgpu_err("%s access to (%x) outside of MMIO range\n",
@@ -873,7 +874,7 @@ static int cmd_reg_handler(struct parser_exec_state *s,
 	if (!intel_gvt_mmio_is_cmd_access(gvt, offset)) {
 		gvt_vgpu_err("%s access to non-render register (%x)\n",
 				cmd, offset);
-		return 0;
+		return -EBADRQC;
 	}
 
 	if (is_shadowed_mmio(offset)) {
@@ -895,6 +896,28 @@ static int cmd_reg_handler(struct parser_exec_state *s,
 		patch_value(s, cmd_ptr(s, index), VGT_PVINFO_PAGE);
 	}
 
+	/* TODO
+	 * Right now only scan LRI command on KBL and in inhibit context.
+	 * It's good enough to support initializing mmio by lri command in
+	 * vgpu inhibit context on KBL.
+	 */
+	if (IS_KABYLAKE(s->vgpu->gvt->dev_priv) &&
+			intel_gvt_mmio_is_in_ctx(gvt, offset) &&
+			!strncmp(cmd, "lri", 3)) {
+		intel_gvt_hypervisor_read_gpa(s->vgpu,
+			s->workload->ring_context_gpa + 12, &ctx_sr_ctl, 4);
+		/* check inhibit context */
+		if (ctx_sr_ctl & 1) {
+			u32 data = cmd_val(s, index + 1);
+
+			if (intel_gvt_mmio_has_mode_mask(s->vgpu->gvt, offset))
+				intel_vgpu_mask_mmio_write(vgpu,
+							offset, &data, 4);
+			else
+				vgpu_vreg(vgpu, offset) = data;
+		}
+	}
+
 	/* TODO: Update the global mask if this MMIO is a masked-MMIO */
 	intel_gvt_mmio_set_cmd_accessed(gvt, offset);
 	return 0;
diff --git a/drivers/gpu/drm/i915/gvt/display.c b/drivers/gpu/drm/i915/gvt/display.c
index 6ee50cb328f8..3019dbc39aef 100644
--- a/drivers/gpu/drm/i915/gvt/display.c
+++ b/drivers/gpu/drm/i915/gvt/display.c
@@ -219,7 +219,7 @@ static void emulate_monitor_status_change(struct intel_vgpu *vgpu)
 			~(TRANS_DDI_BPC_MASK | TRANS_DDI_MODE_SELECT_MASK |
 			TRANS_DDI_PORT_MASK);
 		vgpu_vreg_t(vgpu, TRANS_DDI_FUNC_CTL(TRANSCODER_A)) |=
-			(TRANS_DDI_BPC_8 | TRANS_DDI_MODE_SELECT_DP_SST |
+			(TRANS_DDI_BPC_8 | TRANS_DDI_MODE_SELECT_DVI |
 			(PORT_B << TRANS_DDI_PORT_SHIFT) |
 			TRANS_DDI_FUNC_ENABLE);
 		if (IS_BROADWELL(dev_priv)) {
@@ -239,7 +239,7 @@ static void emulate_monitor_status_change(struct intel_vgpu *vgpu)
 			~(TRANS_DDI_BPC_MASK | TRANS_DDI_MODE_SELECT_MASK |
 			TRANS_DDI_PORT_MASK);
 		vgpu_vreg_t(vgpu, TRANS_DDI_FUNC_CTL(TRANSCODER_A)) |=
-			(TRANS_DDI_BPC_8 | TRANS_DDI_MODE_SELECT_DP_SST |
+			(TRANS_DDI_BPC_8 | TRANS_DDI_MODE_SELECT_DVI |
 			(PORT_C << TRANS_DDI_PORT_SHIFT) |
 			TRANS_DDI_FUNC_ENABLE);
 		if (IS_BROADWELL(dev_priv)) {
@@ -259,7 +259,7 @@ static void emulate_monitor_status_change(struct intel_vgpu *vgpu)
 			~(TRANS_DDI_BPC_MASK | TRANS_DDI_MODE_SELECT_MASK |
 			TRANS_DDI_PORT_MASK);
 		vgpu_vreg_t(vgpu, TRANS_DDI_FUNC_CTL(TRANSCODER_A)) |=
-			(TRANS_DDI_BPC_8 | TRANS_DDI_MODE_SELECT_DP_SST |
+			(TRANS_DDI_BPC_8 | TRANS_DDI_MODE_SELECT_DVI |
 			(PORT_D << TRANS_DDI_PORT_SHIFT) |
 			TRANS_DDI_FUNC_ENABLE);
 		if (IS_BROADWELL(dev_priv)) {
diff --git a/drivers/gpu/drm/i915/gvt/gtt.c b/drivers/gpu/drm/i915/gvt/gtt.c
index 642e216e0a5b..00aad8164dec 100644
--- a/drivers/gpu/drm/i915/gvt/gtt.c
+++ b/drivers/gpu/drm/i915/gvt/gtt.c
@@ -216,16 +216,22 @@ static struct gtt_type_table_entry gtt_type_table[] = {
 			GTT_TYPE_PPGTT_PDE_PT,
 			GTT_TYPE_PPGTT_PTE_PT,
 			GTT_TYPE_PPGTT_PTE_2M_ENTRY),
+	/* We take IPS bit as 'PSE' for PTE level. */
 	GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PTE_PT,
 			GTT_TYPE_PPGTT_PTE_4K_ENTRY,
 			GTT_TYPE_PPGTT_PTE_PT,
 			GTT_TYPE_INVALID,
-			GTT_TYPE_INVALID),
+			GTT_TYPE_PPGTT_PTE_64K_ENTRY),
 	GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PTE_4K_ENTRY,
 			GTT_TYPE_PPGTT_PTE_4K_ENTRY,
 			GTT_TYPE_PPGTT_PTE_PT,
 			GTT_TYPE_INVALID,
-			GTT_TYPE_INVALID),
+			GTT_TYPE_PPGTT_PTE_64K_ENTRY),
+	GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PTE_64K_ENTRY,
+			GTT_TYPE_PPGTT_PTE_4K_ENTRY,
+			GTT_TYPE_PPGTT_PTE_PT,
+			GTT_TYPE_INVALID,
+			GTT_TYPE_PPGTT_PTE_64K_ENTRY),
 	GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PTE_2M_ENTRY,
 			GTT_TYPE_PPGTT_PDE_ENTRY,
 			GTT_TYPE_PPGTT_PDE_PT,
@@ -339,8 +345,14 @@ static inline int gtt_set_entry64(void *pt,
 
 #define ADDR_1G_MASK	GENMASK_ULL(GTT_HAW - 1, 30)
 #define ADDR_2M_MASK	GENMASK_ULL(GTT_HAW - 1, 21)
+#define ADDR_64K_MASK	GENMASK_ULL(GTT_HAW - 1, 16)
 #define ADDR_4K_MASK	GENMASK_ULL(GTT_HAW - 1, 12)
 
+#define GTT_SPTE_FLAG_MASK GENMASK_ULL(62, 52)
+#define GTT_SPTE_FLAG_64K_SPLITED BIT(52) /* splited 64K gtt entry */
+
+#define GTT_64K_PTE_STRIDE 16
+
 static unsigned long gen8_gtt_get_pfn(struct intel_gvt_gtt_entry *e)
 {
 	unsigned long pfn;
@@ -349,6 +361,8 @@ static unsigned long gen8_gtt_get_pfn(struct intel_gvt_gtt_entry *e)
 		pfn = (e->val64 & ADDR_1G_MASK) >> PAGE_SHIFT;
 	else if (e->type == GTT_TYPE_PPGTT_PTE_2M_ENTRY)
 		pfn = (e->val64 & ADDR_2M_MASK) >> PAGE_SHIFT;
+	else if (e->type == GTT_TYPE_PPGTT_PTE_64K_ENTRY)
+		pfn = (e->val64 & ADDR_64K_MASK) >> PAGE_SHIFT;
 	else
 		pfn = (e->val64 & ADDR_4K_MASK) >> PAGE_SHIFT;
 	return pfn;
@@ -362,6 +376,9 @@ static void gen8_gtt_set_pfn(struct intel_gvt_gtt_entry *e, unsigned long pfn)
 	} else if (e->type == GTT_TYPE_PPGTT_PTE_2M_ENTRY) {
 		e->val64 &= ~ADDR_2M_MASK;
 		pfn &= (ADDR_2M_MASK >> PAGE_SHIFT);
+	} else if (e->type == GTT_TYPE_PPGTT_PTE_64K_ENTRY) {
+		e->val64 &= ~ADDR_64K_MASK;
+		pfn &= (ADDR_64K_MASK >> PAGE_SHIFT);
 	} else {
 		e->val64 &= ~ADDR_4K_MASK;
 		pfn &= (ADDR_4K_MASK >> PAGE_SHIFT);
@@ -372,16 +389,41 @@ static void gen8_gtt_set_pfn(struct intel_gvt_gtt_entry *e, unsigned long pfn)
 
 static bool gen8_gtt_test_pse(struct intel_gvt_gtt_entry *e)
 {
-	/* Entry doesn't have PSE bit. */
-	if (get_pse_type(e->type) == GTT_TYPE_INVALID)
-		return false;
+	return !!(e->val64 & _PAGE_PSE);
+}
 
-	e->type = get_entry_type(e->type);
-	if (!(e->val64 & _PAGE_PSE))
+static void gen8_gtt_clear_pse(struct intel_gvt_gtt_entry *e)
+{
+	if (gen8_gtt_test_pse(e)) {
+		switch (e->type) {
+		case GTT_TYPE_PPGTT_PTE_2M_ENTRY:
+			e->val64 &= ~_PAGE_PSE;
+			e->type = GTT_TYPE_PPGTT_PDE_ENTRY;
+			break;
+		case GTT_TYPE_PPGTT_PTE_1G_ENTRY:
+			e->type = GTT_TYPE_PPGTT_PDP_ENTRY;
+			e->val64 &= ~_PAGE_PSE;
+			break;
+		default:
+			WARN_ON(1);
+		}
+	}
+}
+
+static bool gen8_gtt_test_ips(struct intel_gvt_gtt_entry *e)
+{
+	if (GEM_WARN_ON(e->type != GTT_TYPE_PPGTT_PDE_ENTRY))
 		return false;
 
-	e->type = get_pse_type(e->type);
-	return true;
+	return !!(e->val64 & GEN8_PDE_IPS_64K);
+}
+
+static void gen8_gtt_clear_ips(struct intel_gvt_gtt_entry *e)
+{
+	if (GEM_WARN_ON(e->type != GTT_TYPE_PPGTT_PDE_ENTRY))
+		return;
+
+	e->val64 &= ~GEN8_PDE_IPS_64K;
 }
 
 static bool gen8_gtt_test_present(struct intel_gvt_gtt_entry *e)
@@ -408,6 +450,21 @@ static void gtt_entry_set_present(struct intel_gvt_gtt_entry *e)
 	e->val64 |= _PAGE_PRESENT;
 }
 
+static bool gen8_gtt_test_64k_splited(struct intel_gvt_gtt_entry *e)
+{
+	return !!(e->val64 & GTT_SPTE_FLAG_64K_SPLITED);
+}
+
+static void gen8_gtt_set_64k_splited(struct intel_gvt_gtt_entry *e)
+{
+	e->val64 |= GTT_SPTE_FLAG_64K_SPLITED;
+}
+
+static void gen8_gtt_clear_64k_splited(struct intel_gvt_gtt_entry *e)
+{
+	e->val64 &= ~GTT_SPTE_FLAG_64K_SPLITED;
+}
+
 /*
  * Per-platform GMA routines.
  */
@@ -440,6 +497,12 @@ static struct intel_gvt_gtt_pte_ops gen8_gtt_pte_ops = {
 	.set_present = gtt_entry_set_present,
 	.test_present = gen8_gtt_test_present,
 	.test_pse = gen8_gtt_test_pse,
+	.clear_pse = gen8_gtt_clear_pse,
+	.clear_ips = gen8_gtt_clear_ips,
+	.test_ips = gen8_gtt_test_ips,
+	.clear_64k_splited = gen8_gtt_clear_64k_splited,
+	.set_64k_splited = gen8_gtt_set_64k_splited,
+	.test_64k_splited = gen8_gtt_test_64k_splited,
 	.get_pfn = gen8_gtt_get_pfn,
 	.set_pfn = gen8_gtt_set_pfn,
 };
@@ -453,6 +516,27 @@ static struct intel_gvt_gtt_gma_ops gen8_gtt_gma_ops = {
 	.gma_to_pml4_index = gen8_gma_to_pml4_index,
 };
 
+/* Update entry type per pse and ips bit. */
+static void update_entry_type_for_real(struct intel_gvt_gtt_pte_ops *pte_ops,
+	struct intel_gvt_gtt_entry *entry, bool ips)
+{
+	switch (entry->type) {
+	case GTT_TYPE_PPGTT_PDE_ENTRY:
+	case GTT_TYPE_PPGTT_PDP_ENTRY:
+		if (pte_ops->test_pse(entry))
+			entry->type = get_pse_type(entry->type);
+		break;
+	case GTT_TYPE_PPGTT_PTE_4K_ENTRY:
+		if (ips)
+			entry->type = get_pse_type(entry->type);
+		break;
+	default:
+		GEM_BUG_ON(!gtt_type_is_entry(entry->type));
+	}
+
+	GEM_BUG_ON(entry->type == GTT_TYPE_INVALID);
+}
+
 /*
  * MM helpers.
  */
@@ -468,8 +552,7 @@ static void _ppgtt_get_root_entry(struct intel_vgpu_mm *mm,
 	pte_ops->get_entry(guest ? mm->ppgtt_mm.guest_pdps :
 			   mm->ppgtt_mm.shadow_pdps,
 			   entry, index, false, 0, mm->vgpu);
-
-	pte_ops->test_pse(entry);
+	update_entry_type_for_real(pte_ops, entry, false);
 }
 
 static inline void ppgtt_get_guest_root_entry(struct intel_vgpu_mm *mm,
@@ -574,7 +657,8 @@ static inline int ppgtt_spt_get_entry(
 	if (ret)
 		return ret;
 
-	ops->test_pse(e);
+	update_entry_type_for_real(ops, e, guest ?
+				   spt->guest_page.pde_ips : false);
 
 	gvt_vdbg_mm("read ppgtt entry, spt type %d, entry type %d, index %lu, value %llx\n",
 		    type, e->type, index, e->val64);
@@ -653,10 +737,12 @@ static void ppgtt_free_spt(struct intel_vgpu_ppgtt_spt *spt)
 
 	radix_tree_delete(&spt->vgpu->gtt.spt_tree, spt->shadow_page.mfn);
 
-	if (spt->guest_page.oos_page)
-		detach_oos_page(spt->vgpu, spt->guest_page.oos_page);
+	if (spt->guest_page.gfn) {
+		if (spt->guest_page.oos_page)
+			detach_oos_page(spt->vgpu, spt->guest_page.oos_page);
 
-	intel_vgpu_unregister_page_track(spt->vgpu, spt->guest_page.gfn);
+		intel_vgpu_unregister_page_track(spt->vgpu, spt->guest_page.gfn);
+	}
 
 	list_del_init(&spt->post_shadow_list);
 	free_spt(spt);
@@ -717,8 +803,9 @@ static inline struct intel_vgpu_ppgtt_spt *intel_vgpu_find_spt_by_mfn(
 
 static int reclaim_one_ppgtt_mm(struct intel_gvt *gvt);
 
+/* Allocate shadow page table without guest page. */
 static struct intel_vgpu_ppgtt_spt *ppgtt_alloc_spt(
-		struct intel_vgpu *vgpu, int type, unsigned long gfn)
+		struct intel_vgpu *vgpu, intel_gvt_gtt_type_t type)
 {
 	struct device *kdev = &vgpu->gvt->dev_priv->drm.pdev->dev;
 	struct intel_vgpu_ppgtt_spt *spt = NULL;
@@ -753,26 +840,12 @@ retry:
 	spt->shadow_page.vaddr = page_address(spt->shadow_page.page);
 	spt->shadow_page.mfn = daddr >> I915_GTT_PAGE_SHIFT;
 
-	/*
-	 * Init guest_page.
-	 */
-	spt->guest_page.type = type;
-	spt->guest_page.gfn = gfn;
-
-	ret = intel_vgpu_register_page_track(vgpu, spt->guest_page.gfn,
-					ppgtt_write_protection_handler, spt);
-	if (ret)
-		goto err_unmap_dma;
-
 	ret = radix_tree_insert(&vgpu->gtt.spt_tree, spt->shadow_page.mfn, spt);
 	if (ret)
-		goto err_unreg_page_track;
+		goto err_unmap_dma;
 
-	trace_spt_alloc(vgpu->id, spt, type, spt->shadow_page.mfn, gfn);
 	return spt;
 
-err_unreg_page_track:
-	intel_vgpu_unregister_page_track(vgpu, spt->guest_page.gfn);
 err_unmap_dma:
 	dma_unmap_page(kdev, daddr, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
 err_free_spt:
@@ -780,6 +853,37 @@ err_free_spt:
 	return ERR_PTR(ret);
 }
 
+/* Allocate shadow page table associated with specific gfn. */
+static struct intel_vgpu_ppgtt_spt *ppgtt_alloc_spt_gfn(
+		struct intel_vgpu *vgpu, intel_gvt_gtt_type_t type,
+		unsigned long gfn, bool guest_pde_ips)
+{
+	struct intel_vgpu_ppgtt_spt *spt;
+	int ret;
+
+	spt = ppgtt_alloc_spt(vgpu, type);
+	if (IS_ERR(spt))
+		return spt;
+
+	/*
+	 * Init guest_page.
+	 */
+	ret = intel_vgpu_register_page_track(vgpu, gfn,
+			ppgtt_write_protection_handler, spt);
+	if (ret) {
+		ppgtt_free_spt(spt);
+		return ERR_PTR(ret);
+	}
+
+	spt->guest_page.type = type;
+	spt->guest_page.gfn = gfn;
+	spt->guest_page.pde_ips = guest_pde_ips;
+
+	trace_spt_alloc(vgpu->id, spt, type, spt->shadow_page.mfn, gfn);
+
+	return spt;
+}
+
 #define pt_entry_size_shift(spt) \
 	((spt)->vgpu->gvt->device_info.gtt_entry_size_shift)
 
@@ -787,24 +891,38 @@ err_free_spt:
 	(I915_GTT_PAGE_SIZE >> pt_entry_size_shift(spt))
 
 #define for_each_present_guest_entry(spt, e, i) \
-	for (i = 0; i < pt_entries(spt); i++) \
+	for (i = 0; i < pt_entries(spt); \
+	     i += spt->guest_page.pde_ips ? GTT_64K_PTE_STRIDE : 1) \
 		if (!ppgtt_get_guest_entry(spt, e, i) && \
 		    spt->vgpu->gvt->gtt.pte_ops->test_present(e))
 
 #define for_each_present_shadow_entry(spt, e, i) \
-	for (i = 0; i < pt_entries(spt); i++) \
+	for (i = 0; i < pt_entries(spt); \
+	     i += spt->shadow_page.pde_ips ? GTT_64K_PTE_STRIDE : 1) \
 		if (!ppgtt_get_shadow_entry(spt, e, i) && \
 		    spt->vgpu->gvt->gtt.pte_ops->test_present(e))
 
-static void ppgtt_get_spt(struct intel_vgpu_ppgtt_spt *spt)
+#define for_each_shadow_entry(spt, e, i) \
+	for (i = 0; i < pt_entries(spt); \
+	     i += (spt->shadow_page.pde_ips ? GTT_64K_PTE_STRIDE : 1)) \
+		if (!ppgtt_get_shadow_entry(spt, e, i))
+
+static inline void ppgtt_get_spt(struct intel_vgpu_ppgtt_spt *spt)
 {
 	int v = atomic_read(&spt->refcount);
 
 	trace_spt_refcount(spt->vgpu->id, "inc", spt, v, (v + 1));
-
 	atomic_inc(&spt->refcount);
 }
 
+static inline int ppgtt_put_spt(struct intel_vgpu_ppgtt_spt *spt)
+{
+	int v = atomic_read(&spt->refcount);
+
+	trace_spt_refcount(spt->vgpu->id, "dec", spt, v, (v - 1));
+	return atomic_dec_return(&spt->refcount);
+}
+
 static int ppgtt_invalidate_spt(struct intel_vgpu_ppgtt_spt *spt);
 
 static int ppgtt_invalidate_spt_by_shadow_entry(struct intel_vgpu *vgpu,
@@ -843,7 +961,8 @@ static inline void ppgtt_invalidate_pte(struct intel_vgpu_ppgtt_spt *spt,
 	pfn = ops->get_pfn(entry);
 	type = spt->shadow_page.type;
 
-	if (pfn == vgpu->gtt.scratch_pt[type].page_mfn)
+	/* Uninitialized spte or unshadowed spte. */
+	if (!pfn || pfn == vgpu->gtt.scratch_pt[type].page_mfn)
 		return;
 
 	intel_gvt_hypervisor_dma_unmap_guest_page(vgpu, pfn << PAGE_SHIFT);
@@ -855,14 +974,11 @@ static int ppgtt_invalidate_spt(struct intel_vgpu_ppgtt_spt *spt)
 	struct intel_gvt_gtt_entry e;
 	unsigned long index;
 	int ret;
-	int v = atomic_read(&spt->refcount);
 
 	trace_spt_change(spt->vgpu->id, "die", spt,
 			spt->guest_page.gfn, spt->shadow_page.type);
 
-	trace_spt_refcount(spt->vgpu->id, "dec", spt, v, (v - 1));
-
-	if (atomic_dec_return(&spt->refcount) > 0)
+	if (ppgtt_put_spt(spt) > 0)
 		return 0;
 
 	for_each_present_shadow_entry(spt, &e, index) {
@@ -871,9 +987,15 @@ static int ppgtt_invalidate_spt(struct intel_vgpu_ppgtt_spt *spt)
 			gvt_vdbg_mm("invalidate 4K entry\n");
 			ppgtt_invalidate_pte(spt, &e);
 			break;
+		case GTT_TYPE_PPGTT_PTE_64K_ENTRY:
+			/* We don't setup 64K shadow entry so far. */
+			WARN(1, "suspicious 64K gtt entry\n");
+			continue;
 		case GTT_TYPE_PPGTT_PTE_2M_ENTRY:
+			gvt_vdbg_mm("invalidate 2M entry\n");
+			continue;
 		case GTT_TYPE_PPGTT_PTE_1G_ENTRY:
-			WARN(1, "GVT doesn't support 2M/1GB page\n");
+			WARN(1, "GVT doesn't support 1GB page\n");
 			continue;
 		case GTT_TYPE_PPGTT_PML4_ENTRY:
 		case GTT_TYPE_PPGTT_PDP_ENTRY:
@@ -899,6 +1021,22 @@ fail:
 	return ret;
 }
 
+static bool vgpu_ips_enabled(struct intel_vgpu *vgpu)
+{
+	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+
+	if (INTEL_GEN(dev_priv) == 9 || INTEL_GEN(dev_priv) == 10) {
+		u32 ips = vgpu_vreg_t(vgpu, GEN8_GAMW_ECO_DEV_RW_IA) &
+			GAMW_ECO_ENABLE_64K_IPS_FIELD;
+
+		return ips == GAMW_ECO_ENABLE_64K_IPS_FIELD;
+	} else if (INTEL_GEN(dev_priv) >= 11) {
+		/* 64K paging only controlled by IPS bit in PTE now. */
+		return true;
+	} else
+		return false;
+}
+
 static int ppgtt_populate_spt(struct intel_vgpu_ppgtt_spt *spt);
 
 static struct intel_vgpu_ppgtt_spt *ppgtt_populate_spt_by_guest_entry(
@@ -906,35 +1044,54 @@ static struct intel_vgpu_ppgtt_spt *ppgtt_populate_spt_by_guest_entry(
 {
 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
 	struct intel_vgpu_ppgtt_spt *spt = NULL;
+	bool ips = false;
 	int ret;
 
 	GEM_BUG_ON(!gtt_type_is_pt(get_next_pt_type(we->type)));
 
+	if (we->type == GTT_TYPE_PPGTT_PDE_ENTRY)
+		ips = vgpu_ips_enabled(vgpu) && ops->test_ips(we);
+
 	spt = intel_vgpu_find_spt_by_gfn(vgpu, ops->get_pfn(we));
-	if (spt)
+	if (spt) {
 		ppgtt_get_spt(spt);
-	else {
+
+		if (ips != spt->guest_page.pde_ips) {
+			spt->guest_page.pde_ips = ips;
+
+			gvt_dbg_mm("reshadow PDE since ips changed\n");
+			clear_page(spt->shadow_page.vaddr);
+			ret = ppgtt_populate_spt(spt);
+			if (ret) {
+				ppgtt_put_spt(spt);
+				goto err;
+			}
+		}
+	} else {
 		int type = get_next_pt_type(we->type);
 
-		spt = ppgtt_alloc_spt(vgpu, type, ops->get_pfn(we));
+		spt = ppgtt_alloc_spt_gfn(vgpu, type, ops->get_pfn(we), ips);
 		if (IS_ERR(spt)) {
 			ret = PTR_ERR(spt);
-			goto fail;
+			goto err;
 		}
 
 		ret = intel_vgpu_enable_page_track(vgpu, spt->guest_page.gfn);
 		if (ret)
-			goto fail;
+			goto err_free_spt;
 
 		ret = ppgtt_populate_spt(spt);
 		if (ret)
-			goto fail;
+			goto err_free_spt;
 
 		trace_spt_change(vgpu->id, "new", spt, spt->guest_page.gfn,
 				 spt->shadow_page.type);
 	}
 	return spt;
-fail:
+
+err_free_spt:
+	ppgtt_free_spt(spt);
+err:
 	gvt_vgpu_err("fail: shadow page %p guest entry 0x%llx type %d\n",
 		     spt, we->val64, we->type);
 	return ERR_PTR(ret);
@@ -948,16 +1105,118 @@ static inline void ppgtt_generate_shadow_entry(struct intel_gvt_gtt_entry *se,
 	se->type = ge->type;
 	se->val64 = ge->val64;
 
+	/* Because we always split 64KB pages, so clear IPS in shadow PDE. */
+	if (se->type == GTT_TYPE_PPGTT_PDE_ENTRY)
+		ops->clear_ips(se);
+
 	ops->set_pfn(se, s->shadow_page.mfn);
 }
 
+/**
+ * Return 1 if 2MB huge gtt shadowing is possilbe, 0 if miscondition,
+ * negtive if found err.
+ */
+static int is_2MB_gtt_possible(struct intel_vgpu *vgpu,
+	struct intel_gvt_gtt_entry *entry)
+{
+	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
+	unsigned long pfn;
+
+	if (!HAS_PAGE_SIZES(vgpu->gvt->dev_priv, I915_GTT_PAGE_SIZE_2M))
+		return 0;
+
+	pfn = intel_gvt_hypervisor_gfn_to_mfn(vgpu, ops->get_pfn(entry));
+	if (pfn == INTEL_GVT_INVALID_ADDR)
+		return -EINVAL;
+
+	return PageTransHuge(pfn_to_page(pfn));
+}
+
+static int split_2MB_gtt_entry(struct intel_vgpu *vgpu,
+	struct intel_vgpu_ppgtt_spt *spt, unsigned long index,
+	struct intel_gvt_gtt_entry *se)
+{
+	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
+	struct intel_vgpu_ppgtt_spt *sub_spt;
+	struct intel_gvt_gtt_entry sub_se;
+	unsigned long start_gfn;
+	dma_addr_t dma_addr;
+	unsigned long sub_index;
+	int ret;
+
+	gvt_dbg_mm("Split 2M gtt entry, index %lu\n", index);
+
+	start_gfn = ops->get_pfn(se);
+
+	sub_spt = ppgtt_alloc_spt(vgpu, GTT_TYPE_PPGTT_PTE_PT);
+	if (IS_ERR(sub_spt))
+		return PTR_ERR(sub_spt);
+
+	for_each_shadow_entry(sub_spt, &sub_se, sub_index) {
+		ret = intel_gvt_hypervisor_dma_map_guest_page(vgpu,
+				start_gfn + sub_index, PAGE_SIZE, &dma_addr);
+		if (ret) {
+			ppgtt_invalidate_spt(spt);
+			return ret;
+		}
+		sub_se.val64 = se->val64;
+
+		/* Copy the PAT field from PDE. */
+		sub_se.val64 &= ~_PAGE_PAT;
+		sub_se.val64 |= (se->val64 & _PAGE_PAT_LARGE) >> 5;
+
+		ops->set_pfn(&sub_se, dma_addr >> PAGE_SHIFT);
+		ppgtt_set_shadow_entry(sub_spt, &sub_se, sub_index);
+	}
+
+	/* Clear dirty field. */
+	se->val64 &= ~_PAGE_DIRTY;
+
+	ops->clear_pse(se);
+	ops->clear_ips(se);
+	ops->set_pfn(se, sub_spt->shadow_page.mfn);
+	ppgtt_set_shadow_entry(spt, se, index);
+	return 0;
+}
+
+static int split_64KB_gtt_entry(struct intel_vgpu *vgpu,
+	struct intel_vgpu_ppgtt_spt *spt, unsigned long index,
+	struct intel_gvt_gtt_entry *se)
+{
+	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
+	struct intel_gvt_gtt_entry entry = *se;
+	unsigned long start_gfn;
+	dma_addr_t dma_addr;
+	int i, ret;
+
+	gvt_vdbg_mm("Split 64K gtt entry, index %lu\n", index);
+
+	GEM_BUG_ON(index % GTT_64K_PTE_STRIDE);
+
+	start_gfn = ops->get_pfn(se);
+
+	entry.type = GTT_TYPE_PPGTT_PTE_4K_ENTRY;
+	ops->set_64k_splited(&entry);
+
+	for (i = 0; i < GTT_64K_PTE_STRIDE; i++) {
+		ret = intel_gvt_hypervisor_dma_map_guest_page(vgpu,
+					start_gfn + i, PAGE_SIZE, &dma_addr);
+		if (ret)
+			return ret;
+
+		ops->set_pfn(&entry, dma_addr >> PAGE_SHIFT);
+		ppgtt_set_shadow_entry(spt, &entry, index + i);
+	}
+	return 0;
+}
+
 static int ppgtt_populate_shadow_entry(struct intel_vgpu *vgpu,
 	struct intel_vgpu_ppgtt_spt *spt, unsigned long index,
 	struct intel_gvt_gtt_entry *ge)
 {
 	struct intel_gvt_gtt_pte_ops *pte_ops = vgpu->gvt->gtt.pte_ops;
 	struct intel_gvt_gtt_entry se = *ge;
-	unsigned long gfn;
+	unsigned long gfn, page_size = PAGE_SIZE;
 	dma_addr_t dma_addr;
 	int ret;
 
@@ -970,16 +1229,33 @@ static int ppgtt_populate_shadow_entry(struct intel_vgpu *vgpu,
 	case GTT_TYPE_PPGTT_PTE_4K_ENTRY:
 		gvt_vdbg_mm("shadow 4K gtt entry\n");
 		break;
+	case GTT_TYPE_PPGTT_PTE_64K_ENTRY:
+		gvt_vdbg_mm("shadow 64K gtt entry\n");
+		/*
+		 * The layout of 64K page is special, the page size is
+		 * controlled by uper PDE. To be simple, we always split
+		 * 64K page to smaller 4K pages in shadow PT.
+		 */
+		return split_64KB_gtt_entry(vgpu, spt, index, &se);
 	case GTT_TYPE_PPGTT_PTE_2M_ENTRY:
+		gvt_vdbg_mm("shadow 2M gtt entry\n");
+		ret = is_2MB_gtt_possible(vgpu, ge);
+		if (ret == 0)
+			return split_2MB_gtt_entry(vgpu, spt, index, &se);
+		else if (ret < 0)
+			return ret;
+		page_size = I915_GTT_PAGE_SIZE_2M;
+		break;
 	case GTT_TYPE_PPGTT_PTE_1G_ENTRY:
-		gvt_vgpu_err("GVT doesn't support 2M/1GB entry\n");
+		gvt_vgpu_err("GVT doesn't support 1GB entry\n");
 		return -EINVAL;
 	default:
 		GEM_BUG_ON(1);
 	};
 
 	/* direct shadow */
-	ret = intel_gvt_hypervisor_dma_map_guest_page(vgpu, gfn, &dma_addr);
+	ret = intel_gvt_hypervisor_dma_map_guest_page(vgpu, gfn, page_size,
+						      &dma_addr);
 	if (ret)
 		return -ENXIO;
 
@@ -1062,8 +1338,12 @@ static int ppgtt_handle_guest_entry_removal(struct intel_vgpu_ppgtt_spt *spt,
 		ret = ppgtt_invalidate_spt(s);
 		if (ret)
 			goto fail;
-	} else
+	} else {
+		/* We don't setup 64K shadow entry so far. */
+		WARN(se->type == GTT_TYPE_PPGTT_PTE_64K_ENTRY,
+		     "suspicious 64K entry\n");
 		ppgtt_invalidate_pte(spt, se);
+	}
 
 	return 0;
 fail:
@@ -1286,7 +1566,7 @@ static int ppgtt_handle_guest_write_page_table(
 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
 	struct intel_gvt_gtt_entry old_se;
 	int new_present;
-	int ret;
+	int i, ret;
 
 	new_present = ops->test_present(we);
 
@@ -1308,8 +1588,27 @@ static int ppgtt_handle_guest_write_page_table(
 		goto fail;
 
 	if (!new_present) {
-		ops->set_pfn(&old_se, vgpu->gtt.scratch_pt[type].page_mfn);
-		ppgtt_set_shadow_entry(spt, &old_se, index);
+		/* For 64KB splited entries, we need clear them all. */
+		if (ops->test_64k_splited(&old_se) &&
+		    !(index % GTT_64K_PTE_STRIDE)) {
+			gvt_vdbg_mm("remove splited 64K shadow entries\n");
+			for (i = 0; i < GTT_64K_PTE_STRIDE; i++) {
+				ops->clear_64k_splited(&old_se);
+				ops->set_pfn(&old_se,
+					vgpu->gtt.scratch_pt[type].page_mfn);
+				ppgtt_set_shadow_entry(spt, &old_se, index + i);
+			}
+		} else if (old_se.type == GTT_TYPE_PPGTT_PTE_2M_ENTRY ||
+			   old_se.type == GTT_TYPE_PPGTT_PTE_1G_ENTRY) {
+			ops->clear_pse(&old_se);
+			ops->set_pfn(&old_se,
+				     vgpu->gtt.scratch_pt[type].page_mfn);
+			ppgtt_set_shadow_entry(spt, &old_se, index);
+		} else {
+			ops->set_pfn(&old_se,
+				     vgpu->gtt.scratch_pt[type].page_mfn);
+			ppgtt_set_shadow_entry(spt, &old_se, index);
+		}
 	}
 
 	return 0;
@@ -1391,7 +1690,17 @@ static int ppgtt_handle_guest_write_page_table_bytes(
 
 	ppgtt_get_guest_entry(spt, &we, index);
 
-	ops->test_pse(&we);
+	/*
+	 * For page table which has 64K gtt entry, only PTE#0, PTE#16,
+	 * PTE#32, ... PTE#496 are used. Unused PTEs update should be
+	 * ignored.
+	 */
+	if (we.type == GTT_TYPE_PPGTT_PTE_64K_ENTRY &&
+	    (index % GTT_64K_PTE_STRIDE)) {
+		gvt_vdbg_mm("Ignore write to unused PTE entry, index %lu\n",
+			    index);
+		return 0;
+	}
 
 	if (bytes == info->gtt_entry_size) {
 		ret = ppgtt_handle_guest_write_page_table(spt, &we, index);
@@ -1592,6 +1901,7 @@ static struct intel_vgpu_mm *intel_vgpu_create_ggtt_mm(struct intel_vgpu *vgpu)
 		vgpu_free_mm(mm);
 		return ERR_PTR(-ENOMEM);
 	}
+	mm->ggtt_mm.last_partial_off = -1UL;
 
 	return mm;
 }
@@ -1616,6 +1926,7 @@ void _intel_vgpu_mm_release(struct kref *mm_ref)
 		invalidate_ppgtt_mm(mm);
 	} else {
 		vfree(mm->ggtt_mm.virtual_ggtt);
+		mm->ggtt_mm.last_partial_off = -1UL;
 	}
 
 	vgpu_free_mm(mm);
@@ -1868,6 +2179,62 @@ static int emulate_ggtt_mmio_write(struct intel_vgpu *vgpu, unsigned int off,
 	memcpy((void *)&e.val64 + (off & (info->gtt_entry_size - 1)), p_data,
 			bytes);
 
+	/* If ggtt entry size is 8 bytes, and it's split into two 4 bytes
+	 * write, we assume the two 4 bytes writes are consecutive.
+	 * Otherwise, we abort and report error
+	 */
+	if (bytes < info->gtt_entry_size) {
+		if (ggtt_mm->ggtt_mm.last_partial_off == -1UL) {
+			/* the first partial part*/
+			ggtt_mm->ggtt_mm.last_partial_off = off;
+			ggtt_mm->ggtt_mm.last_partial_data = e.val64;
+			return 0;
+		} else if ((g_gtt_index ==
+				(ggtt_mm->ggtt_mm.last_partial_off >>
+				info->gtt_entry_size_shift)) &&
+			(off !=	ggtt_mm->ggtt_mm.last_partial_off)) {
+			/* the second partial part */
+
+			int last_off = ggtt_mm->ggtt_mm.last_partial_off &
+				(info->gtt_entry_size - 1);
+
+			memcpy((void *)&e.val64 + last_off,
+				(void *)&ggtt_mm->ggtt_mm.last_partial_data +
+				last_off, bytes);
+
+			ggtt_mm->ggtt_mm.last_partial_off = -1UL;
+		} else {
+			int last_offset;
+
+			gvt_vgpu_err("failed to populate guest ggtt entry: abnormal ggtt entry write sequence, last_partial_off=%lx, offset=%x, bytes=%d, ggtt entry size=%d\n",
+					ggtt_mm->ggtt_mm.last_partial_off, off,
+					bytes, info->gtt_entry_size);
+
+			/* set host ggtt entry to scratch page and clear
+			 * virtual ggtt entry as not present for last
+			 * partially write offset
+			 */
+			last_offset = ggtt_mm->ggtt_mm.last_partial_off &
+					(~(info->gtt_entry_size - 1));
+
+			ggtt_get_host_entry(ggtt_mm, &m, last_offset);
+			ggtt_invalidate_pte(vgpu, &m);
+			ops->set_pfn(&m, gvt->gtt.scratch_mfn);
+			ops->clear_present(&m);
+			ggtt_set_host_entry(ggtt_mm, &m, last_offset);
+			ggtt_invalidate(gvt->dev_priv);
+
+			ggtt_get_guest_entry(ggtt_mm, &e, last_offset);
+			ops->clear_present(&e);
+			ggtt_set_guest_entry(ggtt_mm, &e, last_offset);
+
+			ggtt_mm->ggtt_mm.last_partial_off = off;
+			ggtt_mm->ggtt_mm.last_partial_data = e.val64;
+
+			return 0;
+		}
+	}
+
 	if (ops->test_present(&e)) {
 		gfn = ops->get_pfn(&e);
 		m = e;
@@ -1881,7 +2248,7 @@ static int emulate_ggtt_mmio_write(struct intel_vgpu *vgpu, unsigned int off,
 		}
 
 		ret = intel_gvt_hypervisor_dma_map_guest_page(vgpu, gfn,
-							      &dma_addr);
+							PAGE_SIZE, &dma_addr);
 		if (ret) {
 			gvt_vgpu_err("fail to populate guest ggtt entry\n");
 			/* guest driver may read/write the entry when partial
diff --git a/drivers/gpu/drm/i915/gvt/gtt.h b/drivers/gpu/drm/i915/gvt/gtt.h
index 3792f2b7f4ff..7a9b36176efb 100644
--- a/drivers/gpu/drm/i915/gvt/gtt.h
+++ b/drivers/gpu/drm/i915/gvt/gtt.h
@@ -63,6 +63,12 @@ struct intel_gvt_gtt_pte_ops {
 	void (*clear_present)(struct intel_gvt_gtt_entry *e);
 	void (*set_present)(struct intel_gvt_gtt_entry *e);
 	bool (*test_pse)(struct intel_gvt_gtt_entry *e);
+	void (*clear_pse)(struct intel_gvt_gtt_entry *e);
+	bool (*test_ips)(struct intel_gvt_gtt_entry *e);
+	void (*clear_ips)(struct intel_gvt_gtt_entry *e);
+	bool (*test_64k_splited)(struct intel_gvt_gtt_entry *e);
+	void (*clear_64k_splited)(struct intel_gvt_gtt_entry *e);
+	void (*set_64k_splited)(struct intel_gvt_gtt_entry *e);
 	void (*set_pfn)(struct intel_gvt_gtt_entry *e, unsigned long pfn);
 	unsigned long (*get_pfn)(struct intel_gvt_gtt_entry *e);
 };
@@ -95,6 +101,7 @@ typedef enum {
 	GTT_TYPE_GGTT_PTE,
 
 	GTT_TYPE_PPGTT_PTE_4K_ENTRY,
+	GTT_TYPE_PPGTT_PTE_64K_ENTRY,
 	GTT_TYPE_PPGTT_PTE_2M_ENTRY,
 	GTT_TYPE_PPGTT_PTE_1G_ENTRY,
 
@@ -150,6 +157,8 @@ struct intel_vgpu_mm {
 		} ppgtt_mm;
 		struct {
 			void *virtual_ggtt;
+			unsigned long last_partial_off;
+			u64 last_partial_data;
 		} ggtt_mm;
 	};
 };
@@ -220,6 +229,7 @@ struct intel_vgpu_ppgtt_spt {
 
 	struct {
 		intel_gvt_gtt_type_t type;
+		bool pde_ips; /* for 64KB PTEs */
 		void *vaddr;
 		struct page *page;
 		unsigned long mfn;
@@ -227,6 +237,7 @@ struct intel_vgpu_ppgtt_spt {
 
 	struct {
 		intel_gvt_gtt_type_t type;
+		bool pde_ips; /* for 64KB PTEs */
 		unsigned long gfn;
 		unsigned long write_cnt;
 		struct intel_vgpu_oos_page *oos_page;
diff --git a/drivers/gpu/drm/i915/gvt/gvt.c b/drivers/gpu/drm/i915/gvt/gvt.c
index 4e65266e7b95..46c8b720e336 100644
--- a/drivers/gpu/drm/i915/gvt/gvt.c
+++ b/drivers/gpu/drm/i915/gvt/gvt.c
@@ -176,6 +176,7 @@ static const struct intel_gvt_ops intel_gvt_ops = {
 	.emulate_mmio_write = intel_vgpu_emulate_mmio_write,
 	.vgpu_create = intel_gvt_create_vgpu,
 	.vgpu_destroy = intel_gvt_destroy_vgpu,
+	.vgpu_release = intel_gvt_release_vgpu,
 	.vgpu_reset = intel_gvt_reset_vgpu,
 	.vgpu_activate = intel_gvt_activate_vgpu,
 	.vgpu_deactivate = intel_gvt_deactivate_vgpu,
@@ -315,6 +316,11 @@ void intel_gvt_clean_device(struct drm_i915_private *dev_priv)
 	if (WARN_ON(!gvt))
 		return;
 
+	intel_gvt_destroy_idle_vgpu(gvt->idle_vgpu);
+	intel_gvt_hypervisor_host_exit(&dev_priv->drm.pdev->dev, gvt);
+	intel_gvt_cleanup_vgpu_type_groups(gvt);
+	intel_gvt_clean_vgpu_types(gvt);
+
 	intel_gvt_debugfs_clean(gvt);
 	clean_service_thread(gvt);
 	intel_gvt_clean_cmd_parser(gvt);
@@ -322,17 +328,10 @@ void intel_gvt_clean_device(struct drm_i915_private *dev_priv)
 	intel_gvt_clean_workload_scheduler(gvt);
 	intel_gvt_clean_gtt(gvt);
 	intel_gvt_clean_irq(gvt);
-	intel_gvt_clean_mmio_info(gvt);
 	intel_gvt_free_firmware(gvt);
-
-	intel_gvt_hypervisor_host_exit(&dev_priv->drm.pdev->dev, gvt);
-	intel_gvt_cleanup_vgpu_type_groups(gvt);
-	intel_gvt_clean_vgpu_types(gvt);
-
+	intel_gvt_clean_mmio_info(gvt);
 	idr_destroy(&gvt->vgpu_idr);
 
-	intel_gvt_destroy_idle_vgpu(gvt->idle_vgpu);
-
 	kfree(dev_priv->gvt);
 	dev_priv->gvt = NULL;
 }
@@ -468,3 +467,7 @@ out_clean_idr:
 	kfree(gvt);
 	return ret;
 }
+
+#if IS_ENABLED(CONFIG_DRM_I915_GVT_KVMGT)
+MODULE_SOFTDEP("pre: kvmgt");
+#endif
diff --git a/drivers/gpu/drm/i915/gvt/gvt.h b/drivers/gpu/drm/i915/gvt/gvt.h
index de2a3a2580be..31f6cdbe5c42 100644
--- a/drivers/gpu/drm/i915/gvt/gvt.h
+++ b/drivers/gpu/drm/i915/gvt/gvt.h
@@ -274,6 +274,8 @@ struct intel_gvt_mmio {
 #define F_CMD_ACCESSED	(1 << 5)
 /* This reg could be accessed by unaligned address */
 #define F_UNALIGN	(1 << 6)
+/* This reg is saved/restored in context */
+#define F_IN_CTX	(1 << 7)
 
 	struct gvt_mmio_block *mmio_block;
 	unsigned int num_mmio_block;
@@ -484,6 +486,7 @@ void intel_gvt_destroy_idle_vgpu(struct intel_vgpu *vgpu);
 struct intel_vgpu *intel_gvt_create_vgpu(struct intel_gvt *gvt,
 					 struct intel_vgpu_type *type);
 void intel_gvt_destroy_vgpu(struct intel_vgpu *vgpu);
+void intel_gvt_release_vgpu(struct intel_vgpu *vgpu);
 void intel_gvt_reset_vgpu_locked(struct intel_vgpu *vgpu, bool dmlr,
 				 unsigned int engine_mask);
 void intel_gvt_reset_vgpu(struct intel_vgpu *vgpu);
@@ -561,7 +564,8 @@ struct intel_gvt_ops {
 				unsigned int);
 	struct intel_vgpu *(*vgpu_create)(struct intel_gvt *,
 				struct intel_vgpu_type *);
-	void (*vgpu_destroy)(struct intel_vgpu *);
+	void (*vgpu_destroy)(struct intel_vgpu *vgpu);
+	void (*vgpu_release)(struct intel_vgpu *vgpu);
 	void (*vgpu_reset)(struct intel_vgpu *);
 	void (*vgpu_activate)(struct intel_vgpu *);
 	void (*vgpu_deactivate)(struct intel_vgpu *);
@@ -655,6 +659,33 @@ static inline bool intel_gvt_mmio_has_mode_mask(
 	return gvt->mmio.mmio_attribute[offset >> 2] & F_MODE_MASK;
 }
 
+/**
+ * intel_gvt_mmio_is_in_ctx - check if a MMIO has in-ctx mask
+ * @gvt: a GVT device
+ * @offset: register offset
+ *
+ * Returns:
+ * True if a MMIO has a in-context mask, false if it isn't.
+ *
+ */
+static inline bool intel_gvt_mmio_is_in_ctx(
+			struct intel_gvt *gvt, unsigned int offset)
+{
+	return gvt->mmio.mmio_attribute[offset >> 2] & F_IN_CTX;
+}
+
+/**
+ * intel_gvt_mmio_set_in_ctx - mask a MMIO in logical context
+ * @gvt: a GVT device
+ * @offset: register offset
+ *
+ */
+static inline void intel_gvt_mmio_set_in_ctx(
+			struct intel_gvt *gvt, unsigned int offset)
+{
+	gvt->mmio.mmio_attribute[offset >> 2] |= F_IN_CTX;
+}
+
 int intel_gvt_debugfs_add_vgpu(struct intel_vgpu *vgpu);
 void intel_gvt_debugfs_remove_vgpu(struct intel_vgpu *vgpu);
 int intel_gvt_debugfs_init(struct intel_gvt *gvt);
diff --git a/drivers/gpu/drm/i915/gvt/handlers.c b/drivers/gpu/drm/i915/gvt/handlers.c
index e39492aaff6c..7a58ca555197 100644
--- a/drivers/gpu/drm/i915/gvt/handlers.c
+++ b/drivers/gpu/drm/i915/gvt/handlers.c
@@ -210,6 +210,31 @@ static int sanitize_fence_mmio_access(struct intel_vgpu *vgpu,
 	return 0;
 }
 
+static int gamw_echo_dev_rw_ia_write(struct intel_vgpu *vgpu,
+		unsigned int offset, void *p_data, unsigned int bytes)
+{
+	u32 ips = (*(u32 *)p_data) & GAMW_ECO_ENABLE_64K_IPS_FIELD;
+
+	if (INTEL_GEN(vgpu->gvt->dev_priv) <= 10) {
+		if (ips == GAMW_ECO_ENABLE_64K_IPS_FIELD)
+			gvt_dbg_core("vgpu%d: ips enabled\n", vgpu->id);
+		else if (!ips)
+			gvt_dbg_core("vgpu%d: ips disabled\n", vgpu->id);
+		else {
+			/* All engines must be enabled together for vGPU,
+			 * since we don't know which engine the ppgtt will
+			 * bind to when shadowing.
+			 */
+			gvt_vgpu_err("Unsupported IPS setting %x, cannot enable 64K gtt.\n",
+				     ips);
+			return -EINVAL;
+		}
+	}
+
+	write_vreg(vgpu, offset, p_data, bytes);
+	return 0;
+}
+
 static int fence_mmio_read(struct intel_vgpu *vgpu, unsigned int off,
 		void *p_data, unsigned int bytes)
 {
@@ -1564,6 +1589,13 @@ static int bxt_gt_disp_pwron_write(struct intel_vgpu *vgpu,
 	return 0;
 }
 
+static int bxt_edp_psr_imr_iir_write(struct intel_vgpu *vgpu,
+		unsigned int offset, void *p_data, unsigned int bytes)
+{
+	vgpu_vreg(vgpu, offset) = 0;
+	return 0;
+}
+
 static int mmio_read_from_hw(struct intel_vgpu *vgpu,
 		unsigned int offset, void *p_data, unsigned int bytes)
 {
@@ -1774,7 +1806,9 @@ static int init_generic_mmio_info(struct intel_gvt *gvt)
 
 	MMIO_RING_DFH(RING_HWSTAM, D_ALL, F_CMD_ACCESS, NULL, NULL);
 
-	MMIO_GM_RDR(RENDER_HWS_PGA_GEN7, D_ALL, NULL, NULL);
+	MMIO_DH(GEN8_GAMW_ECO_DEV_RW_IA, D_BDW_PLUS, NULL,
+		gamw_echo_dev_rw_ia_write);
+
 	MMIO_GM_RDR(BSD_HWS_PGA_GEN7, D_ALL, NULL, NULL);
 	MMIO_GM_RDR(BLT_HWS_PGA_GEN7, D_ALL, NULL, NULL);
 	MMIO_GM_RDR(VEBOX_HWS_PGA_GEN7, D_ALL, NULL, NULL);
@@ -3160,6 +3194,9 @@ static int init_bxt_mmio_info(struct intel_gvt *gvt)
 	MMIO_D(HSW_TVIDEO_DIP_GCP(TRANSCODER_B), D_BXT);
 	MMIO_D(HSW_TVIDEO_DIP_GCP(TRANSCODER_C), D_BXT);
 
+	MMIO_DH(EDP_PSR_IMR, D_BXT, NULL, bxt_edp_psr_imr_iir_write);
+	MMIO_DH(EDP_PSR_IIR, D_BXT, NULL, bxt_edp_psr_imr_iir_write);
+
 	MMIO_D(RC6_CTX_BASE, D_BXT);
 
 	MMIO_D(GEN8_PUSHBUS_CONTROL, D_BXT);
@@ -3351,6 +3388,30 @@ int intel_vgpu_default_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
 }
 
 /**
+ * intel_vgpu_mask_mmio_write - write mask register
+ * @vgpu: a vGPU
+ * @offset: access offset
+ * @p_data: write data buffer
+ * @bytes: access data length
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ */
+int intel_vgpu_mask_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
+		void *p_data, unsigned int bytes)
+{
+	u32 mask, old_vreg;
+
+	old_vreg = vgpu_vreg(vgpu, offset);
+	write_vreg(vgpu, offset, p_data, bytes);
+	mask = vgpu_vreg(vgpu, offset) >> 16;
+	vgpu_vreg(vgpu, offset) = (old_vreg & ~mask) |
+				(vgpu_vreg(vgpu, offset) & mask);
+
+	return 0;
+}
+
+/**
  * intel_gvt_in_force_nonpriv_whitelist - if a mmio is in whitelist to be
  * force-nopriv register
  *
diff --git a/drivers/gpu/drm/i915/gvt/hypercall.h b/drivers/gpu/drm/i915/gvt/hypercall.h
index f6dd9f717888..5af11cf1b482 100644
--- a/drivers/gpu/drm/i915/gvt/hypercall.h
+++ b/drivers/gpu/drm/i915/gvt/hypercall.h
@@ -53,7 +53,7 @@ struct intel_gvt_mpt {
 	unsigned long (*gfn_to_mfn)(unsigned long handle, unsigned long gfn);
 
 	int (*dma_map_guest_page)(unsigned long handle, unsigned long gfn,
-				  dma_addr_t *dma_addr);
+				  unsigned long size, dma_addr_t *dma_addr);
 	void (*dma_unmap_guest_page)(unsigned long handle, dma_addr_t dma_addr);
 
 	int (*map_gfn_to_mfn)(unsigned long handle, unsigned long gfn,
diff --git a/drivers/gpu/drm/i915/gvt/kvmgt.c b/drivers/gpu/drm/i915/gvt/kvmgt.c
index df4e4a07db3d..a45f46d8537f 100644
--- a/drivers/gpu/drm/i915/gvt/kvmgt.c
+++ b/drivers/gpu/drm/i915/gvt/kvmgt.c
@@ -43,6 +43,8 @@
 #include <linux/mdev.h>
 #include <linux/debugfs.h>
 
+#include <linux/nospec.h>
+
 #include "i915_drv.h"
 #include "gvt.h"
 
@@ -94,6 +96,7 @@ struct gvt_dma {
 	struct rb_node dma_addr_node;
 	gfn_t gfn;
 	dma_addr_t dma_addr;
+	unsigned long size;
 	struct kref ref;
 };
 
@@ -106,36 +109,90 @@ static int kvmgt_guest_init(struct mdev_device *mdev);
 static void intel_vgpu_release_work(struct work_struct *work);
 static bool kvmgt_guest_exit(struct kvmgt_guest_info *info);
 
-static int gvt_dma_map_page(struct intel_vgpu *vgpu, unsigned long gfn,
-		dma_addr_t *dma_addr)
+static void gvt_unpin_guest_page(struct intel_vgpu *vgpu, unsigned long gfn,
+		unsigned long size)
 {
-	struct device *dev = &vgpu->gvt->dev_priv->drm.pdev->dev;
-	struct page *page;
-	unsigned long pfn;
+	int total_pages;
+	int npage;
 	int ret;
 
-	/* Pin the page first. */
-	ret = vfio_pin_pages(mdev_dev(vgpu->vdev.mdev), &gfn, 1,
-			     IOMMU_READ | IOMMU_WRITE, &pfn);
-	if (ret != 1) {
-		gvt_vgpu_err("vfio_pin_pages failed for gfn 0x%lx: %d\n",
-			     gfn, ret);
-		return -EINVAL;
+	total_pages = roundup(size, PAGE_SIZE) / PAGE_SIZE;
+
+	for (npage = 0; npage < total_pages; npage++) {
+		unsigned long cur_gfn = gfn + npage;
+
+		ret = vfio_unpin_pages(mdev_dev(vgpu->vdev.mdev), &cur_gfn, 1);
+		WARN_ON(ret != 1);
 	}
+}
 
-	if (!pfn_valid(pfn)) {
-		gvt_vgpu_err("pfn 0x%lx is not mem backed\n", pfn);
-		vfio_unpin_pages(mdev_dev(vgpu->vdev.mdev), &gfn, 1);
-		return -EINVAL;
+/* Pin a normal or compound guest page for dma. */
+static int gvt_pin_guest_page(struct intel_vgpu *vgpu, unsigned long gfn,
+		unsigned long size, struct page **page)
+{
+	unsigned long base_pfn = 0;
+	int total_pages;
+	int npage;
+	int ret;
+
+	total_pages = roundup(size, PAGE_SIZE) / PAGE_SIZE;
+	/*
+	 * We pin the pages one-by-one to avoid allocating a big arrary
+	 * on stack to hold pfns.
+	 */
+	for (npage = 0; npage < total_pages; npage++) {
+		unsigned long cur_gfn = gfn + npage;
+		unsigned long pfn;
+
+		ret = vfio_pin_pages(mdev_dev(vgpu->vdev.mdev), &cur_gfn, 1,
+				     IOMMU_READ | IOMMU_WRITE, &pfn);
+		if (ret != 1) {
+			gvt_vgpu_err("vfio_pin_pages failed for gfn 0x%lx, ret %d\n",
+				     cur_gfn, ret);
+			goto err;
+		}
+
+		if (!pfn_valid(pfn)) {
+			gvt_vgpu_err("pfn 0x%lx is not mem backed\n", pfn);
+			npage++;
+			ret = -EFAULT;
+			goto err;
+		}
+
+		if (npage == 0)
+			base_pfn = pfn;
+		else if (base_pfn + npage != pfn) {
+			gvt_vgpu_err("The pages are not continuous\n");
+			ret = -EINVAL;
+			npage++;
+			goto err;
+		}
 	}
 
+	*page = pfn_to_page(base_pfn);
+	return 0;
+err:
+	gvt_unpin_guest_page(vgpu, gfn, npage * PAGE_SIZE);
+	return ret;
+}
+
+static int gvt_dma_map_page(struct intel_vgpu *vgpu, unsigned long gfn,
+		dma_addr_t *dma_addr, unsigned long size)
+{
+	struct device *dev = &vgpu->gvt->dev_priv->drm.pdev->dev;
+	struct page *page = NULL;
+	int ret;
+
+	ret = gvt_pin_guest_page(vgpu, gfn, size, &page);
+	if (ret)
+		return ret;
+
 	/* Setup DMA mapping. */
-	page = pfn_to_page(pfn);
-	*dma_addr = dma_map_page(dev, page, 0, PAGE_SIZE,
-				 PCI_DMA_BIDIRECTIONAL);
+	*dma_addr = dma_map_page(dev, page, 0, size, PCI_DMA_BIDIRECTIONAL);
 	if (dma_mapping_error(dev, *dma_addr)) {
-		gvt_vgpu_err("DMA mapping failed for gfn 0x%lx\n", gfn);
-		vfio_unpin_pages(mdev_dev(vgpu->vdev.mdev), &gfn, 1);
+		gvt_vgpu_err("DMA mapping failed for pfn 0x%lx, ret %d\n",
+			     page_to_pfn(page), ret);
+		gvt_unpin_guest_page(vgpu, gfn, size);
 		return -ENOMEM;
 	}
 
@@ -143,14 +200,12 @@ static int gvt_dma_map_page(struct intel_vgpu *vgpu, unsigned long gfn,
 }
 
 static void gvt_dma_unmap_page(struct intel_vgpu *vgpu, unsigned long gfn,
-		dma_addr_t dma_addr)
+		dma_addr_t dma_addr, unsigned long size)
 {
 	struct device *dev = &vgpu->gvt->dev_priv->drm.pdev->dev;
-	int ret;
 
-	dma_unmap_page(dev, dma_addr, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
-	ret = vfio_unpin_pages(mdev_dev(vgpu->vdev.mdev), &gfn, 1);
-	WARN_ON(ret != 1);
+	dma_unmap_page(dev, dma_addr, size, PCI_DMA_BIDIRECTIONAL);
+	gvt_unpin_guest_page(vgpu, gfn, size);
 }
 
 static struct gvt_dma *__gvt_cache_find_dma_addr(struct intel_vgpu *vgpu,
@@ -191,7 +246,7 @@ static struct gvt_dma *__gvt_cache_find_gfn(struct intel_vgpu *vgpu, gfn_t gfn)
 }
 
 static int __gvt_cache_add(struct intel_vgpu *vgpu, gfn_t gfn,
-		dma_addr_t dma_addr)
+		dma_addr_t dma_addr, unsigned long size)
 {
 	struct gvt_dma *new, *itr;
 	struct rb_node **link, *parent = NULL;
@@ -203,6 +258,7 @@ static int __gvt_cache_add(struct intel_vgpu *vgpu, gfn_t gfn,
 	new->vgpu = vgpu;
 	new->gfn = gfn;
 	new->dma_addr = dma_addr;
+	new->size = size;
 	kref_init(&new->ref);
 
 	/* gfn_cache maps gfn to struct gvt_dma. */
@@ -260,7 +316,7 @@ static void gvt_cache_destroy(struct intel_vgpu *vgpu)
 			break;
 		}
 		dma = rb_entry(node, struct gvt_dma, gfn_node);
-		gvt_dma_unmap_page(vgpu, dma->gfn, dma->dma_addr);
+		gvt_dma_unmap_page(vgpu, dma->gfn, dma->dma_addr, dma->size);
 		__gvt_cache_remove_entry(vgpu, dma);
 		mutex_unlock(&vgpu->vdev.cache_lock);
 	}
@@ -515,7 +571,8 @@ static int intel_vgpu_iommu_notifier(struct notifier_block *nb,
 			if (!entry)
 				continue;
 
-			gvt_dma_unmap_page(vgpu, entry->gfn, entry->dma_addr);
+			gvt_dma_unmap_page(vgpu, entry->gfn, entry->dma_addr,
+					   entry->size);
 			__gvt_cache_remove_entry(vgpu, entry);
 		}
 		mutex_unlock(&vgpu->vdev.cache_lock);
@@ -611,7 +668,7 @@ static void __intel_vgpu_release(struct intel_vgpu *vgpu)
 	if (atomic_cmpxchg(&vgpu->vdev.released, 0, 1))
 		return;
 
-	intel_gvt_ops->vgpu_deactivate(vgpu);
+	intel_gvt_ops->vgpu_release(vgpu);
 
 	ret = vfio_unregister_notifier(mdev_dev(vgpu->vdev.mdev), VFIO_IOMMU_NOTIFY,
 					&vgpu->vdev.iommu_notifier);
@@ -1084,7 +1141,8 @@ static long intel_vgpu_ioctl(struct mdev_device *mdev, unsigned int cmd,
 	} else if (cmd == VFIO_DEVICE_GET_REGION_INFO) {
 		struct vfio_region_info info;
 		struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
-		int i, ret;
+		unsigned int i;
+		int ret;
 		struct vfio_region_info_cap_sparse_mmap *sparse = NULL;
 		size_t size;
 		int nr_areas = 1;
@@ -1169,6 +1227,10 @@ static long intel_vgpu_ioctl(struct mdev_device *mdev, unsigned int cmd,
 				if (info.index >= VFIO_PCI_NUM_REGIONS +
 						vgpu->vdev.num_regions)
 					return -EINVAL;
+				info.index =
+					array_index_nospec(info.index,
+							VFIO_PCI_NUM_REGIONS +
+							vgpu->vdev.num_regions);
 
 				i = info.index - VFIO_PCI_NUM_REGIONS;
 
@@ -1195,11 +1257,13 @@ static long intel_vgpu_ioctl(struct mdev_device *mdev, unsigned int cmd,
 					&sparse->header, sizeof(*sparse) +
 					(sparse->nr_areas *
 						sizeof(*sparse->areas)));
-				kfree(sparse);
-				if (ret)
+				if (ret) {
+					kfree(sparse);
 					return ret;
+				}
 				break;
 			default:
+				kfree(sparse);
 				return -EINVAL;
 			}
 		}
@@ -1215,6 +1279,7 @@ static long intel_vgpu_ioctl(struct mdev_device *mdev, unsigned int cmd,
 						  sizeof(info), caps.buf,
 						  caps.size)) {
 					kfree(caps.buf);
+					kfree(sparse);
 					return -EFAULT;
 				}
 				info.cap_offset = sizeof(info);
@@ -1223,6 +1288,7 @@ static long intel_vgpu_ioctl(struct mdev_device *mdev, unsigned int cmd,
 			kfree(caps.buf);
 		}
 
+		kfree(sparse);
 		return copy_to_user((void __user *)arg, &info, minsz) ?
 			-EFAULT : 0;
 	} else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) {
@@ -1560,7 +1626,6 @@ static int kvmgt_guest_init(struct mdev_device *mdev)
 	kvmgt_protect_table_init(info);
 	gvt_cache_init(vgpu);
 
-	mutex_init(&vgpu->dmabuf_lock);
 	init_completion(&vgpu->vblank_done);
 
 	info->track_node.track_write = kvmgt_page_track_write;
@@ -1648,7 +1713,7 @@ static unsigned long kvmgt_gfn_to_pfn(unsigned long handle, unsigned long gfn)
 }
 
 int kvmgt_dma_map_guest_page(unsigned long handle, unsigned long gfn,
-		dma_addr_t *dma_addr)
+		unsigned long size, dma_addr_t *dma_addr)
 {
 	struct kvmgt_guest_info *info;
 	struct intel_vgpu *vgpu;
@@ -1665,11 +1730,11 @@ int kvmgt_dma_map_guest_page(unsigned long handle, unsigned long gfn,
 
 	entry = __gvt_cache_find_gfn(info->vgpu, gfn);
 	if (!entry) {
-		ret = gvt_dma_map_page(vgpu, gfn, dma_addr);
+		ret = gvt_dma_map_page(vgpu, gfn, dma_addr, size);
 		if (ret)
 			goto err_unlock;
 
-		ret = __gvt_cache_add(info->vgpu, gfn, *dma_addr);
+		ret = __gvt_cache_add(info->vgpu, gfn, *dma_addr, size);
 		if (ret)
 			goto err_unmap;
 	} else {
@@ -1681,7 +1746,7 @@ int kvmgt_dma_map_guest_page(unsigned long handle, unsigned long gfn,
 	return 0;
 
 err_unmap:
-	gvt_dma_unmap_page(vgpu, gfn, *dma_addr);
+	gvt_dma_unmap_page(vgpu, gfn, *dma_addr, size);
 err_unlock:
 	mutex_unlock(&info->vgpu->vdev.cache_lock);
 	return ret;
@@ -1691,7 +1756,8 @@ static void __gvt_dma_release(struct kref *ref)
 {
 	struct gvt_dma *entry = container_of(ref, typeof(*entry), ref);
 
-	gvt_dma_unmap_page(entry->vgpu, entry->gfn, entry->dma_addr);
+	gvt_dma_unmap_page(entry->vgpu, entry->gfn, entry->dma_addr,
+			   entry->size);
 	__gvt_cache_remove_entry(entry->vgpu, entry);
 }
 
diff --git a/drivers/gpu/drm/i915/gvt/mmio.h b/drivers/gpu/drm/i915/gvt/mmio.h
index e474188b46d2..1ffc69eba30e 100644
--- a/drivers/gpu/drm/i915/gvt/mmio.h
+++ b/drivers/gpu/drm/i915/gvt/mmio.h
@@ -99,4 +99,6 @@ bool intel_gvt_in_force_nonpriv_whitelist(struct intel_gvt *gvt,
 int intel_vgpu_mmio_reg_rw(struct intel_vgpu *vgpu, unsigned int offset,
 			   void *pdata, unsigned int bytes, bool is_read);
 
+int intel_vgpu_mask_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
+				  void *p_data, unsigned int bytes);
 #endif
diff --git a/drivers/gpu/drm/i915/gvt/mmio_context.c b/drivers/gpu/drm/i915/gvt/mmio_context.c
index 20be9a92600f..42e1e6bdcc2c 100644
--- a/drivers/gpu/drm/i915/gvt/mmio_context.c
+++ b/drivers/gpu/drm/i915/gvt/mmio_context.c
@@ -587,7 +587,9 @@ void intel_gvt_init_engine_mmio_context(struct intel_gvt *gvt)
 
 	for (mmio = gvt->engine_mmio_list.mmio;
 	     i915_mmio_reg_valid(mmio->reg); mmio++) {
-		if (mmio->in_context)
+		if (mmio->in_context) {
 			gvt->engine_mmio_list.ctx_mmio_count[mmio->ring_id]++;
+			intel_gvt_mmio_set_in_ctx(gvt, mmio->reg.reg);
+		}
 	}
 }
diff --git a/drivers/gpu/drm/i915/gvt/mpt.h b/drivers/gpu/drm/i915/gvt/mpt.h
index 32ffcd566cdd..67f19992b226 100644
--- a/drivers/gpu/drm/i915/gvt/mpt.h
+++ b/drivers/gpu/drm/i915/gvt/mpt.h
@@ -230,17 +230,18 @@ static inline unsigned long intel_gvt_hypervisor_gfn_to_mfn(
 /**
  * intel_gvt_hypervisor_dma_map_guest_page - setup dma map for guest page
  * @vgpu: a vGPU
- * @gpfn: guest pfn
+ * @gfn: guest pfn
+ * @size: page size
  * @dma_addr: retrieve allocated dma addr
  *
  * Returns:
  * 0 on success, negative error code if failed.
  */
 static inline int intel_gvt_hypervisor_dma_map_guest_page(
-		struct intel_vgpu *vgpu, unsigned long gfn,
+		struct intel_vgpu *vgpu, unsigned long gfn, unsigned long size,
 		dma_addr_t *dma_addr)
 {
-	return intel_gvt_host.mpt->dma_map_guest_page(vgpu->handle, gfn,
+	return intel_gvt_host.mpt->dma_map_guest_page(vgpu->handle, gfn, size,
 						      dma_addr);
 }
 
diff --git a/drivers/gpu/drm/i915/gvt/scheduler.c b/drivers/gpu/drm/i915/gvt/scheduler.c
index 928818f218f7..43aa058e29fc 100644
--- a/drivers/gpu/drm/i915/gvt/scheduler.c
+++ b/drivers/gpu/drm/i915/gvt/scheduler.c
@@ -476,7 +476,11 @@ static int prepare_shadow_batch_buffer(struct intel_vgpu_workload *workload)
 			i915_gem_obj_finish_shmem_access(bb->obj);
 			bb->accessing = false;
 
-			i915_vma_move_to_active(bb->vma, workload->req, 0);
+			ret = i915_vma_move_to_active(bb->vma,
+						      workload->req,
+						      0);
+			if (ret)
+				goto err;
 		}
 	}
 	return 0;
@@ -780,7 +784,8 @@ static void update_guest_context(struct intel_vgpu_workload *workload)
 	kunmap(page);
 }
 
-static void clean_workloads(struct intel_vgpu *vgpu, unsigned long engine_mask)
+void intel_vgpu_clean_workloads(struct intel_vgpu *vgpu,
+				unsigned long engine_mask)
 {
 	struct intel_vgpu_submission *s = &vgpu->submission;
 	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
@@ -875,7 +880,7 @@ static void complete_current_workload(struct intel_gvt *gvt, int ring_id)
 		 * cleaned up during the resetting process later, so doing
 		 * the workload clean up here doesn't have any impact.
 		 **/
-		clean_workloads(vgpu, ENGINE_MASK(ring_id));
+		intel_vgpu_clean_workloads(vgpu, ENGINE_MASK(ring_id));
 	}
 
 	workload->complete(workload);
@@ -1077,7 +1082,7 @@ void intel_vgpu_reset_submission(struct intel_vgpu *vgpu,
 	if (!s->active)
 		return;
 
-	clean_workloads(vgpu, engine_mask);
+	intel_vgpu_clean_workloads(vgpu, engine_mask);
 	s->ops->reset(vgpu, engine_mask);
 }
 
diff --git a/drivers/gpu/drm/i915/gvt/scheduler.h b/drivers/gpu/drm/i915/gvt/scheduler.h
index 21eddab4a9cd..ca5529d0e48e 100644
--- a/drivers/gpu/drm/i915/gvt/scheduler.h
+++ b/drivers/gpu/drm/i915/gvt/scheduler.h
@@ -158,4 +158,7 @@ intel_vgpu_create_workload(struct intel_vgpu *vgpu, int ring_id,
 
 void intel_vgpu_destroy_workload(struct intel_vgpu_workload *workload);
 
+void intel_vgpu_clean_workloads(struct intel_vgpu *vgpu,
+				unsigned long engine_mask);
+
 #endif
diff --git a/drivers/gpu/drm/i915/gvt/vgpu.c b/drivers/gpu/drm/i915/gvt/vgpu.c
index 83a43970783f..a4e8e3cf74fd 100644
--- a/drivers/gpu/drm/i915/gvt/vgpu.c
+++ b/drivers/gpu/drm/i915/gvt/vgpu.c
@@ -46,6 +46,7 @@ void populate_pvinfo_page(struct intel_vgpu *vgpu)
 
 	vgpu_vreg_t(vgpu, vgtif_reg(vgt_caps)) = VGT_CAPS_FULL_48BIT_PPGTT;
 	vgpu_vreg_t(vgpu, vgtif_reg(vgt_caps)) |= VGT_CAPS_HWSP_EMULATION;
+	vgpu_vreg_t(vgpu, vgtif_reg(vgt_caps)) |= VGT_CAPS_HUGE_GTT;
 
 	vgpu_vreg_t(vgpu, vgtif_reg(avail_rs.mappable_gmadr.base)) =
 		vgpu_aperture_gmadr_base(vgpu);
@@ -221,7 +222,7 @@ void intel_gvt_activate_vgpu(struct intel_vgpu *vgpu)
  * @vgpu: virtual GPU
  *
  * This function is called when user wants to deactivate a virtual GPU.
- * All virtual GPU runtime information will be destroyed.
+ * The virtual GPU will be stopped.
  *
  */
 void intel_gvt_deactivate_vgpu(struct intel_vgpu *vgpu)
@@ -237,12 +238,30 @@ void intel_gvt_deactivate_vgpu(struct intel_vgpu *vgpu)
 	}
 
 	intel_vgpu_stop_schedule(vgpu);
-	intel_vgpu_dmabuf_cleanup(vgpu);
 
 	mutex_unlock(&vgpu->vgpu_lock);
 }
 
 /**
+ * intel_gvt_release_vgpu - release a virtual GPU
+ * @vgpu: virtual GPU
+ *
+ * This function is called when user wants to release a virtual GPU.
+ * The virtual GPU will be stopped and all runtime information will be
+ * destroyed.
+ *
+ */
+void intel_gvt_release_vgpu(struct intel_vgpu *vgpu)
+{
+	intel_gvt_deactivate_vgpu(vgpu);
+
+	mutex_lock(&vgpu->vgpu_lock);
+	intel_vgpu_clean_workloads(vgpu, ALL_ENGINES);
+	intel_vgpu_dmabuf_cleanup(vgpu);
+	mutex_unlock(&vgpu->vgpu_lock);
+}
+
+/**
  * intel_gvt_destroy_vgpu - destroy a virtual GPU
  * @vgpu: virtual GPU
  *
@@ -360,6 +379,7 @@ static struct intel_vgpu *__intel_gvt_create_vgpu(struct intel_gvt *gvt,
 	vgpu->gvt = gvt;
 	vgpu->sched_ctl.weight = param->weight;
 	mutex_init(&vgpu->vgpu_lock);
+	mutex_init(&vgpu->dmabuf_lock);
 	INIT_LIST_HEAD(&vgpu->dmabuf_obj_list_head);
 	INIT_RADIX_TREE(&vgpu->page_track_tree, GFP_KERNEL);
 	idr_init(&vgpu->object_idr);
diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c
index c400f42a54ec..f9ce35da4123 100644
--- a/drivers/gpu/drm/i915/i915_debugfs.c
+++ b/drivers/gpu/drm/i915/i915_debugfs.c
@@ -1218,7 +1218,8 @@ static int i915_frequency_info(struct seq_file *m, void *unused)
 			   rpcurup, GT_PM_INTERVAL_TO_US(dev_priv, rpcurup));
 		seq_printf(m, "RP PREV UP: %d (%dus)\n",
 			   rpprevup, GT_PM_INTERVAL_TO_US(dev_priv, rpprevup));
-		seq_printf(m, "Up threshold: %d%%\n", rps->up_threshold);
+		seq_printf(m, "Up threshold: %d%%\n",
+			   rps->power.up_threshold);
 
 		seq_printf(m, "RP CUR DOWN EI: %d (%dus)\n",
 			   rpdownei, GT_PM_INTERVAL_TO_US(dev_priv, rpdownei));
@@ -1226,7 +1227,8 @@ static int i915_frequency_info(struct seq_file *m, void *unused)
 			   rpcurdown, GT_PM_INTERVAL_TO_US(dev_priv, rpcurdown));
 		seq_printf(m, "RP PREV DOWN: %d (%dus)\n",
 			   rpprevdown, GT_PM_INTERVAL_TO_US(dev_priv, rpprevdown));
-		seq_printf(m, "Down threshold: %d%%\n", rps->down_threshold);
+		seq_printf(m, "Down threshold: %d%%\n",
+			   rps->power.down_threshold);
 
 		max_freq = (IS_GEN9_LP(dev_priv) ? rp_state_cap >> 0 :
 			    rp_state_cap >> 16) & 0xff;
@@ -1659,11 +1661,6 @@ static int i915_fbc_status(struct seq_file *m, void *unused)
 	else
 		seq_printf(m, "FBC disabled: %s\n", fbc->no_fbc_reason);
 
-	if (fbc->work.scheduled)
-		seq_printf(m, "FBC worker scheduled on vblank %llu, now %llu\n",
-			   fbc->work.scheduled_vblank,
-			   drm_crtc_vblank_count(&fbc->crtc->base));
-
 	if (intel_fbc_is_active(dev_priv)) {
 		u32 mask;
 
@@ -2223,6 +2220,7 @@ static int i915_rps_boost_info(struct seq_file *m, void *data)
 	seq_printf(m, "CPU waiting? %d\n", count_irq_waiters(dev_priv));
 	seq_printf(m, "Boosts outstanding? %d\n",
 		   atomic_read(&rps->num_waiters));
+	seq_printf(m, "Interactive? %d\n", READ_ONCE(rps->power.interactive));
 	seq_printf(m, "Frequency requested %d\n",
 		   intel_gpu_freq(dev_priv, rps->cur_freq));
 	seq_printf(m, "  min hard:%d, soft:%d; max soft:%d, hard:%d\n",
@@ -2266,13 +2264,13 @@ static int i915_rps_boost_info(struct seq_file *m, void *data)
 		intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
 
 		seq_printf(m, "\nRPS Autotuning (current \"%s\" window):\n",
-			   rps_power_to_str(rps->power));
+			   rps_power_to_str(rps->power.mode));
 		seq_printf(m, "  Avg. up: %d%% [above threshold? %d%%]\n",
 			   rpup && rpupei ? 100 * rpup / rpupei : 0,
-			   rps->up_threshold);
+			   rps->power.up_threshold);
 		seq_printf(m, "  Avg. down: %d%% [below threshold? %d%%]\n",
 			   rpdown && rpdownei ? 100 * rpdown / rpdownei : 0,
-			   rps->down_threshold);
+			   rps->power.down_threshold);
 	} else {
 		seq_puts(m, "\nRPS Autotuning inactive\n");
 	}
@@ -2597,31 +2595,9 @@ static const struct file_operations i915_guc_log_relay_fops = {
 	.release = i915_guc_log_relay_release,
 };
 
-static const char *psr2_live_status(u32 val)
-{
-	static const char * const live_status[] = {
-		"IDLE",
-		"CAPTURE",
-		"CAPTURE_FS",
-		"SLEEP",
-		"BUFON_FW",
-		"ML_UP",
-		"SU_STANDBY",
-		"FAST_SLEEP",
-		"DEEP_SLEEP",
-		"BUF_ON",
-		"TG_ON"
-	};
-
-	val = (val & EDP_PSR2_STATUS_STATE_MASK) >> EDP_PSR2_STATUS_STATE_SHIFT;
-	if (val < ARRAY_SIZE(live_status))
-		return live_status[val];
-
-	return "unknown";
-}
-
-static const char *psr_sink_status(u8 val)
+static int i915_psr_sink_status_show(struct seq_file *m, void *data)
 {
+	u8 val;
 	static const char * const sink_status[] = {
 		"inactive",
 		"transition to active, capture and display",
@@ -2630,14 +2606,88 @@ static const char *psr_sink_status(u8 val)
 		"transition to inactive, capture and display, timing re-sync",
 		"reserved",
 		"reserved",
-		"sink internal error"
+		"sink internal error",
 	};
+	struct drm_connector *connector = m->private;
+	struct drm_i915_private *dev_priv = to_i915(connector->dev);
+	struct intel_dp *intel_dp =
+		enc_to_intel_dp(&intel_attached_encoder(connector)->base);
+	int ret;
 
-	val &= DP_PSR_SINK_STATE_MASK;
-	if (val < ARRAY_SIZE(sink_status))
-		return sink_status[val];
+	if (!CAN_PSR(dev_priv)) {
+		seq_puts(m, "PSR Unsupported\n");
+		return -ENODEV;
+	}
 
-	return "unknown";
+	if (connector->status != connector_status_connected)
+		return -ENODEV;
+
+	ret = drm_dp_dpcd_readb(&intel_dp->aux, DP_PSR_STATUS, &val);
+
+	if (ret == 1) {
+		const char *str = "unknown";
+
+		val &= DP_PSR_SINK_STATE_MASK;
+		if (val < ARRAY_SIZE(sink_status))
+			str = sink_status[val];
+		seq_printf(m, "Sink PSR status: 0x%x [%s]\n", val, str);
+	} else {
+		return ret;
+	}
+
+	return 0;
+}
+DEFINE_SHOW_ATTRIBUTE(i915_psr_sink_status);
+
+static void
+psr_source_status(struct drm_i915_private *dev_priv, struct seq_file *m)
+{
+	u32 val, psr_status;
+
+	if (dev_priv->psr.psr2_enabled) {
+		static const char * const live_status[] = {
+			"IDLE",
+			"CAPTURE",
+			"CAPTURE_FS",
+			"SLEEP",
+			"BUFON_FW",
+			"ML_UP",
+			"SU_STANDBY",
+			"FAST_SLEEP",
+			"DEEP_SLEEP",
+			"BUF_ON",
+			"TG_ON"
+		};
+		psr_status = I915_READ(EDP_PSR2_STATUS);
+		val = (psr_status & EDP_PSR2_STATUS_STATE_MASK) >>
+			EDP_PSR2_STATUS_STATE_SHIFT;
+		if (val < ARRAY_SIZE(live_status)) {
+			seq_printf(m, "Source PSR status: 0x%x [%s]\n",
+				   psr_status, live_status[val]);
+			return;
+		}
+	} else {
+		static const char * const live_status[] = {
+			"IDLE",
+			"SRDONACK",
+			"SRDENT",
+			"BUFOFF",
+			"BUFON",
+			"AUXACK",
+			"SRDOFFACK",
+			"SRDENT_ON",
+		};
+		psr_status = I915_READ(EDP_PSR_STATUS);
+		val = (psr_status & EDP_PSR_STATUS_STATE_MASK) >>
+			EDP_PSR_STATUS_STATE_SHIFT;
+		if (val < ARRAY_SIZE(live_status)) {
+			seq_printf(m, "Source PSR status: 0x%x [%s]\n",
+				   psr_status, live_status[val]);
+			return;
+		}
+	}
+
+	seq_printf(m, "Source PSR status: 0x%x [%s]\n", psr_status, "unknown");
 }
 
 static int i915_edp_psr_status(struct seq_file *m, void *data)
@@ -2681,21 +2731,8 @@ static int i915_edp_psr_status(struct seq_file *m, void *data)
 
 		seq_printf(m, "Performance_Counter: %u\n", psrperf);
 	}
-	if (dev_priv->psr.psr2_enabled) {
-		u32 psr2 = I915_READ(EDP_PSR2_STATUS);
 
-		seq_printf(m, "EDP_PSR2_STATUS: %x [%s]\n",
-			   psr2, psr2_live_status(psr2));
-	}
-
-	if (dev_priv->psr.enabled) {
-		struct drm_dp_aux *aux = &dev_priv->psr.enabled->aux;
-		u8 val;
-
-		if (drm_dp_dpcd_readb(aux, DP_PSR_STATUS, &val) == 1)
-			seq_printf(m, "Sink PSR status: 0x%x [%s]\n", val,
-				   psr_sink_status(val));
-	}
+	psr_source_status(dev_priv, m);
 	mutex_unlock(&dev_priv->psr.lock);
 
 	if (READ_ONCE(dev_priv->psr.debug)) {
@@ -2742,86 +2779,6 @@ DEFINE_SIMPLE_ATTRIBUTE(i915_edp_psr_debug_fops,
 			i915_edp_psr_debug_get, i915_edp_psr_debug_set,
 			"%llu\n");
 
-static int i915_sink_crc(struct seq_file *m, void *data)
-{
-	struct drm_i915_private *dev_priv = node_to_i915(m->private);
-	struct drm_device *dev = &dev_priv->drm;
-	struct intel_connector *connector;
-	struct drm_connector_list_iter conn_iter;
-	struct intel_dp *intel_dp = NULL;
-	struct drm_modeset_acquire_ctx ctx;
-	int ret;
-	u8 crc[6];
-
-	drm_modeset_acquire_init(&ctx, DRM_MODESET_ACQUIRE_INTERRUPTIBLE);
-
-	drm_connector_list_iter_begin(dev, &conn_iter);
-
-	for_each_intel_connector_iter(connector, &conn_iter) {
-		struct drm_crtc *crtc;
-		struct drm_connector_state *state;
-		struct intel_crtc_state *crtc_state;
-
-		if (connector->base.connector_type != DRM_MODE_CONNECTOR_eDP)
-			continue;
-
-retry:
-		ret = drm_modeset_lock(&dev->mode_config.connection_mutex, &ctx);
-		if (ret)
-			goto err;
-
-		state = connector->base.state;
-		if (!state->best_encoder)
-			continue;
-
-		crtc = state->crtc;
-		ret = drm_modeset_lock(&crtc->mutex, &ctx);
-		if (ret)
-			goto err;
-
-		crtc_state = to_intel_crtc_state(crtc->state);
-		if (!crtc_state->base.active)
-			continue;
-
-		/*
-		 * We need to wait for all crtc updates to complete, to make
-		 * sure any pending modesets and plane updates are completed.
-		 */
-		if (crtc_state->base.commit) {
-			ret = wait_for_completion_interruptible(&crtc_state->base.commit->hw_done);
-
-			if (ret)
-				goto err;
-		}
-
-		intel_dp = enc_to_intel_dp(state->best_encoder);
-
-		ret = intel_dp_sink_crc(intel_dp, crtc_state, crc);
-		if (ret)
-			goto err;
-
-		seq_printf(m, "%02x%02x%02x%02x%02x%02x\n",
-			   crc[0], crc[1], crc[2],
-			   crc[3], crc[4], crc[5]);
-		goto out;
-
-err:
-		if (ret == -EDEADLK) {
-			ret = drm_modeset_backoff(&ctx);
-			if (!ret)
-				goto retry;
-		}
-		goto out;
-	}
-	ret = -ENODEV;
-out:
-	drm_connector_list_iter_end(&conn_iter);
-	drm_modeset_drop_locks(&ctx);
-	drm_modeset_acquire_fini(&ctx);
-
-	return ret;
-}
-
 static int i915_energy_uJ(struct seq_file *m, void *data)
 {
 	struct drm_i915_private *dev_priv = node_to_i915(m->private);
@@ -4086,7 +4043,8 @@ fault_irq_set(struct drm_i915_private *i915,
 
 	err = i915_gem_wait_for_idle(i915,
 				     I915_WAIT_LOCKED |
-				     I915_WAIT_INTERRUPTIBLE);
+				     I915_WAIT_INTERRUPTIBLE,
+				     MAX_SCHEDULE_TIMEOUT);
 	if (err)
 		goto err_unlock;
 
@@ -4191,7 +4149,8 @@ i915_drop_caches_set(void *data, u64 val)
 		if (val & DROP_ACTIVE)
 			ret = i915_gem_wait_for_idle(dev_priv,
 						     I915_WAIT_INTERRUPTIBLE |
-						     I915_WAIT_LOCKED);
+						     I915_WAIT_LOCKED,
+						     MAX_SCHEDULE_TIMEOUT);
 
 		if (val & DROP_RETIRE)
 			i915_retire_requests(dev_priv);
@@ -4765,7 +4724,6 @@ static const struct drm_info_list i915_debugfs_list[] = {
 	{"i915_ppgtt_info", i915_ppgtt_info, 0},
 	{"i915_llc", i915_llc, 0},
 	{"i915_edp_psr_status", i915_edp_psr_status, 0},
-	{"i915_sink_crc_eDP1", i915_sink_crc, 0},
 	{"i915_energy_uJ", i915_energy_uJ, 0},
 	{"i915_runtime_pm_status", i915_runtime_pm_status, 0},
 	{"i915_power_domain_info", i915_power_domain_info, 0},
@@ -4799,7 +4757,6 @@ static const struct i915_debugfs_files {
 #endif
 	{"i915_fifo_underrun_reset", &i915_fifo_underrun_reset_ops},
 	{"i915_next_seqno", &i915_next_seqno_fops},
-	{"i915_display_crc_ctl", &i915_display_crc_ctl_fops},
 	{"i915_pri_wm_latency", &i915_pri_wm_latency_fops},
 	{"i915_spr_wm_latency", &i915_spr_wm_latency_fops},
 	{"i915_cur_wm_latency", &i915_cur_wm_latency_fops},
@@ -4819,7 +4776,7 @@ int i915_debugfs_register(struct drm_i915_private *dev_priv)
 {
 	struct drm_minor *minor = dev_priv->drm.primary;
 	struct dentry *ent;
-	int ret, i;
+	int i;
 
 	ent = debugfs_create_file("i915_forcewake_user", S_IRUSR,
 				  minor->debugfs_root, to_i915(minor->dev),
@@ -4827,10 +4784,6 @@ int i915_debugfs_register(struct drm_i915_private *dev_priv)
 	if (!ent)
 		return -ENOMEM;
 
-	ret = intel_pipe_crc_create(minor);
-	if (ret)
-		return ret;
-
 	for (i = 0; i < ARRAY_SIZE(i915_debugfs_files); i++) {
 		ent = debugfs_create_file(i915_debugfs_files[i].name,
 					  S_IRUGO | S_IWUSR,
@@ -4952,9 +4905,12 @@ int i915_debugfs_connector_add(struct drm_connector *connector)
 		debugfs_create_file("i915_dpcd", S_IRUGO, root,
 				    connector, &i915_dpcd_fops);
 
-	if (connector->connector_type == DRM_MODE_CONNECTOR_eDP)
+	if (connector->connector_type == DRM_MODE_CONNECTOR_eDP) {
 		debugfs_create_file("i915_panel_timings", S_IRUGO, root,
 				    connector, &i915_panel_fops);
+		debugfs_create_file("i915_psr_sink_status", S_IRUGO, root,
+				    connector, &i915_psr_sink_status_fops);
+	}
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c
index beb0951001ce..f8cfd16be534 100644
--- a/drivers/gpu/drm/i915/i915_drv.c
+++ b/drivers/gpu/drm/i915/i915_drv.c
@@ -104,8 +104,13 @@ __i915_printk(struct drm_i915_private *dev_priv, const char *level,
 	vaf.fmt = fmt;
 	vaf.va = &args;
 
-	dev_printk(level, kdev, "[" DRM_NAME ":%ps] %pV",
-		   __builtin_return_address(0), &vaf);
+	if (is_error)
+		dev_printk(level, kdev, "%pV", &vaf);
+	else
+		dev_printk(level, kdev, "[" DRM_NAME ":%ps] %pV",
+			   __builtin_return_address(0), &vaf);
+
+	va_end(args);
 
 	if (is_error && !shown_bug_once) {
 		/*
@@ -117,8 +122,6 @@ __i915_printk(struct drm_i915_private *dev_priv, const char *level,
 			dev_notice(kdev, "%s", FDO_BUG_MSG);
 		shown_bug_once = true;
 	}
-
-	va_end(args);
 }
 
 /* Map PCH device id to PCH type, or PCH_NONE if unknown. */
@@ -679,7 +682,7 @@ static int i915_load_modeset_init(struct drm_device *dev)
 
 	ret = i915_gem_init(dev_priv);
 	if (ret)
-		goto cleanup_irq;
+		goto cleanup_modeset;
 
 	intel_setup_overlay(dev_priv);
 
@@ -699,6 +702,8 @@ cleanup_gem:
 	if (i915_gem_suspend(dev_priv))
 		DRM_ERROR("failed to idle hardware; continuing to unload!\n");
 	i915_gem_fini(dev_priv);
+cleanup_modeset:
+	intel_modeset_cleanup(dev);
 cleanup_irq:
 	drm_irq_uninstall(dev);
 	intel_teardown_gmbus(dev_priv);
@@ -895,7 +900,6 @@ static int i915_driver_init_early(struct drm_i915_private *dev_priv,
 	spin_lock_init(&dev_priv->uncore.lock);
 
 	mutex_init(&dev_priv->sb_lock);
-	mutex_init(&dev_priv->modeset_restore_lock);
 	mutex_init(&dev_priv->av_mutex);
 	mutex_init(&dev_priv->wm.wm_mutex);
 	mutex_init(&dev_priv->pps_mutex);
@@ -1149,8 +1153,6 @@ static int i915_driver_init_hw(struct drm_i915_private *dev_priv)
 
 	intel_uncore_sanitize(dev_priv);
 
-	intel_opregion_setup(dev_priv);
-
 	i915_gem_load_init_fences(dev_priv);
 
 	/* On the 945G/GM, the chipset reports the MSI capability on the
@@ -1165,6 +1167,12 @@ static int i915_driver_init_hw(struct drm_i915_private *dev_priv)
 	 * get lost on g4x as well, and interrupt delivery seems to stay
 	 * properly dead afterwards. So we'll just disable them for all
 	 * pre-gen5 chipsets.
+	 *
+	 * dp aux and gmbus irq on gen4 seems to be able to generate legacy
+	 * interrupts even when in MSI mode. This results in spurious
+	 * interrupt warnings if the legacy irq no. is shared with another
+	 * device. The kernel then disables that interrupt source and so
+	 * prevents the other device from working properly.
 	 */
 	if (INTEL_GEN(dev_priv) >= 5) {
 		if (pci_enable_msi(pdev) < 0)
@@ -1173,10 +1181,16 @@ static int i915_driver_init_hw(struct drm_i915_private *dev_priv)
 
 	ret = intel_gvt_init(dev_priv);
 	if (ret)
-		goto err_ggtt;
+		goto err_msi;
+
+	intel_opregion_setup(dev_priv);
 
 	return 0;
 
+err_msi:
+	if (pdev->msi_enabled)
+		pci_disable_msi(pdev);
+	pm_qos_remove_request(&dev_priv->pm_qos);
 err_ggtt:
 	i915_ggtt_cleanup_hw(dev_priv);
 err_perf:
@@ -1409,6 +1423,7 @@ out_fini:
 	drm_dev_fini(&dev_priv->drm);
 out_free:
 	kfree(dev_priv);
+	pci_set_drvdata(pdev, NULL);
 	return ret;
 }
 
@@ -1554,11 +1569,6 @@ static int i915_drm_suspend(struct drm_device *dev)
 	struct pci_dev *pdev = dev_priv->drm.pdev;
 	pci_power_t opregion_target_state;
 
-	/* ignore lid events during suspend */
-	mutex_lock(&dev_priv->modeset_restore_lock);
-	dev_priv->modeset_restore = MODESET_SUSPENDED;
-	mutex_unlock(&dev_priv->modeset_restore_lock);
-
 	disable_rpm_wakeref_asserts(dev_priv);
 
 	/* We do a lot of poking in a lot of registers, make sure they work
@@ -1571,7 +1581,7 @@ static int i915_drm_suspend(struct drm_device *dev)
 
 	intel_display_suspend(dev);
 
-	intel_dp_mst_suspend(dev);
+	intel_dp_mst_suspend(dev_priv);
 
 	intel_runtime_pm_disable_interrupts(dev_priv);
 	intel_hpd_cancel_work(dev_priv);
@@ -1736,7 +1746,7 @@ static int i915_drm_resume(struct drm_device *dev)
 		dev_priv->display.hpd_irq_setup(dev_priv);
 	spin_unlock_irq(&dev_priv->irq_lock);
 
-	intel_dp_mst_resume(dev);
+	intel_dp_mst_resume(dev_priv);
 
 	intel_display_resume(dev);
 
@@ -1754,10 +1764,6 @@ static int i915_drm_resume(struct drm_device *dev)
 
 	intel_fbdev_set_suspend(dev, FBINFO_STATE_RUNNING, false);
 
-	mutex_lock(&dev_priv->modeset_restore_lock);
-	dev_priv->modeset_restore = MODESET_DONE;
-	mutex_unlock(&dev_priv->modeset_restore_lock);
-
 	intel_opregion_notify_adapter(dev_priv, PCI_D0);
 
 	enable_rpm_wakeref_asserts(dev_priv);
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index f4751b383858..4aca5344863d 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -86,8 +86,8 @@
 
 #define DRIVER_NAME		"i915"
 #define DRIVER_DESC		"Intel Graphics"
-#define DRIVER_DATE		"20180620"
-#define DRIVER_TIMESTAMP	1529529048
+#define DRIVER_DATE		"20180719"
+#define DRIVER_TIMESTAMP	1532015279
 
 /* Use I915_STATE_WARN(x) and I915_STATE_WARN_ON() (rather than WARN() and
  * WARN_ON()) for hw state sanity checks to check for unexpected conditions
@@ -299,7 +299,6 @@ struct i915_hotplug {
 	u32 event_bits;
 	struct delayed_work reenable_work;
 
-	struct intel_digital_port *irq_port[I915_MAX_PORTS];
 	u32 long_port_mask;
 	u32 short_port_mask;
 	struct work_struct dig_port_work;
@@ -512,6 +511,7 @@ struct intel_fbc {
 
 	bool enabled;
 	bool active;
+	bool flip_pending;
 
 	bool underrun_detected;
 	struct work_struct underrun_work;
@@ -579,12 +579,6 @@ struct intel_fbc {
 		unsigned int gen9_wa_cfb_stride;
 	} params;
 
-	struct intel_fbc_work {
-		bool scheduled;
-		u64 scheduled_vblank;
-		struct work_struct work;
-	} work;
-
 	const char *no_fbc_reason;
 };
 
@@ -631,14 +625,6 @@ struct i915_psr {
 	bool debug;
 	ktime_t last_entry_attempt;
 	ktime_t last_exit;
-
-	void (*enable_source)(struct intel_dp *,
-			      const struct intel_crtc_state *);
-	void (*disable_source)(struct intel_dp *,
-			       const struct intel_crtc_state *);
-	void (*enable_sink)(struct intel_dp *);
-	void (*activate)(struct intel_dp *);
-	void (*setup_vsc)(struct intel_dp *, const struct intel_crtc_state *);
 };
 
 enum intel_pch {
@@ -663,6 +649,7 @@ enum intel_sbi_destination {
 #define QUIRK_BACKLIGHT_PRESENT (1<<3)
 #define QUIRK_PIN_SWIZZLED_PAGES (1<<5)
 #define QUIRK_INCREASE_T12_DELAY (1<<6)
+#define QUIRK_INCREASE_DDI_DISABLED_TIME (1<<7)
 
 struct intel_fbdev;
 struct intel_fbc_work;
@@ -792,11 +779,17 @@ struct intel_rps {
 	u8 rp0_freq;		/* Non-overclocked max frequency. */
 	u16 gpll_ref_freq;	/* vlv/chv GPLL reference frequency */
 
-	u8 up_threshold; /* Current %busy required to uplock */
-	u8 down_threshold; /* Current %busy required to downclock */
-
 	int last_adj;
-	enum { LOW_POWER, BETWEEN, HIGH_POWER } power;
+
+	struct {
+		struct mutex mutex;
+
+		enum { LOW_POWER, BETWEEN, HIGH_POWER } mode;
+		unsigned int interactive;
+
+		u8 up_threshold; /* Current %busy required to uplock */
+		u8 down_threshold; /* Current %busy required to downclock */
+	} power;
 
 	bool enabled;
 	atomic_t num_waiters;
@@ -965,7 +958,7 @@ struct i915_gem_mm {
 	/**
 	 * Small stash of WC pages
 	 */
-	struct pagevec wc_stash;
+	struct pagestash wc_stash;
 
 	/**
 	 * tmpfs instance used for shmem backed objects
@@ -1015,12 +1008,6 @@ struct i915_gem_mm {
 
 #define I915_ENGINE_WEDGED_TIMEOUT  (60 * HZ)  /* Reset but no recovery? */
 
-enum modeset_restore {
-	MODESET_ON_LID_OPEN,
-	MODESET_DONE,
-	MODESET_SUSPENDED,
-};
-
 #define DP_AUX_A 0x40
 #define DP_AUX_B 0x10
 #define DP_AUX_C 0x20
@@ -1284,20 +1271,11 @@ enum intel_pipe_crc_source {
 	INTEL_PIPE_CRC_SOURCE_MAX,
 };
 
-struct intel_pipe_crc_entry {
-	uint32_t frame;
-	uint32_t crc[5];
-};
-
 #define INTEL_PIPE_CRC_ENTRIES_NR	128
 struct intel_pipe_crc {
 	spinlock_t lock;
-	bool opened;		/* exclusive access to the result file */
-	struct intel_pipe_crc_entry *entries;
-	enum intel_pipe_crc_source source;
-	int head, tail;
-	wait_queue_head_t wq;
 	int skipped;
+	enum intel_pipe_crc_source source;
 };
 
 struct i915_frontbuffer_tracking {
@@ -1752,12 +1730,9 @@ struct drm_i915_private {
 
 	unsigned long quirks;
 
-	enum modeset_restore modeset_restore;
-	struct mutex modeset_restore_lock;
 	struct drm_atomic_state *modeset_restore_state;
 	struct drm_modeset_acquire_ctx reset_ctx;
 
-	struct list_head vm_list; /* Global list of all address spaces */
 	struct i915_ggtt ggtt; /* VM representing the global address space */
 
 	struct i915_gem_mm mm;
@@ -2326,6 +2301,7 @@ intel_info(const struct drm_i915_private *dev_priv)
 }
 
 #define INTEL_INFO(dev_priv)	intel_info((dev_priv))
+#define DRIVER_CAPS(dev_priv)	(&(dev_priv)->caps)
 
 #define INTEL_GEN(dev_priv)	((dev_priv)->info.gen)
 #define INTEL_DEVID(dev_priv)	((dev_priv)->info.device_id)
@@ -2578,17 +2554,10 @@ intel_info(const struct drm_i915_private *dev_priv)
 	(IS_CANNONLAKE(dev_priv) || \
 	 IS_SKL_GT3(dev_priv) || IS_SKL_GT4(dev_priv))
 
-/*
- * dp aux and gmbus irq on gen4 seems to be able to generate legacy interrupts
- * even when in MSI mode. This results in spurious interrupt warnings if the
- * legacy irq no. is shared with another device. The kernel then disables that
- * interrupt source and so prevents the other device from working properly.
- *
- * Since we don't enable MSI anymore on gen4, we can always use GMBUS/AUX
- * interrupts.
- */
-#define HAS_AUX_IRQ(dev_priv)   true
 #define HAS_GMBUS_IRQ(dev_priv) (INTEL_GEN(dev_priv) >= 4)
+#define HAS_GMBUS_BURST_READ(dev_priv) (INTEL_GEN(dev_priv) >= 10 || \
+					IS_GEMINILAKE(dev_priv) || \
+					IS_KABYLAKE(dev_priv))
 
 /* With the 945 and later, Y tiling got adjusted so that it was 32 128-byte
  * rows, which changed the alignment requirements and fence programming.
@@ -2771,8 +2740,6 @@ void intel_hpd_irq_handler(struct drm_i915_private *dev_priv,
 void intel_hpd_init(struct drm_i915_private *dev_priv);
 void intel_hpd_init_work(struct drm_i915_private *dev_priv);
 void intel_hpd_cancel_work(struct drm_i915_private *dev_priv);
-enum port intel_hpd_pin_to_port(struct drm_i915_private *dev_priv,
-				enum hpd_pin pin);
 enum hpd_pin intel_hpd_pin_default(struct drm_i915_private *dev_priv,
 				   enum port port);
 bool intel_hpd_disable(struct drm_i915_private *dev_priv, enum hpd_pin pin);
@@ -3119,9 +3086,6 @@ i915_gem_obj_finish_shmem_access(struct drm_i915_gem_object *obj)
 }
 
 int __must_check i915_mutex_lock_interruptible(struct drm_device *dev);
-void i915_vma_move_to_active(struct i915_vma *vma,
-			     struct i915_request *rq,
-			     unsigned int flags);
 int i915_gem_dumb_create(struct drm_file *file_priv,
 			 struct drm_device *dev,
 			 struct drm_mode_create_dumb *args);
@@ -3189,7 +3153,7 @@ void i915_gem_init_swizzling(struct drm_i915_private *dev_priv);
 void i915_gem_fini(struct drm_i915_private *dev_priv);
 void i915_gem_cleanup_engines(struct drm_i915_private *dev_priv);
 int i915_gem_wait_for_idle(struct drm_i915_private *dev_priv,
-			   unsigned int flags);
+			   unsigned int flags, long timeout);
 int __must_check i915_gem_suspend(struct drm_i915_private *dev_priv);
 void i915_gem_suspend_late(struct drm_i915_private *dev_priv);
 void i915_gem_resume(struct drm_i915_private *dev_priv);
@@ -3339,7 +3303,7 @@ unsigned long i915_gem_shrink(struct drm_i915_private *i915,
 unsigned long i915_gem_shrink_all(struct drm_i915_private *i915);
 void i915_gem_shrinker_register(struct drm_i915_private *i915);
 void i915_gem_shrinker_unregister(struct drm_i915_private *i915);
-
+void i915_gem_shrinker_taints_mutex(struct mutex *mutex);
 
 /* i915_gem_tiling.c */
 static inline bool i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
@@ -3464,6 +3428,8 @@ extern void i915_redisable_vga_power_on(struct drm_i915_private *dev_priv);
 extern bool ironlake_set_drps(struct drm_i915_private *dev_priv, u8 val);
 extern void intel_init_pch_refclk(struct drm_i915_private *dev_priv);
 extern int intel_set_rps(struct drm_i915_private *dev_priv, u8 val);
+extern void intel_rps_mark_interactive(struct drm_i915_private *i915,
+				       bool interactive);
 extern bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv,
 				  bool enable);
 
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 858d188dd33b..fcc73a6ab503 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -802,7 +802,7 @@ void i915_gem_flush_ggtt_writes(struct drm_i915_private *dev_priv)
 	 * that was!).
 	 */
 
-	wmb();
+	i915_gem_chipset_flush(dev_priv);
 
 	intel_runtime_pm_get(dev_priv);
 	spin_lock_irq(&dev_priv->uncore.lock);
@@ -837,6 +837,10 @@ flush_write_domain(struct drm_i915_gem_object *obj, unsigned int flush_domains)
 		}
 		break;
 
+	case I915_GEM_DOMAIN_WC:
+		wmb();
+		break;
+
 	case I915_GEM_DOMAIN_CPU:
 		i915_gem_clflush_object(obj, I915_CLFLUSH_SYNC);
 		break;
@@ -1623,6 +1627,12 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 		goto err;
 	}
 
+	/* Writes not allowed into this read-only object */
+	if (i915_gem_object_is_readonly(obj)) {
+		ret = -EINVAL;
+		goto err;
+	}
+
 	trace_i915_gem_object_pwrite(obj, args->offset, args->size);
 
 	ret = -ENODEV;
@@ -2006,9 +2016,12 @@ vm_fault_t i915_gem_fault(struct vm_fault *vmf)
 	bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
 	struct i915_vma *vma;
 	pgoff_t page_offset;
-	unsigned int flags;
 	int ret;
 
+	/* Sanity check that we allow writing into this object */
+	if (i915_gem_object_is_readonly(obj) && write)
+		return VM_FAULT_SIGBUS;
+
 	/* We don't use vmf->pgoff since that has the fake offset */
 	page_offset = (vmf->address - area->vm_start) >> PAGE_SHIFT;
 
@@ -2042,27 +2055,34 @@ vm_fault_t i915_gem_fault(struct vm_fault *vmf)
 		goto err_unlock;
 	}
 
-	/* If the object is smaller than a couple of partial vma, it is
-	 * not worth only creating a single partial vma - we may as well
-	 * clear enough space for the full object.
-	 */
-	flags = PIN_MAPPABLE;
-	if (obj->base.size > 2 * MIN_CHUNK_PAGES << PAGE_SHIFT)
-		flags |= PIN_NONBLOCK | PIN_NONFAULT;
 
 	/* Now pin it into the GTT as needed */
-	vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, flags);
+	vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
+				       PIN_MAPPABLE |
+				       PIN_NONBLOCK |
+				       PIN_NONFAULT);
 	if (IS_ERR(vma)) {
 		/* Use a partial view if it is bigger than available space */
 		struct i915_ggtt_view view =
 			compute_partial_view(obj, page_offset, MIN_CHUNK_PAGES);
+		unsigned int flags;
+
+		flags = PIN_MAPPABLE;
+		if (view.type == I915_GGTT_VIEW_NORMAL)
+			flags |= PIN_NONBLOCK; /* avoid warnings for pinned */
 
-		/* Userspace is now writing through an untracked VMA, abandon
+		/*
+		 * Userspace is now writing through an untracked VMA, abandon
 		 * all hope that the hardware is able to track future writes.
 		 */
 		obj->frontbuffer_ggtt_origin = ORIGIN_CPU;
 
-		vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, PIN_MAPPABLE);
+		vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, flags);
+		if (IS_ERR(vma) && !view.type) {
+			flags = PIN_MAPPABLE;
+			view.type = I915_GGTT_VIEW_PARTIAL;
+			vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, flags);
+		}
 	}
 	if (IS_ERR(vma)) {
 		ret = PTR_ERR(vma);
@@ -2114,6 +2134,7 @@ err:
 		 */
 		if (!i915_terminally_wedged(&dev_priv->gpu_error))
 			return VM_FAULT_SIGBUS;
+		/* else: fall through */
 	case -EAGAIN:
 		/*
 		 * EAGAIN means the gpu is hung and we'll wait for the error
@@ -2256,7 +2277,9 @@ static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj)
 
 	/* Attempt to reap some mmap space from dead objects */
 	do {
-		err = i915_gem_wait_for_idle(dev_priv, I915_WAIT_INTERRUPTIBLE);
+		err = i915_gem_wait_for_idle(dev_priv,
+					     I915_WAIT_INTERRUPTIBLE,
+					     MAX_SCHEDULE_TIMEOUT);
 		if (err)
 			break;
 
@@ -3074,25 +3097,6 @@ int i915_gem_reset_prepare(struct drm_i915_private *dev_priv)
 	return err;
 }
 
-static void skip_request(struct i915_request *request)
-{
-	void *vaddr = request->ring->vaddr;
-	u32 head;
-
-	/* As this request likely depends on state from the lost
-	 * context, clear out all the user operations leaving the
-	 * breadcrumb at the end (so we get the fence notifications).
-	 */
-	head = request->head;
-	if (request->postfix < head) {
-		memset(vaddr + head, 0, request->ring->size - head);
-		head = 0;
-	}
-	memset(vaddr + head, 0, request->postfix - head);
-
-	dma_fence_set_error(&request->fence, -EIO);
-}
-
 static void engine_skip_context(struct i915_request *request)
 {
 	struct intel_engine_cs *engine = request->engine;
@@ -3103,14 +3107,14 @@ static void engine_skip_context(struct i915_request *request)
 	GEM_BUG_ON(timeline == &engine->timeline);
 
 	spin_lock_irqsave(&engine->timeline.lock, flags);
-	spin_lock_nested(&timeline->lock, SINGLE_DEPTH_NESTING);
+	spin_lock(&timeline->lock);
 
 	list_for_each_entry_continue(request, &engine->timeline.requests, link)
 		if (request->gem_context == hung_ctx)
-			skip_request(request);
+			i915_request_skip(request, -EIO);
 
 	list_for_each_entry(request, &timeline->requests, link)
-		skip_request(request);
+		i915_request_skip(request, -EIO);
 
 	spin_unlock(&timeline->lock);
 	spin_unlock_irqrestore(&engine->timeline.lock, flags);
@@ -3153,7 +3157,7 @@ i915_gem_reset_request(struct intel_engine_cs *engine,
 
 	if (stalled) {
 		i915_gem_context_mark_guilty(request->gem_context);
-		skip_request(request);
+		i915_request_skip(request, -EIO);
 
 		/* If this context is now banned, skip all pending requests. */
 		if (i915_gem_context_is_banned(request->gem_context))
@@ -3750,14 +3754,14 @@ i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 	return ret;
 }
 
-static int wait_for_timeline(struct i915_timeline *tl, unsigned int flags)
+static long wait_for_timeline(struct i915_timeline *tl,
+			      unsigned int flags, long timeout)
 {
 	struct i915_request *rq;
-	long ret;
 
 	rq = i915_gem_active_get_unlocked(&tl->last_request);
 	if (!rq)
-		return 0;
+		return timeout;
 
 	/*
 	 * "Race-to-idle".
@@ -3771,10 +3775,10 @@ static int wait_for_timeline(struct i915_timeline *tl, unsigned int flags)
 	if (flags & I915_WAIT_FOR_IDLE_BOOST)
 		gen6_rps_boost(rq, NULL);
 
-	ret = i915_request_wait(rq, flags, MAX_SCHEDULE_TIMEOUT);
+	timeout = i915_request_wait(rq, flags, timeout);
 	i915_request_put(rq);
 
-	return ret < 0 ? ret : 0;
+	return timeout;
 }
 
 static int wait_for_engines(struct drm_i915_private *i915)
@@ -3790,10 +3794,12 @@ static int wait_for_engines(struct drm_i915_private *i915)
 	return 0;
 }
 
-int i915_gem_wait_for_idle(struct drm_i915_private *i915, unsigned int flags)
+int i915_gem_wait_for_idle(struct drm_i915_private *i915,
+			   unsigned int flags, long timeout)
 {
-	GEM_TRACE("flags=%x (%s)\n",
-		  flags, flags & I915_WAIT_LOCKED ? "locked" : "unlocked");
+	GEM_TRACE("flags=%x (%s), timeout=%ld%s\n",
+		  flags, flags & I915_WAIT_LOCKED ? "locked" : "unlocked",
+		  timeout, timeout == MAX_SCHEDULE_TIMEOUT ? " (forever)" : "");
 
 	/* If the device is asleep, we have no requests outstanding */
 	if (!READ_ONCE(i915->gt.awake))
@@ -3806,27 +3812,31 @@ int i915_gem_wait_for_idle(struct drm_i915_private *i915, unsigned int flags)
 		lockdep_assert_held(&i915->drm.struct_mutex);
 
 		list_for_each_entry(tl, &i915->gt.timelines, link) {
-			err = wait_for_timeline(tl, flags);
-			if (err)
-				return err;
+			timeout = wait_for_timeline(tl, flags, timeout);
+			if (timeout < 0)
+				return timeout;
 		}
+
+		err = wait_for_engines(i915);
+		if (err)
+			return err;
+
 		i915_retire_requests(i915);
 		GEM_BUG_ON(i915->gt.active_requests);
-
-		return wait_for_engines(i915);
 	} else {
 		struct intel_engine_cs *engine;
 		enum intel_engine_id id;
-		int err;
 
 		for_each_engine(engine, i915, id) {
-			err = wait_for_timeline(&engine->timeline, flags);
-			if (err)
-				return err;
-		}
+			struct i915_timeline *tl = &engine->timeline;
 
-		return 0;
+			timeout = wait_for_timeline(tl, flags, timeout);
+			if (timeout < 0)
+				return timeout;
+		}
 	}
+
+	return 0;
 }
 
 static void __i915_gem_object_flush_for_display(struct drm_i915_gem_object *obj)
@@ -5029,65 +5039,70 @@ void i915_gem_sanitize(struct drm_i915_private *i915)
 	mutex_unlock(&i915->drm.struct_mutex);
 }
 
-int i915_gem_suspend(struct drm_i915_private *dev_priv)
+int i915_gem_suspend(struct drm_i915_private *i915)
 {
-	struct drm_device *dev = &dev_priv->drm;
 	int ret;
 
 	GEM_TRACE("\n");
 
-	intel_runtime_pm_get(dev_priv);
-	intel_suspend_gt_powersave(dev_priv);
+	intel_runtime_pm_get(i915);
+	intel_suspend_gt_powersave(i915);
 
-	mutex_lock(&dev->struct_mutex);
+	mutex_lock(&i915->drm.struct_mutex);
 
-	/* We have to flush all the executing contexts to main memory so
+	/*
+	 * We have to flush all the executing contexts to main memory so
 	 * that they can saved in the hibernation image. To ensure the last
 	 * context image is coherent, we have to switch away from it. That
-	 * leaves the dev_priv->kernel_context still active when
+	 * leaves the i915->kernel_context still active when
 	 * we actually suspend, and its image in memory may not match the GPU
 	 * state. Fortunately, the kernel_context is disposable and we do
 	 * not rely on its state.
 	 */
-	if (!i915_terminally_wedged(&dev_priv->gpu_error)) {
-		ret = i915_gem_switch_to_kernel_context(dev_priv);
+	if (!i915_terminally_wedged(&i915->gpu_error)) {
+		ret = i915_gem_switch_to_kernel_context(i915);
 		if (ret)
 			goto err_unlock;
 
-		ret = i915_gem_wait_for_idle(dev_priv,
+		ret = i915_gem_wait_for_idle(i915,
 					     I915_WAIT_INTERRUPTIBLE |
 					     I915_WAIT_LOCKED |
-					     I915_WAIT_FOR_IDLE_BOOST);
+					     I915_WAIT_FOR_IDLE_BOOST,
+					     MAX_SCHEDULE_TIMEOUT);
 		if (ret && ret != -EIO)
 			goto err_unlock;
 
-		assert_kernel_context_is_current(dev_priv);
+		assert_kernel_context_is_current(i915);
 	}
-	mutex_unlock(&dev->struct_mutex);
+	i915_retire_requests(i915); /* ensure we flush after wedging */
 
-	intel_uc_suspend(dev_priv);
+	mutex_unlock(&i915->drm.struct_mutex);
+
+	intel_uc_suspend(i915);
 
-	cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
-	cancel_delayed_work_sync(&dev_priv->gt.retire_work);
+	cancel_delayed_work_sync(&i915->gpu_error.hangcheck_work);
+	cancel_delayed_work_sync(&i915->gt.retire_work);
 
-	/* As the idle_work is rearming if it detects a race, play safe and
+	/*
+	 * As the idle_work is rearming if it detects a race, play safe and
 	 * repeat the flush until it is definitely idle.
 	 */
-	drain_delayed_work(&dev_priv->gt.idle_work);
+	drain_delayed_work(&i915->gt.idle_work);
 
-	/* Assert that we sucessfully flushed all the work and
+	/*
+	 * Assert that we successfully flushed all the work and
 	 * reset the GPU back to its idle, low power state.
 	 */
-	WARN_ON(dev_priv->gt.awake);
-	if (WARN_ON(!intel_engines_are_idle(dev_priv)))
-		i915_gem_set_wedged(dev_priv); /* no hope, discard everything */
+	WARN_ON(i915->gt.awake);
+	if (WARN_ON(!intel_engines_are_idle(i915)))
+		i915_gem_set_wedged(i915); /* no hope, discard everything */
 
-	intel_runtime_pm_put(dev_priv);
+	intel_runtime_pm_put(i915);
 	return 0;
 
 err_unlock:
-	mutex_unlock(&dev->struct_mutex);
-	intel_runtime_pm_put(dev_priv);
+	mutex_unlock(&i915->drm.struct_mutex);
+	intel_runtime_pm_put(i915);
 	return ret;
 }
 
@@ -5310,13 +5325,17 @@ int i915_gem_init_hw(struct drm_i915_private *dev_priv)
 	ret = __i915_gem_restart_engines(dev_priv);
 	if (ret)
 		goto cleanup_uc;
-out:
+
 	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
-	return ret;
+
+	return 0;
 
 cleanup_uc:
 	intel_uc_fini_hw(dev_priv);
-	goto out;
+out:
+	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
+
+	return ret;
 }
 
 static int __intel_engines_record_defaults(struct drm_i915_private *i915)
@@ -5361,9 +5380,11 @@ static int __intel_engines_record_defaults(struct drm_i915_private *i915)
 	if (err)
 		goto err_active;
 
-	err = i915_gem_wait_for_idle(i915, I915_WAIT_LOCKED);
-	if (err)
+	if (i915_gem_wait_for_idle(i915, I915_WAIT_LOCKED, HZ / 5)) {
+		i915_gem_set_wedged(i915);
+		err = -EIO; /* Caller will declare us wedged */
 		goto err_active;
+	}
 
 	assert_kernel_context_is_current(i915);
 
@@ -5426,7 +5447,9 @@ err_active:
 	if (WARN_ON(i915_gem_switch_to_kernel_context(i915)))
 		goto out_ctx;
 
-	if (WARN_ON(i915_gem_wait_for_idle(i915, I915_WAIT_LOCKED)))
+	if (WARN_ON(i915_gem_wait_for_idle(i915,
+					   I915_WAIT_LOCKED,
+					   MAX_SCHEDULE_TIMEOUT)))
 		goto out_ctx;
 
 	i915_gem_contexts_lost(i915);
@@ -5456,13 +5479,13 @@ int i915_gem_init(struct drm_i915_private *dev_priv)
 	if (ret)
 		return ret;
 
-	ret = intel_wopcm_init(&dev_priv->wopcm);
+	ret = intel_uc_init_misc(dev_priv);
 	if (ret)
 		return ret;
 
-	ret = intel_uc_init_misc(dev_priv);
+	ret = intel_wopcm_init(&dev_priv->wopcm);
 	if (ret)
-		return ret;
+		goto err_uc_misc;
 
 	/* This is just a security blanket to placate dragons.
 	 * On some systems, we very sporadically observe that the first TLBs
@@ -5543,6 +5566,8 @@ err_init_hw:
 	WARN_ON(i915_gem_suspend(dev_priv));
 	i915_gem_suspend_late(dev_priv);
 
+	i915_gem_drain_workqueue(dev_priv);
+
 	mutex_lock(&dev_priv->drm.struct_mutex);
 	intel_uc_fini_hw(dev_priv);
 err_uc_init:
@@ -5560,6 +5585,7 @@ err_unlock:
 	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
 	mutex_unlock(&dev_priv->drm.struct_mutex);
 
+err_uc_misc:
 	intel_uc_fini_misc(dev_priv);
 
 	if (ret != -EIO)
diff --git a/drivers/gpu/drm/i915/i915_gem.h b/drivers/gpu/drm/i915/i915_gem.h
index 261da577829a..e46592956872 100644
--- a/drivers/gpu/drm/i915/i915_gem.h
+++ b/drivers/gpu/drm/i915/i915_gem.h
@@ -88,4 +88,9 @@ static inline void __tasklet_enable_sync_once(struct tasklet_struct *t)
 		tasklet_kill(t);
 }
 
+static inline bool __tasklet_is_enabled(const struct tasklet_struct *t)
+{
+	return !atomic_read(&t->count);
+}
+
 #endif /* __I915_GEM_H__ */
diff --git a/drivers/gpu/drm/i915/i915_gem_context.c b/drivers/gpu/drm/i915/i915_gem_context.c
index ccf463ab6562..b10770cfccd2 100644
--- a/drivers/gpu/drm/i915/i915_gem_context.c
+++ b/drivers/gpu/drm/i915/i915_gem_context.c
@@ -374,7 +374,7 @@ i915_gem_create_context(struct drm_i915_private *dev_priv,
 	if (USES_FULL_PPGTT(dev_priv)) {
 		struct i915_hw_ppgtt *ppgtt;
 
-		ppgtt = i915_ppgtt_create(dev_priv, file_priv, ctx->name);
+		ppgtt = i915_ppgtt_create(dev_priv, file_priv);
 		if (IS_ERR(ppgtt)) {
 			DRM_DEBUG_DRIVER("PPGTT setup failed (%ld)\n",
 					 PTR_ERR(ppgtt));
@@ -512,8 +512,8 @@ int i915_gem_contexts_init(struct drm_i915_private *dev_priv)
 	}
 
 	DRM_DEBUG_DRIVER("%s context support initialized\n",
-			 dev_priv->engine[RCS]->context_size ? "logical" :
-			 "fake");
+			 DRIVER_CAPS(dev_priv)->has_logical_contexts ?
+			 "logical" : "fake");
 	return 0;
 }
 
@@ -720,7 +720,7 @@ int i915_gem_context_create_ioctl(struct drm_device *dev, void *data,
 	struct i915_gem_context *ctx;
 	int ret;
 
-	if (!dev_priv->engine[RCS]->context_size)
+	if (!DRIVER_CAPS(dev_priv)->has_logical_contexts)
 		return -ENODEV;
 
 	if (args->pad != 0)
diff --git a/drivers/gpu/drm/i915/i915_gem_evict.c b/drivers/gpu/drm/i915/i915_gem_evict.c
index 54814a196ee4..02b83a5ed96c 100644
--- a/drivers/gpu/drm/i915/i915_gem_evict.c
+++ b/drivers/gpu/drm/i915/i915_gem_evict.c
@@ -69,7 +69,8 @@ static int ggtt_flush(struct drm_i915_private *i915)
 
 	err = i915_gem_wait_for_idle(i915,
 				     I915_WAIT_INTERRUPTIBLE |
-				     I915_WAIT_LOCKED);
+				     I915_WAIT_LOCKED,
+				     MAX_SCHEDULE_TIMEOUT);
 	if (err)
 		return err;
 
diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
index 60dc2a865f5f..3f0c612d42e7 100644
--- a/drivers/gpu/drm/i915/i915_gem_execbuffer.c
+++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
@@ -66,6 +66,15 @@ enum {
 #define __I915_EXEC_ILLEGAL_FLAGS \
 	(__I915_EXEC_UNKNOWN_FLAGS | I915_EXEC_CONSTANTS_MASK)
 
+/* Catch emission of unexpected errors for CI! */
+#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)
+#undef EINVAL
+#define EINVAL ({ \
+	DRM_DEBUG_DRIVER("EINVAL at %s:%d\n", __func__, __LINE__); \
+	22; \
+})
+#endif
+
 /**
  * DOC: User command execution
  *
@@ -534,7 +543,8 @@ eb_add_vma(struct i915_execbuffer *eb,
 	 * paranoia do it everywhere.
 	 */
 	if (i == batch_idx) {
-		if (!(eb->flags[i] & EXEC_OBJECT_PINNED))
+		if (entry->relocation_count &&
+		    !(eb->flags[i] & EXEC_OBJECT_PINNED))
 			eb->flags[i] |= __EXEC_OBJECT_NEEDS_BIAS;
 		if (eb->reloc_cache.has_fence)
 			eb->flags[i] |= EXEC_OBJECT_NEEDS_FENCE;
@@ -1155,18 +1165,16 @@ static int __reloc_gpu_alloc(struct i915_execbuffer *eb,
 		goto err_request;
 
 	GEM_BUG_ON(!reservation_object_test_signaled_rcu(batch->resv, true));
-	i915_vma_move_to_active(batch, rq, 0);
-	reservation_object_lock(batch->resv, NULL);
-	reservation_object_add_excl_fence(batch->resv, &rq->fence);
-	reservation_object_unlock(batch->resv);
-	i915_vma_unpin(batch);
+	err = i915_vma_move_to_active(batch, rq, 0);
+	if (err)
+		goto skip_request;
 
-	i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
-	reservation_object_lock(vma->resv, NULL);
-	reservation_object_add_excl_fence(vma->resv, &rq->fence);
-	reservation_object_unlock(vma->resv);
+	err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
+	if (err)
+		goto skip_request;
 
 	rq->batch = batch;
+	i915_vma_unpin(batch);
 
 	cache->rq = rq;
 	cache->rq_cmd = cmd;
@@ -1175,6 +1183,8 @@ static int __reloc_gpu_alloc(struct i915_execbuffer *eb,
 	/* Return with batch mapping (cmd) still pinned */
 	return 0;
 
+skip_request:
+	i915_request_skip(rq, err);
 err_request:
 	i915_request_add(rq);
 err_unpin:
@@ -1761,25 +1771,6 @@ slow:
 	return eb_relocate_slow(eb);
 }
 
-static void eb_export_fence(struct i915_vma *vma,
-			    struct i915_request *rq,
-			    unsigned int flags)
-{
-	struct reservation_object *resv = vma->resv;
-
-	/*
-	 * Ignore errors from failing to allocate the new fence, we can't
-	 * handle an error right now. Worst case should be missed
-	 * synchronisation leading to rendering corruption.
-	 */
-	reservation_object_lock(resv, NULL);
-	if (flags & EXEC_OBJECT_WRITE)
-		reservation_object_add_excl_fence(resv, &rq->fence);
-	else if (reservation_object_reserve_shared(resv) == 0)
-		reservation_object_add_shared_fence(resv, &rq->fence);
-	reservation_object_unlock(resv);
-}
-
 static int eb_move_to_gpu(struct i915_execbuffer *eb)
 {
 	const unsigned int count = eb->buffer_count;
@@ -1833,8 +1824,11 @@ static int eb_move_to_gpu(struct i915_execbuffer *eb)
 		unsigned int flags = eb->flags[i];
 		struct i915_vma *vma = eb->vma[i];
 
-		i915_vma_move_to_active(vma, eb->request, flags);
-		eb_export_fence(vma, eb->request, flags);
+		err = i915_vma_move_to_active(vma, eb->request, flags);
+		if (unlikely(err)) {
+			i915_request_skip(eb->request, err);
+			return err;
+		}
 
 		__eb_unreserve_vma(vma, flags);
 		vma->exec_flags = NULL;
@@ -1874,45 +1868,6 @@ static bool i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec)
 	return true;
 }
 
-void i915_vma_move_to_active(struct i915_vma *vma,
-			     struct i915_request *rq,
-			     unsigned int flags)
-{
-	struct drm_i915_gem_object *obj = vma->obj;
-	const unsigned int idx = rq->engine->id;
-
-	lockdep_assert_held(&rq->i915->drm.struct_mutex);
-	GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
-
-	/*
-	 * Add a reference if we're newly entering the active list.
-	 * The order in which we add operations to the retirement queue is
-	 * vital here: mark_active adds to the start of the callback list,
-	 * such that subsequent callbacks are called first. Therefore we
-	 * add the active reference first and queue for it to be dropped
-	 * *last*.
-	 */
-	if (!i915_vma_is_active(vma))
-		obj->active_count++;
-	i915_vma_set_active(vma, idx);
-	i915_gem_active_set(&vma->last_read[idx], rq);
-	list_move_tail(&vma->vm_link, &vma->vm->active_list);
-
-	obj->write_domain = 0;
-	if (flags & EXEC_OBJECT_WRITE) {
-		obj->write_domain = I915_GEM_DOMAIN_RENDER;
-
-		if (intel_fb_obj_invalidate(obj, ORIGIN_CS))
-			i915_gem_active_set(&obj->frontbuffer_write, rq);
-
-		obj->read_domains = 0;
-	}
-	obj->read_domains |= I915_GEM_GPU_DOMAINS;
-
-	if (flags & EXEC_OBJECT_NEEDS_FENCE)
-		i915_gem_active_set(&vma->last_fence, rq);
-}
-
 static int i915_reset_gen7_sol_offsets(struct i915_request *rq)
 {
 	u32 *cs;
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
index c6aa761ca085..f00c7fbef79e 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -204,9 +204,9 @@ static int ppgtt_bind_vma(struct i915_vma *vma,
 			return err;
 	}
 
-	/* Currently applicable only to VLV */
+	/* Applicable to VLV, and gen8+ */
 	pte_flags = 0;
-	if (vma->obj->gt_ro)
+	if (i915_gem_object_is_readonly(vma->obj))
 		pte_flags |= PTE_READ_ONLY;
 
 	vma->vm->insert_entries(vma->vm, vma, cache_level, pte_flags);
@@ -244,10 +244,13 @@ static void clear_pages(struct i915_vma *vma)
 }
 
 static gen8_pte_t gen8_pte_encode(dma_addr_t addr,
-				  enum i915_cache_level level)
+				  enum i915_cache_level level,
+				  u32 flags)
 {
-	gen8_pte_t pte = _PAGE_PRESENT | _PAGE_RW;
-	pte |= addr;
+	gen8_pte_t pte = addr | _PAGE_PRESENT | _PAGE_RW;
+
+	if (unlikely(flags & PTE_READ_ONLY))
+		pte &= ~_PAGE_RW;
 
 	switch (level) {
 	case I915_CACHE_NONE:
@@ -375,37 +378,70 @@ static gen6_pte_t iris_pte_encode(dma_addr_t addr,
 	return pte;
 }
 
+static void stash_init(struct pagestash *stash)
+{
+	pagevec_init(&stash->pvec);
+	spin_lock_init(&stash->lock);
+}
+
+static struct page *stash_pop_page(struct pagestash *stash)
+{
+	struct page *page = NULL;
+
+	spin_lock(&stash->lock);
+	if (likely(stash->pvec.nr))
+		page = stash->pvec.pages[--stash->pvec.nr];
+	spin_unlock(&stash->lock);
+
+	return page;
+}
+
+static void stash_push_pagevec(struct pagestash *stash, struct pagevec *pvec)
+{
+	int nr;
+
+	spin_lock_nested(&stash->lock, SINGLE_DEPTH_NESTING);
+
+	nr = min_t(int, pvec->nr, pagevec_space(&stash->pvec));
+	memcpy(stash->pvec.pages + stash->pvec.nr,
+	       pvec->pages + pvec->nr - nr,
+	       sizeof(pvec->pages[0]) * nr);
+	stash->pvec.nr += nr;
+
+	spin_unlock(&stash->lock);
+
+	pvec->nr -= nr;
+}
+
 static struct page *vm_alloc_page(struct i915_address_space *vm, gfp_t gfp)
 {
-	struct pagevec *pvec = &vm->free_pages;
-	struct pagevec stash;
+	struct pagevec stack;
+	struct page *page;
 
 	if (I915_SELFTEST_ONLY(should_fail(&vm->fault_attr, 1)))
 		i915_gem_shrink_all(vm->i915);
 
-	if (likely(pvec->nr))
-		return pvec->pages[--pvec->nr];
+	page = stash_pop_page(&vm->free_pages);
+	if (page)
+		return page;
 
 	if (!vm->pt_kmap_wc)
 		return alloc_page(gfp);
 
-	/* A placeholder for a specific mutex to guard the WC stash */
-	lockdep_assert_held(&vm->i915->drm.struct_mutex);
-
 	/* Look in our global stash of WC pages... */
-	pvec = &vm->i915->mm.wc_stash;
-	if (likely(pvec->nr))
-		return pvec->pages[--pvec->nr];
+	page = stash_pop_page(&vm->i915->mm.wc_stash);
+	if (page)
+		return page;
 
 	/*
-	 * Otherwise batch allocate pages to amoritize cost of set_pages_wc.
+	 * Otherwise batch allocate pages to amortize cost of set_pages_wc.
 	 *
 	 * We have to be careful as page allocation may trigger the shrinker
 	 * (via direct reclaim) which will fill up the WC stash underneath us.
 	 * So we add our WB pages into a temporary pvec on the stack and merge
 	 * them into the WC stash after all the allocations are complete.
 	 */
-	pagevec_init(&stash);
+	pagevec_init(&stack);
 	do {
 		struct page *page;
 
@@ -413,59 +449,67 @@ static struct page *vm_alloc_page(struct i915_address_space *vm, gfp_t gfp)
 		if (unlikely(!page))
 			break;
 
-		stash.pages[stash.nr++] = page;
-	} while (stash.nr < pagevec_space(pvec));
+		stack.pages[stack.nr++] = page;
+	} while (pagevec_space(&stack));
 
-	if (stash.nr) {
-		int nr = min_t(int, stash.nr, pagevec_space(pvec));
-		struct page **pages = stash.pages + stash.nr - nr;
+	if (stack.nr && !set_pages_array_wc(stack.pages, stack.nr)) {
+		page = stack.pages[--stack.nr];
 
-		if (nr && !set_pages_array_wc(pages, nr)) {
-			memcpy(pvec->pages + pvec->nr,
-			       pages, sizeof(pages[0]) * nr);
-			pvec->nr += nr;
-			stash.nr -= nr;
-		}
+		/* Merge spare WC pages to the global stash */
+		stash_push_pagevec(&vm->i915->mm.wc_stash, &stack);
+
+		/* Push any surplus WC pages onto the local VM stash */
+		if (stack.nr)
+			stash_push_pagevec(&vm->free_pages, &stack);
+	}
 
-		pagevec_release(&stash);
+	/* Return unwanted leftovers */
+	if (unlikely(stack.nr)) {
+		WARN_ON_ONCE(set_pages_array_wb(stack.pages, stack.nr));
+		__pagevec_release(&stack);
 	}
 
-	return likely(pvec->nr) ? pvec->pages[--pvec->nr] : NULL;
+	return page;
 }
 
 static void vm_free_pages_release(struct i915_address_space *vm,
 				  bool immediate)
 {
-	struct pagevec *pvec = &vm->free_pages;
+	struct pagevec *pvec = &vm->free_pages.pvec;
+	struct pagevec stack;
 
+	lockdep_assert_held(&vm->free_pages.lock);
 	GEM_BUG_ON(!pagevec_count(pvec));
 
 	if (vm->pt_kmap_wc) {
-		struct pagevec *stash = &vm->i915->mm.wc_stash;
-
-		/* When we use WC, first fill up the global stash and then
+		/*
+		 * When we use WC, first fill up the global stash and then
 		 * only if full immediately free the overflow.
 		 */
+		stash_push_pagevec(&vm->i915->mm.wc_stash, pvec);
 
-		lockdep_assert_held(&vm->i915->drm.struct_mutex);
-		if (pagevec_space(stash)) {
-			do {
-				stash->pages[stash->nr++] =
-					pvec->pages[--pvec->nr];
-				if (!pvec->nr)
-					return;
-			} while (pagevec_space(stash));
-
-			/* As we have made some room in the VM's free_pages,
-			 * we can wait for it to fill again. Unless we are
-			 * inside i915_address_space_fini() and must
-			 * immediately release the pages!
-			 */
-			if (!immediate)
-				return;
-		}
+		/*
+		 * As we have made some room in the VM's free_pages,
+		 * we can wait for it to fill again. Unless we are
+		 * inside i915_address_space_fini() and must
+		 * immediately release the pages!
+		 */
+		if (pvec->nr <= (immediate ? 0 : PAGEVEC_SIZE - 1))
+			return;
 
+		/*
+		 * We have to drop the lock to allow ourselves to sleep,
+		 * so take a copy of the pvec and clear the stash for
+		 * others to use it as we sleep.
+		 */
+		stack = *pvec;
+		pagevec_reinit(pvec);
+		spin_unlock(&vm->free_pages.lock);
+
+		pvec = &stack;
 		set_pages_array_wb(pvec->pages, pvec->nr);
+
+		spin_lock(&vm->free_pages.lock);
 	}
 
 	__pagevec_release(pvec);
@@ -481,8 +525,45 @@ static void vm_free_page(struct i915_address_space *vm, struct page *page)
 	 * unconditional might_sleep() for everybody.
 	 */
 	might_sleep();
-	if (!pagevec_add(&vm->free_pages, page))
+	spin_lock(&vm->free_pages.lock);
+	if (!pagevec_add(&vm->free_pages.pvec, page))
 		vm_free_pages_release(vm, false);
+	spin_unlock(&vm->free_pages.lock);
+}
+
+static void i915_address_space_init(struct i915_address_space *vm,
+				    struct drm_i915_private *dev_priv)
+{
+	/*
+	 * The vm->mutex must be reclaim safe (for use in the shrinker).
+	 * Do a dummy acquire now under fs_reclaim so that any allocation
+	 * attempt holding the lock is immediately reported by lockdep.
+	 */
+	mutex_init(&vm->mutex);
+	i915_gem_shrinker_taints_mutex(&vm->mutex);
+
+	GEM_BUG_ON(!vm->total);
+	drm_mm_init(&vm->mm, 0, vm->total);
+	vm->mm.head_node.color = I915_COLOR_UNEVICTABLE;
+
+	stash_init(&vm->free_pages);
+
+	INIT_LIST_HEAD(&vm->active_list);
+	INIT_LIST_HEAD(&vm->inactive_list);
+	INIT_LIST_HEAD(&vm->unbound_list);
+}
+
+static void i915_address_space_fini(struct i915_address_space *vm)
+{
+	spin_lock(&vm->free_pages.lock);
+	if (pagevec_count(&vm->free_pages.pvec))
+		vm_free_pages_release(vm, true);
+	GEM_BUG_ON(pagevec_count(&vm->free_pages.pvec));
+	spin_unlock(&vm->free_pages.lock);
+
+	drm_mm_takedown(&vm->mm);
+
+	mutex_destroy(&vm->mutex);
 }
 
 static int __setup_page_dma(struct i915_address_space *vm,
@@ -493,8 +574,11 @@ static int __setup_page_dma(struct i915_address_space *vm,
 	if (unlikely(!p->page))
 		return -ENOMEM;
 
-	p->daddr = dma_map_page(vm->dma, p->page, 0, PAGE_SIZE,
-				PCI_DMA_BIDIRECTIONAL);
+	p->daddr = dma_map_page_attrs(vm->dma,
+				      p->page, 0, PAGE_SIZE,
+				      PCI_DMA_BIDIRECTIONAL,
+				      DMA_ATTR_SKIP_CPU_SYNC |
+				      DMA_ATTR_NO_WARN);
 	if (unlikely(dma_mapping_error(vm->dma, p->daddr))) {
 		vm_free_page(vm, p->page);
 		return -ENOMEM;
@@ -575,8 +659,11 @@ setup_scratch_page(struct i915_address_space *vm, gfp_t gfp)
 		if (unlikely(!page))
 			goto skip;
 
-		addr = dma_map_page(vm->dma, page, 0, size,
-				    PCI_DMA_BIDIRECTIONAL);
+		addr = dma_map_page_attrs(vm->dma,
+					  page, 0, size,
+					  PCI_DMA_BIDIRECTIONAL,
+					  DMA_ATTR_SKIP_CPU_SYNC |
+					  DMA_ATTR_NO_WARN);
 		if (unlikely(dma_mapping_error(vm->dma, addr)))
 			goto free_page;
 
@@ -637,7 +724,7 @@ static void gen8_initialize_pt(struct i915_address_space *vm,
 			       struct i915_page_table *pt)
 {
 	fill_px(vm, pt,
-		gen8_pte_encode(vm->scratch_page.daddr, I915_CACHE_LLC));
+		gen8_pte_encode(vm->scratch_page.daddr, I915_CACHE_LLC, 0));
 }
 
 static void gen6_initialize_pt(struct gen6_hw_ppgtt *ppgtt,
@@ -785,7 +872,7 @@ static bool gen8_ppgtt_clear_pt(struct i915_address_space *vm,
 	unsigned int pte = gen8_pte_index(start);
 	unsigned int pte_end = pte + num_entries;
 	const gen8_pte_t scratch_pte =
-		gen8_pte_encode(vm->scratch_page.daddr, I915_CACHE_LLC);
+		gen8_pte_encode(vm->scratch_page.daddr, I915_CACHE_LLC, 0);
 	gen8_pte_t *vaddr;
 
 	GEM_BUG_ON(num_entries > pt->used_ptes);
@@ -957,10 +1044,11 @@ gen8_ppgtt_insert_pte_entries(struct i915_hw_ppgtt *ppgtt,
 			      struct i915_page_directory_pointer *pdp,
 			      struct sgt_dma *iter,
 			      struct gen8_insert_pte *idx,
-			      enum i915_cache_level cache_level)
+			      enum i915_cache_level cache_level,
+			      u32 flags)
 {
 	struct i915_page_directory *pd;
-	const gen8_pte_t pte_encode = gen8_pte_encode(0, cache_level);
+	const gen8_pte_t pte_encode = gen8_pte_encode(0, cache_level, flags);
 	gen8_pte_t *vaddr;
 	bool ret;
 
@@ -1011,14 +1099,14 @@ gen8_ppgtt_insert_pte_entries(struct i915_hw_ppgtt *ppgtt,
 static void gen8_ppgtt_insert_3lvl(struct i915_address_space *vm,
 				   struct i915_vma *vma,
 				   enum i915_cache_level cache_level,
-				   u32 unused)
+				   u32 flags)
 {
 	struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
 	struct sgt_dma iter = sgt_dma(vma);
 	struct gen8_insert_pte idx = gen8_insert_pte(vma->node.start);
 
 	gen8_ppgtt_insert_pte_entries(ppgtt, &ppgtt->pdp, &iter, &idx,
-				      cache_level);
+				      cache_level, flags);
 
 	vma->page_sizes.gtt = I915_GTT_PAGE_SIZE;
 }
@@ -1026,9 +1114,10 @@ static void gen8_ppgtt_insert_3lvl(struct i915_address_space *vm,
 static void gen8_ppgtt_insert_huge_entries(struct i915_vma *vma,
 					   struct i915_page_directory_pointer **pdps,
 					   struct sgt_dma *iter,
-					   enum i915_cache_level cache_level)
+					   enum i915_cache_level cache_level,
+					   u32 flags)
 {
-	const gen8_pte_t pte_encode = gen8_pte_encode(0, cache_level);
+	const gen8_pte_t pte_encode = gen8_pte_encode(0, cache_level, flags);
 	u64 start = vma->node.start;
 	dma_addr_t rem = iter->sg->length;
 
@@ -1144,19 +1233,21 @@ static void gen8_ppgtt_insert_huge_entries(struct i915_vma *vma,
 static void gen8_ppgtt_insert_4lvl(struct i915_address_space *vm,
 				   struct i915_vma *vma,
 				   enum i915_cache_level cache_level,
-				   u32 unused)
+				   u32 flags)
 {
 	struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
 	struct sgt_dma iter = sgt_dma(vma);
 	struct i915_page_directory_pointer **pdps = ppgtt->pml4.pdps;
 
 	if (vma->page_sizes.sg > I915_GTT_PAGE_SIZE) {
-		gen8_ppgtt_insert_huge_entries(vma, pdps, &iter, cache_level);
+		gen8_ppgtt_insert_huge_entries(vma, pdps, &iter, cache_level,
+					       flags);
 	} else {
 		struct gen8_insert_pte idx = gen8_insert_pte(vma->node.start);
 
 		while (gen8_ppgtt_insert_pte_entries(ppgtt, pdps[idx.pml4e++],
-						     &iter, &idx, cache_level))
+						     &iter, &idx, cache_level,
+						     flags))
 			GEM_BUG_ON(idx.pml4e >= GEN8_PML4ES_PER_PML4);
 
 		vma->page_sizes.gtt = I915_GTT_PAGE_SIZE;
@@ -1494,7 +1585,7 @@ static void gen8_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)
 {
 	struct i915_address_space *vm = &ppgtt->vm;
 	const gen8_pte_t scratch_pte =
-		gen8_pte_encode(vm->scratch_page.daddr, I915_CACHE_LLC);
+		gen8_pte_encode(vm->scratch_page.daddr, I915_CACHE_LLC, 0);
 	u64 start = 0, length = ppgtt->vm.total;
 
 	if (use_4lvl(vm)) {
@@ -1562,6 +1653,8 @@ static struct i915_hw_ppgtt *gen8_ppgtt_create(struct drm_i915_private *i915)
 	if (!ppgtt)
 		return ERR_PTR(-ENOMEM);
 
+	kref_init(&ppgtt->ref);
+
 	ppgtt->vm.i915 = i915;
 	ppgtt->vm.dma = &i915->drm.pdev->dev;
 
@@ -1569,6 +1662,15 @@ static struct i915_hw_ppgtt *gen8_ppgtt_create(struct drm_i915_private *i915)
 		1ULL << 48 :
 		1ULL << 32;
 
+	/*
+	 * From bdw, there is support for read-only pages in the PPGTT.
+	 *
+	 * XXX GVT is not honouring the lack of RW in the PTE bits.
+	 */
+	ppgtt->vm.has_read_only = !intel_vgpu_active(i915);
+
+	i915_address_space_init(&ppgtt->vm, i915);
+
 	/* There are only few exceptions for gen >=6. chv and bxt.
 	 * And we are not sure about the latter so play safe for now.
 	 */
@@ -1996,7 +2098,6 @@ static struct i915_vma *pd_vma_create(struct gen6_hw_ppgtt *ppgtt, int size)
 	struct drm_i915_private *i915 = ppgtt->base.vm.i915;
 	struct i915_ggtt *ggtt = &i915->ggtt;
 	struct i915_vma *vma;
-	int i;
 
 	GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
 	GEM_BUG_ON(size > ggtt->vm.total);
@@ -2005,14 +2106,14 @@ static struct i915_vma *pd_vma_create(struct gen6_hw_ppgtt *ppgtt, int size)
 	if (!vma)
 		return ERR_PTR(-ENOMEM);
 
-	for (i = 0; i < ARRAY_SIZE(vma->last_read); i++)
-		init_request_active(&vma->last_read[i], NULL);
 	init_request_active(&vma->last_fence, NULL);
 
 	vma->vm = &ggtt->vm;
 	vma->ops = &pd_vma_ops;
 	vma->private = ppgtt;
 
+	vma->active = RB_ROOT;
+
 	vma->size = size;
 	vma->fence_size = size;
 	vma->flags = I915_VMA_GGTT;
@@ -2068,11 +2169,15 @@ static struct i915_hw_ppgtt *gen6_ppgtt_create(struct drm_i915_private *i915)
 	if (!ppgtt)
 		return ERR_PTR(-ENOMEM);
 
+	kref_init(&ppgtt->base.ref);
+
 	ppgtt->base.vm.i915 = i915;
 	ppgtt->base.vm.dma = &i915->drm.pdev->dev;
 
 	ppgtt->base.vm.total = I915_PDES * GEN6_PTES * PAGE_SIZE;
 
+	i915_address_space_init(&ppgtt->base.vm, i915);
+
 	ppgtt->base.vm.allocate_va_range = gen6_alloc_va_range;
 	ppgtt->base.vm.clear_range = gen6_ppgtt_clear_range;
 	ppgtt->base.vm.insert_entries = gen6_ppgtt_insert_entries;
@@ -2105,30 +2210,6 @@ err_free:
 	return ERR_PTR(err);
 }
 
-static void i915_address_space_init(struct i915_address_space *vm,
-				    struct drm_i915_private *dev_priv,
-				    const char *name)
-{
-	drm_mm_init(&vm->mm, 0, vm->total);
-	vm->mm.head_node.color = I915_COLOR_UNEVICTABLE;
-
-	INIT_LIST_HEAD(&vm->active_list);
-	INIT_LIST_HEAD(&vm->inactive_list);
-	INIT_LIST_HEAD(&vm->unbound_list);
-
-	list_add_tail(&vm->global_link, &dev_priv->vm_list);
-	pagevec_init(&vm->free_pages);
-}
-
-static void i915_address_space_fini(struct i915_address_space *vm)
-{
-	if (pagevec_count(&vm->free_pages))
-		vm_free_pages_release(vm, true);
-
-	drm_mm_takedown(&vm->mm);
-	list_del(&vm->global_link);
-}
-
 static void gtt_write_workarounds(struct drm_i915_private *dev_priv)
 {
 	/* This function is for gtt related workarounds. This function is
@@ -2199,8 +2280,7 @@ __hw_ppgtt_create(struct drm_i915_private *i915)
 
 struct i915_hw_ppgtt *
 i915_ppgtt_create(struct drm_i915_private *i915,
-		  struct drm_i915_file_private *fpriv,
-		  const char *name)
+		  struct drm_i915_file_private *fpriv)
 {
 	struct i915_hw_ppgtt *ppgtt;
 
@@ -2208,8 +2288,6 @@ i915_ppgtt_create(struct drm_i915_private *i915,
 	if (IS_ERR(ppgtt))
 		return ppgtt;
 
-	kref_init(&ppgtt->ref);
-	i915_address_space_init(&ppgtt->vm, i915, name);
 	ppgtt->vm.file = fpriv;
 
 	trace_i915_ppgtt_create(&ppgtt->vm);
@@ -2397,7 +2475,7 @@ static void gen8_ggtt_insert_page(struct i915_address_space *vm,
 	gen8_pte_t __iomem *pte =
 		(gen8_pte_t __iomem *)ggtt->gsm + (offset >> PAGE_SHIFT);
 
-	gen8_set_pte(pte, gen8_pte_encode(addr, level));
+	gen8_set_pte(pte, gen8_pte_encode(addr, level, 0));
 
 	ggtt->invalidate(vm->i915);
 }
@@ -2405,14 +2483,19 @@ static void gen8_ggtt_insert_page(struct i915_address_space *vm,
 static void gen8_ggtt_insert_entries(struct i915_address_space *vm,
 				     struct i915_vma *vma,
 				     enum i915_cache_level level,
-				     u32 unused)
+				     u32 flags)
 {
 	struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
 	struct sgt_iter sgt_iter;
 	gen8_pte_t __iomem *gtt_entries;
-	const gen8_pte_t pte_encode = gen8_pte_encode(0, level);
+	const gen8_pte_t pte_encode = gen8_pte_encode(0, level, 0);
 	dma_addr_t addr;
 
+	/*
+	 * Note that we ignore PTE_READ_ONLY here. The caller must be careful
+	 * not to allow the user to override access to a read only page.
+	 */
+
 	gtt_entries = (gen8_pte_t __iomem *)ggtt->gsm;
 	gtt_entries += vma->node.start >> PAGE_SHIFT;
 	for_each_sgt_dma(addr, sgt_iter, vma->pages)
@@ -2478,7 +2561,7 @@ static void gen8_ggtt_clear_range(struct i915_address_space *vm,
 	unsigned first_entry = start >> PAGE_SHIFT;
 	unsigned num_entries = length >> PAGE_SHIFT;
 	const gen8_pte_t scratch_pte =
-		gen8_pte_encode(vm->scratch_page.daddr, I915_CACHE_LLC);
+		gen8_pte_encode(vm->scratch_page.daddr, I915_CACHE_LLC, 0);
 	gen8_pte_t __iomem *gtt_base =
 		(gen8_pte_t __iomem *)ggtt->gsm + first_entry;
 	const int max_entries = ggtt_total_entries(ggtt) - first_entry;
@@ -2539,13 +2622,14 @@ struct insert_entries {
 	struct i915_address_space *vm;
 	struct i915_vma *vma;
 	enum i915_cache_level level;
+	u32 flags;
 };
 
 static int bxt_vtd_ggtt_insert_entries__cb(void *_arg)
 {
 	struct insert_entries *arg = _arg;
 
-	gen8_ggtt_insert_entries(arg->vm, arg->vma, arg->level, 0);
+	gen8_ggtt_insert_entries(arg->vm, arg->vma, arg->level, arg->flags);
 	bxt_vtd_ggtt_wa(arg->vm);
 
 	return 0;
@@ -2554,9 +2638,9 @@ static int bxt_vtd_ggtt_insert_entries__cb(void *_arg)
 static void bxt_vtd_ggtt_insert_entries__BKL(struct i915_address_space *vm,
 					     struct i915_vma *vma,
 					     enum i915_cache_level level,
-					     u32 unused)
+					     u32 flags)
 {
-	struct insert_entries arg = { vm, vma, level };
+	struct insert_entries arg = { vm, vma, level, flags };
 
 	stop_machine(bxt_vtd_ggtt_insert_entries__cb, &arg, NULL);
 }
@@ -2647,9 +2731,9 @@ static int ggtt_bind_vma(struct i915_vma *vma,
 	struct drm_i915_gem_object *obj = vma->obj;
 	u32 pte_flags;
 
-	/* Currently applicable only to VLV */
+	/* Applicable to VLV (gen8+ do not support RO in the GGTT) */
 	pte_flags = 0;
-	if (obj->gt_ro)
+	if (i915_gem_object_is_readonly(obj))
 		pte_flags |= PTE_READ_ONLY;
 
 	intel_runtime_pm_get(i915);
@@ -2687,7 +2771,7 @@ static int aliasing_gtt_bind_vma(struct i915_vma *vma,
 
 	/* Currently applicable only to VLV */
 	pte_flags = 0;
-	if (vma->obj->gt_ro)
+	if (i915_gem_object_is_readonly(vma->obj))
 		pte_flags |= PTE_READ_ONLY;
 
 	if (flags & I915_VMA_LOCAL_BIND) {
@@ -2739,7 +2823,7 @@ void i915_gem_gtt_finish_pages(struct drm_i915_gem_object *obj,
 	struct i915_ggtt *ggtt = &dev_priv->ggtt;
 
 	if (unlikely(ggtt->do_idle_maps)) {
-		if (i915_gem_wait_for_idle(dev_priv, 0)) {
+		if (i915_gem_wait_for_idle(dev_priv, 0, MAX_SCHEDULE_TIMEOUT)) {
 			DRM_ERROR("Failed to wait for idle; VT'd may hang.\n");
 			/* Wait a bit, in hopes it avoids the hang */
 			udelay(10);
@@ -2788,7 +2872,7 @@ int i915_gem_init_aliasing_ppgtt(struct drm_i915_private *i915)
 	struct i915_hw_ppgtt *ppgtt;
 	int err;
 
-	ppgtt = i915_ppgtt_create(i915, ERR_PTR(-EPERM), "[alias]");
+	ppgtt = i915_ppgtt_create(i915, ERR_PTR(-EPERM));
 	if (IS_ERR(ppgtt))
 		return PTR_ERR(ppgtt);
 
@@ -2918,7 +3002,7 @@ void i915_ggtt_cleanup_hw(struct drm_i915_private *dev_priv)
 
 	ggtt->vm.cleanup(&ggtt->vm);
 
-	pvec = &dev_priv->mm.wc_stash;
+	pvec = &dev_priv->mm.wc_stash.pvec;
 	if (pvec->nr) {
 		set_pages_array_wb(pvec->pages, pvec->nr);
 		__pagevec_release(pvec);
@@ -3518,7 +3602,7 @@ int i915_ggtt_init_hw(struct drm_i915_private *dev_priv)
 	struct i915_ggtt *ggtt = &dev_priv->ggtt;
 	int ret;
 
-	INIT_LIST_HEAD(&dev_priv->vm_list);
+	stash_init(&dev_priv->mm.wc_stash);
 
 	/* Note that we use page colouring to enforce a guard page at the
 	 * end of the address space. This is required as the CS may prefetch
@@ -3526,7 +3610,11 @@ int i915_ggtt_init_hw(struct drm_i915_private *dev_priv)
 	 * and beyond the end of the GTT if we do not provide a guard.
 	 */
 	mutex_lock(&dev_priv->drm.struct_mutex);
-	i915_address_space_init(&ggtt->vm, dev_priv, "[global]");
+	i915_address_space_init(&ggtt->vm, dev_priv);
+
+	/* Only VLV supports read-only GGTT mappings */
+	ggtt->vm.has_read_only = IS_VALLEYVIEW(dev_priv);
+
 	if (!HAS_LLC(dev_priv) && !USES_PPGTT(dev_priv))
 		ggtt->vm.mm.color_adjust = i915_gtt_color_adjust;
 	mutex_unlock(&dev_priv->drm.struct_mutex);
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.h b/drivers/gpu/drm/i915/i915_gem_gtt.h
index 9a4824cae68d..2a116a91420b 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.h
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.h
@@ -270,6 +270,11 @@ struct i915_vma_ops {
 	void (*clear_pages)(struct i915_vma *vma);
 };
 
+struct pagestash {
+	spinlock_t lock;
+	struct pagevec pvec;
+};
+
 struct i915_address_space {
 	struct drm_mm mm;
 	struct drm_i915_private *i915;
@@ -283,12 +288,13 @@ struct i915_address_space {
 	 * assign blame.
 	 */
 	struct drm_i915_file_private *file;
-	struct list_head global_link;
 	u64 total;		/* size addr space maps (ex. 2GB for ggtt) */
 	u64 reserved;		/* size addr space reserved */
 
 	bool closed;
 
+	struct mutex mutex; /* protects vma and our lists */
+
 	struct i915_page_dma scratch_page;
 	struct i915_page_table *scratch_pt;
 	struct i915_page_directory *scratch_pd;
@@ -324,8 +330,13 @@ struct i915_address_space {
 	 */
 	struct list_head unbound_list;
 
-	struct pagevec free_pages;
-	bool pt_kmap_wc;
+	struct pagestash free_pages;
+
+	/* Some systems require uncached updates of the page directories */
+	bool pt_kmap_wc:1;
+
+	/* Some systems support read-only mappings for GGTT and/or PPGTT */
+	bool has_read_only:1;
 
 	/* FIXME: Need a more generic return type */
 	gen6_pte_t (*pte_encode)(dma_addr_t addr,
@@ -615,8 +626,7 @@ void i915_ggtt_cleanup_hw(struct drm_i915_private *dev_priv);
 int i915_ppgtt_init_hw(struct drm_i915_private *dev_priv);
 void i915_ppgtt_release(struct kref *kref);
 struct i915_hw_ppgtt *i915_ppgtt_create(struct drm_i915_private *dev_priv,
-					struct drm_i915_file_private *fpriv,
-					const char *name);
+					struct drm_i915_file_private *fpriv);
 void i915_ppgtt_close(struct i915_address_space *vm);
 static inline void i915_ppgtt_get(struct i915_hw_ppgtt *ppgtt)
 {
diff --git a/drivers/gpu/drm/i915/i915_gem_object.h b/drivers/gpu/drm/i915/i915_gem_object.h
index 54f00b350779..83e5e01fa9ea 100644
--- a/drivers/gpu/drm/i915/i915_gem_object.h
+++ b/drivers/gpu/drm/i915/i915_gem_object.h
@@ -141,7 +141,6 @@ struct drm_i915_gem_object {
 	 * Is the object to be mapped as read-only to the GPU
 	 * Only honoured if hardware has relevant pte bit
 	 */
-	unsigned long gt_ro:1;
 	unsigned int cache_level:3;
 	unsigned int cache_coherent:2;
 #define I915_BO_CACHE_COHERENT_FOR_READ BIT(0)
@@ -268,7 +267,6 @@ struct drm_i915_gem_object {
 	union {
 		struct i915_gem_userptr {
 			uintptr_t ptr;
-			unsigned read_only :1;
 
 			struct i915_mm_struct *mm;
 			struct i915_mmu_object *mmu_object;
@@ -337,26 +335,17 @@ __attribute__((nonnull))
 static inline struct drm_i915_gem_object *
 i915_gem_object_get(struct drm_i915_gem_object *obj)
 {
-	drm_gem_object_reference(&obj->base);
+	drm_gem_object_get(&obj->base);
 	return obj;
 }
 
-__deprecated
-extern void drm_gem_object_reference(struct drm_gem_object *);
-
 __attribute__((nonnull))
 static inline void
 i915_gem_object_put(struct drm_i915_gem_object *obj)
 {
-	__drm_gem_object_unreference(&obj->base);
+	__drm_gem_object_put(&obj->base);
 }
 
-__deprecated
-extern void drm_gem_object_unreference(struct drm_gem_object *);
-
-__deprecated
-extern void drm_gem_object_unreference_unlocked(struct drm_gem_object *);
-
 static inline void i915_gem_object_lock(struct drm_i915_gem_object *obj)
 {
 	reservation_object_lock(obj->resv, NULL);
@@ -367,6 +356,18 @@ static inline void i915_gem_object_unlock(struct drm_i915_gem_object *obj)
 	reservation_object_unlock(obj->resv);
 }
 
+static inline void
+i915_gem_object_set_readonly(struct drm_i915_gem_object *obj)
+{
+	obj->base.vma_node.readonly = true;
+}
+
+static inline bool
+i915_gem_object_is_readonly(const struct drm_i915_gem_object *obj)
+{
+	return obj->base.vma_node.readonly;
+}
+
 static inline bool
 i915_gem_object_has_struct_page(const struct drm_i915_gem_object *obj)
 {
diff --git a/drivers/gpu/drm/i915/i915_gem_render_state.c b/drivers/gpu/drm/i915/i915_gem_render_state.c
index 3210cedfa46c..90baf9086d0a 100644
--- a/drivers/gpu/drm/i915/i915_gem_render_state.c
+++ b/drivers/gpu/drm/i915/i915_gem_render_state.c
@@ -222,7 +222,7 @@ int i915_gem_render_state_emit(struct i915_request *rq)
 			goto err_unpin;
 	}
 
-	i915_vma_move_to_active(so.vma, rq, 0);
+	err = i915_vma_move_to_active(so.vma, rq, 0);
 err_unpin:
 	i915_vma_unpin(so.vma);
 err_vma:
diff --git a/drivers/gpu/drm/i915/i915_gem_shrinker.c b/drivers/gpu/drm/i915/i915_gem_shrinker.c
index 55e84e71f526..ea90d3a0d511 100644
--- a/drivers/gpu/drm/i915/i915_gem_shrinker.c
+++ b/drivers/gpu/drm/i915/i915_gem_shrinker.c
@@ -23,6 +23,7 @@
  */
 
 #include <linux/oom.h>
+#include <linux/sched/mm.h>
 #include <linux/shmem_fs.h>
 #include <linux/slab.h>
 #include <linux/swap.h>
@@ -172,7 +173,9 @@ i915_gem_shrink(struct drm_i915_private *i915,
 	 * we will free as much as we can and hope to get a second chance.
 	 */
 	if (flags & I915_SHRINK_ACTIVE)
-		i915_gem_wait_for_idle(i915, I915_WAIT_LOCKED);
+		i915_gem_wait_for_idle(i915,
+				       I915_WAIT_LOCKED,
+				       MAX_SCHEDULE_TIMEOUT);
 
 	trace_i915_gem_shrink(i915, target, flags);
 	i915_retire_requests(i915);
@@ -392,7 +395,8 @@ shrinker_lock_uninterruptible(struct drm_i915_private *i915, bool *unlock,
 	unsigned long timeout = jiffies + msecs_to_jiffies_timeout(timeout_ms);
 
 	do {
-		if (i915_gem_wait_for_idle(i915, 0) == 0 &&
+		if (i915_gem_wait_for_idle(i915,
+					   0, MAX_SCHEDULE_TIMEOUT) == 0 &&
 		    shrinker_lock(i915, unlock))
 			break;
 
@@ -466,7 +470,9 @@ i915_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr
 		return NOTIFY_DONE;
 
 	/* Force everything onto the inactive lists */
-	ret = i915_gem_wait_for_idle(i915, I915_WAIT_LOCKED);
+	ret = i915_gem_wait_for_idle(i915,
+				     I915_WAIT_LOCKED,
+				     MAX_SCHEDULE_TIMEOUT);
 	if (ret)
 		goto out;
 
@@ -526,3 +532,14 @@ void i915_gem_shrinker_unregister(struct drm_i915_private *i915)
 	WARN_ON(unregister_oom_notifier(&i915->mm.oom_notifier));
 	unregister_shrinker(&i915->mm.shrinker);
 }
+
+void i915_gem_shrinker_taints_mutex(struct mutex *mutex)
+{
+	if (!IS_ENABLED(CONFIG_LOCKDEP))
+		return;
+
+	fs_reclaim_acquire(GFP_KERNEL);
+	mutex_lock(mutex);
+	mutex_unlock(mutex);
+	fs_reclaim_release(GFP_KERNEL);
+}
diff --git a/drivers/gpu/drm/i915/i915_gem_stolen.c b/drivers/gpu/drm/i915/i915_gem_stolen.c
index 79a347295e00..53440bf87650 100644
--- a/drivers/gpu/drm/i915/i915_gem_stolen.c
+++ b/drivers/gpu/drm/i915/i915_gem_stolen.c
@@ -254,6 +254,7 @@ static void vlv_get_stolen_reserved(struct drm_i915_private *dev_priv,
 	switch (reg_val & GEN7_STOLEN_RESERVED_SIZE_MASK) {
 	default:
 		MISSING_CASE(reg_val & GEN7_STOLEN_RESERVED_SIZE_MASK);
+		/* fall through */
 	case GEN7_STOLEN_RESERVED_1M:
 		*size = 1024 * 1024;
 		break;
@@ -343,6 +344,35 @@ static void bdw_get_stolen_reserved(struct drm_i915_private *dev_priv,
 	*size = stolen_top - *base;
 }
 
+static void icl_get_stolen_reserved(struct drm_i915_private *dev_priv,
+				    resource_size_t *base,
+				    resource_size_t *size)
+{
+	u64 reg_val = I915_READ64(GEN6_STOLEN_RESERVED);
+
+	DRM_DEBUG_DRIVER("GEN6_STOLEN_RESERVED = 0x%016llx\n", reg_val);
+
+	*base = reg_val & GEN11_STOLEN_RESERVED_ADDR_MASK;
+
+	switch (reg_val & GEN8_STOLEN_RESERVED_SIZE_MASK) {
+	case GEN8_STOLEN_RESERVED_1M:
+		*size = 1024 * 1024;
+		break;
+	case GEN8_STOLEN_RESERVED_2M:
+		*size = 2 * 1024 * 1024;
+		break;
+	case GEN8_STOLEN_RESERVED_4M:
+		*size = 4 * 1024 * 1024;
+		break;
+	case GEN8_STOLEN_RESERVED_8M:
+		*size = 8 * 1024 * 1024;
+		break;
+	default:
+		*size = 8 * 1024 * 1024;
+		MISSING_CASE(reg_val & GEN8_STOLEN_RESERVED_SIZE_MASK);
+	}
+}
+
 int i915_gem_init_stolen(struct drm_i915_private *dev_priv)
 {
 	resource_size_t reserved_base, stolen_top;
@@ -399,7 +429,9 @@ int i915_gem_init_stolen(struct drm_i915_private *dev_priv)
 			gen7_get_stolen_reserved(dev_priv,
 						 &reserved_base, &reserved_size);
 		break;
-	default:
+	case 8:
+	case 9:
+	case 10:
 		if (IS_LP(dev_priv))
 			chv_get_stolen_reserved(dev_priv,
 						&reserved_base, &reserved_size);
@@ -407,6 +439,11 @@ int i915_gem_init_stolen(struct drm_i915_private *dev_priv)
 			bdw_get_stolen_reserved(dev_priv,
 						&reserved_base, &reserved_size);
 		break;
+	case 11:
+	default:
+		icl_get_stolen_reserved(dev_priv, &reserved_base,
+					&reserved_size);
+		break;
 	}
 
 	/*
diff --git a/drivers/gpu/drm/i915/i915_gem_userptr.c b/drivers/gpu/drm/i915/i915_gem_userptr.c
index 854bd51b9478..2c9b284036d1 100644
--- a/drivers/gpu/drm/i915/i915_gem_userptr.c
+++ b/drivers/gpu/drm/i915/i915_gem_userptr.c
@@ -112,10 +112,11 @@ static void del_object(struct i915_mmu_object *mo)
 	mo->attached = false;
 }
 
-static void i915_gem_userptr_mn_invalidate_range_start(struct mmu_notifier *_mn,
+static int i915_gem_userptr_mn_invalidate_range_start(struct mmu_notifier *_mn,
 						       struct mm_struct *mm,
 						       unsigned long start,
-						       unsigned long end)
+						       unsigned long end,
+						       bool blockable)
 {
 	struct i915_mmu_notifier *mn =
 		container_of(_mn, struct i915_mmu_notifier, mn);
@@ -124,7 +125,7 @@ static void i915_gem_userptr_mn_invalidate_range_start(struct mmu_notifier *_mn,
 	LIST_HEAD(cancelled);
 
 	if (RB_EMPTY_ROOT(&mn->objects.rb_root))
-		return;
+		return 0;
 
 	/* interval ranges are inclusive, but invalidate range is exclusive */
 	end--;
@@ -132,6 +133,10 @@ static void i915_gem_userptr_mn_invalidate_range_start(struct mmu_notifier *_mn,
 	spin_lock(&mn->lock);
 	it = interval_tree_iter_first(&mn->objects, start, end);
 	while (it) {
+		if (!blockable) {
+			spin_unlock(&mn->lock);
+			return -EAGAIN;
+		}
 		/* The mmu_object is released late when destroying the
 		 * GEM object so it is entirely possible to gain a
 		 * reference on an object in the process of being freed
@@ -154,6 +159,8 @@ static void i915_gem_userptr_mn_invalidate_range_start(struct mmu_notifier *_mn,
 
 	if (!list_empty(&cancelled))
 		flush_workqueue(mn->wq);
+
+	return 0;
 }
 
 static const struct mmu_notifier_ops i915_gem_userptr_notifier = {
@@ -507,7 +514,7 @@ __i915_gem_userptr_get_pages_worker(struct work_struct *_work)
 		struct mm_struct *mm = obj->userptr.mm->mm;
 		unsigned int flags = 0;
 
-		if (!obj->userptr.read_only)
+		if (!i915_gem_object_is_readonly(obj))
 			flags |= FOLL_WRITE;
 
 		ret = -EFAULT;
@@ -643,7 +650,7 @@ static int i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj)
 		if (pvec) /* defer to worker if malloc fails */
 			pinned = __get_user_pages_fast(obj->userptr.ptr,
 						       num_pages,
-						       !obj->userptr.read_only,
+						       !i915_gem_object_is_readonly(obj),
 						       pvec);
 	}
 
@@ -789,10 +796,15 @@ i915_gem_userptr_ioctl(struct drm_device *dev,
 		return -EFAULT;
 
 	if (args->flags & I915_USERPTR_READ_ONLY) {
-		/* On almost all of the current hw, we cannot tell the GPU that a
-		 * page is readonly, so this is just a placeholder in the uAPI.
+		struct i915_hw_ppgtt *ppgtt;
+
+		/*
+		 * On almost all of the older hw, we cannot tell the GPU that
+		 * a page is readonly.
 		 */
-		return -ENODEV;
+		ppgtt = dev_priv->kernel_context->ppgtt;
+		if (!ppgtt || !ppgtt->vm.has_read_only)
+			return -ENODEV;
 	}
 
 	obj = i915_gem_object_alloc(dev_priv);
@@ -806,7 +818,8 @@ i915_gem_userptr_ioctl(struct drm_device *dev,
 	i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);
 
 	obj->userptr.ptr = args->user_ptr;
-	obj->userptr.read_only = !!(args->flags & I915_USERPTR_READ_ONLY);
+	if (args->flags & I915_USERPTR_READ_ONLY)
+		i915_gem_object_set_readonly(obj);
 
 	/* And keep a pointer to the current->mm for resolving the user pages
 	 * at binding. This means that we need to hook into the mmu_notifier
diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c
index df524c9cad40..f7f2aa71d8d9 100644
--- a/drivers/gpu/drm/i915/i915_gpu_error.c
+++ b/drivers/gpu/drm/i915/i915_gpu_error.c
@@ -31,6 +31,7 @@
 #include <linux/stop_machine.h>
 #include <linux/zlib.h>
 #include <drm/drm_print.h>
+#include <linux/ascii85.h>
 
 #include "i915_gpu_error.h"
 #include "i915_drv.h"
@@ -335,21 +336,16 @@ static void print_error_buffers(struct drm_i915_error_state_buf *m,
 				struct drm_i915_error_buffer *err,
 				int count)
 {
-	int i;
-
 	err_printf(m, "%s [%d]:\n", name, count);
 
 	while (count--) {
-		err_printf(m, "    %08x_%08x %8u %02x %02x [ ",
+		err_printf(m, "    %08x_%08x %8u %02x %02x %02x",
 			   upper_32_bits(err->gtt_offset),
 			   lower_32_bits(err->gtt_offset),
 			   err->size,
 			   err->read_domains,
-			   err->write_domain);
-		for (i = 0; i < I915_NUM_ENGINES; i++)
-			err_printf(m, "%02x ", err->rseqno[i]);
-
-		err_printf(m, "] %02x", err->wseqno);
+			   err->write_domain,
+			   err->wseqno);
 		err_puts(m, tiling_flag(err->tiling));
 		err_puts(m, dirty_flag(err->dirty));
 		err_puts(m, purgeable_flag(err->purgeable));
@@ -522,35 +518,12 @@ void i915_error_printf(struct drm_i915_error_state_buf *e, const char *f, ...)
 	va_end(args);
 }
 
-static int
-ascii85_encode_len(int len)
-{
-	return DIV_ROUND_UP(len, 4);
-}
-
-static bool
-ascii85_encode(u32 in, char *out)
-{
-	int i;
-
-	if (in == 0)
-		return false;
-
-	out[5] = '\0';
-	for (i = 5; i--; ) {
-		out[i] = '!' + in % 85;
-		in /= 85;
-	}
-
-	return true;
-}
-
 static void print_error_obj(struct drm_i915_error_state_buf *m,
 			    struct intel_engine_cs *engine,
 			    const char *name,
 			    struct drm_i915_error_object *obj)
 {
-	char out[6];
+	char out[ASCII85_BUFSZ];
 	int page;
 
 	if (!obj)
@@ -572,12 +545,8 @@ static void print_error_obj(struct drm_i915_error_state_buf *m,
 			len -= obj->unused;
 		len = ascii85_encode_len(len);
 
-		for (i = 0; i < len; i++) {
-			if (ascii85_encode(obj->pages[page][i], out))
-				err_puts(m, out);
-			else
-				err_puts(m, "z");
-		}
+		for (i = 0; i < len; i++)
+			err_puts(m, ascii85_encode(obj->pages[page][i], out));
 	}
 	err_puts(m, "\n");
 }
@@ -1021,13 +990,10 @@ static void capture_bo(struct drm_i915_error_buffer *err,
 		       struct i915_vma *vma)
 {
 	struct drm_i915_gem_object *obj = vma->obj;
-	int i;
 
 	err->size = obj->base.size;
 	err->name = obj->base.name;
 
-	for (i = 0; i < I915_NUM_ENGINES; i++)
-		err->rseqno[i] = __active_get_seqno(&vma->last_read[i]);
 	err->wseqno = __active_get_seqno(&obj->frontbuffer_write);
 	err->engine = __active_get_engine_id(&obj->frontbuffer_write);
 
diff --git a/drivers/gpu/drm/i915/i915_gpu_error.h b/drivers/gpu/drm/i915/i915_gpu_error.h
index 58910f1dc67c..f893a4e8b783 100644
--- a/drivers/gpu/drm/i915/i915_gpu_error.h
+++ b/drivers/gpu/drm/i915/i915_gpu_error.h
@@ -177,7 +177,7 @@ struct i915_gpu_state {
 	struct drm_i915_error_buffer {
 		u32 size;
 		u32 name;
-		u32 rseqno[I915_NUM_ENGINES], wseqno;
+		u32 wseqno;
 		u64 gtt_offset;
 		u32 read_domains;
 		u32 write_domain;
diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c
index 46aaef5c1851..90628a47ae17 100644
--- a/drivers/gpu/drm/i915/i915_irq.c
+++ b/drivers/gpu/drm/i915/i915_irq.c
@@ -122,6 +122,15 @@ static const u32 hpd_gen11[HPD_NUM_PINS] = {
 	[HPD_PORT_F] = GEN11_TC4_HOTPLUG | GEN11_TBT4_HOTPLUG
 };
 
+static const u32 hpd_icp[HPD_NUM_PINS] = {
+	[HPD_PORT_A] = SDE_DDIA_HOTPLUG_ICP,
+	[HPD_PORT_B] = SDE_DDIB_HOTPLUG_ICP,
+	[HPD_PORT_C] = SDE_TC1_HOTPLUG_ICP,
+	[HPD_PORT_D] = SDE_TC2_HOTPLUG_ICP,
+	[HPD_PORT_E] = SDE_TC3_HOTPLUG_ICP,
+	[HPD_PORT_F] = SDE_TC4_HOTPLUG_ICP
+};
+
 /* IIR can theoretically queue up two events. Be paranoid. */
 #define GEN8_IRQ_RESET_NDX(type, which) do { \
 	I915_WRITE(GEN8_##type##_IMR(which), 0xffffffff); \
@@ -254,9 +263,9 @@ static u32
 gen11_gt_engine_identity(struct drm_i915_private * const i915,
 			 const unsigned int bank, const unsigned int bit);
 
-bool gen11_reset_one_iir(struct drm_i915_private * const i915,
-			 const unsigned int bank,
-			 const unsigned int bit)
+static bool gen11_reset_one_iir(struct drm_i915_private * const i915,
+				const unsigned int bank,
+				const unsigned int bit)
 {
 	void __iomem * const regs = i915->regs;
 	u32 dw;
@@ -1145,21 +1154,21 @@ static void ironlake_rps_change_irq_handler(struct drm_i915_private *dev_priv)
 
 static void notify_ring(struct intel_engine_cs *engine)
 {
+	const u32 seqno = intel_engine_get_seqno(engine);
 	struct i915_request *rq = NULL;
+	struct task_struct *tsk = NULL;
 	struct intel_wait *wait;
 
-	if (!engine->breadcrumbs.irq_armed)
+	if (unlikely(!engine->breadcrumbs.irq_armed))
 		return;
 
-	atomic_inc(&engine->irq_count);
-	set_bit(ENGINE_IRQ_BREADCRUMB, &engine->irq_posted);
+	rcu_read_lock();
 
 	spin_lock(&engine->breadcrumbs.irq_lock);
 	wait = engine->breadcrumbs.irq_wait;
 	if (wait) {
-		bool wakeup = engine->irq_seqno_barrier;
-
-		/* We use a callback from the dma-fence to submit
+		/*
+		 * We use a callback from the dma-fence to submit
 		 * requests after waiting on our own requests. To
 		 * ensure minimum delay in queuing the next request to
 		 * hardware, signal the fence now rather than wait for
@@ -1170,19 +1179,26 @@ static void notify_ring(struct intel_engine_cs *engine)
 		 * and to handle coalescing of multiple seqno updates
 		 * and many waiters.
 		 */
-		if (i915_seqno_passed(intel_engine_get_seqno(engine),
-				      wait->seqno)) {
+		if (i915_seqno_passed(seqno, wait->seqno)) {
 			struct i915_request *waiter = wait->request;
 
-			wakeup = true;
-			if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
+			if (waiter &&
+			    !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
 				      &waiter->fence.flags) &&
 			    intel_wait_check_request(wait, waiter))
 				rq = i915_request_get(waiter);
+
+			tsk = wait->tsk;
+		} else {
+			if (engine->irq_seqno_barrier &&
+			    i915_seqno_passed(seqno, wait->seqno - 1)) {
+				set_bit(ENGINE_IRQ_BREADCRUMB,
+					&engine->irq_posted);
+				tsk = wait->tsk;
+			}
 		}
 
-		if (wakeup)
-			wake_up_process(wait->tsk);
+		engine->breadcrumbs.irq_count++;
 	} else {
 		if (engine->breadcrumbs.irq_armed)
 			__intel_engine_disarm_breadcrumbs(engine);
@@ -1190,11 +1206,19 @@ static void notify_ring(struct intel_engine_cs *engine)
 	spin_unlock(&engine->breadcrumbs.irq_lock);
 
 	if (rq) {
-		dma_fence_signal(&rq->fence);
+		spin_lock(&rq->lock);
+		dma_fence_signal_locked(&rq->fence);
 		GEM_BUG_ON(!i915_request_completed(rq));
+		spin_unlock(&rq->lock);
+
 		i915_request_put(rq);
 	}
 
+	if (tsk && tsk->state & TASK_NORMAL)
+		wake_up_process(tsk);
+
+	rcu_read_unlock();
+
 	trace_intel_engine_notify(engine, wait);
 }
 
@@ -1241,9 +1265,9 @@ static u32 vlv_wa_c0_ei(struct drm_i915_private *dev_priv, u32 pm_iir)
 		c0 = max(render, media);
 		c0 *= 1000 * 100 << 8; /* to usecs and scale to threshold% */
 
-		if (c0 > time * rps->up_threshold)
+		if (c0 > time * rps->power.up_threshold)
 			events = GEN6_PM_RP_UP_THRESHOLD;
-		else if (c0 < time * rps->down_threshold)
+		else if (c0 < time * rps->power.down_threshold)
 			events = GEN6_PM_RP_DOWN_THRESHOLD;
 	}
 
@@ -1469,14 +1493,10 @@ static void snb_gt_irq_handler(struct drm_i915_private *dev_priv,
 static void
 gen8_cs_irq_handler(struct intel_engine_cs *engine, u32 iir)
 {
-	struct intel_engine_execlists * const execlists = &engine->execlists;
 	bool tasklet = false;
 
-	if (iir & GT_CONTEXT_SWITCH_INTERRUPT) {
-		if (READ_ONCE(engine->execlists.active))
-			tasklet = !test_and_set_bit(ENGINE_IRQ_EXECLIST,
-						    &engine->irq_posted);
-	}
+	if (iir & GT_CONTEXT_SWITCH_INTERRUPT)
+		tasklet = true;
 
 	if (iir & GT_RENDER_USER_INTERRUPT) {
 		notify_ring(engine);
@@ -1484,7 +1504,7 @@ gen8_cs_irq_handler(struct intel_engine_cs *engine, u32 iir)
 	}
 
 	if (tasklet)
-		tasklet_hi_schedule(&execlists->tasklet);
+		tasklet_hi_schedule(&engine->execlists.tasklet);
 }
 
 static void gen8_gt_irq_ack(struct drm_i915_private *i915,
@@ -1556,94 +1576,122 @@ static void gen8_gt_irq_handler(struct drm_i915_private *i915,
 	}
 }
 
-static bool gen11_port_hotplug_long_detect(enum port port, u32 val)
+static bool gen11_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
 {
-	switch (port) {
-	case PORT_C:
+	switch (pin) {
+	case HPD_PORT_C:
 		return val & GEN11_HOTPLUG_CTL_LONG_DETECT(PORT_TC1);
-	case PORT_D:
+	case HPD_PORT_D:
 		return val & GEN11_HOTPLUG_CTL_LONG_DETECT(PORT_TC2);
-	case PORT_E:
+	case HPD_PORT_E:
 		return val & GEN11_HOTPLUG_CTL_LONG_DETECT(PORT_TC3);
-	case PORT_F:
+	case HPD_PORT_F:
 		return val & GEN11_HOTPLUG_CTL_LONG_DETECT(PORT_TC4);
 	default:
 		return false;
 	}
 }
 
-static bool bxt_port_hotplug_long_detect(enum port port, u32 val)
+static bool bxt_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
 {
-	switch (port) {
-	case PORT_A:
+	switch (pin) {
+	case HPD_PORT_A:
 		return val & PORTA_HOTPLUG_LONG_DETECT;
-	case PORT_B:
+	case HPD_PORT_B:
 		return val & PORTB_HOTPLUG_LONG_DETECT;
-	case PORT_C:
+	case HPD_PORT_C:
 		return val & PORTC_HOTPLUG_LONG_DETECT;
 	default:
 		return false;
 	}
 }
 
-static bool spt_port_hotplug2_long_detect(enum port port, u32 val)
+static bool icp_ddi_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
 {
-	switch (port) {
-	case PORT_E:
+	switch (pin) {
+	case HPD_PORT_A:
+		return val & ICP_DDIA_HPD_LONG_DETECT;
+	case HPD_PORT_B:
+		return val & ICP_DDIB_HPD_LONG_DETECT;
+	default:
+		return false;
+	}
+}
+
+static bool icp_tc_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
+{
+	switch (pin) {
+	case HPD_PORT_C:
+		return val & ICP_TC_HPD_LONG_DETECT(PORT_TC1);
+	case HPD_PORT_D:
+		return val & ICP_TC_HPD_LONG_DETECT(PORT_TC2);
+	case HPD_PORT_E:
+		return val & ICP_TC_HPD_LONG_DETECT(PORT_TC3);
+	case HPD_PORT_F:
+		return val & ICP_TC_HPD_LONG_DETECT(PORT_TC4);
+	default:
+		return false;
+	}
+}
+
+static bool spt_port_hotplug2_long_detect(enum hpd_pin pin, u32 val)
+{
+	switch (pin) {
+	case HPD_PORT_E:
 		return val & PORTE_HOTPLUG_LONG_DETECT;
 	default:
 		return false;
 	}
 }
 
-static bool spt_port_hotplug_long_detect(enum port port, u32 val)
+static bool spt_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
 {
-	switch (port) {
-	case PORT_A:
+	switch (pin) {
+	case HPD_PORT_A:
 		return val & PORTA_HOTPLUG_LONG_DETECT;
-	case PORT_B:
+	case HPD_PORT_B:
 		return val & PORTB_HOTPLUG_LONG_DETECT;
-	case PORT_C:
+	case HPD_PORT_C:
 		return val & PORTC_HOTPLUG_LONG_DETECT;
-	case PORT_D:
+	case HPD_PORT_D:
 		return val & PORTD_HOTPLUG_LONG_DETECT;
 	default:
 		return false;
 	}
 }
 
-static bool ilk_port_hotplug_long_detect(enum port port, u32 val)
+static bool ilk_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
 {
-	switch (port) {
-	case PORT_A:
+	switch (pin) {
+	case HPD_PORT_A:
 		return val & DIGITAL_PORTA_HOTPLUG_LONG_DETECT;
 	default:
 		return false;
 	}
 }
 
-static bool pch_port_hotplug_long_detect(enum port port, u32 val)
+static bool pch_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
 {
-	switch (port) {
-	case PORT_B:
+	switch (pin) {
+	case HPD_PORT_B:
 		return val & PORTB_HOTPLUG_LONG_DETECT;
-	case PORT_C:
+	case HPD_PORT_C:
 		return val & PORTC_HOTPLUG_LONG_DETECT;
-	case PORT_D:
+	case HPD_PORT_D:
 		return val & PORTD_HOTPLUG_LONG_DETECT;
 	default:
 		return false;
 	}
 }
 
-static bool i9xx_port_hotplug_long_detect(enum port port, u32 val)
+static bool i9xx_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
 {
-	switch (port) {
-	case PORT_B:
+	switch (pin) {
+	case HPD_PORT_B:
 		return val & PORTB_HOTPLUG_INT_LONG_PULSE;
-	case PORT_C:
+	case HPD_PORT_C:
 		return val & PORTC_HOTPLUG_INT_LONG_PULSE;
-	case PORT_D:
+	case HPD_PORT_D:
 		return val & PORTD_HOTPLUG_INT_LONG_PULSE;
 	default:
 		return false;
@@ -1661,27 +1709,22 @@ static void intel_get_hpd_pins(struct drm_i915_private *dev_priv,
 			       u32 *pin_mask, u32 *long_mask,
 			       u32 hotplug_trigger, u32 dig_hotplug_reg,
 			       const u32 hpd[HPD_NUM_PINS],
-			       bool long_pulse_detect(enum port port, u32 val))
+			       bool long_pulse_detect(enum hpd_pin pin, u32 val))
 {
-	enum port port;
-	int i;
+	enum hpd_pin pin;
 
-	for_each_hpd_pin(i) {
-		if ((hpd[i] & hotplug_trigger) == 0)
+	for_each_hpd_pin(pin) {
+		if ((hpd[pin] & hotplug_trigger) == 0)
 			continue;
 
-		*pin_mask |= BIT(i);
-
-		port = intel_hpd_pin_to_port(dev_priv, i);
-		if (port == PORT_NONE)
-			continue;
+		*pin_mask |= BIT(pin);
 
-		if (long_pulse_detect(port, dig_hotplug_reg))
-			*long_mask |= BIT(i);
+		if (long_pulse_detect(pin, dig_hotplug_reg))
+			*long_mask |= BIT(pin);
 	}
 
-	DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x, dig 0x%08x, pins 0x%08x\n",
-			 hotplug_trigger, dig_hotplug_reg, *pin_mask);
+	DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x, dig 0x%08x, pins 0x%08x, long 0x%08x\n",
+			 hotplug_trigger, dig_hotplug_reg, *pin_mask, *long_mask);
 
 }
 
@@ -1703,69 +1746,34 @@ static void display_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
 					 uint32_t crc4)
 {
 	struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe];
-	struct intel_pipe_crc_entry *entry;
 	struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
-	struct drm_driver *driver = dev_priv->drm.driver;
 	uint32_t crcs[5];
-	int head, tail;
 
 	spin_lock(&pipe_crc->lock);
-	if (pipe_crc->source && !crtc->base.crc.opened) {
-		if (!pipe_crc->entries) {
-			spin_unlock(&pipe_crc->lock);
-			DRM_DEBUG_KMS("spurious interrupt\n");
-			return;
-		}
-
-		head = pipe_crc->head;
-		tail = pipe_crc->tail;
-
-		if (CIRC_SPACE(head, tail, INTEL_PIPE_CRC_ENTRIES_NR) < 1) {
-			spin_unlock(&pipe_crc->lock);
-			DRM_ERROR("CRC buffer overflowing\n");
-			return;
-		}
-
-		entry = &pipe_crc->entries[head];
-
-		entry->frame = driver->get_vblank_counter(&dev_priv->drm, pipe);
-		entry->crc[0] = crc0;
-		entry->crc[1] = crc1;
-		entry->crc[2] = crc2;
-		entry->crc[3] = crc3;
-		entry->crc[4] = crc4;
-
-		head = (head + 1) & (INTEL_PIPE_CRC_ENTRIES_NR - 1);
-		pipe_crc->head = head;
-
-		spin_unlock(&pipe_crc->lock);
-
-		wake_up_interruptible(&pipe_crc->wq);
-	} else {
-		/*
-		 * For some not yet identified reason, the first CRC is
-		 * bonkers. So let's just wait for the next vblank and read
-		 * out the buggy result.
-		 *
-		 * On GEN8+ sometimes the second CRC is bonkers as well, so
-		 * don't trust that one either.
-		 */
-		if (pipe_crc->skipped <= 0 ||
-		    (INTEL_GEN(dev_priv) >= 8 && pipe_crc->skipped == 1)) {
-			pipe_crc->skipped++;
-			spin_unlock(&pipe_crc->lock);
-			return;
-		}
+	/*
+	 * For some not yet identified reason, the first CRC is
+	 * bonkers. So let's just wait for the next vblank and read
+	 * out the buggy result.
+	 *
+	 * On GEN8+ sometimes the second CRC is bonkers as well, so
+	 * don't trust that one either.
+	 */
+	if (pipe_crc->skipped <= 0 ||
+	    (INTEL_GEN(dev_priv) >= 8 && pipe_crc->skipped == 1)) {
+		pipe_crc->skipped++;
 		spin_unlock(&pipe_crc->lock);
-		crcs[0] = crc0;
-		crcs[1] = crc1;
-		crcs[2] = crc2;
-		crcs[3] = crc3;
-		crcs[4] = crc4;
-		drm_crtc_add_crc_entry(&crtc->base, true,
-				       drm_crtc_accurate_vblank_count(&crtc->base),
-				       crcs);
+		return;
 	}
+	spin_unlock(&pipe_crc->lock);
+
+	crcs[0] = crc0;
+	crcs[1] = crc1;
+	crcs[2] = crc2;
+	crcs[3] = crc3;
+	crcs[4] = crc4;
+	drm_crtc_add_crc_entry(&crtc->base, true,
+				drm_crtc_accurate_vblank_count(&crtc->base),
+				crcs);
 }
 #else
 static inline void
@@ -2021,10 +2029,38 @@ static void valleyview_pipestat_irq_handler(struct drm_i915_private *dev_priv,
 
 static u32 i9xx_hpd_irq_ack(struct drm_i915_private *dev_priv)
 {
-	u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);
+	u32 hotplug_status = 0, hotplug_status_mask;
+	int i;
+
+	if (IS_G4X(dev_priv) ||
+	    IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
+		hotplug_status_mask = HOTPLUG_INT_STATUS_G4X |
+			DP_AUX_CHANNEL_MASK_INT_STATUS_G4X;
+	else
+		hotplug_status_mask = HOTPLUG_INT_STATUS_I915;
+
+	/*
+	 * We absolutely have to clear all the pending interrupt
+	 * bits in PORT_HOTPLUG_STAT. Otherwise the ISR port
+	 * interrupt bit won't have an edge, and the i965/g4x
+	 * edge triggered IIR will not notice that an interrupt
+	 * is still pending. We can't use PORT_HOTPLUG_EN to
+	 * guarantee the edge as the act of toggling the enable
+	 * bits can itself generate a new hotplug interrupt :(
+	 */
+	for (i = 0; i < 10; i++) {
+		u32 tmp = I915_READ(PORT_HOTPLUG_STAT) & hotplug_status_mask;
+
+		if (tmp == 0)
+			return hotplug_status;
 
-	if (hotplug_status)
+		hotplug_status |= tmp;
 		I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
+	}
+
+	WARN_ONCE(1,
+		  "PORT_HOTPLUG_STAT did not clear (0x%08x)\n",
+		  I915_READ(PORT_HOTPLUG_STAT));
 
 	return hotplug_status;
 }
@@ -2131,7 +2167,6 @@ static irqreturn_t valleyview_irq_handler(int irq, void *arg)
 
 		I915_WRITE(VLV_IER, ier);
 		I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
-		POSTING_READ(VLV_MASTER_IER);
 
 		if (gt_iir)
 			snb_gt_irq_handler(dev_priv, gt_iir);
@@ -2216,7 +2251,6 @@ static irqreturn_t cherryview_irq_handler(int irq, void *arg)
 
 		I915_WRITE(VLV_IER, ier);
 		I915_WRITE(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
-		POSTING_READ(GEN8_MASTER_IRQ);
 
 		gen8_gt_irq_handler(dev_priv, master_ctl, gt_iir);
 
@@ -2385,6 +2419,43 @@ static void cpt_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
 		cpt_serr_int_handler(dev_priv);
 }
 
+static void icp_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
+{
+	u32 ddi_hotplug_trigger = pch_iir & SDE_DDI_MASK_ICP;
+	u32 tc_hotplug_trigger = pch_iir & SDE_TC_MASK_ICP;
+	u32 pin_mask = 0, long_mask = 0;
+
+	if (ddi_hotplug_trigger) {
+		u32 dig_hotplug_reg;
+
+		dig_hotplug_reg = I915_READ(SHOTPLUG_CTL_DDI);
+		I915_WRITE(SHOTPLUG_CTL_DDI, dig_hotplug_reg);
+
+		intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
+				   ddi_hotplug_trigger,
+				   dig_hotplug_reg, hpd_icp,
+				   icp_ddi_port_hotplug_long_detect);
+	}
+
+	if (tc_hotplug_trigger) {
+		u32 dig_hotplug_reg;
+
+		dig_hotplug_reg = I915_READ(SHOTPLUG_CTL_TC);
+		I915_WRITE(SHOTPLUG_CTL_TC, dig_hotplug_reg);
+
+		intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
+				   tc_hotplug_trigger,
+				   dig_hotplug_reg, hpd_icp,
+				   icp_tc_port_hotplug_long_detect);
+	}
+
+	if (pin_mask)
+		intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
+
+	if (pch_iir & SDE_GMBUS_ICP)
+		gmbus_irq_handler(dev_priv);
+}
+
 static void spt_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
 {
 	u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_SPT &
@@ -2548,7 +2619,6 @@ static irqreturn_t ironlake_irq_handler(int irq, void *arg)
 	/* disable master interrupt before clearing iir  */
 	de_ier = I915_READ(DEIER);
 	I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
-	POSTING_READ(DEIER);
 
 	/* Disable south interrupts. We'll only write to SDEIIR once, so further
 	 * interrupts will will be stored on its back queue, and then we'll be
@@ -2558,7 +2628,6 @@ static irqreturn_t ironlake_irq_handler(int irq, void *arg)
 	if (!HAS_PCH_NOP(dev_priv)) {
 		sde_ier = I915_READ(SDEIER);
 		I915_WRITE(SDEIER, 0);
-		POSTING_READ(SDEIER);
 	}
 
 	/* Find, clear, then process each source of interrupt */
@@ -2593,11 +2662,8 @@ static irqreturn_t ironlake_irq_handler(int irq, void *arg)
 	}
 
 	I915_WRITE(DEIER, de_ier);
-	POSTING_READ(DEIER);
-	if (!HAS_PCH_NOP(dev_priv)) {
+	if (!HAS_PCH_NOP(dev_priv))
 		I915_WRITE(SDEIER, sde_ier);
-		POSTING_READ(SDEIER);
-	}
 
 	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
 	enable_rpm_wakeref_asserts(dev_priv);
@@ -2804,8 +2870,11 @@ gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl)
 			I915_WRITE(SDEIIR, iir);
 			ret = IRQ_HANDLED;
 
-			if (HAS_PCH_SPT(dev_priv) || HAS_PCH_KBP(dev_priv) ||
-			    HAS_PCH_CNP(dev_priv))
+			if (HAS_PCH_ICP(dev_priv))
+				icp_irq_handler(dev_priv, iir);
+			else if (HAS_PCH_SPT(dev_priv) ||
+				 HAS_PCH_KBP(dev_priv) ||
+				 HAS_PCH_CNP(dev_priv))
 				spt_irq_handler(dev_priv, iir);
 			else
 				cpt_irq_handler(dev_priv, iir);
@@ -3170,7 +3239,7 @@ static void i915_clear_error_registers(struct drm_i915_private *dev_priv)
 		 */
 		DRM_DEBUG_DRIVER("EIR stuck: 0x%08x, masking\n", eir);
 		I915_WRITE(EMR, I915_READ(EMR) | eir);
-		I915_WRITE(IIR, I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
+		I915_WRITE(IIR, I915_MASTER_ERROR_INTERRUPT);
 	}
 }
 
@@ -3584,6 +3653,9 @@ static void gen11_irq_reset(struct drm_device *dev)
 	GEN3_IRQ_RESET(GEN11_DE_HPD_);
 	GEN3_IRQ_RESET(GEN11_GU_MISC_);
 	GEN3_IRQ_RESET(GEN8_PCU_);
+
+	if (HAS_PCH_ICP(dev_priv))
+		GEN3_IRQ_RESET(SDE);
 }
 
 void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv,
@@ -3700,6 +3772,35 @@ static void ibx_hpd_irq_setup(struct drm_i915_private *dev_priv)
 	ibx_hpd_detection_setup(dev_priv);
 }
 
+static void icp_hpd_detection_setup(struct drm_i915_private *dev_priv)
+{
+	u32 hotplug;
+
+	hotplug = I915_READ(SHOTPLUG_CTL_DDI);
+	hotplug |= ICP_DDIA_HPD_ENABLE |
+		   ICP_DDIB_HPD_ENABLE;
+	I915_WRITE(SHOTPLUG_CTL_DDI, hotplug);
+
+	hotplug = I915_READ(SHOTPLUG_CTL_TC);
+	hotplug |= ICP_TC_HPD_ENABLE(PORT_TC1) |
+		   ICP_TC_HPD_ENABLE(PORT_TC2) |
+		   ICP_TC_HPD_ENABLE(PORT_TC3) |
+		   ICP_TC_HPD_ENABLE(PORT_TC4);
+	I915_WRITE(SHOTPLUG_CTL_TC, hotplug);
+}
+
+static void icp_hpd_irq_setup(struct drm_i915_private *dev_priv)
+{
+	u32 hotplug_irqs, enabled_irqs;
+
+	hotplug_irqs = SDE_DDI_MASK_ICP | SDE_TC_MASK_ICP;
+	enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_icp);
+
+	ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
+
+	icp_hpd_detection_setup(dev_priv);
+}
+
 static void gen11_hpd_detection_setup(struct drm_i915_private *dev_priv)
 {
 	u32 hotplug;
@@ -3733,6 +3834,9 @@ static void gen11_hpd_irq_setup(struct drm_i915_private *dev_priv)
 	POSTING_READ(GEN11_DE_HPD_IMR);
 
 	gen11_hpd_detection_setup(dev_priv);
+
+	if (HAS_PCH_ICP(dev_priv))
+		icp_hpd_irq_setup(dev_priv);
 }
 
 static void spt_hpd_detection_setup(struct drm_i915_private *dev_priv)
@@ -4168,11 +4272,29 @@ static void gen11_gt_irq_postinstall(struct drm_i915_private *dev_priv)
 	I915_WRITE(GEN11_GPM_WGBOXPERF_INTR_MASK,  ~0);
 }
 
+static void icp_irq_postinstall(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = to_i915(dev);
+	u32 mask = SDE_GMBUS_ICP;
+
+	WARN_ON(I915_READ(SDEIER) != 0);
+	I915_WRITE(SDEIER, 0xffffffff);
+	POSTING_READ(SDEIER);
+
+	gen3_assert_iir_is_zero(dev_priv, SDEIIR);
+	I915_WRITE(SDEIMR, ~mask);
+
+	icp_hpd_detection_setup(dev_priv);
+}
+
 static int gen11_irq_postinstall(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	u32 gu_misc_masked = GEN11_GU_MISC_GSE;
 
+	if (HAS_PCH_ICP(dev_priv))
+		icp_irq_postinstall(dev);
+
 	gen11_gt_irq_postinstall(dev_priv);
 	gen8_de_irq_postinstall(dev_priv);
 
@@ -4225,11 +4347,13 @@ static int i8xx_irq_postinstall(struct drm_device *dev)
 	/* Unmask the interrupts that we always want on. */
 	dev_priv->irq_mask =
 		~(I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
-		  I915_DISPLAY_PIPE_B_EVENT_INTERRUPT);
+		  I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
+		  I915_MASTER_ERROR_INTERRUPT);
 
 	enable_mask =
 		I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
 		I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
+		I915_MASTER_ERROR_INTERRUPT |
 		I915_USER_INTERRUPT;
 
 	GEN2_IRQ_INIT(, dev_priv->irq_mask, enable_mask);
@@ -4244,6 +4368,81 @@ static int i8xx_irq_postinstall(struct drm_device *dev)
 	return 0;
 }
 
+static void i8xx_error_irq_ack(struct drm_i915_private *dev_priv,
+			       u16 *eir, u16 *eir_stuck)
+{
+	u16 emr;
+
+	*eir = I915_READ16(EIR);
+
+	if (*eir)
+		I915_WRITE16(EIR, *eir);
+
+	*eir_stuck = I915_READ16(EIR);
+	if (*eir_stuck == 0)
+		return;
+
+	/*
+	 * Toggle all EMR bits to make sure we get an edge
+	 * in the ISR master error bit if we don't clear
+	 * all the EIR bits. Otherwise the edge triggered
+	 * IIR on i965/g4x wouldn't notice that an interrupt
+	 * is still pending. Also some EIR bits can't be
+	 * cleared except by handling the underlying error
+	 * (or by a GPU reset) so we mask any bit that
+	 * remains set.
+	 */
+	emr = I915_READ16(EMR);
+	I915_WRITE16(EMR, 0xffff);
+	I915_WRITE16(EMR, emr | *eir_stuck);
+}
+
+static void i8xx_error_irq_handler(struct drm_i915_private *dev_priv,
+				   u16 eir, u16 eir_stuck)
+{
+	DRM_DEBUG("Master Error: EIR 0x%04x\n", eir);
+
+	if (eir_stuck)
+		DRM_DEBUG_DRIVER("EIR stuck: 0x%04x, masked\n", eir_stuck);
+}
+
+static void i9xx_error_irq_ack(struct drm_i915_private *dev_priv,
+			       u32 *eir, u32 *eir_stuck)
+{
+	u32 emr;
+
+	*eir = I915_READ(EIR);
+
+	I915_WRITE(EIR, *eir);
+
+	*eir_stuck = I915_READ(EIR);
+	if (*eir_stuck == 0)
+		return;
+
+	/*
+	 * Toggle all EMR bits to make sure we get an edge
+	 * in the ISR master error bit if we don't clear
+	 * all the EIR bits. Otherwise the edge triggered
+	 * IIR on i965/g4x wouldn't notice that an interrupt
+	 * is still pending. Also some EIR bits can't be
+	 * cleared except by handling the underlying error
+	 * (or by a GPU reset) so we mask any bit that
+	 * remains set.
+	 */
+	emr = I915_READ(EMR);
+	I915_WRITE(EMR, 0xffffffff);
+	I915_WRITE(EMR, emr | *eir_stuck);
+}
+
+static void i9xx_error_irq_handler(struct drm_i915_private *dev_priv,
+				   u32 eir, u32 eir_stuck)
+{
+	DRM_DEBUG("Master Error, EIR 0x%08x\n", eir);
+
+	if (eir_stuck)
+		DRM_DEBUG_DRIVER("EIR stuck: 0x%08x, masked\n", eir_stuck);
+}
+
 static irqreturn_t i8xx_irq_handler(int irq, void *arg)
 {
 	struct drm_device *dev = arg;
@@ -4258,6 +4457,7 @@ static irqreturn_t i8xx_irq_handler(int irq, void *arg)
 
 	do {
 		u32 pipe_stats[I915_MAX_PIPES] = {};
+		u16 eir = 0, eir_stuck = 0;
 		u16 iir;
 
 		iir = I915_READ16(IIR);
@@ -4270,13 +4470,16 @@ static irqreturn_t i8xx_irq_handler(int irq, void *arg)
 		 * signalled in iir */
 		i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
 
+		if (iir & I915_MASTER_ERROR_INTERRUPT)
+			i8xx_error_irq_ack(dev_priv, &eir, &eir_stuck);
+
 		I915_WRITE16(IIR, iir);
 
 		if (iir & I915_USER_INTERRUPT)
 			notify_ring(dev_priv->engine[RCS]);
 
-		if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
-			DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
+		if (iir & I915_MASTER_ERROR_INTERRUPT)
+			i8xx_error_irq_handler(dev_priv, eir, eir_stuck);
 
 		i8xx_pipestat_irq_handler(dev_priv, iir, pipe_stats);
 	} while (0);
@@ -4314,12 +4517,14 @@ static int i915_irq_postinstall(struct drm_device *dev)
 	dev_priv->irq_mask =
 		~(I915_ASLE_INTERRUPT |
 		  I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
-		  I915_DISPLAY_PIPE_B_EVENT_INTERRUPT);
+		  I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
+		  I915_MASTER_ERROR_INTERRUPT);
 
 	enable_mask =
 		I915_ASLE_INTERRUPT |
 		I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
 		I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
+		I915_MASTER_ERROR_INTERRUPT |
 		I915_USER_INTERRUPT;
 
 	if (I915_HAS_HOTPLUG(dev_priv)) {
@@ -4357,6 +4562,7 @@ static irqreturn_t i915_irq_handler(int irq, void *arg)
 
 	do {
 		u32 pipe_stats[I915_MAX_PIPES] = {};
+		u32 eir = 0, eir_stuck = 0;
 		u32 hotplug_status = 0;
 		u32 iir;
 
@@ -4374,13 +4580,16 @@ static irqreturn_t i915_irq_handler(int irq, void *arg)
 		 * signalled in iir */
 		i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
 
+		if (iir & I915_MASTER_ERROR_INTERRUPT)
+			i9xx_error_irq_ack(dev_priv, &eir, &eir_stuck);
+
 		I915_WRITE(IIR, iir);
 
 		if (iir & I915_USER_INTERRUPT)
 			notify_ring(dev_priv->engine[RCS]);
 
-		if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
-			DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
+		if (iir & I915_MASTER_ERROR_INTERRUPT)
+			i9xx_error_irq_handler(dev_priv, eir, eir_stuck);
 
 		if (hotplug_status)
 			i9xx_hpd_irq_handler(dev_priv, hotplug_status);
@@ -4434,14 +4643,14 @@ static int i965_irq_postinstall(struct drm_device *dev)
 		  I915_DISPLAY_PORT_INTERRUPT |
 		  I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
 		  I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
-		  I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
+		  I915_MASTER_ERROR_INTERRUPT);
 
 	enable_mask =
 		I915_ASLE_INTERRUPT |
 		I915_DISPLAY_PORT_INTERRUPT |
 		I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
 		I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
-		I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT |
+		I915_MASTER_ERROR_INTERRUPT |
 		I915_USER_INTERRUPT;
 
 	if (IS_G4X(dev_priv))
@@ -4501,6 +4710,7 @@ static irqreturn_t i965_irq_handler(int irq, void *arg)
 
 	do {
 		u32 pipe_stats[I915_MAX_PIPES] = {};
+		u32 eir = 0, eir_stuck = 0;
 		u32 hotplug_status = 0;
 		u32 iir;
 
@@ -4517,6 +4727,9 @@ static irqreturn_t i965_irq_handler(int irq, void *arg)
 		 * signalled in iir */
 		i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
 
+		if (iir & I915_MASTER_ERROR_INTERRUPT)
+			i9xx_error_irq_ack(dev_priv, &eir, &eir_stuck);
+
 		I915_WRITE(IIR, iir);
 
 		if (iir & I915_USER_INTERRUPT)
@@ -4525,8 +4738,8 @@ static irqreturn_t i965_irq_handler(int irq, void *arg)
 		if (iir & I915_BSD_USER_INTERRUPT)
 			notify_ring(dev_priv->engine[VCS]);
 
-		if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
-			DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
+		if (iir & I915_MASTER_ERROR_INTERRUPT)
+			i9xx_error_irq_handler(dev_priv, eir, eir_stuck);
 
 		if (hotplug_status)
 			i9xx_hpd_irq_handler(dev_priv, hotplug_status);
diff --git a/drivers/gpu/drm/i915/i915_params.c b/drivers/gpu/drm/i915/i915_params.c
index 49fcc4679db6..295e981e4a39 100644
--- a/drivers/gpu/drm/i915/i915_params.c
+++ b/drivers/gpu/drm/i915/i915_params.c
@@ -44,10 +44,6 @@ i915_param_named(modeset, int, 0400,
 	"Use kernel modesetting [KMS] (0=disable, "
 	"1=on, -1=force vga console preference [default])");
 
-i915_param_named_unsafe(panel_ignore_lid, int, 0600,
-	"Override lid status (0=autodetect, 1=autodetect disabled [default], "
-	"-1=force lid closed, -2=force lid open)");
-
 i915_param_named_unsafe(enable_dc, int, 0400,
 	"Enable power-saving display C-states. "
 	"(-1=auto [default]; 0=disable; 1=up to DC5; 2=up to DC6)");
@@ -92,7 +88,7 @@ i915_param_named_unsafe(enable_ppgtt, int, 0400,
 
 i915_param_named_unsafe(enable_psr, int, 0600,
 	"Enable PSR "
-	"(0=disabled, 1=enabled - link mode chosen per-platform, 2=force link-standby mode, 3=force link-off mode) "
+	"(0=disabled, 1=enabled) "
 	"Default: -1 (use per-chip default)");
 
 i915_param_named_unsafe(alpha_support, bool, 0400,
diff --git a/drivers/gpu/drm/i915/i915_params.h b/drivers/gpu/drm/i915/i915_params.h
index aebe0469ddaa..6c4d4a21474b 100644
--- a/drivers/gpu/drm/i915/i915_params.h
+++ b/drivers/gpu/drm/i915/i915_params.h
@@ -36,7 +36,6 @@ struct drm_printer;
 #define I915_PARAMS_FOR_EACH(param) \
 	param(char *, vbt_firmware, NULL) \
 	param(int, modeset, -1) \
-	param(int, panel_ignore_lid, 1) \
 	param(int, lvds_channel_mode, 0) \
 	param(int, panel_use_ssc, -1) \
 	param(int, vbt_sdvo_panel_type, -1) \
diff --git a/drivers/gpu/drm/i915/i915_pci.c b/drivers/gpu/drm/i915/i915_pci.c
index 55543f1b0236..6a4d1388ad2d 100644
--- a/drivers/gpu/drm/i915/i915_pci.c
+++ b/drivers/gpu/drm/i915/i915_pci.c
@@ -674,10 +674,16 @@ MODULE_DEVICE_TABLE(pci, pciidlist);
 
 static void i915_pci_remove(struct pci_dev *pdev)
 {
-	struct drm_device *dev = pci_get_drvdata(pdev);
+	struct drm_device *dev;
+
+	dev = pci_get_drvdata(pdev);
+	if (!dev) /* driver load aborted, nothing to cleanup */
+		return;
 
 	i915_driver_unload(dev);
 	drm_dev_put(dev);
+
+	pci_set_drvdata(pdev, NULL);
 }
 
 static int i915_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
@@ -712,6 +718,11 @@ static int i915_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	if (err)
 		return err;
 
+	if (i915_inject_load_failure()) {
+		i915_pci_remove(pdev);
+		return -ENODEV;
+	}
+
 	err = i915_live_selftests(pdev);
 	if (err) {
 		i915_pci_remove(pdev);
diff --git a/drivers/gpu/drm/i915/i915_perf.c b/drivers/gpu/drm/i915/i915_perf.c
index 447407fee3b8..6bf10952c724 100644
--- a/drivers/gpu/drm/i915/i915_perf.c
+++ b/drivers/gpu/drm/i915/i915_perf.c
@@ -1836,7 +1836,9 @@ static int gen8_configure_all_contexts(struct drm_i915_private *dev_priv,
 	 * So far the best way to work around this issue seems to be draining
 	 * the GPU from any submitted work.
 	 */
-	ret = i915_gem_wait_for_idle(dev_priv, wait_flags);
+	ret = i915_gem_wait_for_idle(dev_priv,
+				     wait_flags,
+				     MAX_SCHEDULE_TIMEOUT);
 	if (ret)
 		goto out;
 
diff --git a/drivers/gpu/drm/i915/i915_pmu.c b/drivers/gpu/drm/i915/i915_pmu.c
index c39541ed2219..d6c8f8fdfda5 100644
--- a/drivers/gpu/drm/i915/i915_pmu.c
+++ b/drivers/gpu/drm/i915/i915_pmu.c
@@ -4,6 +4,7 @@
  * Copyright © 2017-2018 Intel Corporation
  */
 
+#include <linux/irq.h>
 #include "i915_pmu.h"
 #include "intel_ringbuffer.h"
 #include "i915_drv.h"
diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h
index 4bfd7a9bd75f..08ec7446282e 100644
--- a/drivers/gpu/drm/i915/i915_reg.h
+++ b/drivers/gpu/drm/i915/i915_reg.h
@@ -139,19 +139,35 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg)
 	return !i915_mmio_reg_equal(reg, INVALID_MMIO_REG);
 }
 
+/*
+ * Given the first two numbers __a and __b of arbitrarily many evenly spaced
+ * numbers, pick the 0-based __index'th value.
+ *
+ * Always prefer this over _PICK() if the numbers are evenly spaced.
+ */
+#define _PICK_EVEN(__index, __a, __b) ((__a) + (__index) * ((__b) - (__a)))
+
+/*
+ * Given the arbitrary numbers in varargs, pick the 0-based __index'th number.
+ *
+ * Always prefer _PICK_EVEN() over this if the numbers are evenly spaced.
+ */
 #define _PICK(__index, ...) (((const u32 []){ __VA_ARGS__ })[__index])
 
-#define _PIPE(pipe, a, b) ((a) + (pipe) * ((b) - (a)))
+/*
+ * Named helper wrappers around _PICK_EVEN() and _PICK().
+ */
+#define _PIPE(pipe, a, b) _PICK_EVEN(pipe, a, b)
 #define _MMIO_PIPE(pipe, a, b) _MMIO(_PIPE(pipe, a, b))
-#define _PLANE(plane, a, b) _PIPE(plane, a, b)
+#define _PLANE(plane, a, b) _PICK_EVEN(plane, a, b)
 #define _MMIO_PLANE(plane, a, b) _MMIO_PIPE(plane, a, b)
-#define _TRANS(tran, a, b) ((a) + (tran) * ((b) - (a)))
+#define _TRANS(tran, a, b) _PICK_EVEN(tran, a, b)
 #define _MMIO_TRANS(tran, a, b) _MMIO(_TRANS(tran, a, b))
-#define _PORT(port, a, b) ((a) + (port) * ((b) - (a)))
+#define _PORT(port, a, b) _PICK_EVEN(port, a, b)
 #define _MMIO_PORT(port, a, b) _MMIO(_PORT(port, a, b))
 #define _MMIO_PIPE3(pipe, a, b, c) _MMIO(_PICK(pipe, a, b, c))
 #define _MMIO_PORT3(pipe, a, b, c) _MMIO(_PICK(pipe, a, b, c))
-#define _PLL(pll, a, b) ((a) + (pll) * ((b) - (a)))
+#define _PLL(pll, a, b) _PICK_EVEN(pll, a, b)
 #define _MMIO_PLL(pll, a, b) _MMIO(_PLL(pll, a, b))
 #define _PHY3(phy, ...) _PICK(phy, __VA_ARGS__)
 #define _MMIO_PHY3(phy, a, b, c) _MMIO(_PHY3(phy, a, b, c))
@@ -396,6 +412,7 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg)
 #define GEN8_STOLEN_RESERVED_4M		(2 << 7)
 #define GEN8_STOLEN_RESERVED_8M		(3 << 7)
 #define GEN6_STOLEN_RESERVED_ENABLE	(1 << 0)
+#define GEN11_STOLEN_RESERVED_ADDR_MASK	(0xFFFFFFFFFFFULL << 20)
 
 /* VGA stuff */
 
@@ -1045,13 +1062,13 @@ enum i915_power_well_id {
 
 	/*
 	 * HSW/BDW
-	 *  - HSW_PWR_WELL_CTL_DRIVER(0) (status bit: id*2, req bit: id*2+1)
+	 *  - _HSW_PWR_WELL_CTL1-4 (status bit: id*2, req bit: id*2+1)
 	 */
 	HSW_DISP_PW_GLOBAL = 15,
 
 	/*
 	 * GEN9+
-	 *  - HSW_PWR_WELL_CTL_DRIVER(0) (status bit: id*2, req bit: id*2+1)
+	 *  - _HSW_PWR_WELL_CTL1-4 (status bit: id*2, req bit: id*2+1)
 	 */
 	SKL_DISP_PW_MISC_IO = 0,
 	SKL_DISP_PW_DDI_A_E,
@@ -1075,17 +1092,54 @@ enum i915_power_well_id {
 	SKL_DISP_PW_2,
 
 	/* - custom power wells */
-	SKL_DISP_PW_DC_OFF,
 	BXT_DPIO_CMN_A,
 	BXT_DPIO_CMN_BC,
-	GLK_DPIO_CMN_C,			/* 19 */
+	GLK_DPIO_CMN_C,			/* 18 */
+
+	/*
+	 * GEN11+
+	 *  - _HSW_PWR_WELL_CTL1-4
+	 *    (status bit: (id&15)*2, req bit:(id&15)*2+1)
+	 */
+	ICL_DISP_PW_1 = 0,
+	ICL_DISP_PW_2,
+	ICL_DISP_PW_3,
+	ICL_DISP_PW_4,
+
+	/*
+	 *  - _HSW_PWR_WELL_CTL_AUX1/2/4
+	 *    (status bit: (id&15)*2, req bit:(id&15)*2+1)
+	 */
+	ICL_DISP_PW_AUX_A = 16,
+	ICL_DISP_PW_AUX_B,
+	ICL_DISP_PW_AUX_C,
+	ICL_DISP_PW_AUX_D,
+	ICL_DISP_PW_AUX_E,
+	ICL_DISP_PW_AUX_F,
+
+	ICL_DISP_PW_AUX_TBT1 = 24,
+	ICL_DISP_PW_AUX_TBT2,
+	ICL_DISP_PW_AUX_TBT3,
+	ICL_DISP_PW_AUX_TBT4,
+
+	/*
+	 *  - _HSW_PWR_WELL_CTL_DDI1/2/4
+	 *    (status bit: (id&15)*2, req bit:(id&15)*2+1)
+	 */
+	ICL_DISP_PW_DDI_A = 32,
+	ICL_DISP_PW_DDI_B,
+	ICL_DISP_PW_DDI_C,
+	ICL_DISP_PW_DDI_D,
+	ICL_DISP_PW_DDI_E,
+	ICL_DISP_PW_DDI_F,                      /* 37 */
 
 	/*
 	 * Multiple platforms.
 	 * Must start following the highest ID of any platform.
 	 * - custom power wells
 	 */
-	I915_DISP_PW_ALWAYS_ON = 20,
+	SKL_DISP_PW_DC_OFF = 38,
+	I915_DISP_PW_ALWAYS_ON,
 };
 
 #define PUNIT_REG_PWRGT_CTRL			0x60
@@ -1667,6 +1721,26 @@ enum i915_power_well_id {
 #define ICL_PORT_CL_DW5(port)	_MMIO_PORT(port, _ICL_PORT_CL_DW5_A, \
 						 _ICL_PORT_CL_DW5_B)
 
+#define _CNL_PORT_CL_DW10_A		0x162028
+#define _ICL_PORT_CL_DW10_B		0x6c028
+#define ICL_PORT_CL_DW10(port)		_MMIO_PORT(port,	\
+						   _CNL_PORT_CL_DW10_A, \
+						   _ICL_PORT_CL_DW10_B)
+#define  PG_SEQ_DELAY_OVERRIDE_MASK	(3 << 25)
+#define  PG_SEQ_DELAY_OVERRIDE_SHIFT	25
+#define  PG_SEQ_DELAY_OVERRIDE_ENABLE	(1 << 24)
+#define  PWR_UP_ALL_LANES		(0x0 << 4)
+#define  PWR_DOWN_LN_3_2_1		(0xe << 4)
+#define  PWR_DOWN_LN_3_2		(0xc << 4)
+#define  PWR_DOWN_LN_3			(0x8 << 4)
+#define  PWR_DOWN_LN_2_1_0		(0x7 << 4)
+#define  PWR_DOWN_LN_1_0		(0x3 << 4)
+#define  PWR_DOWN_LN_1			(0x2 << 4)
+#define  PWR_DOWN_LN_3_1		(0xa << 4)
+#define  PWR_DOWN_LN_3_1_0		(0xb << 4)
+#define  PWR_DOWN_LN_MASK		(0xf << 4)
+#define  PWR_DOWN_LN_SHIFT		4
+
 #define _PORT_CL1CM_DW9_A		0x162024
 #define _PORT_CL1CM_DW9_BC		0x6C024
 #define   IREF0RC_OFFSET_SHIFT		8
@@ -1679,6 +1753,13 @@ enum i915_power_well_id {
 #define   IREF1RC_OFFSET_MASK		(0xFF << IREF1RC_OFFSET_SHIFT)
 #define BXT_PORT_CL1CM_DW10(phy)	_BXT_PHY((phy), _PORT_CL1CM_DW10_BC)
 
+#define _ICL_PORT_CL_DW12_A		0x162030
+#define _ICL_PORT_CL_DW12_B		0x6C030
+#define   ICL_LANE_ENABLE_AUX		(1 << 0)
+#define ICL_PORT_CL_DW12(port)		_MMIO_PORT((port),		\
+						   _ICL_PORT_CL_DW12_A, \
+						   _ICL_PORT_CL_DW12_B)
+
 #define _PORT_CL1CM_DW28_A		0x162070
 #define _PORT_CL1CM_DW28_BC		0x6C070
 #define   OCL1_POWER_DOWN_EN		(1 << 23)
@@ -1716,16 +1797,22 @@ enum i915_power_well_id {
 						    _CNL_PORT_PCS_DW1_LN0_D, \
 						    _CNL_PORT_PCS_DW1_LN0_AE, \
 						    _CNL_PORT_PCS_DW1_LN0_F))
+
 #define _ICL_PORT_PCS_DW1_GRP_A		0x162604
 #define _ICL_PORT_PCS_DW1_GRP_B		0x6C604
 #define _ICL_PORT_PCS_DW1_LN0_A		0x162804
 #define _ICL_PORT_PCS_DW1_LN0_B		0x6C804
+#define _ICL_PORT_PCS_DW1_AUX_A		0x162304
+#define _ICL_PORT_PCS_DW1_AUX_B		0x6c304
 #define ICL_PORT_PCS_DW1_GRP(port)	_MMIO_PORT(port,\
 						   _ICL_PORT_PCS_DW1_GRP_A, \
 						   _ICL_PORT_PCS_DW1_GRP_B)
 #define ICL_PORT_PCS_DW1_LN0(port)	_MMIO_PORT(port, \
 						   _ICL_PORT_PCS_DW1_LN0_A, \
 						   _ICL_PORT_PCS_DW1_LN0_B)
+#define ICL_PORT_PCS_DW1_AUX(port)	_MMIO_PORT(port, \
+						   _ICL_PORT_PCS_DW1_AUX_A, \
+						   _ICL_PORT_PCS_DW1_AUX_B)
 #define   COMMON_KEEPER_EN		(1 << 26)
 
 /* CNL Port TX registers */
@@ -1762,16 +1849,23 @@ enum i915_power_well_id {
 #define _ICL_PORT_TX_DW2_GRP_B		0x6C688
 #define _ICL_PORT_TX_DW2_LN0_A		0x162888
 #define _ICL_PORT_TX_DW2_LN0_B		0x6C888
+#define _ICL_PORT_TX_DW2_AUX_A		0x162388
+#define _ICL_PORT_TX_DW2_AUX_B		0x6c388
 #define ICL_PORT_TX_DW2_GRP(port)	_MMIO_PORT(port, \
 						   _ICL_PORT_TX_DW2_GRP_A, \
 						   _ICL_PORT_TX_DW2_GRP_B)
 #define ICL_PORT_TX_DW2_LN0(port)	_MMIO_PORT(port, \
 						   _ICL_PORT_TX_DW2_LN0_A, \
 						   _ICL_PORT_TX_DW2_LN0_B)
+#define ICL_PORT_TX_DW2_AUX(port)	_MMIO_PORT(port, \
+						   _ICL_PORT_TX_DW2_AUX_A, \
+						   _ICL_PORT_TX_DW2_AUX_B)
 #define   SWING_SEL_UPPER(x)		(((x) >> 3) << 15)
 #define   SWING_SEL_UPPER_MASK		(1 << 15)
 #define   SWING_SEL_LOWER(x)		(((x) & 0x7) << 11)
 #define   SWING_SEL_LOWER_MASK		(0x7 << 11)
+#define   FRC_LATENCY_OPTIM_MASK	(0x7 << 8)
+#define   FRC_LATENCY_OPTIM_VAL(x)	((x) << 8)
 #define   RCOMP_SCALAR(x)		((x) << 0)
 #define   RCOMP_SCALAR_MASK		(0xFF << 0)
 
@@ -1787,6 +1881,8 @@ enum i915_power_well_id {
 #define _ICL_PORT_TX_DW4_LN0_A		0x162890
 #define _ICL_PORT_TX_DW4_LN1_A		0x162990
 #define _ICL_PORT_TX_DW4_LN0_B		0x6C890
+#define _ICL_PORT_TX_DW4_AUX_A		0x162390
+#define _ICL_PORT_TX_DW4_AUX_B		0x6c390
 #define ICL_PORT_TX_DW4_GRP(port)	_MMIO_PORT(port, \
 						   _ICL_PORT_TX_DW4_GRP_A, \
 						   _ICL_PORT_TX_DW4_GRP_B)
@@ -1795,6 +1891,9 @@ enum i915_power_well_id {
 						   _ICL_PORT_TX_DW4_LN0_B) + \
 					     ((ln) * (_ICL_PORT_TX_DW4_LN1_A - \
 						      _ICL_PORT_TX_DW4_LN0_A)))
+#define ICL_PORT_TX_DW4_AUX(port)	_MMIO_PORT(port, \
+						   _ICL_PORT_TX_DW4_AUX_A, \
+						   _ICL_PORT_TX_DW4_AUX_B)
 #define   LOADGEN_SELECT		(1 << 31)
 #define   POST_CURSOR_1(x)		((x) << 12)
 #define   POST_CURSOR_1_MASK		(0x3F << 12)
@@ -1809,12 +1908,17 @@ enum i915_power_well_id {
 #define _ICL_PORT_TX_DW5_GRP_B		0x6C694
 #define _ICL_PORT_TX_DW5_LN0_A		0x162894
 #define _ICL_PORT_TX_DW5_LN0_B		0x6C894
+#define _ICL_PORT_TX_DW5_AUX_A		0x162394
+#define _ICL_PORT_TX_DW5_AUX_B		0x6c394
 #define ICL_PORT_TX_DW5_GRP(port)	_MMIO_PORT(port, \
 						   _ICL_PORT_TX_DW5_GRP_A, \
 						   _ICL_PORT_TX_DW5_GRP_B)
 #define ICL_PORT_TX_DW5_LN0(port)	_MMIO_PORT(port, \
 						   _ICL_PORT_TX_DW5_LN0_A, \
 						   _ICL_PORT_TX_DW5_LN0_B)
+#define ICL_PORT_TX_DW5_AUX(port)	_MMIO_PORT(port, \
+						   _ICL_PORT_TX_DW5_AUX_A, \
+						   _ICL_PORT_TX_DW5_AUX_B)
 #define   TX_TRAINING_EN		(1 << 31)
 #define   TAP2_DISABLE			(1 << 30)
 #define   TAP3_DISABLE			(1 << 29)
@@ -2676,9 +2780,6 @@ enum i915_power_well_id {
 #define   GEN8_4x4_STC_OPTIMIZATION_DISABLE	(1 << 6)
 #define   GEN9_PARTIAL_RESOLVE_IN_VC_DISABLE	(1 << 1)
 
-#define GEN10_CACHE_MODE_SS			_MMIO(0xe420)
-#define   FLOAT_BLEND_OPTIMIZATION_ENABLE	(1 << 4)
-
 #define GEN6_BLITTER_ECOSKPD	_MMIO(0x221d0)
 #define   GEN6_BLITTER_LOCK_SHIFT			16
 #define   GEN6_BLITTER_FBC_NOTIFY			(1 << 3)
@@ -2811,7 +2912,6 @@ enum i915_power_well_id {
 #define I915_DISPLAY_PORT_INTERRUPT			(1 << 17)
 #define I915_DISPLAY_PIPE_C_HBLANK_INTERRUPT		(1 << 16)
 #define I915_MASTER_ERROR_INTERRUPT			(1 << 15)
-#define I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT	(1 << 15)
 #define I915_DISPLAY_PIPE_B_HBLANK_INTERRUPT		(1 << 14)
 #define I915_GMCH_THERMAL_SENSOR_EVENT_INTERRUPT	(1 << 14) /* p-state */
 #define I915_DISPLAY_PIPE_A_HBLANK_INTERRUPT		(1 << 13)
@@ -3020,6 +3120,7 @@ enum i915_power_well_id {
 #define   GMBUS_RATE_400KHZ	(2 << 8) /* reserved on Pineview */
 #define   GMBUS_RATE_1MHZ	(3 << 8) /* reserved on Pineview */
 #define   GMBUS_HOLD_EXT	(1 << 7) /* 300ns hold time, rsvd on Pineview */
+#define   GMBUS_BYTE_CNT_OVERRIDE (1 << 6)
 #define   GMBUS_PIN_DISABLED	0
 #define   GMBUS_PIN_SSC		1
 #define   GMBUS_PIN_VGADDC	2
@@ -3049,6 +3150,7 @@ enum i915_power_well_id {
 #define   GMBUS_CYCLE_STOP	(4 << 25)
 #define   GMBUS_BYTE_COUNT_SHIFT 16
 #define   GMBUS_BYTE_COUNT_MAX   256U
+#define   GEN9_GMBUS_BYTE_COUNT_MAX 511U
 #define   GMBUS_SLAVE_INDEX_SHIFT 8
 #define   GMBUS_SLAVE_ADDR_SHIFT 1
 #define   GMBUS_SLAVE_READ	(1 << 0)
@@ -4044,6 +4146,7 @@ enum {
 #define   EDP_PSR_SKIP_AUX_EXIT			(1 << 12)
 #define   EDP_PSR_TP1_TP2_SEL			(0 << 11)
 #define   EDP_PSR_TP1_TP3_SEL			(1 << 11)
+#define   EDP_PSR_CRC_ENABLE			(1 << 10) /* BDW+ */
 #define   EDP_PSR_TP2_TP3_TIME_500us		(0 << 8)
 #define   EDP_PSR_TP2_TP3_TIME_100us		(1 << 8)
 #define   EDP_PSR_TP2_TP3_TIME_2500us		(2 << 8)
@@ -4072,6 +4175,7 @@ enum {
 
 #define EDP_PSR_STATUS				_MMIO(dev_priv->psr_mmio_base + 0x40)
 #define   EDP_PSR_STATUS_STATE_MASK		(7 << 29)
+#define   EDP_PSR_STATUS_STATE_SHIFT		29
 #define   EDP_PSR_STATUS_STATE_IDLE		(0 << 29)
 #define   EDP_PSR_STATUS_STATE_SRDONACK		(1 << 29)
 #define   EDP_PSR_STATUS_STATE_SRDENT		(2 << 29)
@@ -4498,6 +4602,16 @@ enum {
 #define   VIDEO_DIP_ENABLE_GMP_HSW	(1 << 4)
 #define   VIDEO_DIP_ENABLE_SPD_HSW	(1 << 0)
 
+#define  DRM_DIP_ENABLE			(1 << 28)
+#define  PSR_VSC_BIT_7_SET		(1 << 27)
+#define  VSC_SELECT_MASK		(0x3 << 26)
+#define  VSC_SELECT_SHIFT		26
+#define  VSC_DIP_HW_HEA_DATA		(0 << 26)
+#define  VSC_DIP_HW_HEA_SW_DATA		(1 << 26)
+#define  VSC_DIP_HW_DATA_SW_HEA		(2 << 26)
+#define  VSC_DIP_SW_HEA_DATA		(3 << 26)
+#define  VDIP_ENABLE_PPS		(1 << 24)
+
 /* Panel power sequencing */
 #define PPS_BASE			0x61200
 #define VLV_PPS_BASE			(VLV_DISPLAY_BASE + PPS_BASE)
@@ -6829,7 +6943,7 @@ enum {
 #define _PS_ECC_STAT_2B     0x68AD0
 #define _PS_ECC_STAT_1C     0x691D0
 
-#define _ID(id, a, b) ((a) + (id) * ((b) - (a)))
+#define _ID(id, a, b) _PICK_EVEN(id, a, b)
 #define SKL_PS_CTRL(pipe, id) _MMIO_PIPE(pipe,        \
 			_ID(id, _PS_1A_CTRL, _PS_2A_CTRL),       \
 			_ID(id, _PS_1B_CTRL, _PS_2B_CTRL))
@@ -7366,6 +7480,14 @@ enum {
 #define BDW_SCRATCH1					_MMIO(0xb11c)
 #define  GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE	(1 << 2)
 
+/*GEN11 chicken */
+#define _PIPEA_CHICKEN			0x70038
+#define _PIPEB_CHICKEN			0x71038
+#define _PIPEC_CHICKEN			0x72038
+#define  PER_PIXEL_ALPHA_BYPASS_EN	(1 << 7)
+#define PIPE_CHICKEN(pipe)		_MMIO_PIPE(pipe, _PIPEA_CHICKEN,\
+						   _PIPEB_CHICKEN)
+
 /* PCH */
 
 /* south display engine interrupt: IBX */
@@ -7409,7 +7531,7 @@ enum {
 #define SDE_TRANSA_FIFO_UNDER	(1 << 0)
 #define SDE_TRANS_MASK		(0x3f)
 
-/* south display engine interrupt: CPT/PPT */
+/* south display engine interrupt: CPT - CNP */
 #define SDE_AUDIO_POWER_D_CPT	(1 << 31)
 #define SDE_AUDIO_POWER_C_CPT	(1 << 30)
 #define SDE_AUDIO_POWER_B_CPT	(1 << 29)
@@ -7457,6 +7579,21 @@ enum {
 				 SDE_FDI_RXB_CPT | \
 				 SDE_FDI_RXA_CPT)
 
+/* south display engine interrupt: ICP */
+#define SDE_TC4_HOTPLUG_ICP		(1 << 27)
+#define SDE_TC3_HOTPLUG_ICP		(1 << 26)
+#define SDE_TC2_HOTPLUG_ICP		(1 << 25)
+#define SDE_TC1_HOTPLUG_ICP		(1 << 24)
+#define SDE_GMBUS_ICP			(1 << 23)
+#define SDE_DDIB_HOTPLUG_ICP		(1 << 17)
+#define SDE_DDIA_HOTPLUG_ICP		(1 << 16)
+#define SDE_DDI_MASK_ICP		(SDE_DDIB_HOTPLUG_ICP |	\
+					 SDE_DDIA_HOTPLUG_ICP)
+#define SDE_TC_MASK_ICP			(SDE_TC4_HOTPLUG_ICP |	\
+					 SDE_TC3_HOTPLUG_ICP |	\
+					 SDE_TC2_HOTPLUG_ICP |	\
+					 SDE_TC1_HOTPLUG_ICP)
+
 #define SDEISR  _MMIO(0xc4000)
 #define SDEIMR  _MMIO(0xc4004)
 #define SDEIIR  _MMIO(0xc4008)
@@ -7517,6 +7654,134 @@ enum {
 #define  PORTE_HOTPLUG_SHORT_DETECT	(1 << 0)
 #define  PORTE_HOTPLUG_LONG_DETECT	(2 << 0)
 
+/* This register is a reuse of PCH_PORT_HOTPLUG register. The
+ * functionality covered in PCH_PORT_HOTPLUG is split into
+ * SHOTPLUG_CTL_DDI and SHOTPLUG_CTL_TC.
+ */
+
+#define SHOTPLUG_CTL_DDI			_MMIO(0xc4030)
+#define   ICP_DDIB_HPD_ENABLE			(1 << 7)
+#define   ICP_DDIB_HPD_STATUS_MASK		(3 << 4)
+#define   ICP_DDIB_HPD_NO_DETECT		(0 << 4)
+#define   ICP_DDIB_HPD_SHORT_DETECT		(1 << 4)
+#define   ICP_DDIB_HPD_LONG_DETECT		(2 << 4)
+#define   ICP_DDIB_HPD_SHORT_LONG_DETECT	(3 << 4)
+#define   ICP_DDIA_HPD_ENABLE			(1 << 3)
+#define   ICP_DDIA_HPD_STATUS_MASK		(3 << 0)
+#define   ICP_DDIA_HPD_NO_DETECT		(0 << 0)
+#define   ICP_DDIA_HPD_SHORT_DETECT		(1 << 0)
+#define   ICP_DDIA_HPD_LONG_DETECT		(2 << 0)
+#define   ICP_DDIA_HPD_SHORT_LONG_DETECT	(3 << 0)
+
+#define SHOTPLUG_CTL_TC				_MMIO(0xc4034)
+#define   ICP_TC_HPD_ENABLE(tc_port)		(8 << (tc_port) * 4)
+/* Icelake DSC Rate Control Range Parameter Registers */
+#define DSCA_RC_RANGE_PARAMETERS_0		_MMIO(0x6B240)
+#define DSCA_RC_RANGE_PARAMETERS_0_UDW		_MMIO(0x6B240 + 4)
+#define DSCC_RC_RANGE_PARAMETERS_0		_MMIO(0x6BA40)
+#define DSCC_RC_RANGE_PARAMETERS_0_UDW		_MMIO(0x6BA40 + 4)
+#define _ICL_DSC0_RC_RANGE_PARAMETERS_0_PB	(0x78208)
+#define _ICL_DSC0_RC_RANGE_PARAMETERS_0_UDW_PB	(0x78208 + 4)
+#define _ICL_DSC1_RC_RANGE_PARAMETERS_0_PB	(0x78308)
+#define _ICL_DSC1_RC_RANGE_PARAMETERS_0_UDW_PB	(0x78308 + 4)
+#define _ICL_DSC0_RC_RANGE_PARAMETERS_0_PC	(0x78408)
+#define _ICL_DSC0_RC_RANGE_PARAMETERS_0_UDW_PC	(0x78408 + 4)
+#define _ICL_DSC1_RC_RANGE_PARAMETERS_0_PC	(0x78508)
+#define _ICL_DSC1_RC_RANGE_PARAMETERS_0_UDW_PC	(0x78508 + 4)
+#define ICL_DSC0_RC_RANGE_PARAMETERS_0(pipe)		_MMIO_PIPE((pipe) - PIPE_B, \
+							_ICL_DSC0_RC_RANGE_PARAMETERS_0_PB, \
+							_ICL_DSC0_RC_RANGE_PARAMETERS_0_PC)
+#define ICL_DSC0_RC_RANGE_PARAMETERS_0_UDW(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							_ICL_DSC0_RC_RANGE_PARAMETERS_0_UDW_PB, \
+							_ICL_DSC0_RC_RANGE_PARAMETERS_0_UDW_PC)
+#define ICL_DSC1_RC_RANGE_PARAMETERS_0(pipe)		_MMIO_PIPE((pipe) - PIPE_B, \
+							_ICL_DSC1_RC_RANGE_PARAMETERS_0_PB, \
+							_ICL_DSC1_RC_RANGE_PARAMETERS_0_PC)
+#define ICL_DSC1_RC_RANGE_PARAMETERS_0_UDW(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							_ICL_DSC1_RC_RANGE_PARAMETERS_0_UDW_PB, \
+							_ICL_DSC1_RC_RANGE_PARAMETERS_0_UDW_PC)
+#define RC_BPG_OFFSET_SHIFT			10
+#define RC_MAX_QP_SHIFT				5
+#define RC_MIN_QP_SHIFT				0
+
+#define DSCA_RC_RANGE_PARAMETERS_1		_MMIO(0x6B248)
+#define DSCA_RC_RANGE_PARAMETERS_1_UDW		_MMIO(0x6B248 + 4)
+#define DSCC_RC_RANGE_PARAMETERS_1		_MMIO(0x6BA48)
+#define DSCC_RC_RANGE_PARAMETERS_1_UDW		_MMIO(0x6BA48 + 4)
+#define _ICL_DSC0_RC_RANGE_PARAMETERS_1_PB	(0x78210)
+#define _ICL_DSC0_RC_RANGE_PARAMETERS_1_UDW_PB	(0x78210 + 4)
+#define _ICL_DSC1_RC_RANGE_PARAMETERS_1_PB	(0x78310)
+#define _ICL_DSC1_RC_RANGE_PARAMETERS_1_UDW_PB	(0x78310 + 4)
+#define _ICL_DSC0_RC_RANGE_PARAMETERS_1_PC	(0x78410)
+#define _ICL_DSC0_RC_RANGE_PARAMETERS_1_UDW_PC	(0x78410 + 4)
+#define _ICL_DSC1_RC_RANGE_PARAMETERS_1_PC	(0x78510)
+#define _ICL_DSC1_RC_RANGE_PARAMETERS_1_UDW_PC	(0x78510 + 4)
+#define ICL_DSC0_RC_RANGE_PARAMETERS_1(pipe)		_MMIO_PIPE((pipe) - PIPE_B, \
+							_ICL_DSC0_RC_RANGE_PARAMETERS_1_PB, \
+							_ICL_DSC0_RC_RANGE_PARAMETERS_1_PC)
+#define ICL_DSC0_RC_RANGE_PARAMETERS_1_UDW(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							_ICL_DSC0_RC_RANGE_PARAMETERS_1_UDW_PB, \
+							_ICL_DSC0_RC_RANGE_PARAMETERS_1_UDW_PC)
+#define ICL_DSC1_RC_RANGE_PARAMETERS_1(pipe)		_MMIO_PIPE((pipe) - PIPE_B, \
+							_ICL_DSC1_RC_RANGE_PARAMETERS_1_PB, \
+							_ICL_DSC1_RC_RANGE_PARAMETERS_1_PC)
+#define ICL_DSC1_RC_RANGE_PARAMETERS_1_UDW(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							_ICL_DSC1_RC_RANGE_PARAMETERS_1_UDW_PB, \
+							_ICL_DSC1_RC_RANGE_PARAMETERS_1_UDW_PC)
+
+#define DSCA_RC_RANGE_PARAMETERS_2		_MMIO(0x6B250)
+#define DSCA_RC_RANGE_PARAMETERS_2_UDW		_MMIO(0x6B250 + 4)
+#define DSCC_RC_RANGE_PARAMETERS_2		_MMIO(0x6BA50)
+#define DSCC_RC_RANGE_PARAMETERS_2_UDW		_MMIO(0x6BA50 + 4)
+#define _ICL_DSC0_RC_RANGE_PARAMETERS_2_PB	(0x78218)
+#define _ICL_DSC0_RC_RANGE_PARAMETERS_2_UDW_PB	(0x78218 + 4)
+#define _ICL_DSC1_RC_RANGE_PARAMETERS_2_PB	(0x78318)
+#define _ICL_DSC1_RC_RANGE_PARAMETERS_2_UDW_PB	(0x78318 + 4)
+#define _ICL_DSC0_RC_RANGE_PARAMETERS_2_PC	(0x78418)
+#define _ICL_DSC0_RC_RANGE_PARAMETERS_2_UDW_PC	(0x78418 + 4)
+#define _ICL_DSC1_RC_RANGE_PARAMETERS_2_PC	(0x78518)
+#define _ICL_DSC1_RC_RANGE_PARAMETERS_2_UDW_PC	(0x78518 + 4)
+#define ICL_DSC0_RC_RANGE_PARAMETERS_2(pipe)		_MMIO_PIPE((pipe) - PIPE_B, \
+							_ICL_DSC0_RC_RANGE_PARAMETERS_2_PB, \
+							_ICL_DSC0_RC_RANGE_PARAMETERS_2_PC)
+#define ICL_DSC0_RC_RANGE_PARAMETERS_2_UDW(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							_ICL_DSC0_RC_RANGE_PARAMETERS_2_UDW_PB, \
+							_ICL_DSC0_RC_RANGE_PARAMETERS_2_UDW_PC)
+#define ICL_DSC1_RC_RANGE_PARAMETERS_2(pipe)		_MMIO_PIPE((pipe) - PIPE_B, \
+							_ICL_DSC1_RC_RANGE_PARAMETERS_2_PB, \
+							_ICL_DSC1_RC_RANGE_PARAMETERS_2_PC)
+#define ICL_DSC1_RC_RANGE_PARAMETERS_2_UDW(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							_ICL_DSC1_RC_RANGE_PARAMETERS_2_UDW_PB, \
+							_ICL_DSC1_RC_RANGE_PARAMETERS_2_UDW_PC)
+
+#define DSCA_RC_RANGE_PARAMETERS_3		_MMIO(0x6B258)
+#define DSCA_RC_RANGE_PARAMETERS_3_UDW		_MMIO(0x6B258 + 4)
+#define DSCC_RC_RANGE_PARAMETERS_3		_MMIO(0x6BA58)
+#define DSCC_RC_RANGE_PARAMETERS_3_UDW		_MMIO(0x6BA58 + 4)
+#define _ICL_DSC0_RC_RANGE_PARAMETERS_3_PB	(0x78220)
+#define _ICL_DSC0_RC_RANGE_PARAMETERS_3_UDW_PB	(0x78220 + 4)
+#define _ICL_DSC1_RC_RANGE_PARAMETERS_3_PB	(0x78320)
+#define _ICL_DSC1_RC_RANGE_PARAMETERS_3_UDW_PB	(0x78320 + 4)
+#define _ICL_DSC0_RC_RANGE_PARAMETERS_3_PC	(0x78420)
+#define _ICL_DSC0_RC_RANGE_PARAMETERS_3_UDW_PC	(0x78420 + 4)
+#define _ICL_DSC1_RC_RANGE_PARAMETERS_3_PC	(0x78520)
+#define _ICL_DSC1_RC_RANGE_PARAMETERS_3_UDW_PC	(0x78520 + 4)
+#define ICL_DSC0_RC_RANGE_PARAMETERS_3(pipe)		_MMIO_PIPE((pipe) - PIPE_B, \
+							_ICL_DSC0_RC_RANGE_PARAMETERS_3_PB, \
+							_ICL_DSC0_RC_RANGE_PARAMETERS_3_PC)
+#define ICL_DSC0_RC_RANGE_PARAMETERS_3_UDW(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							_ICL_DSC0_RC_RANGE_PARAMETERS_3_UDW_PB, \
+							_ICL_DSC0_RC_RANGE_PARAMETERS_3_UDW_PC)
+#define ICL_DSC1_RC_RANGE_PARAMETERS_3(pipe)		_MMIO_PIPE((pipe) - PIPE_B, \
+							_ICL_DSC1_RC_RANGE_PARAMETERS_3_PB, \
+							_ICL_DSC1_RC_RANGE_PARAMETERS_3_PC)
+#define ICL_DSC1_RC_RANGE_PARAMETERS_3_UDW(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							_ICL_DSC1_RC_RANGE_PARAMETERS_3_UDW_PB, \
+							_ICL_DSC1_RC_RANGE_PARAMETERS_3_UDW_PC)
+
+#define   ICP_TC_HPD_LONG_DETECT(tc_port)	(2 << (tc_port) * 4)
+#define   ICP_TC_HPD_SHORT_DETECT(tc_port)	(1 << (tc_port) * 4)
+
 #define PCH_GPIOA               _MMIO(0xc5010)
 #define PCH_GPIOB               _MMIO(0xc5014)
 #define PCH_GPIOC               _MMIO(0xc5018)
@@ -7689,12 +7954,25 @@ enum {
 #define _HSW_VIDEO_DIP_VSC_ECC_B	0x61344
 #define _HSW_VIDEO_DIP_GCP_B		0x61210
 
+/* Icelake PPS_DATA and _ECC DIP Registers.
+ * These are available for transcoders B,C and eDP.
+ * Adding the _A so as to reuse the _MMIO_TRANS2
+ * definition, with which it offsets to the right location.
+ */
+
+#define _ICL_VIDEO_DIP_PPS_DATA_A	0x60350
+#define _ICL_VIDEO_DIP_PPS_DATA_B	0x61350
+#define _ICL_VIDEO_DIP_PPS_ECC_A	0x603D4
+#define _ICL_VIDEO_DIP_PPS_ECC_B	0x613D4
+
 #define HSW_TVIDEO_DIP_CTL(trans)		_MMIO_TRANS2(trans, _HSW_VIDEO_DIP_CTL_A)
 #define HSW_TVIDEO_DIP_AVI_DATA(trans, i)	_MMIO_TRANS2(trans, _HSW_VIDEO_DIP_AVI_DATA_A + (i) * 4)
 #define HSW_TVIDEO_DIP_VS_DATA(trans, i)	_MMIO_TRANS2(trans, _HSW_VIDEO_DIP_VS_DATA_A + (i) * 4)
 #define HSW_TVIDEO_DIP_SPD_DATA(trans, i)	_MMIO_TRANS2(trans, _HSW_VIDEO_DIP_SPD_DATA_A + (i) * 4)
 #define HSW_TVIDEO_DIP_GCP(trans)		_MMIO_TRANS2(trans, _HSW_VIDEO_DIP_GCP_A)
 #define HSW_TVIDEO_DIP_VSC_DATA(trans, i)	_MMIO_TRANS2(trans, _HSW_VIDEO_DIP_VSC_DATA_A + (i) * 4)
+#define ICL_VIDEO_DIP_PPS_DATA(trans, i)	_MMIO_TRANS2(trans, _ICL_VIDEO_DIP_PPS_DATA_A + (i) * 4)
+#define ICL_VIDEO_DIP_PPS_ECC(trans, i)		_MMIO_TRANS2(trans, _ICL_VIDEO_DIP_PPS_ECC_A + (i) * 4)
 
 #define _HSW_STEREO_3D_CTL_A		0x70020
 #define   S3D_ENABLE			(1 << 31)
@@ -8555,6 +8833,14 @@ enum {
 #define _HSW_PWR_WELL_CTL3			0x45408
 #define _HSW_PWR_WELL_CTL4			0x4540C
 
+#define _ICL_PWR_WELL_CTL_AUX1			0x45440
+#define _ICL_PWR_WELL_CTL_AUX2			0x45444
+#define _ICL_PWR_WELL_CTL_AUX4			0x4544C
+
+#define _ICL_PWR_WELL_CTL_DDI1			0x45450
+#define _ICL_PWR_WELL_CTL_DDI2			0x45454
+#define _ICL_PWR_WELL_CTL_DDI4			0x4545C
+
 /*
  * Each power well control register contains up to 16 (request, status) HW
  * flag tuples. The register index and HW flag shift is determined by the
@@ -8564,14 +8850,20 @@ enum {
  */
 #define _HSW_PW_REG_IDX(pw)			((pw) >> 4)
 #define _HSW_PW_SHIFT(pw)			(((pw) & 0xf) * 2)
-/* TODO: Add all PWR_WELL_CTL registers below for new platforms */
 #define HSW_PWR_WELL_CTL_BIOS(pw)	_MMIO(_PICK(_HSW_PW_REG_IDX(pw),       \
-						    _HSW_PWR_WELL_CTL1))
+						    _HSW_PWR_WELL_CTL1,	       \
+						    _ICL_PWR_WELL_CTL_AUX1,    \
+						    _ICL_PWR_WELL_CTL_DDI1))
 #define HSW_PWR_WELL_CTL_DRIVER(pw)	_MMIO(_PICK(_HSW_PW_REG_IDX(pw),       \
-						    _HSW_PWR_WELL_CTL2))
+						    _HSW_PWR_WELL_CTL2,	       \
+						    _ICL_PWR_WELL_CTL_AUX2,    \
+						    _ICL_PWR_WELL_CTL_DDI2))
+/* KVMR doesn't have a reg for AUX or DDI power well control */
 #define HSW_PWR_WELL_CTL_KVMR		_MMIO(_HSW_PWR_WELL_CTL3)
 #define HSW_PWR_WELL_CTL_DEBUG(pw)	_MMIO(_PICK(_HSW_PW_REG_IDX(pw),       \
-						    _HSW_PWR_WELL_CTL4))
+						    _HSW_PWR_WELL_CTL4,	       \
+						    _ICL_PWR_WELL_CTL_AUX4,    \
+						    _ICL_PWR_WELL_CTL_DDI4))
 
 #define   HSW_PWR_WELL_CTL_REQ(pw)		(1 << (_HSW_PW_SHIFT(pw) + 1))
 #define   HSW_PWR_WELL_CTL_STATE(pw)		(1 << _HSW_PW_SHIFT(pw))
@@ -8592,6 +8884,8 @@ enum skl_power_gate {
 #define  SKL_FUSE_DOWNLOAD_STATUS		(1 << 31)
 /* PG0 (HW control->no power well ID), PG1..PG2 (SKL_DISP_PW1..SKL_DISP_PW2) */
 #define  SKL_PW_TO_PG(pw)			((pw) - SKL_DISP_PW_1 + SKL_PG1)
+/* PG0 (HW control->no power well ID), PG1..PG4 (ICL_DISP_PW1..ICL_DISP_PW4) */
+#define  ICL_PW_TO_PG(pw)			((pw) - ICL_DISP_PW_1 + SKL_PG1)
 #define  SKL_FUSE_PG_DIST_STATUS(pg)		(1 << (27 - (pg)))
 
 #define _CNL_AUX_REG_IDX(pw)		((pw) - 9)
@@ -8907,6 +9201,7 @@ enum skl_power_gate {
 #define  TRANS_MSA_10_BPC		(2 << 5)
 #define  TRANS_MSA_12_BPC		(3 << 5)
 #define  TRANS_MSA_16_BPC		(4 << 5)
+#define  TRANS_MSA_CEA_RANGE		(1 << 3)
 
 /* LCPLL Control */
 #define LCPLL_CTL			_MMIO(0x130040)
@@ -9047,6 +9342,7 @@ enum skl_power_gate {
 #define _MG_REFCLKIN_CTL_PORT3				0x16A92C
 #define _MG_REFCLKIN_CTL_PORT4				0x16B92C
 #define   MG_REFCLKIN_CTL_OD_2_MUX(x)			((x) << 8)
+#define   MG_REFCLKIN_CTL_OD_2_MUX_MASK			(0x7 << 8)
 #define MG_REFCLKIN_CTL(port) _MMIO_PORT((port) - PORT_C, \
 					 _MG_REFCLKIN_CTL_PORT1, \
 					 _MG_REFCLKIN_CTL_PORT2)
@@ -9056,7 +9352,9 @@ enum skl_power_gate {
 #define _MG_CLKTOP2_CORECLKCTL1_PORT3			0x16A8D8
 #define _MG_CLKTOP2_CORECLKCTL1_PORT4			0x16B8D8
 #define   MG_CLKTOP2_CORECLKCTL1_B_DIVRATIO(x)		((x) << 16)
+#define   MG_CLKTOP2_CORECLKCTL1_B_DIVRATIO_MASK	(0xff << 16)
 #define   MG_CLKTOP2_CORECLKCTL1_A_DIVRATIO(x)		((x) << 8)
+#define   MG_CLKTOP2_CORECLKCTL1_A_DIVRATIO_MASK	(0xff << 8)
 #define MG_CLKTOP2_CORECLKCTL1(port) _MMIO_PORT((port) - PORT_C, \
 						_MG_CLKTOP2_CORECLKCTL1_PORT1, \
 						_MG_CLKTOP2_CORECLKCTL1_PORT2)
@@ -9066,9 +9364,13 @@ enum skl_power_gate {
 #define _MG_CLKTOP2_HSCLKCTL_PORT3			0x16A8D4
 #define _MG_CLKTOP2_HSCLKCTL_PORT4			0x16B8D4
 #define   MG_CLKTOP2_HSCLKCTL_CORE_INPUTSEL(x)		((x) << 16)
+#define   MG_CLKTOP2_HSCLKCTL_CORE_INPUTSEL_MASK	(0x1 << 16)
 #define   MG_CLKTOP2_HSCLKCTL_TLINEDRV_CLKSEL(x)	((x) << 14)
+#define   MG_CLKTOP2_HSCLKCTL_TLINEDRV_CLKSEL_MASK	(0x3 << 14)
 #define   MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO(x)		((x) << 12)
+#define   MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO_MASK		(0x3 << 12)
 #define   MG_CLKTOP2_HSCLKCTL_DSDIV_RATIO(x)		((x) << 8)
+#define   MG_CLKTOP2_HSCLKCTL_DSDIV_RATIO_MASK		(0xf << 8)
 #define MG_CLKTOP2_HSCLKCTL(port) _MMIO_PORT((port) - PORT_C, \
 					     _MG_CLKTOP2_HSCLKCTL_PORT1, \
 					     _MG_CLKTOP2_HSCLKCTL_PORT2)
@@ -9142,12 +9444,18 @@ enum skl_power_gate {
 #define _MG_PLL_BIAS_PORT3				0x16AA14
 #define _MG_PLL_BIAS_PORT4				0x16BA14
 #define   MG_PLL_BIAS_BIAS_GB_SEL(x)			((x) << 30)
+#define   MG_PLL_BIAS_BIAS_GB_SEL_MASK			(0x3 << 30)
 #define   MG_PLL_BIAS_INIT_DCOAMP(x)			((x) << 24)
+#define   MG_PLL_BIAS_INIT_DCOAMP_MASK			(0x3f << 24)
 #define   MG_PLL_BIAS_BIAS_BONUS(x)			((x) << 16)
+#define   MG_PLL_BIAS_BIAS_BONUS_MASK			(0xff << 16)
 #define   MG_PLL_BIAS_BIASCAL_EN			(1 << 15)
 #define   MG_PLL_BIAS_CTRIM(x)				((x) << 8)
+#define   MG_PLL_BIAS_CTRIM_MASK			(0x1f << 8)
 #define   MG_PLL_BIAS_VREF_RDAC(x)			((x) << 5)
+#define   MG_PLL_BIAS_VREF_RDAC_MASK			(0x7 << 5)
 #define   MG_PLL_BIAS_IREFTRIM(x)			((x) << 0)
+#define   MG_PLL_BIAS_IREFTRIM_MASK			(0x1f << 0)
 #define MG_PLL_BIAS(port) _MMIO_PORT((port) - PORT_C, _MG_PLL_BIAS_PORT1, \
 				     _MG_PLL_BIAS_PORT2)
 
@@ -9401,6 +9709,22 @@ enum skl_power_gate {
 #define MIPIO_TXESC_CLK_DIV2			_MMIO(0x160008)
 #define  GLK_TX_ESC_CLK_DIV2_MASK			0x3FF
 
+#define _ICL_DSI_ESC_CLK_DIV0		0x6b090
+#define _ICL_DSI_ESC_CLK_DIV1		0x6b890
+#define ICL_DSI_ESC_CLK_DIV(port)	_MMIO_PORT((port),	\
+							_ICL_DSI_ESC_CLK_DIV0, \
+							_ICL_DSI_ESC_CLK_DIV1)
+#define _ICL_DPHY_ESC_CLK_DIV0		0x162190
+#define _ICL_DPHY_ESC_CLK_DIV1		0x6C190
+#define ICL_DPHY_ESC_CLK_DIV(port)	_MMIO_PORT((port),	\
+						_ICL_DPHY_ESC_CLK_DIV0, \
+						_ICL_DPHY_ESC_CLK_DIV1)
+#define  ICL_BYTE_CLK_PER_ESC_CLK_MASK		(0x1f << 16)
+#define  ICL_BYTE_CLK_PER_ESC_CLK_SHIFT	16
+#define  ICL_ESC_CLK_DIV_MASK			0x1ff
+#define  ICL_ESC_CLK_DIV_SHIFT			0
+#define DSI_MAX_ESC_CLK			20000		/* in KHz */
+
 /* Gen4+ Timestamp and Pipe Frame time stamp registers */
 #define GEN4_TIMESTAMP		_MMIO(0x2358)
 #define ILK_TIMESTAMP_HI	_MMIO(0x70070)
@@ -9535,6 +9859,14 @@ enum skl_power_gate {
 #define _BXT_MIPIC_PORT_CTRL				0x6B8C0
 #define BXT_MIPI_PORT_CTRL(tc)	_MMIO_MIPI(tc, _BXT_MIPIA_PORT_CTRL, _BXT_MIPIC_PORT_CTRL)
 
+/* ICL DSI MODE control */
+#define _ICL_DSI_IO_MODECTL_0				0x6B094
+#define _ICL_DSI_IO_MODECTL_1				0x6B894
+#define ICL_DSI_IO_MODECTL(port)	_MMIO_PORT(port,	\
+						    _ICL_DSI_IO_MODECTL_0, \
+						    _ICL_DSI_IO_MODECTL_1)
+#define  COMBO_PHY_MODE_DSI				(1 << 0)
+
 #define BXT_P_DSI_REGULATOR_CFG			_MMIO(0x160020)
 #define  STAP_SELECT					(1 << 0)
 
@@ -10014,4 +10346,310 @@ enum skl_power_gate {
 						 _ICL_PHY_MISC_B)
 #define  ICL_PHY_MISC_DE_IO_COMP_PWR_DOWN	(1 << 23)
 
+/* Icelake Display Stream Compression Registers */
+#define DSCA_PICTURE_PARAMETER_SET_0		0x6B200
+#define DSCC_PICTURE_PARAMETER_SET_0		0x6BA00
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_0_PB	0x78270
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_0_PB	0x78370
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_0_PC	0x78470
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_0_PC	0x78570
+#define ICL_DSC0_PICTURE_PARAMETER_SET_0(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_0_PB, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_0_PC)
+#define ICL_DSC1_PICTURE_PARAMETER_SET_0(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_0_PB, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_0_PC)
+#define  DSC_VBR_ENABLE			(1 << 19)
+#define  DSC_422_ENABLE			(1 << 18)
+#define  DSC_COLOR_SPACE_CONVERSION	(1 << 17)
+#define  DSC_BLOCK_PREDICTION		(1 << 16)
+#define  DSC_LINE_BUF_DEPTH_SHIFT	12
+#define  DSC_BPC_SHIFT			8
+#define  DSC_VER_MIN_SHIFT		4
+#define  DSC_VER_MAJ			(0x1 << 0)
+
+#define DSCA_PICTURE_PARAMETER_SET_1		0x6B204
+#define DSCC_PICTURE_PARAMETER_SET_1		0x6BA04
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_1_PB	0x78274
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_1_PB	0x78374
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_1_PC	0x78474
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_1_PC	0x78574
+#define ICL_DSC0_PICTURE_PARAMETER_SET_1(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_1_PB, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_1_PC)
+#define ICL_DSC1_PICTURE_PARAMETER_SET_1(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_1_PB, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_1_PC)
+#define  DSC_BPP(bpp)				((bpp) << 0)
+
+#define DSCA_PICTURE_PARAMETER_SET_2		0x6B208
+#define DSCC_PICTURE_PARAMETER_SET_2		0x6BA08
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_2_PB	0x78278
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_2_PB	0x78378
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_2_PC	0x78478
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_2_PC	0x78578
+#define ICL_DSC0_PICTURE_PARAMETER_SET_2(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_2_PB, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_2_PC)
+#define ICL_DSC1_PICTURE_PARAMETER_SET_2(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+					    _ICL_DSC1_PICTURE_PARAMETER_SET_2_PB, \
+					    _ICL_DSC1_PICTURE_PARAMETER_SET_2_PC)
+#define  DSC_PIC_WIDTH(pic_width)	((pic_width) << 16)
+#define  DSC_PIC_HEIGHT(pic_height)	((pic_height) << 0)
+
+#define DSCA_PICTURE_PARAMETER_SET_3		0x6B20C
+#define DSCC_PICTURE_PARAMETER_SET_3		0x6BA0C
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_3_PB	0x7827C
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_3_PB	0x7837C
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_3_PC	0x7847C
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_3_PC	0x7857C
+#define ICL_DSC0_PICTURE_PARAMETER_SET_3(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_3_PB, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_3_PC)
+#define ICL_DSC1_PICTURE_PARAMETER_SET_3(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_3_PB, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_3_PC)
+#define  DSC_SLICE_WIDTH(slice_width)   ((slice_width) << 16)
+#define  DSC_SLICE_HEIGHT(slice_height) ((slice_height) << 0)
+
+#define DSCA_PICTURE_PARAMETER_SET_4		0x6B210
+#define DSCC_PICTURE_PARAMETER_SET_4		0x6BA10
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_4_PB	0x78280
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_4_PB	0x78380
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_4_PC	0x78480
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_4_PC	0x78580
+#define ICL_DSC0_PICTURE_PARAMETER_SET_4(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_4_PB, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_4_PC)
+#define ICL_DSC1_PICTURE_PARAMETER_SET_4(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_4_PB, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_4_PC)
+#define  DSC_INITIAL_DEC_DELAY(dec_delay)       ((dec_delay) << 16)
+#define  DSC_INITIAL_XMIT_DELAY(xmit_delay)     ((xmit_delay) << 0)
+
+#define DSCA_PICTURE_PARAMETER_SET_5		0x6B214
+#define DSCC_PICTURE_PARAMETER_SET_5		0x6BA14
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_5_PB	0x78284
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_5_PB	0x78384
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_5_PC	0x78484
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_5_PC	0x78584
+#define ICL_DSC0_PICTURE_PARAMETER_SET_5(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_5_PB, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_5_PC)
+#define ICL_DSC1_PICTURE_PARAMETER_SET_5(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_5_PC, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_5_PC)
+#define  DSC_SCALE_DEC_INTINT(scale_dec)	((scale_dec) << 16)
+#define  DSC_SCALE_INC_INT(scale_inc)		((scale_inc) << 0)
+
+#define DSCA_PICTURE_PARAMETER_SET_6		0x6B218
+#define DSCC_PICTURE_PARAMETER_SET_6		0x6BA18
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_6_PB	0x78288
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_6_PB	0x78388
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_6_PC	0x78488
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_6_PC	0x78588
+#define ICL_DSC0_PICTURE_PARAMETER_SET_6(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_6_PB, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_6_PC)
+#define ICL_DSC1_PICTURE_PARAMETER_SET_6(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_6_PB, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_6_PC)
+#define  DSC_FLATNESS_MAX_QP(max_qp)		(qp << 24)
+#define  DSC_FLATNESS_MIN_QP(min_qp)		(qp << 16)
+#define  DSC_FIRST_LINE_BPG_OFFSET(offset)	((offset) << 8)
+#define  DSC_INITIAL_SCALE_VALUE(value)		((value) << 0)
+
+#define DSCA_PICTURE_PARAMETER_SET_7		0x6B21C
+#define DSCC_PICTURE_PARAMETER_SET_7		0x6BA1C
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_7_PB	0x7828C
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_7_PB	0x7838C
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_7_PC	0x7848C
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_7_PC	0x7858C
+#define ICL_DSC0_PICTURE_PARAMETER_SET_7(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							    _ICL_DSC0_PICTURE_PARAMETER_SET_7_PB, \
+							    _ICL_DSC0_PICTURE_PARAMETER_SET_7_PC)
+#define ICL_DSC1_PICTURE_PARAMETER_SET_7(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							    _ICL_DSC1_PICTURE_PARAMETER_SET_7_PB, \
+							    _ICL_DSC1_PICTURE_PARAMETER_SET_7_PC)
+#define  DSC_NFL_BPG_OFFSET(bpg_offset)		((bpg_offset) << 16)
+#define  DSC_SLICE_BPG_OFFSET(bpg_offset)	((bpg_offset) << 0)
+
+#define DSCA_PICTURE_PARAMETER_SET_8		0x6B220
+#define DSCC_PICTURE_PARAMETER_SET_8		0x6BA20
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_8_PB	0x78290
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_8_PB	0x78390
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_8_PC	0x78490
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_8_PC	0x78590
+#define ICL_DSC0_PICTURE_PARAMETER_SET_8(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_8_PB, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_8_PC)
+#define ICL_DSC1_PICTURE_PARAMETER_SET_8(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_8_PB, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_8_PC)
+#define  DSC_INITIAL_OFFSET(initial_offset)		((initial_offset) << 16)
+#define  DSC_FINAL_OFFSET(final_offset)			((final_offset) << 0)
+
+#define DSCA_PICTURE_PARAMETER_SET_9		0x6B224
+#define DSCC_PICTURE_PARAMETER_SET_9		0x6BA24
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_9_PB	0x78294
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_9_PB	0x78394
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_9_PC	0x78494
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_9_PC	0x78594
+#define ICL_DSC0_PICTURE_PARAMETER_SET_9(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_9_PB, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_9_PC)
+#define ICL_DSC1_PICTURE_PARAMETER_SET_9(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_9_PB, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_9_PC)
+#define  DSC_RC_EDGE_FACTOR(rc_edge_fact)	((rc_edge_fact) << 16)
+#define  DSC_RC_MODEL_SIZE(rc_model_size)	((rc_model_size) << 0)
+
+#define DSCA_PICTURE_PARAMETER_SET_10		0x6B228
+#define DSCC_PICTURE_PARAMETER_SET_10		0x6BA28
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_10_PB	0x78298
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_10_PB	0x78398
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_10_PC	0x78498
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_10_PC	0x78598
+#define ICL_DSC0_PICTURE_PARAMETER_SET_10(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_10_PB, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_10_PC)
+#define ICL_DSC1_PICTURE_PARAMETER_SET_10(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_10_PB, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_10_PC)
+#define  DSC_RC_TARGET_OFF_LOW(rc_tgt_off_low)		((rc_tgt_off_low) << 20)
+#define  DSC_RC_TARGET_OFF_HIGH(rc_tgt_off_high)	((rc_tgt_off_high) << 16)
+#define  DSC_RC_QUANT_INC_LIMIT1(lim)			((lim) << 8)
+#define  DSC_RC_QUANT_INC_LIMIT0(lim)			((lim) << 0)
+
+#define DSCA_PICTURE_PARAMETER_SET_11		0x6B22C
+#define DSCC_PICTURE_PARAMETER_SET_11		0x6BA2C
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_11_PB	0x7829C
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_11_PB	0x7839C
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_11_PC	0x7849C
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_11_PC	0x7859C
+#define ICL_DSC0_PICTURE_PARAMETER_SET_11(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_11_PB, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_11_PC)
+#define ICL_DSC1_PICTURE_PARAMETER_SET_11(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_11_PB, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_11_PC)
+
+#define DSCA_PICTURE_PARAMETER_SET_12		0x6B260
+#define DSCC_PICTURE_PARAMETER_SET_12		0x6BA60
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_12_PB	0x782A0
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_12_PB	0x783A0
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_12_PC	0x784A0
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_12_PC	0x785A0
+#define ICL_DSC0_PICTURE_PARAMETER_SET_12(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_12_PB, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_12_PC)
+#define ICL_DSC1_PICTURE_PARAMETER_SET_12(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_12_PB, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_12_PC)
+
+#define DSCA_PICTURE_PARAMETER_SET_13		0x6B264
+#define DSCC_PICTURE_PARAMETER_SET_13		0x6BA64
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_13_PB	0x782A4
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_13_PB	0x783A4
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_13_PC	0x784A4
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_13_PC	0x785A4
+#define ICL_DSC0_PICTURE_PARAMETER_SET_13(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_13_PB, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_13_PC)
+#define ICL_DSC1_PICTURE_PARAMETER_SET_13(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_13_PB, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_13_PC)
+
+#define DSCA_PICTURE_PARAMETER_SET_14		0x6B268
+#define DSCC_PICTURE_PARAMETER_SET_14		0x6BA68
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_14_PB	0x782A8
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_14_PB	0x783A8
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_14_PC	0x784A8
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_14_PC	0x785A8
+#define ICL_DSC0_PICTURE_PARAMETER_SET_14(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_14_PB, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_14_PC)
+#define ICL_DSC1_PICTURE_PARAMETER_SET_14(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_14_PB, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_14_PC)
+
+#define DSCA_PICTURE_PARAMETER_SET_15		0x6B26C
+#define DSCC_PICTURE_PARAMETER_SET_15		0x6BA6C
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_15_PB	0x782AC
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_15_PB	0x783AC
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_15_PC	0x784AC
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_15_PC	0x785AC
+#define ICL_DSC0_PICTURE_PARAMETER_SET_15(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_15_PB, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_15_PC)
+#define ICL_DSC1_PICTURE_PARAMETER_SET_15(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_15_PB, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_15_PC)
+
+#define DSCA_PICTURE_PARAMETER_SET_16		0x6B270
+#define DSCC_PICTURE_PARAMETER_SET_16		0x6BA70
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_16_PB	0x782B0
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_16_PB	0x783B0
+#define _ICL_DSC0_PICTURE_PARAMETER_SET_16_PC	0x784B0
+#define _ICL_DSC1_PICTURE_PARAMETER_SET_16_PC	0x785B0
+#define ICL_DSC0_PICTURE_PARAMETER_SET_16(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_16_PB, \
+							   _ICL_DSC0_PICTURE_PARAMETER_SET_16_PC)
+#define ICL_DSC1_PICTURE_PARAMETER_SET_16(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_16_PB, \
+							   _ICL_DSC1_PICTURE_PARAMETER_SET_16_PC)
+#define  DSC_SLICE_PER_LINE(slice_per_line)		((slice_per_line) << 16)
+#define  DSC_SLICE_CHUNK_SIZE(slice_chunk_aize)		(slice_chunk_size << 0)
+
+/* Icelake Rate Control Buffer Threshold Registers */
+#define DSCA_RC_BUF_THRESH_0			_MMIO(0x6B230)
+#define DSCA_RC_BUF_THRESH_0_UDW		_MMIO(0x6B230 + 4)
+#define DSCC_RC_BUF_THRESH_0			_MMIO(0x6BA30)
+#define DSCC_RC_BUF_THRESH_0_UDW		_MMIO(0x6BA30 + 4)
+#define _ICL_DSC0_RC_BUF_THRESH_0_PB		(0x78254)
+#define _ICL_DSC0_RC_BUF_THRESH_0_UDW_PB	(0x78254 + 4)
+#define _ICL_DSC1_RC_BUF_THRESH_0_PB		(0x78354)
+#define _ICL_DSC1_RC_BUF_THRESH_0_UDW_PB	(0x78354 + 4)
+#define _ICL_DSC0_RC_BUF_THRESH_0_PC		(0x78454)
+#define _ICL_DSC0_RC_BUF_THRESH_0_UDW_PC	(0x78454 + 4)
+#define _ICL_DSC1_RC_BUF_THRESH_0_PC		(0x78554)
+#define _ICL_DSC1_RC_BUF_THRESH_0_UDW_PC	(0x78554 + 4)
+#define ICL_DSC0_RC_BUF_THRESH_0(pipe)		_MMIO_PIPE((pipe) - PIPE_B, \
+						_ICL_DSC0_RC_BUF_THRESH_0_PB, \
+						_ICL_DSC0_RC_BUF_THRESH_0_PC)
+#define ICL_DSC0_RC_BUF_THRESH_0_UDW(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+						_ICL_DSC0_RC_BUF_THRESH_0_UDW_PB, \
+						_ICL_DSC0_RC_BUF_THRESH_0_UDW_PC)
+#define ICL_DSC1_RC_BUF_THRESH_0(pipe)		_MMIO_PIPE((pipe) - PIPE_B, \
+						_ICL_DSC1_RC_BUF_THRESH_0_PB, \
+						_ICL_DSC1_RC_BUF_THRESH_0_PC)
+#define ICL_DSC1_RC_BUF_THRESH_0_UDW(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+						_ICL_DSC1_RC_BUF_THRESH_0_UDW_PB, \
+						_ICL_DSC1_RC_BUF_THRESH_0_UDW_PC)
+
+#define DSCA_RC_BUF_THRESH_1			_MMIO(0x6B238)
+#define DSCA_RC_BUF_THRESH_1_UDW		_MMIO(0x6B238 + 4)
+#define DSCC_RC_BUF_THRESH_1			_MMIO(0x6BA38)
+#define DSCC_RC_BUF_THRESH_1_UDW		_MMIO(0x6BA38 + 4)
+#define _ICL_DSC0_RC_BUF_THRESH_1_PB		(0x7825C)
+#define _ICL_DSC0_RC_BUF_THRESH_1_UDW_PB	(0x7825C + 4)
+#define _ICL_DSC1_RC_BUF_THRESH_1_PB		(0x7835C)
+#define _ICL_DSC1_RC_BUF_THRESH_1_UDW_PB	(0x7835C + 4)
+#define _ICL_DSC0_RC_BUF_THRESH_1_PC		(0x7845C)
+#define _ICL_DSC0_RC_BUF_THRESH_1_UDW_PC	(0x7845C + 4)
+#define _ICL_DSC1_RC_BUF_THRESH_1_PC		(0x7855C)
+#define _ICL_DSC1_RC_BUF_THRESH_1_UDW_PC	(0x7855C + 4)
+#define ICL_DSC0_RC_BUF_THRESH_1(pipe)		_MMIO_PIPE((pipe) - PIPE_B, \
+						_ICL_DSC0_RC_BUF_THRESH_1_PB, \
+						_ICL_DSC0_RC_BUF_THRESH_1_PC)
+#define ICL_DSC0_RC_BUF_THRESH_1_UDW(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+						_ICL_DSC0_RC_BUF_THRESH_1_UDW_PB, \
+						_ICL_DSC0_RC_BUF_THRESH_1_UDW_PC)
+#define ICL_DSC1_RC_BUF_THRESH_1(pipe)		_MMIO_PIPE((pipe) - PIPE_B, \
+						_ICL_DSC1_RC_BUF_THRESH_1_PB, \
+						_ICL_DSC1_RC_BUF_THRESH_1_PC)
+#define ICL_DSC1_RC_BUF_THRESH_1_UDW(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
+						_ICL_DSC1_RC_BUF_THRESH_1_UDW_PB, \
+						_ICL_DSC1_RC_BUF_THRESH_1_UDW_PC)
+
 #endif /* _I915_REG_H_ */
diff --git a/drivers/gpu/drm/i915/i915_request.c b/drivers/gpu/drm/i915/i915_request.c
index e1dbb544046f..5c2c93cbab12 100644
--- a/drivers/gpu/drm/i915/i915_request.c
+++ b/drivers/gpu/drm/i915/i915_request.c
@@ -206,7 +206,8 @@ static int reset_all_global_seqno(struct drm_i915_private *i915, u32 seqno)
 	/* Carefully retire all requests without writing to the rings */
 	ret = i915_gem_wait_for_idle(i915,
 				     I915_WAIT_INTERRUPTIBLE |
-				     I915_WAIT_LOCKED);
+				     I915_WAIT_LOCKED,
+				     MAX_SCHEDULE_TIMEOUT);
 	if (ret)
 		return ret;
 
@@ -503,7 +504,7 @@ static void move_to_timeline(struct i915_request *request,
 	GEM_BUG_ON(request->timeline == &request->engine->timeline);
 	lockdep_assert_held(&request->engine->timeline.lock);
 
-	spin_lock_nested(&request->timeline->lock, SINGLE_DEPTH_NESTING);
+	spin_lock(&request->timeline->lock);
 	list_move_tail(&request->link, &timeline->requests);
 	spin_unlock(&request->timeline->lock);
 }
@@ -735,7 +736,8 @@ i915_request_alloc(struct intel_engine_cs *engine, struct i915_gem_context *ctx)
 		/* Ratelimit ourselves to prevent oom from malicious clients */
 		ret = i915_gem_wait_for_idle(i915,
 					     I915_WAIT_LOCKED |
-					     I915_WAIT_INTERRUPTIBLE);
+					     I915_WAIT_INTERRUPTIBLE,
+					     MAX_SCHEDULE_TIMEOUT);
 		if (ret)
 			goto err_unreserve;
 
@@ -1013,6 +1015,27 @@ i915_request_await_object(struct i915_request *to,
 	return ret;
 }
 
+void i915_request_skip(struct i915_request *rq, int error)
+{
+	void *vaddr = rq->ring->vaddr;
+	u32 head;
+
+	GEM_BUG_ON(!IS_ERR_VALUE((long)error));
+	dma_fence_set_error(&rq->fence, error);
+
+	/*
+	 * As this request likely depends on state from the lost
+	 * context, clear out all the user operations leaving the
+	 * breadcrumb at the end (so we get the fence notifications).
+	 */
+	head = rq->infix;
+	if (rq->postfix < head) {
+		memset(vaddr + head, 0, rq->ring->size - head);
+		head = 0;
+	}
+	memset(vaddr + head, 0, rq->postfix - head);
+}
+
 /*
  * NB: This function is not allowed to fail. Doing so would mean the the
  * request is not being tracked for completion but the work itself is
@@ -1196,7 +1219,7 @@ static bool __i915_spin_request(const struct i915_request *rq,
 	 * takes to sleep on a request, on the order of a microsecond.
 	 */
 
-	irq = atomic_read(&engine->irq_count);
+	irq = READ_ONCE(engine->breadcrumbs.irq_count);
 	timeout_us += local_clock_us(&cpu);
 	do {
 		if (i915_seqno_passed(intel_engine_get_seqno(engine), seqno))
@@ -1208,7 +1231,7 @@ static bool __i915_spin_request(const struct i915_request *rq,
 		 * assume we won't see one in the near future but require
 		 * the engine->seqno_barrier() to fixup coherency.
 		 */
-		if (atomic_read(&engine->irq_count) != irq)
+		if (READ_ONCE(engine->breadcrumbs.irq_count) != irq)
 			break;
 
 		if (signal_pending_state(state, current))
@@ -1285,7 +1308,7 @@ long i915_request_wait(struct i915_request *rq,
 	if (flags & I915_WAIT_LOCKED)
 		add_wait_queue(errq, &reset);
 
-	intel_wait_init(&wait, rq);
+	intel_wait_init(&wait);
 
 restart:
 	do {
diff --git a/drivers/gpu/drm/i915/i915_request.h b/drivers/gpu/drm/i915/i915_request.h
index 7ee220ded9c9..e1c9365dfefb 100644
--- a/drivers/gpu/drm/i915/i915_request.h
+++ b/drivers/gpu/drm/i915/i915_request.h
@@ -258,6 +258,8 @@ void i915_request_add(struct i915_request *rq);
 void __i915_request_submit(struct i915_request *request);
 void i915_request_submit(struct i915_request *request);
 
+void i915_request_skip(struct i915_request *request, int error);
+
 void __i915_request_unsubmit(struct i915_request *request);
 void i915_request_unsubmit(struct i915_request *request);
 
@@ -378,6 +380,7 @@ static inline void
 init_request_active(struct i915_gem_active *active,
 		    i915_gem_retire_fn retire)
 {
+	RCU_INIT_POINTER(active->request, NULL);
 	INIT_LIST_HEAD(&active->link);
 	active->retire = retire ?: i915_gem_retire_noop;
 }
diff --git a/drivers/gpu/drm/i915/i915_selftest.h b/drivers/gpu/drm/i915/i915_selftest.h
index 9766e806dce6..a73472dd12fd 100644
--- a/drivers/gpu/drm/i915/i915_selftest.h
+++ b/drivers/gpu/drm/i915/i915_selftest.h
@@ -99,6 +99,6 @@ __printf(2, 3)
 bool __igt_timeout(unsigned long timeout, const char *fmt, ...);
 
 #define igt_timeout(t, fmt, ...) \
-	__igt_timeout((t), KERN_NOTICE pr_fmt(fmt), ##__VA_ARGS__)
+	__igt_timeout((t), KERN_DEBUG pr_fmt(fmt), ##__VA_ARGS__)
 
 #endif /* !__I915_SELFTEST_H__ */
diff --git a/drivers/gpu/drm/i915/i915_timeline.h b/drivers/gpu/drm/i915/i915_timeline.h
index dc2a4632faa7..a2c2c3ab5fb0 100644
--- a/drivers/gpu/drm/i915/i915_timeline.h
+++ b/drivers/gpu/drm/i915/i915_timeline.h
@@ -37,6 +37,8 @@ struct i915_timeline {
 	u32 seqno;
 
 	spinlock_t lock;
+#define TIMELINE_CLIENT 0 /* default subclass */
+#define TIMELINE_ENGINE 1
 
 	/**
 	 * List of breadcrumbs associated with GPU requests currently
diff --git a/drivers/gpu/drm/i915/i915_utils.h b/drivers/gpu/drm/i915/i915_utils.h
index 00165ad55fb3..395dd2511568 100644
--- a/drivers/gpu/drm/i915/i915_utils.h
+++ b/drivers/gpu/drm/i915/i915_utils.h
@@ -43,7 +43,7 @@
 #define MISSING_CASE(x) WARN(1, "Missing case (%s == %ld)\n", \
 			     __stringify(x), (long)(x))
 
-#if GCC_VERSION >= 70000
+#if defined(GCC_VERSION) && GCC_VERSION >= 70000
 #define add_overflows(A, B) \
 	__builtin_add_overflow_p((A), (B), (typeof((A) + (B)))0)
 #else
diff --git a/drivers/gpu/drm/i915/i915_vma.c b/drivers/gpu/drm/i915/i915_vma.c
index e82aa804cdba..11d834f94220 100644
--- a/drivers/gpu/drm/i915/i915_vma.c
+++ b/drivers/gpu/drm/i915/i915_vma.c
@@ -21,7 +21,7 @@
  * IN THE SOFTWARE.
  *
  */
- 
+
 #include "i915_vma.h"
 
 #include "i915_drv.h"
@@ -30,18 +30,53 @@
 
 #include <drm/drm_gem.h>
 
+#if IS_ENABLED(CONFIG_DRM_I915_ERRLOG_GEM) && IS_ENABLED(CONFIG_DRM_DEBUG_MM)
+
+#include <linux/stackdepot.h>
+
+static void vma_print_allocator(struct i915_vma *vma, const char *reason)
+{
+	unsigned long entries[12];
+	struct stack_trace trace = {
+		.entries = entries,
+		.max_entries = ARRAY_SIZE(entries),
+	};
+	char buf[512];
+
+	if (!vma->node.stack) {
+		DRM_DEBUG_DRIVER("vma.node [%08llx + %08llx] %s: unknown owner\n",
+				 vma->node.start, vma->node.size, reason);
+		return;
+	}
+
+	depot_fetch_stack(vma->node.stack, &trace);
+	snprint_stack_trace(buf, sizeof(buf), &trace, 0);
+	DRM_DEBUG_DRIVER("vma.node [%08llx + %08llx] %s: inserted at %s\n",
+			 vma->node.start, vma->node.size, reason, buf);
+}
+
+#else
+
+static void vma_print_allocator(struct i915_vma *vma, const char *reason)
+{
+}
+
+#endif
+
+struct i915_vma_active {
+	struct i915_gem_active base;
+	struct i915_vma *vma;
+	struct rb_node node;
+	u64 timeline;
+};
+
 static void
-i915_vma_retire(struct i915_gem_active *active, struct i915_request *rq)
+__i915_vma_retire(struct i915_vma *vma, struct i915_request *rq)
 {
-	const unsigned int idx = rq->engine->id;
-	struct i915_vma *vma =
-		container_of(active, struct i915_vma, last_read[idx]);
 	struct drm_i915_gem_object *obj = vma->obj;
 
-	GEM_BUG_ON(!i915_vma_has_active_engine(vma, idx));
-
-	i915_vma_clear_active(vma, idx);
-	if (i915_vma_is_active(vma))
+	GEM_BUG_ON(!i915_vma_is_active(vma));
+	if (--vma->active_count)
 		return;
 
 	GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
@@ -75,6 +110,21 @@ i915_vma_retire(struct i915_gem_active *active, struct i915_request *rq)
 	}
 }
 
+static void
+i915_vma_retire(struct i915_gem_active *base, struct i915_request *rq)
+{
+	struct i915_vma_active *active =
+		container_of(base, typeof(*active), base);
+
+	__i915_vma_retire(active->vma, rq);
+}
+
+static void
+i915_vma_last_retire(struct i915_gem_active *base, struct i915_request *rq)
+{
+	__i915_vma_retire(container_of(base, struct i915_vma, last_active), rq);
+}
+
 static struct i915_vma *
 vma_create(struct drm_i915_gem_object *obj,
 	   struct i915_address_space *vm,
@@ -82,7 +132,6 @@ vma_create(struct drm_i915_gem_object *obj,
 {
 	struct i915_vma *vma;
 	struct rb_node *rb, **p;
-	int i;
 
 	/* The aliasing_ppgtt should never be used directly! */
 	GEM_BUG_ON(vm == &vm->i915->mm.aliasing_ppgtt->vm);
@@ -91,8 +140,9 @@ vma_create(struct drm_i915_gem_object *obj,
 	if (vma == NULL)
 		return ERR_PTR(-ENOMEM);
 
-	for (i = 0; i < ARRAY_SIZE(vma->last_read); i++)
-		init_request_active(&vma->last_read[i], i915_vma_retire);
+	vma->active = RB_ROOT;
+
+	init_request_active(&vma->last_active, i915_vma_last_retire);
 	init_request_active(&vma->last_fence, NULL);
 	vma->vm = vm;
 	vma->ops = &vm->vma_ops;
@@ -110,7 +160,7 @@ vma_create(struct drm_i915_gem_object *obj,
 						     obj->base.size >> PAGE_SHIFT));
 			vma->size = view->partial.size;
 			vma->size <<= PAGE_SHIFT;
-			GEM_BUG_ON(vma->size >= obj->base.size);
+			GEM_BUG_ON(vma->size > obj->base.size);
 		} else if (view->type == I915_GGTT_VIEW_ROTATED) {
 			vma->size = intel_rotation_info_size(&view->rotated);
 			vma->size <<= PAGE_SHIFT;
@@ -745,13 +795,11 @@ void i915_vma_reopen(struct i915_vma *vma)
 static void __i915_vma_destroy(struct i915_vma *vma)
 {
 	struct drm_i915_private *i915 = vma->vm->i915;
-	int i;
+	struct i915_vma_active *iter, *n;
 
 	GEM_BUG_ON(vma->node.allocated);
 	GEM_BUG_ON(vma->fence);
 
-	for (i = 0; i < ARRAY_SIZE(vma->last_read); i++)
-		GEM_BUG_ON(i915_gem_active_isset(&vma->last_read[i]));
 	GEM_BUG_ON(i915_gem_active_isset(&vma->last_fence));
 
 	list_del(&vma->obj_link);
@@ -762,6 +810,11 @@ static void __i915_vma_destroy(struct i915_vma *vma)
 	if (!i915_vma_is_ggtt(vma))
 		i915_ppgtt_put(i915_vm_to_ppgtt(vma->vm));
 
+	rbtree_postorder_for_each_entry_safe(iter, n, &vma->active, node) {
+		GEM_BUG_ON(i915_gem_active_isset(&iter->base));
+		kfree(iter);
+	}
+
 	kmem_cache_free(i915->vmas, vma);
 }
 
@@ -826,9 +879,159 @@ void i915_vma_revoke_mmap(struct i915_vma *vma)
 		list_del(&vma->obj->userfault_link);
 }
 
+static void export_fence(struct i915_vma *vma,
+			 struct i915_request *rq,
+			 unsigned int flags)
+{
+	struct reservation_object *resv = vma->resv;
+
+	/*
+	 * Ignore errors from failing to allocate the new fence, we can't
+	 * handle an error right now. Worst case should be missed
+	 * synchronisation leading to rendering corruption.
+	 */
+	reservation_object_lock(resv, NULL);
+	if (flags & EXEC_OBJECT_WRITE)
+		reservation_object_add_excl_fence(resv, &rq->fence);
+	else if (reservation_object_reserve_shared(resv) == 0)
+		reservation_object_add_shared_fence(resv, &rq->fence);
+	reservation_object_unlock(resv);
+}
+
+static struct i915_gem_active *active_instance(struct i915_vma *vma, u64 idx)
+{
+	struct i915_vma_active *active;
+	struct rb_node **p, *parent;
+	struct i915_request *old;
+
+	/*
+	 * We track the most recently used timeline to skip a rbtree search
+	 * for the common case, under typical loads we never need the rbtree
+	 * at all. We can reuse the last_active slot if it is empty, that is
+	 * after the previous activity has been retired, or if the active
+	 * matches the current timeline.
+	 *
+	 * Note that we allow the timeline to be active simultaneously in
+	 * the rbtree and the last_active cache. We do this to avoid having
+	 * to search and replace the rbtree element for a new timeline, with
+	 * the cost being that we must be aware that the vma may be retired
+	 * twice for the same timeline (as the older rbtree element will be
+	 * retired before the new request added to last_active).
+	 */
+	old = i915_gem_active_raw(&vma->last_active,
+				  &vma->vm->i915->drm.struct_mutex);
+	if (!old || old->fence.context == idx)
+		goto out;
+
+	/* Move the currently active fence into the rbtree */
+	idx = old->fence.context;
+
+	parent = NULL;
+	p = &vma->active.rb_node;
+	while (*p) {
+		parent = *p;
+
+		active = rb_entry(parent, struct i915_vma_active, node);
+		if (active->timeline == idx)
+			goto replace;
+
+		if (active->timeline < idx)
+			p = &parent->rb_right;
+		else
+			p = &parent->rb_left;
+	}
+
+	active = kmalloc(sizeof(*active), GFP_KERNEL);
+
+	/* kmalloc may retire the vma->last_active request (thanks shrinker)! */
+	if (unlikely(!i915_gem_active_raw(&vma->last_active,
+					  &vma->vm->i915->drm.struct_mutex))) {
+		kfree(active);
+		goto out;
+	}
+
+	if (unlikely(!active))
+		return ERR_PTR(-ENOMEM);
+
+	init_request_active(&active->base, i915_vma_retire);
+	active->vma = vma;
+	active->timeline = idx;
+
+	rb_link_node(&active->node, parent, p);
+	rb_insert_color(&active->node, &vma->active);
+
+replace:
+	/*
+	 * Overwrite the previous active slot in the rbtree with last_active,
+	 * leaving last_active zeroed. If the previous slot is still active,
+	 * we must be careful as we now only expect to receive one retire
+	 * callback not two, and so much undo the active counting for the
+	 * overwritten slot.
+	 */
+	if (i915_gem_active_isset(&active->base)) {
+		/* Retire ourselves from the old rq->active_list */
+		__list_del_entry(&active->base.link);
+		vma->active_count--;
+		GEM_BUG_ON(!vma->active_count);
+	}
+	GEM_BUG_ON(list_empty(&vma->last_active.link));
+	list_replace_init(&vma->last_active.link, &active->base.link);
+	active->base.request = fetch_and_zero(&vma->last_active.request);
+
+out:
+	return &vma->last_active;
+}
+
+int i915_vma_move_to_active(struct i915_vma *vma,
+			    struct i915_request *rq,
+			    unsigned int flags)
+{
+	struct drm_i915_gem_object *obj = vma->obj;
+	struct i915_gem_active *active;
+
+	lockdep_assert_held(&rq->i915->drm.struct_mutex);
+	GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
+
+	active = active_instance(vma, rq->fence.context);
+	if (IS_ERR(active))
+		return PTR_ERR(active);
+
+	/*
+	 * Add a reference if we're newly entering the active list.
+	 * The order in which we add operations to the retirement queue is
+	 * vital here: mark_active adds to the start of the callback list,
+	 * such that subsequent callbacks are called first. Therefore we
+	 * add the active reference first and queue for it to be dropped
+	 * *last*.
+	 */
+	if (!i915_gem_active_isset(active) && !vma->active_count++) {
+		list_move_tail(&vma->vm_link, &vma->vm->active_list);
+		obj->active_count++;
+	}
+	i915_gem_active_set(active, rq);
+	GEM_BUG_ON(!i915_vma_is_active(vma));
+	GEM_BUG_ON(!obj->active_count);
+
+	obj->write_domain = 0;
+	if (flags & EXEC_OBJECT_WRITE) {
+		obj->write_domain = I915_GEM_DOMAIN_RENDER;
+
+		if (intel_fb_obj_invalidate(obj, ORIGIN_CS))
+			i915_gem_active_set(&obj->frontbuffer_write, rq);
+
+		obj->read_domains = 0;
+	}
+	obj->read_domains |= I915_GEM_GPU_DOMAINS;
+
+	if (flags & EXEC_OBJECT_NEEDS_FENCE)
+		i915_gem_active_set(&vma->last_fence, rq);
+
+	export_fence(vma, rq, flags);
+	return 0;
+}
+
 int i915_vma_unbind(struct i915_vma *vma)
 {
-	unsigned long active;
 	int ret;
 
 	lockdep_assert_held(&vma->vm->i915->drm.struct_mutex);
@@ -838,9 +1041,8 @@ int i915_vma_unbind(struct i915_vma *vma)
 	 * have side-effects such as unpinning or even unbinding this vma.
 	 */
 	might_sleep();
-	active = i915_vma_get_active(vma);
-	if (active) {
-		int idx;
+	if (i915_vma_is_active(vma)) {
+		struct i915_vma_active *active, *n;
 
 		/*
 		 * When a closed VMA is retired, it is unbound - eek.
@@ -857,26 +1059,32 @@ int i915_vma_unbind(struct i915_vma *vma)
 		 */
 		__i915_vma_pin(vma);
 
-		for_each_active(active, idx) {
-			ret = i915_gem_active_retire(&vma->last_read[idx],
-						     &vma->vm->i915->drm.struct_mutex);
-			if (ret)
-				break;
-		}
+		ret = i915_gem_active_retire(&vma->last_active,
+					     &vma->vm->i915->drm.struct_mutex);
+		if (ret)
+			goto unpin;
 
-		if (!ret) {
-			ret = i915_gem_active_retire(&vma->last_fence,
+		rbtree_postorder_for_each_entry_safe(active, n,
+						     &vma->active, node) {
+			ret = i915_gem_active_retire(&active->base,
 						     &vma->vm->i915->drm.struct_mutex);
+			if (ret)
+				goto unpin;
 		}
 
+		ret = i915_gem_active_retire(&vma->last_fence,
+					     &vma->vm->i915->drm.struct_mutex);
+unpin:
 		__i915_vma_unpin(vma);
 		if (ret)
 			return ret;
 	}
 	GEM_BUG_ON(i915_vma_is_active(vma));
 
-	if (i915_vma_is_pinned(vma))
+	if (i915_vma_is_pinned(vma)) {
+		vma_print_allocator(vma, "is pinned");
 		return -EBUSY;
+	}
 
 	if (!drm_mm_node_allocated(&vma->node))
 		return 0;
diff --git a/drivers/gpu/drm/i915/i915_vma.h b/drivers/gpu/drm/i915/i915_vma.h
index 66a228931517..f06d66377107 100644
--- a/drivers/gpu/drm/i915/i915_vma.h
+++ b/drivers/gpu/drm/i915/i915_vma.h
@@ -26,6 +26,7 @@
 #define __I915_VMA_H__
 
 #include <linux/io-mapping.h>
+#include <linux/rbtree.h>
 
 #include <drm/drm_mm.h>
 
@@ -94,8 +95,9 @@ struct i915_vma {
 #define I915_VMA_USERFAULT	BIT(I915_VMA_USERFAULT_BIT)
 #define I915_VMA_GGTT_WRITE	BIT(12)
 
-	unsigned int active;
-	struct i915_gem_active last_read[I915_NUM_ENGINES];
+	unsigned int active_count;
+	struct rb_root active;
+	struct i915_gem_active last_active;
 	struct i915_gem_active last_fence;
 
 	/**
@@ -138,6 +140,15 @@ i915_vma_instance(struct drm_i915_gem_object *obj,
 
 void i915_vma_unpin_and_release(struct i915_vma **p_vma);
 
+static inline bool i915_vma_is_active(struct i915_vma *vma)
+{
+	return vma->active_count;
+}
+
+int __must_check i915_vma_move_to_active(struct i915_vma *vma,
+					 struct i915_request *rq,
+					 unsigned int flags);
+
 static inline bool i915_vma_is_ggtt(const struct i915_vma *vma)
 {
 	return vma->flags & I915_VMA_GGTT;
@@ -187,34 +198,6 @@ static inline bool i915_vma_has_userfault(const struct i915_vma *vma)
 	return test_bit(I915_VMA_USERFAULT_BIT, &vma->flags);
 }
 
-static inline unsigned int i915_vma_get_active(const struct i915_vma *vma)
-{
-	return vma->active;
-}
-
-static inline bool i915_vma_is_active(const struct i915_vma *vma)
-{
-	return i915_vma_get_active(vma);
-}
-
-static inline void i915_vma_set_active(struct i915_vma *vma,
-				       unsigned int engine)
-{
-	vma->active |= BIT(engine);
-}
-
-static inline void i915_vma_clear_active(struct i915_vma *vma,
-					 unsigned int engine)
-{
-	vma->active &= ~BIT(engine);
-}
-
-static inline bool i915_vma_has_active_engine(const struct i915_vma *vma,
-					      unsigned int engine)
-{
-	return vma->active & BIT(engine);
-}
-
 static inline u32 i915_ggtt_offset(const struct i915_vma *vma)
 {
 	GEM_BUG_ON(!i915_vma_is_ggtt(vma));
diff --git a/drivers/gpu/drm/i915/icl_dsi.c b/drivers/gpu/drm/i915/icl_dsi.c
new file mode 100644
index 000000000000..13830e43a4d1
--- /dev/null
+++ b/drivers/gpu/drm/i915/icl_dsi.c
@@ -0,0 +1,127 @@
+/*
+ * Copyright © 2018 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ *
+ * Authors:
+ *   Madhav Chauhan <madhav.chauhan@intel.com>
+ *   Jani Nikula <jani.nikula@intel.com>
+ */
+
+#include "intel_dsi.h"
+
+static void gen11_dsi_program_esc_clk_div(struct intel_encoder *encoder)
+{
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
+	enum port port;
+	u32 bpp = mipi_dsi_pixel_format_to_bpp(intel_dsi->pixel_format);
+	u32 afe_clk_khz; /* 8X Clock */
+	u32 esc_clk_div_m;
+
+	afe_clk_khz = DIV_ROUND_CLOSEST(intel_dsi->pclk * bpp,
+					intel_dsi->lane_count);
+
+	esc_clk_div_m = DIV_ROUND_UP(afe_clk_khz, DSI_MAX_ESC_CLK);
+
+	for_each_dsi_port(port, intel_dsi->ports) {
+		I915_WRITE(ICL_DSI_ESC_CLK_DIV(port),
+			   esc_clk_div_m & ICL_ESC_CLK_DIV_MASK);
+		POSTING_READ(ICL_DSI_ESC_CLK_DIV(port));
+	}
+
+	for_each_dsi_port(port, intel_dsi->ports) {
+		I915_WRITE(ICL_DPHY_ESC_CLK_DIV(port),
+			   esc_clk_div_m & ICL_ESC_CLK_DIV_MASK);
+		POSTING_READ(ICL_DPHY_ESC_CLK_DIV(port));
+	}
+}
+
+static void gen11_dsi_enable_io_power(struct intel_encoder *encoder)
+{
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
+	enum port port;
+	u32 tmp;
+
+	for_each_dsi_port(port, intel_dsi->ports) {
+		tmp = I915_READ(ICL_DSI_IO_MODECTL(port));
+		tmp |= COMBO_PHY_MODE_DSI;
+		I915_WRITE(ICL_DSI_IO_MODECTL(port), tmp);
+	}
+
+	for_each_dsi_port(port, intel_dsi->ports) {
+		intel_display_power_get(dev_priv, port == PORT_A ?
+					POWER_DOMAIN_PORT_DDI_A_IO :
+					POWER_DOMAIN_PORT_DDI_B_IO);
+	}
+}
+
+static void gen11_dsi_power_up_lanes(struct intel_encoder *encoder)
+{
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
+	enum port port;
+	u32 tmp;
+	u32 lane_mask;
+
+	switch (intel_dsi->lane_count) {
+	case 1:
+		lane_mask = PWR_DOWN_LN_3_1_0;
+		break;
+	case 2:
+		lane_mask = PWR_DOWN_LN_3_1;
+		break;
+	case 3:
+		lane_mask = PWR_DOWN_LN_3;
+		break;
+	case 4:
+	default:
+		lane_mask = PWR_UP_ALL_LANES;
+		break;
+	}
+
+	for_each_dsi_port(port, intel_dsi->ports) {
+		tmp = I915_READ(ICL_PORT_CL_DW10(port));
+		tmp &= ~PWR_DOWN_LN_MASK;
+		I915_WRITE(ICL_PORT_CL_DW10(port), tmp | lane_mask);
+	}
+}
+
+static void gen11_dsi_enable_port_and_phy(struct intel_encoder *encoder)
+{
+	/* step 4a: power up all lanes of the DDI used by DSI */
+	gen11_dsi_power_up_lanes(encoder);
+}
+
+static void __attribute__((unused))
+gen11_dsi_pre_enable(struct intel_encoder *encoder,
+		     const struct intel_crtc_state *pipe_config,
+		     const struct drm_connector_state *conn_state)
+{
+	/* step2: enable IO power */
+	gen11_dsi_enable_io_power(encoder);
+
+	/* step3: enable DSI PLL */
+	gen11_dsi_program_esc_clk_div(encoder);
+
+	/* step4: enable DSI port and DPHY */
+	gen11_dsi_enable_port_and_phy(encoder);
+}
diff --git a/drivers/gpu/drm/i915/intel_audio.c b/drivers/gpu/drm/i915/intel_audio.c
index bb94172ffc07..b725835b47ef 100644
--- a/drivers/gpu/drm/i915/intel_audio.c
+++ b/drivers/gpu/drm/i915/intel_audio.c
@@ -649,11 +649,12 @@ void intel_audio_codec_enable(struct intel_encoder *encoder,
 	dev_priv->av_enc_map[pipe] = encoder;
 	mutex_unlock(&dev_priv->av_mutex);
 
-	if (acomp && acomp->audio_ops && acomp->audio_ops->pin_eld_notify) {
+	if (acomp && acomp->base.audio_ops &&
+	    acomp->base.audio_ops->pin_eld_notify) {
 		/* audio drivers expect pipe = -1 to indicate Non-MST cases */
 		if (!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DP_MST))
 			pipe = -1;
-		acomp->audio_ops->pin_eld_notify(acomp->audio_ops->audio_ptr,
+		acomp->base.audio_ops->pin_eld_notify(acomp->base.audio_ops->audio_ptr,
 						 (int) port, (int) pipe);
 	}
 
@@ -691,11 +692,12 @@ void intel_audio_codec_disable(struct intel_encoder *encoder,
 	dev_priv->av_enc_map[pipe] = NULL;
 	mutex_unlock(&dev_priv->av_mutex);
 
-	if (acomp && acomp->audio_ops && acomp->audio_ops->pin_eld_notify) {
+	if (acomp && acomp->base.audio_ops &&
+	    acomp->base.audio_ops->pin_eld_notify) {
 		/* audio drivers expect pipe = -1 to indicate Non-MST cases */
 		if (!intel_crtc_has_type(old_crtc_state, INTEL_OUTPUT_DP_MST))
 			pipe = -1;
-		acomp->audio_ops->pin_eld_notify(acomp->audio_ops->audio_ptr,
+		acomp->base.audio_ops->pin_eld_notify(acomp->base.audio_ops->audio_ptr,
 						 (int) port, (int) pipe);
 	}
 
@@ -890,7 +892,7 @@ static int i915_audio_component_get_eld(struct device *kdev, int port,
 	return ret;
 }
 
-static const struct i915_audio_component_ops i915_audio_component_ops = {
+static const struct drm_audio_component_ops i915_audio_component_ops = {
 	.owner		= THIS_MODULE,
 	.get_power	= i915_audio_component_get_power,
 	.put_power	= i915_audio_component_put_power,
@@ -907,12 +909,12 @@ static int i915_audio_component_bind(struct device *i915_kdev,
 	struct drm_i915_private *dev_priv = kdev_to_i915(i915_kdev);
 	int i;
 
-	if (WARN_ON(acomp->ops || acomp->dev))
+	if (WARN_ON(acomp->base.ops || acomp->base.dev))
 		return -EEXIST;
 
 	drm_modeset_lock_all(&dev_priv->drm);
-	acomp->ops = &i915_audio_component_ops;
-	acomp->dev = i915_kdev;
+	acomp->base.ops = &i915_audio_component_ops;
+	acomp->base.dev = i915_kdev;
 	BUILD_BUG_ON(MAX_PORTS != I915_MAX_PORTS);
 	for (i = 0; i < ARRAY_SIZE(acomp->aud_sample_rate); i++)
 		acomp->aud_sample_rate[i] = 0;
@@ -929,8 +931,8 @@ static void i915_audio_component_unbind(struct device *i915_kdev,
 	struct drm_i915_private *dev_priv = kdev_to_i915(i915_kdev);
 
 	drm_modeset_lock_all(&dev_priv->drm);
-	acomp->ops = NULL;
-	acomp->dev = NULL;
+	acomp->base.ops = NULL;
+	acomp->base.dev = NULL;
 	dev_priv->audio_component = NULL;
 	drm_modeset_unlock_all(&dev_priv->drm);
 }
diff --git a/drivers/gpu/drm/i915/intel_breadcrumbs.c b/drivers/gpu/drm/i915/intel_breadcrumbs.c
index 86a987b8ac66..1db6ba7d926e 100644
--- a/drivers/gpu/drm/i915/intel_breadcrumbs.c
+++ b/drivers/gpu/drm/i915/intel_breadcrumbs.c
@@ -98,12 +98,14 @@ static void intel_breadcrumbs_hangcheck(struct timer_list *t)
 	struct intel_engine_cs *engine =
 		from_timer(engine, t, breadcrumbs.hangcheck);
 	struct intel_breadcrumbs *b = &engine->breadcrumbs;
+	unsigned int irq_count;
 
 	if (!b->irq_armed)
 		return;
 
-	if (b->hangcheck_interrupts != atomic_read(&engine->irq_count)) {
-		b->hangcheck_interrupts = atomic_read(&engine->irq_count);
+	irq_count = READ_ONCE(b->irq_count);
+	if (b->hangcheck_interrupts != irq_count) {
+		b->hangcheck_interrupts = irq_count;
 		mod_timer(&b->hangcheck, wait_timeout());
 		return;
 	}
@@ -272,13 +274,14 @@ static bool use_fake_irq(const struct intel_breadcrumbs *b)
 	if (!test_bit(engine->id, &engine->i915->gpu_error.missed_irq_rings))
 		return false;
 
-	/* Only start with the heavy weight fake irq timer if we have not
+	/*
+	 * Only start with the heavy weight fake irq timer if we have not
 	 * seen any interrupts since enabling it the first time. If the
 	 * interrupts are still arriving, it means we made a mistake in our
 	 * engine->seqno_barrier(), a timing error that should be transient
 	 * and unlikely to reoccur.
 	 */
-	return atomic_read(&engine->irq_count) == b->hangcheck_interrupts;
+	return READ_ONCE(b->irq_count) == b->hangcheck_interrupts;
 }
 
 static void enable_fake_irq(struct intel_breadcrumbs *b)
diff --git a/drivers/gpu/drm/i915/intel_cdclk.c b/drivers/gpu/drm/i915/intel_cdclk.c
index bf9433d7964d..29075c763428 100644
--- a/drivers/gpu/drm/i915/intel_cdclk.c
+++ b/drivers/gpu/drm/i915/intel_cdclk.c
@@ -316,6 +316,7 @@ static void pnv_get_cdclk(struct drm_i915_private *dev_priv,
 		break;
 	default:
 		DRM_ERROR("Unknown pnv display core clock 0x%04x\n", gcfgc);
+		/* fall through */
 	case GC_DISPLAY_CLOCK_133_MHZ_PNV:
 		cdclk_state->cdclk = 133333;
 		break;
@@ -1797,6 +1798,7 @@ static int icl_calc_cdclk(int min_cdclk, unsigned int ref)
 	switch (ref) {
 	default:
 		MISSING_CASE(ref);
+		/* fall through */
 	case 24000:
 		ranges = ranges_24;
 		break;
@@ -1824,6 +1826,7 @@ static int icl_calc_cdclk_pll_vco(struct drm_i915_private *dev_priv, int cdclk)
 	switch (cdclk) {
 	default:
 		MISSING_CASE(cdclk);
+		/* fall through */
 	case 307200:
 	case 556800:
 	case 652800:
@@ -1896,6 +1899,7 @@ static u8 icl_calc_voltage_level(int cdclk)
 		return 1;
 	default:
 		MISSING_CASE(cdclk);
+		/* fall through */
 	case 652800:
 	case 648000:
 		return 2;
@@ -1913,6 +1917,7 @@ static void icl_get_cdclk(struct drm_i915_private *dev_priv,
 	switch (val & ICL_DSSM_CDCLK_PLL_REFCLK_MASK) {
 	default:
 		MISSING_CASE(val);
+		/* fall through */
 	case ICL_DSSM_CDCLK_PLL_REFCLK_24MHz:
 		cdclk_state->ref = 24000;
 		break;
diff --git a/drivers/gpu/drm/i915/intel_ddi.c b/drivers/gpu/drm/i915/intel_ddi.c
index 044fe1fb9872..8761513f3532 100644
--- a/drivers/gpu/drm/i915/intel_ddi.c
+++ b/drivers/gpu/drm/i915/intel_ddi.c
@@ -1069,6 +1069,7 @@ static uint32_t icl_pll_to_ddi_pll_sel(struct intel_encoder *encoder,
 	switch (id) {
 	default:
 		MISSING_CASE(id);
+		/* fall through */
 	case DPLL_ID_ICL_DPLL0:
 	case DPLL_ID_ICL_DPLL1:
 		return DDI_CLK_SEL_NONE;
@@ -1684,6 +1685,10 @@ void intel_ddi_set_pipe_settings(const struct intel_crtc_state *crtc_state)
 	WARN_ON(transcoder_is_dsi(cpu_transcoder));
 
 	temp = TRANS_MSA_SYNC_CLK;
+
+	if (crtc_state->limited_color_range)
+		temp |= TRANS_MSA_CEA_RANGE;
+
 	switch (crtc_state->pipe_bpp) {
 	case 18:
 		temp |= TRANS_MSA_6_BPC;
@@ -1807,15 +1812,24 @@ void intel_ddi_enable_transcoder_func(const struct intel_crtc_state *crtc_state)
 	I915_WRITE(TRANS_DDI_FUNC_CTL(cpu_transcoder), temp);
 }
 
-void intel_ddi_disable_transcoder_func(struct drm_i915_private *dev_priv,
-				       enum transcoder cpu_transcoder)
+void intel_ddi_disable_transcoder_func(const struct intel_crtc_state *crtc_state)
 {
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
+	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
+	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
 	i915_reg_t reg = TRANS_DDI_FUNC_CTL(cpu_transcoder);
 	uint32_t val = I915_READ(reg);
 
 	val &= ~(TRANS_DDI_FUNC_ENABLE | TRANS_DDI_PORT_MASK | TRANS_DDI_DP_VC_PAYLOAD_ALLOC);
 	val |= TRANS_DDI_PORT_NONE;
 	I915_WRITE(reg, val);
+
+	if (dev_priv->quirks & QUIRK_INCREASE_DDI_DISABLED_TIME &&
+	    intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) {
+		DRM_DEBUG_KMS("Quirk Increase DDI disabled time\n");
+		/* Quirk time at 100ms for reliable operation */
+		msleep(100);
+	}
 }
 
 int intel_ddi_toggle_hdcp_signalling(struct intel_encoder *intel_encoder,
@@ -1983,15 +1997,50 @@ out:
 	return ret;
 }
 
-static u64 intel_ddi_get_power_domains(struct intel_encoder *encoder)
+static inline enum intel_display_power_domain
+intel_ddi_main_link_aux_domain(struct intel_dp *intel_dp)
+{
+	/* CNL HW requires corresponding AUX IOs to be powered up for PSR with
+	 * DC states enabled at the same time, while for driver initiated AUX
+	 * transfers we need the same AUX IOs to be powered but with DC states
+	 * disabled. Accordingly use the AUX power domain here which leaves DC
+	 * states enabled.
+	 * However, for non-A AUX ports the corresponding non-EDP transcoders
+	 * would have already enabled power well 2 and DC_OFF. This means we can
+	 * acquire a wider POWER_DOMAIN_AUX_{B,C,D,F} reference instead of a
+	 * specific AUX_IO reference without powering up any extra wells.
+	 * Note that PSR is enabled only on Port A even though this function
+	 * returns the correct domain for other ports too.
+	 */
+	return intel_dp->aux_ch == AUX_CH_A ? POWER_DOMAIN_AUX_IO_A :
+					      intel_dp->aux_power_domain;
+}
+
+static u64 intel_ddi_get_power_domains(struct intel_encoder *encoder,
+				       struct intel_crtc_state *crtc_state)
 {
-	struct intel_digital_port *dig_port = enc_to_dig_port(&encoder->base);
-	enum pipe pipe;
+	struct intel_digital_port *dig_port;
+	u64 domains;
+
+	/*
+	 * TODO: Add support for MST encoders. Atm, the following should never
+	 * happen since fake-MST encoders don't set their get_power_domains()
+	 * hook.
+	 */
+	if (WARN_ON(intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DP_MST)))
+		return 0;
 
-	if (intel_ddi_get_hw_state(encoder, &pipe))
-		return BIT_ULL(dig_port->ddi_io_power_domain);
+	dig_port = enc_to_dig_port(&encoder->base);
+	domains = BIT_ULL(dig_port->ddi_io_power_domain);
 
-	return 0;
+	/* AUX power is only needed for (e)DP mode, not for HDMI. */
+	if (intel_crtc_has_dp_encoder(crtc_state)) {
+		struct intel_dp *intel_dp = &dig_port->dp;
+
+		domains |= BIT_ULL(intel_ddi_main_link_aux_domain(intel_dp));
+	}
+
+	return domains;
 }
 
 void intel_ddi_enable_pipe_clock(const struct intel_crtc_state *crtc_state)
@@ -2631,6 +2680,9 @@ static void intel_ddi_pre_enable_dp(struct intel_encoder *encoder,
 
 	WARN_ON(is_mst && (port == PORT_A || port == PORT_E));
 
+	intel_display_power_get(dev_priv,
+				intel_ddi_main_link_aux_domain(intel_dp));
+
 	intel_dp_set_link_params(intel_dp, crtc_state->port_clock,
 				 crtc_state->lane_count, is_mst);
 
@@ -2775,6 +2827,9 @@ static void intel_ddi_post_disable_dp(struct intel_encoder *encoder,
 	intel_display_power_put(dev_priv, dig_port->ddi_io_power_domain);
 
 	intel_ddi_clk_disable(encoder);
+
+	intel_display_power_put(dev_priv,
+				intel_ddi_main_link_aux_domain(intel_dp));
 }
 
 static void intel_ddi_post_disable_hdmi(struct intel_encoder *encoder,
@@ -3588,7 +3643,6 @@ void intel_ddi_init(struct drm_i915_private *dev_priv, enum port port)
 			goto err;
 
 		intel_dig_port->hpd_pulse = intel_dp_hpd_pulse;
-		dev_priv->hotplug.irq_port[port] = intel_dig_port;
 	}
 
 	/* In theory we don't need the encoder->type check, but leave it just in
diff --git a/drivers/gpu/drm/i915/intel_device_info.c b/drivers/gpu/drm/i915/intel_device_info.c
index 0fd13df424cf..0ef0c6448d53 100644
--- a/drivers/gpu/drm/i915/intel_device_info.c
+++ b/drivers/gpu/drm/i915/intel_device_info.c
@@ -858,6 +858,8 @@ void intel_device_info_runtime_init(struct intel_device_info *info)
 void intel_driver_caps_print(const struct intel_driver_caps *caps,
 			     struct drm_printer *p)
 {
+	drm_printf(p, "Has logical contexts? %s\n",
+		   yesno(caps->has_logical_contexts));
 	drm_printf(p, "scheduler: %x\n", caps->scheduler);
 }
 
diff --git a/drivers/gpu/drm/i915/intel_device_info.h b/drivers/gpu/drm/i915/intel_device_info.h
index 933e31669557..633f9fbf72ea 100644
--- a/drivers/gpu/drm/i915/intel_device_info.h
+++ b/drivers/gpu/drm/i915/intel_device_info.h
@@ -186,6 +186,7 @@ struct intel_device_info {
 
 struct intel_driver_caps {
 	unsigned int scheduler;
+	bool has_logical_contexts:1;
 };
 
 static inline unsigned int sseu_subslice_total(const struct sseu_dev_info *sseu)
diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c
index bd8956f2544d..9382375d33b2 100644
--- a/drivers/gpu/drm/i915/intel_display.c
+++ b/drivers/gpu/drm/i915/intel_display.c
@@ -5632,6 +5632,7 @@ static void haswell_crtc_enable(struct intel_crtc_state *pipe_config,
 	struct intel_atomic_state *old_intel_state =
 		to_intel_atomic_state(old_state);
 	bool psl_clkgate_wa;
+	u32 pipe_chicken;
 
 	if (WARN_ON(intel_crtc->active))
 		return;
@@ -5691,6 +5692,17 @@ static void haswell_crtc_enable(struct intel_crtc_state *pipe_config,
 	 */
 	intel_color_load_luts(&pipe_config->base);
 
+	/*
+	 * Display WA #1153: enable hardware to bypass the alpha math
+	 * and rounding for per-pixel values 00 and 0xff
+	 */
+	if (INTEL_GEN(dev_priv) >= 11) {
+		pipe_chicken = I915_READ(PIPE_CHICKEN(pipe));
+		if (!(pipe_chicken & PER_PIXEL_ALPHA_BYPASS_EN))
+			I915_WRITE_FW(PIPE_CHICKEN(pipe),
+				      pipe_chicken | PER_PIXEL_ALPHA_BYPASS_EN);
+	}
+
 	intel_ddi_set_pipe_settings(pipe_config);
 	if (!transcoder_is_dsi(cpu_transcoder))
 		intel_ddi_enable_transcoder_func(pipe_config);
@@ -5825,7 +5837,7 @@ static void haswell_crtc_disable(struct intel_crtc_state *old_crtc_state,
 		intel_ddi_set_vc_payload_alloc(old_crtc_state, false);
 
 	if (!transcoder_is_dsi(cpu_transcoder))
-		intel_ddi_disable_transcoder_func(dev_priv, cpu_transcoder);
+		intel_ddi_disable_transcoder_func(old_crtc_state);
 
 	if (INTEL_GEN(dev_priv) >= 9)
 		skylake_scaler_disable(intel_crtc);
@@ -9347,6 +9359,7 @@ static bool hsw_get_transcoder_state(struct intel_crtc *crtc,
 		switch (tmp & TRANS_DDI_EDP_INPUT_MASK) {
 		default:
 			WARN(1, "unknown pipe linked to edp transcoder\n");
+			/* fall through */
 		case TRANS_DDI_EDP_INPUT_A_ONOFF:
 		case TRANS_DDI_EDP_INPUT_A_ON:
 			trans_edp_pipe = PIPE_A;
@@ -9402,7 +9415,7 @@ static bool bxt_get_dsi_transcoder_state(struct intel_crtc *crtc,
 		 * registers/MIPI[BXT]. We can break out here early, since we
 		 * need the same DSI PLL to be enabled for both DSI ports.
 		 */
-		if (!intel_dsi_pll_is_enabled(dev_priv))
+		if (!bxt_dsi_pll_is_enabled(dev_priv))
 			break;
 
 		/* XXX: this works for video mode only */
@@ -10724,7 +10737,7 @@ static void intel_modeset_update_connector_atomic_state(struct drm_device *dev)
 	drm_connector_list_iter_begin(dev, &conn_iter);
 	for_each_intel_connector_iter(connector, &conn_iter) {
 		if (connector->base.state->crtc)
-			drm_connector_unreference(&connector->base);
+			drm_connector_put(&connector->base);
 
 		if (connector->base.encoder) {
 			connector->base.state->best_encoder =
@@ -10732,7 +10745,7 @@ static void intel_modeset_update_connector_atomic_state(struct drm_device *dev)
 			connector->base.state->crtc =
 				connector->base.encoder->crtc;
 
-			drm_connector_reference(&connector->base);
+			drm_connector_get(&connector->base);
 		} else {
 			connector->base.state->best_encoder = NULL;
 			connector->base.state->crtc = NULL;
@@ -11011,6 +11024,7 @@ static bool check_digital_port_conflicts(struct drm_atomic_state *state)
 		case INTEL_OUTPUT_DDI:
 			if (WARN_ON(!HAS_DDI(to_i915(dev))))
 				break;
+			/* else: fall through */
 		case INTEL_OUTPUT_DP:
 		case INTEL_OUTPUT_HDMI:
 		case INTEL_OUTPUT_EDP:
@@ -12542,6 +12556,19 @@ static void intel_atomic_commit_fence_wait(struct intel_atomic_state *intel_stat
 	finish_wait(&dev_priv->gpu_error.wait_queue, &wait_reset);
 }
 
+static void intel_atomic_cleanup_work(struct work_struct *work)
+{
+	struct drm_atomic_state *state =
+		container_of(work, struct drm_atomic_state, commit_work);
+	struct drm_i915_private *i915 = to_i915(state->dev);
+
+	drm_atomic_helper_cleanup_planes(&i915->drm, state);
+	drm_atomic_helper_commit_cleanup_done(state);
+	drm_atomic_state_put(state);
+
+	intel_atomic_helper_free_state(i915);
+}
+
 static void intel_atomic_commit_tail(struct drm_atomic_state *state)
 {
 	struct drm_device *dev = state->dev;
@@ -12702,13 +12729,16 @@ static void intel_atomic_commit_tail(struct drm_atomic_state *state)
 		intel_display_power_put(dev_priv, POWER_DOMAIN_MODESET);
 	}
 
-	drm_atomic_helper_cleanup_planes(dev, state);
-
-	drm_atomic_helper_commit_cleanup_done(state);
-
-	drm_atomic_state_put(state);
-
-	intel_atomic_helper_free_state(dev_priv);
+	/*
+	 * Defer the cleanup of the old state to a separate worker to not
+	 * impede the current task (userspace for blocking modesets) that
+	 * are executed inline. For out-of-line asynchronous modesets/flips,
+	 * deferring to a new worker seems overkill, but we would place a
+	 * schedule point (cond_resched()) here anyway to keep latencies
+	 * down.
+	 */
+	INIT_WORK(&state->commit_work, intel_atomic_cleanup_work);
+	schedule_work(&state->commit_work);
 }
 
 static void intel_atomic_commit_work(struct work_struct *work)
@@ -13076,6 +13106,19 @@ intel_prepare_plane_fb(struct drm_plane *plane,
 		add_rps_boost_after_vblank(new_state->crtc, new_state->fence);
 	}
 
+	/*
+	 * We declare pageflips to be interactive and so merit a small bias
+	 * towards upclocking to deliver the frame on time. By only changing
+	 * the RPS thresholds to sample more regularly and aim for higher
+	 * clocks we can hopefully deliver low power workloads (like kodi)
+	 * that are not quite steady state without resorting to forcing
+	 * maximum clocks following a vblank miss (see do_rps_boost()).
+	 */
+	if (!intel_state->rps_interactive) {
+		intel_rps_mark_interactive(dev_priv, true);
+		intel_state->rps_interactive = true;
+	}
+
 	return 0;
 }
 
@@ -13092,8 +13135,15 @@ void
 intel_cleanup_plane_fb(struct drm_plane *plane,
 		       struct drm_plane_state *old_state)
 {
+	struct intel_atomic_state *intel_state =
+		to_intel_atomic_state(old_state->state);
 	struct drm_i915_private *dev_priv = to_i915(plane->dev);
 
+	if (intel_state->rps_interactive) {
+		intel_rps_mark_interactive(dev_priv, false);
+		intel_state->rps_interactive = false;
+	}
+
 	/* Should only be called after a successful intel_prepare_plane_fb()! */
 	mutex_lock(&dev_priv->drm.struct_mutex);
 	intel_plane_unpin_fb(to_intel_plane_state(old_state));
@@ -14107,7 +14157,7 @@ static void intel_setup_outputs(struct drm_i915_private *dev_priv)
 		intel_ddi_init(dev_priv, PORT_B);
 		intel_ddi_init(dev_priv, PORT_C);
 
-		intel_dsi_init(dev_priv);
+		vlv_dsi_init(dev_priv);
 	} else if (HAS_DDI(dev_priv)) {
 		int found;
 
@@ -14213,7 +14263,7 @@ static void intel_setup_outputs(struct drm_i915_private *dev_priv)
 				intel_hdmi_init(dev_priv, CHV_HDMID, PORT_D);
 		}
 
-		intel_dsi_init(dev_priv);
+		vlv_dsi_init(dev_priv);
 	} else if (!IS_GEN2(dev_priv) && !IS_PINEVIEW(dev_priv)) {
 		bool found = false;
 
@@ -14495,11 +14545,6 @@ static int intel_framebuffer_init(struct intel_framebuffer *intel_fb,
 		}
 		break;
 	case DRM_FORMAT_NV12:
-		if (mode_cmd->modifier[0] == I915_FORMAT_MOD_Y_TILED_CCS ||
-		    mode_cmd->modifier[0] == I915_FORMAT_MOD_Yf_TILED_CCS) {
-			DRM_DEBUG_KMS("RC not to be enabled with NV12\n");
-			goto err;
-		}
 		if (INTEL_GEN(dev_priv) < 9 || IS_SKYLAKE(dev_priv) ||
 		    IS_BROXTON(dev_priv)) {
 			DRM_DEBUG_KMS("unsupported pixel format: %s\n",
@@ -14826,6 +14871,18 @@ static void quirk_increase_t12_delay(struct drm_device *dev)
 	DRM_INFO("Applying T12 delay quirk\n");
 }
 
+/*
+ * GeminiLake NUC HDMI outputs require additional off time
+ * this allows the onboard retimer to correctly sync to signal
+ */
+static void quirk_increase_ddi_disabled_time(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = to_i915(dev);
+
+	dev_priv->quirks |= QUIRK_INCREASE_DDI_DISABLED_TIME;
+	DRM_INFO("Applying Increase DDI Disabled quirk\n");
+}
+
 struct intel_quirk {
 	int device;
 	int subsystem_vendor;
@@ -14912,6 +14969,13 @@ static struct intel_quirk intel_quirks[] = {
 
 	/* Toshiba Satellite P50-C-18C */
 	{ 0x191B, 0x1179, 0xF840, quirk_increase_t12_delay },
+
+	/* GeminiLake NUC */
+	{ 0x3185, 0x8086, 0x2072, quirk_increase_ddi_disabled_time },
+	{ 0x3184, 0x8086, 0x2072, quirk_increase_ddi_disabled_time },
+	/* ASRock ITX*/
+	{ 0x3185, 0x1849, 0x2212, quirk_increase_ddi_disabled_time },
+	{ 0x3184, 0x1849, 0x2212, quirk_increase_ddi_disabled_time },
 };
 
 static void intel_init_quirks(struct drm_device *dev)
@@ -15678,11 +15742,20 @@ get_encoder_power_domains(struct drm_i915_private *dev_priv)
 	for_each_intel_encoder(&dev_priv->drm, encoder) {
 		u64 get_domains;
 		enum intel_display_power_domain domain;
+		struct intel_crtc_state *crtc_state;
 
 		if (!encoder->get_power_domains)
 			continue;
 
-		get_domains = encoder->get_power_domains(encoder);
+		/*
+		 * MST-primary and inactive encoders don't have a crtc state
+		 * and neither of these require any power domain references.
+		 */
+		if (!encoder->base.crtc)
+			continue;
+
+		crtc_state = to_intel_crtc_state(encoder->base.crtc->state);
+		get_domains = encoder->get_power_domains(encoder, crtc_state);
 		for_each_power_domain(domain, get_domains)
 			intel_display_power_get(dev_priv, domain);
 	}
@@ -15858,6 +15931,8 @@ void intel_modeset_cleanup(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
 
+	flush_workqueue(dev_priv->modeset_wq);
+
 	flush_work(&dev_priv->atomic_helper.free_work);
 	WARN_ON(!llist_empty(&dev_priv->atomic_helper.free_list));
 
diff --git a/drivers/gpu/drm/i915/intel_display.h b/drivers/gpu/drm/i915/intel_display.h
index dd30cae5eb00..138a1bc1818c 100644
--- a/drivers/gpu/drm/i915/intel_display.h
+++ b/drivers/gpu/drm/i915/intel_display.h
@@ -126,6 +126,30 @@ enum port {
 
 #define port_name(p) ((p) + 'A')
 
+/*
+ * Ports identifier referenced from other drivers.
+ * Expected to remain stable over time
+ */
+static inline const char *port_identifier(enum port port)
+{
+	switch (port) {
+	case PORT_A:
+		return "Port A";
+	case PORT_B:
+		return "Port B";
+	case PORT_C:
+		return "Port C";
+	case PORT_D:
+		return "Port D";
+	case PORT_E:
+		return "Port E";
+	case PORT_F:
+		return "Port F";
+	default:
+		return "<invalid>";
+	}
+}
+
 enum tc_port {
 	PORT_TC_NONE = -1,
 
@@ -199,6 +223,10 @@ enum intel_display_power_domain {
 	POWER_DOMAIN_AUX_E,
 	POWER_DOMAIN_AUX_F,
 	POWER_DOMAIN_AUX_IO_A,
+	POWER_DOMAIN_AUX_TBT1,
+	POWER_DOMAIN_AUX_TBT2,
+	POWER_DOMAIN_AUX_TBT3,
+	POWER_DOMAIN_AUX_TBT4,
 	POWER_DOMAIN_GMBUS,
 	POWER_DOMAIN_MODESET,
 	POWER_DOMAIN_GT_IRQ,
@@ -285,6 +313,10 @@ struct intel_link_m_n {
 			    &(dev)->mode_config.encoder_list,	\
 			    base.head)
 
+#define for_each_intel_dp(dev, intel_encoder)			\
+	for_each_intel_encoder(dev, intel_encoder)		\
+		for_each_if(intel_encoder_is_dp(intel_encoder))
+
 #define for_each_intel_connector_iter(intel_connector, iter) \
 	while ((intel_connector = to_intel_connector(drm_connector_list_iter_next(iter))))
 
diff --git a/drivers/gpu/drm/i915/intel_dp.c b/drivers/gpu/drm/i915/intel_dp.c
index 8caf20fb6587..cd0f649b57a5 100644
--- a/drivers/gpu/drm/i915/intel_dp.c
+++ b/drivers/gpu/drm/i915/intel_dp.c
@@ -600,14 +600,8 @@ static enum pipe vlv_find_free_pps(struct drm_i915_private *dev_priv)
 	 * We don't have power sequencer currently.
 	 * Pick one that's not used by other ports.
 	 */
-	for_each_intel_encoder(&dev_priv->drm, encoder) {
-		struct intel_dp *intel_dp;
-
-		if (encoder->type != INTEL_OUTPUT_DP &&
-		    encoder->type != INTEL_OUTPUT_EDP)
-			continue;
-
-		intel_dp = enc_to_intel_dp(&encoder->base);
+	for_each_intel_dp(&dev_priv->drm, encoder) {
+		struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
 
 		if (encoder->type == INTEL_OUTPUT_EDP) {
 			WARN_ON(intel_dp->active_pipe != INVALID_PIPE &&
@@ -799,19 +793,8 @@ void intel_power_sequencer_reset(struct drm_i915_private *dev_priv)
 	 * should use them always.
 	 */
 
-	for_each_intel_encoder(&dev_priv->drm, encoder) {
-		struct intel_dp *intel_dp;
-
-		if (encoder->type != INTEL_OUTPUT_DP &&
-		    encoder->type != INTEL_OUTPUT_EDP &&
-		    encoder->type != INTEL_OUTPUT_DDI)
-			continue;
-
-		intel_dp = enc_to_intel_dp(&encoder->base);
-
-		/* Skip pure DVI/HDMI DDI encoders */
-		if (!i915_mmio_reg_valid(intel_dp->output_reg))
-			continue;
+	for_each_intel_dp(&dev_priv->drm, encoder) {
+		struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
 
 		WARN_ON(intel_dp->active_pipe != INVALID_PIPE);
 
@@ -953,7 +936,7 @@ intel_dp_check_edp(struct intel_dp *intel_dp)
 }
 
 static uint32_t
-intel_dp_aux_wait_done(struct intel_dp *intel_dp, bool has_aux_irq)
+intel_dp_aux_wait_done(struct intel_dp *intel_dp)
 {
 	struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
 	i915_reg_t ch_ctl = intel_dp->aux_ch_ctl_reg(intel_dp);
@@ -961,14 +944,10 @@ intel_dp_aux_wait_done(struct intel_dp *intel_dp, bool has_aux_irq)
 	bool done;
 
 #define C (((status = I915_READ_NOTRACE(ch_ctl)) & DP_AUX_CH_CTL_SEND_BUSY) == 0)
-	if (has_aux_irq)
-		done = wait_event_timeout(dev_priv->gmbus_wait_queue, C,
-					  msecs_to_jiffies_timeout(10));
-	else
-		done = wait_for(C, 10) == 0;
+	done = wait_event_timeout(dev_priv->gmbus_wait_queue, C,
+				  msecs_to_jiffies_timeout(10));
 	if (!done)
-		DRM_ERROR("dp aux hw did not signal timeout (has irq: %i)!\n",
-			  has_aux_irq);
+		DRM_ERROR("dp aux hw did not signal timeout!\n");
 #undef C
 
 	return status;
@@ -1033,7 +1012,6 @@ static uint32_t skl_get_aux_clock_divider(struct intel_dp *intel_dp, int index)
 }
 
 static uint32_t g4x_get_aux_send_ctl(struct intel_dp *intel_dp,
-				     bool has_aux_irq,
 				     int send_bytes,
 				     uint32_t aux_clock_divider)
 {
@@ -1054,7 +1032,7 @@ static uint32_t g4x_get_aux_send_ctl(struct intel_dp *intel_dp,
 
 	return DP_AUX_CH_CTL_SEND_BUSY |
 	       DP_AUX_CH_CTL_DONE |
-	       (has_aux_irq ? DP_AUX_CH_CTL_INTERRUPT : 0) |
+	       DP_AUX_CH_CTL_INTERRUPT |
 	       DP_AUX_CH_CTL_TIME_OUT_ERROR |
 	       timeout |
 	       DP_AUX_CH_CTL_RECEIVE_ERROR |
@@ -1064,13 +1042,12 @@ static uint32_t g4x_get_aux_send_ctl(struct intel_dp *intel_dp,
 }
 
 static uint32_t skl_get_aux_send_ctl(struct intel_dp *intel_dp,
-				      bool has_aux_irq,
 				      int send_bytes,
 				      uint32_t unused)
 {
 	return DP_AUX_CH_CTL_SEND_BUSY |
 	       DP_AUX_CH_CTL_DONE |
-	       (has_aux_irq ? DP_AUX_CH_CTL_INTERRUPT : 0) |
+	       DP_AUX_CH_CTL_INTERRUPT |
 	       DP_AUX_CH_CTL_TIME_OUT_ERROR |
 	       DP_AUX_CH_CTL_TIME_OUT_MAX |
 	       DP_AUX_CH_CTL_RECEIVE_ERROR |
@@ -1093,7 +1070,6 @@ intel_dp_aux_xfer(struct intel_dp *intel_dp,
 	int i, ret, recv_bytes;
 	uint32_t status;
 	int try, clock = 0;
-	bool has_aux_irq = HAS_AUX_IRQ(dev_priv);
 	bool vdd;
 
 	ch_ctl = intel_dp->aux_ch_ctl_reg(intel_dp);
@@ -1148,7 +1124,6 @@ intel_dp_aux_xfer(struct intel_dp *intel_dp,
 
 	while ((aux_clock_divider = intel_dp->get_aux_clock_divider(intel_dp, clock++))) {
 		u32 send_ctl = intel_dp->get_aux_send_ctl(intel_dp,
-							  has_aux_irq,
 							  send_bytes,
 							  aux_clock_divider);
 
@@ -1165,7 +1140,7 @@ intel_dp_aux_xfer(struct intel_dp *intel_dp,
 			/* Send the command and wait for it to complete */
 			I915_WRITE(ch_ctl, send_ctl);
 
-			status = intel_dp_aux_wait_done(intel_dp, has_aux_irq);
+			status = intel_dp_aux_wait_done(intel_dp);
 
 			/* Clear done status and any errors */
 			I915_WRITE(ch_ctl,
@@ -2838,10 +2813,6 @@ static void vlv_disable_dp(struct intel_encoder *encoder,
 			   const struct intel_crtc_state *old_crtc_state,
 			   const struct drm_connector_state *old_conn_state)
 {
-	struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
-
-	intel_psr_disable(intel_dp, old_crtc_state);
-
 	intel_disable_dp(encoder, old_crtc_state, old_conn_state);
 }
 
@@ -3054,10 +3025,7 @@ static void vlv_enable_dp(struct intel_encoder *encoder,
 			  const struct intel_crtc_state *pipe_config,
 			  const struct drm_connector_state *conn_state)
 {
-	struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
-
 	intel_edp_backlight_on(pipe_config, conn_state);
-	intel_psr_enable(intel_dp, pipe_config);
 }
 
 static void g4x_pre_enable_dp(struct intel_encoder *encoder,
@@ -3112,16 +3080,9 @@ static void vlv_steal_power_sequencer(struct drm_i915_private *dev_priv,
 
 	lockdep_assert_held(&dev_priv->pps_mutex);
 
-	for_each_intel_encoder(&dev_priv->drm, encoder) {
-		struct intel_dp *intel_dp;
-		enum port port;
-
-		if (encoder->type != INTEL_OUTPUT_DP &&
-		    encoder->type != INTEL_OUTPUT_EDP)
-			continue;
-
-		intel_dp = enc_to_intel_dp(&encoder->base);
-		port = dp_to_dig_port(intel_dp)->base.port;
+	for_each_intel_dp(&dev_priv->drm, encoder) {
+		struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
+		enum port port = encoder->port;
 
 		WARN(intel_dp->active_pipe == pipe,
 		     "stealing pipe %c power sequencer from active (e)DP port %c\n",
@@ -3913,129 +3874,6 @@ intel_dp_configure_mst(struct intel_dp *intel_dp)
 					intel_dp->is_mst);
 }
 
-static int intel_dp_sink_crc_stop(struct intel_dp *intel_dp,
-				  struct intel_crtc_state *crtc_state, bool disable_wa)
-{
-	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
-	struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev);
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
-	u8 buf;
-	int ret = 0;
-	int count = 0;
-	int attempts = 10;
-
-	if (drm_dp_dpcd_readb(&intel_dp->aux, DP_TEST_SINK, &buf) < 0) {
-		DRM_DEBUG_KMS("Sink CRC couldn't be stopped properly\n");
-		ret = -EIO;
-		goto out;
-	}
-
-	if (drm_dp_dpcd_writeb(&intel_dp->aux, DP_TEST_SINK,
-			       buf & ~DP_TEST_SINK_START) < 0) {
-		DRM_DEBUG_KMS("Sink CRC couldn't be stopped properly\n");
-		ret = -EIO;
-		goto out;
-	}
-
-	do {
-		intel_wait_for_vblank(dev_priv, intel_crtc->pipe);
-
-		if (drm_dp_dpcd_readb(&intel_dp->aux,
-				      DP_TEST_SINK_MISC, &buf) < 0) {
-			ret = -EIO;
-			goto out;
-		}
-		count = buf & DP_TEST_COUNT_MASK;
-	} while (--attempts && count);
-
-	if (attempts == 0) {
-		DRM_DEBUG_KMS("TIMEOUT: Sink CRC counter is not zeroed after calculation is stopped\n");
-		ret = -ETIMEDOUT;
-	}
-
- out:
-	if (disable_wa)
-		hsw_enable_ips(crtc_state);
-	return ret;
-}
-
-static int intel_dp_sink_crc_start(struct intel_dp *intel_dp,
-				   struct intel_crtc_state *crtc_state)
-{
-	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
-	struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev);
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
-	u8 buf;
-	int ret;
-
-	if (drm_dp_dpcd_readb(&intel_dp->aux, DP_TEST_SINK_MISC, &buf) < 0)
-		return -EIO;
-
-	if (!(buf & DP_TEST_CRC_SUPPORTED))
-		return -ENOTTY;
-
-	if (drm_dp_dpcd_readb(&intel_dp->aux, DP_TEST_SINK, &buf) < 0)
-		return -EIO;
-
-	if (buf & DP_TEST_SINK_START) {
-		ret = intel_dp_sink_crc_stop(intel_dp, crtc_state, false);
-		if (ret)
-			return ret;
-	}
-
-	hsw_disable_ips(crtc_state);
-
-	if (drm_dp_dpcd_writeb(&intel_dp->aux, DP_TEST_SINK,
-			       buf | DP_TEST_SINK_START) < 0) {
-		hsw_enable_ips(crtc_state);
-		return -EIO;
-	}
-
-	intel_wait_for_vblank(dev_priv, intel_crtc->pipe);
-	return 0;
-}
-
-int intel_dp_sink_crc(struct intel_dp *intel_dp, struct intel_crtc_state *crtc_state, u8 *crc)
-{
-	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
-	struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev);
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
-	u8 buf;
-	int count, ret;
-	int attempts = 6;
-
-	ret = intel_dp_sink_crc_start(intel_dp, crtc_state);
-	if (ret)
-		return ret;
-
-	do {
-		intel_wait_for_vblank(dev_priv, intel_crtc->pipe);
-
-		if (drm_dp_dpcd_readb(&intel_dp->aux,
-				      DP_TEST_SINK_MISC, &buf) < 0) {
-			ret = -EIO;
-			goto stop;
-		}
-		count = buf & DP_TEST_COUNT_MASK;
-
-	} while (--attempts && count == 0);
-
-	if (attempts == 0) {
-		DRM_ERROR("Panel is unable to calculate any CRC after 6 vblanks\n");
-		ret = -ETIMEDOUT;
-		goto stop;
-	}
-
-	if (drm_dp_dpcd_read(&intel_dp->aux, DP_TEST_CRC_R_CR, crc, 6) < 0) {
-		ret = -EIO;
-		goto stop;
-	}
-
-stop:
-	intel_dp_sink_crc_stop(intel_dp, crtc_state, true);
-	return ret;
-}
-
 static bool
 intel_dp_get_sink_irq(struct intel_dp *intel_dp, u8 *sink_irq_vector)
 {
@@ -4502,6 +4340,8 @@ intel_dp_short_pulse(struct intel_dp *intel_dp)
 	if (intel_dp_needs_link_retrain(intel_dp))
 		return false;
 
+	intel_psr_short_pulse(intel_dp);
+
 	if (intel_dp->compliance.test_type == DP_TEST_LINK_TRAINING) {
 		DRM_DEBUG_KMS("Link Training Compliance Test requested\n");
 		/* Send a Hotplug Uevent to userspace to start modeset */
@@ -4569,14 +4409,7 @@ intel_dp_detect_dpcd(struct intel_dp *intel_dp)
 static enum drm_connector_status
 edp_detect(struct intel_dp *intel_dp)
 {
-	struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
-	enum drm_connector_status status;
-
-	status = intel_panel_detect(dev_priv);
-	if (status == connector_status_unknown)
-		status = connector_status_connected;
-
-	return status;
+	return connector_status_connected;
 }
 
 static bool ibx_digital_port_connected(struct intel_encoder *encoder)
@@ -4839,7 +4672,7 @@ intel_dp_long_pulse(struct intel_connector *connector)
 
 	intel_display_power_get(dev_priv, intel_dp->aux_power_domain);
 
-	/* Can't disconnect eDP, but you can close the lid... */
+	/* Can't disconnect eDP */
 	if (intel_dp_is_edp(intel_dp))
 		status = edp_detect(intel_dp);
 	else if (intel_digital_port_connected(&dp_to_dig_port(intel_dp)->base))
@@ -6514,7 +6347,6 @@ bool intel_dp_init(struct drm_i915_private *dev_priv,
 	intel_encoder->port = port;
 
 	intel_dig_port->hpd_pulse = intel_dp_hpd_pulse;
-	dev_priv->hotplug.irq_port[port] = intel_dig_port;
 
 	if (port != PORT_A)
 		intel_infoframe_init(intel_dig_port);
@@ -6533,37 +6365,44 @@ err_connector_alloc:
 	return false;
 }
 
-void intel_dp_mst_suspend(struct drm_device *dev)
+void intel_dp_mst_suspend(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	int i;
+	struct intel_encoder *encoder;
 
-	/* disable MST */
-	for (i = 0; i < I915_MAX_PORTS; i++) {
-		struct intel_digital_port *intel_dig_port = dev_priv->hotplug.irq_port[i];
+	for_each_intel_encoder(&dev_priv->drm, encoder) {
+		struct intel_dp *intel_dp;
 
-		if (!intel_dig_port || !intel_dig_port->dp.can_mst)
+		if (encoder->type != INTEL_OUTPUT_DDI)
 			continue;
 
-		if (intel_dig_port->dp.is_mst)
-			drm_dp_mst_topology_mgr_suspend(&intel_dig_port->dp.mst_mgr);
+		intel_dp = enc_to_intel_dp(&encoder->base);
+
+		if (!intel_dp->can_mst)
+			continue;
+
+		if (intel_dp->is_mst)
+			drm_dp_mst_topology_mgr_suspend(&intel_dp->mst_mgr);
 	}
 }
 
-void intel_dp_mst_resume(struct drm_device *dev)
+void intel_dp_mst_resume(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	int i;
+	struct intel_encoder *encoder;
 
-	for (i = 0; i < I915_MAX_PORTS; i++) {
-		struct intel_digital_port *intel_dig_port = dev_priv->hotplug.irq_port[i];
+	for_each_intel_encoder(&dev_priv->drm, encoder) {
+		struct intel_dp *intel_dp;
 		int ret;
 
-		if (!intel_dig_port || !intel_dig_port->dp.can_mst)
+		if (encoder->type != INTEL_OUTPUT_DDI)
+			continue;
+
+		intel_dp = enc_to_intel_dp(&encoder->base);
+
+		if (!intel_dp->can_mst)
 			continue;
 
-		ret = drm_dp_mst_topology_mgr_resume(&intel_dig_port->dp.mst_mgr);
+		ret = drm_dp_mst_topology_mgr_resume(&intel_dp->mst_mgr);
 		if (ret)
-			intel_dp_check_mst_status(&intel_dig_port->dp);
+			intel_dp_check_mst_status(intel_dp);
 	}
 }
diff --git a/drivers/gpu/drm/i915/intel_dp_mst.c b/drivers/gpu/drm/i915/intel_dp_mst.c
index 0095c197f155..7e3e01607643 100644
--- a/drivers/gpu/drm/i915/intel_dp_mst.c
+++ b/drivers/gpu/drm/i915/intel_dp_mst.c
@@ -513,7 +513,7 @@ static void intel_dp_destroy_mst_connector(struct drm_dp_mst_topology_mgr *mgr,
 	intel_connector->mst_port = NULL;
 	drm_modeset_unlock(&connector->dev->mode_config.connection_mutex);
 
-	drm_connector_unreference(connector);
+	drm_connector_put(connector);
 }
 
 static void intel_dp_mst_hotplug(struct drm_dp_mst_topology_mgr *mgr)
diff --git a/drivers/gpu/drm/i915/intel_dpll_mgr.c b/drivers/gpu/drm/i915/intel_dpll_mgr.c
index 156f8e4cbe4c..b51ad2917dbe 100644
--- a/drivers/gpu/drm/i915/intel_dpll_mgr.c
+++ b/drivers/gpu/drm/i915/intel_dpll_mgr.c
@@ -2566,6 +2566,7 @@ int icl_calc_dp_combo_pll_link(struct drm_i915_private *dev_priv,
 	switch (index) {
 	default:
 		MISSING_CASE(index);
+		/* fall through */
 	case 0:
 		link_clock = 540000;
 		break;
@@ -2639,6 +2640,7 @@ static bool icl_mg_pll_find_divisors(int clock_khz, bool is_dp, bool use_ssc,
 			switch (div1) {
 			default:
 				MISSING_CASE(div1);
+				/* fall through */
 			case 2:
 				hsdiv = 0;
 				break;
@@ -2812,25 +2814,31 @@ static bool icl_calc_mg_pll_state(struct intel_crtc_state *crtc_state,
 				MG_PLL_SSC_FLLEN |
 				MG_PLL_SSC_STEPSIZE(ssc_stepsize);
 
-	pll_state->mg_pll_tdc_coldst_bias = MG_PLL_TDC_COLDST_COLDSTART;
-
-	if (refclk_khz != 38400) {
-		pll_state->mg_pll_tdc_coldst_bias |=
-			MG_PLL_TDC_COLDST_IREFINT_EN |
-			MG_PLL_TDC_COLDST_REFBIAS_START_PULSE_W(iref_pulse_w) |
-			MG_PLL_TDC_COLDST_COLDSTART |
-			MG_PLL_TDC_TDCOVCCORR_EN |
-			MG_PLL_TDC_TDCSEL(3);
-
-		pll_state->mg_pll_bias = MG_PLL_BIAS_BIAS_GB_SEL(3) |
-					 MG_PLL_BIAS_INIT_DCOAMP(0x3F) |
-					 MG_PLL_BIAS_BIAS_BONUS(10) |
-					 MG_PLL_BIAS_BIASCAL_EN |
-					 MG_PLL_BIAS_CTRIM(12) |
-					 MG_PLL_BIAS_VREF_RDAC(4) |
-					 MG_PLL_BIAS_IREFTRIM(iref_trim);
+	pll_state->mg_pll_tdc_coldst_bias = MG_PLL_TDC_COLDST_COLDSTART |
+					    MG_PLL_TDC_COLDST_IREFINT_EN |
+					    MG_PLL_TDC_COLDST_REFBIAS_START_PULSE_W(iref_pulse_w) |
+					    MG_PLL_TDC_TDCOVCCORR_EN |
+					    MG_PLL_TDC_TDCSEL(3);
+
+	pll_state->mg_pll_bias = MG_PLL_BIAS_BIAS_GB_SEL(3) |
+				 MG_PLL_BIAS_INIT_DCOAMP(0x3F) |
+				 MG_PLL_BIAS_BIAS_BONUS(10) |
+				 MG_PLL_BIAS_BIASCAL_EN |
+				 MG_PLL_BIAS_CTRIM(12) |
+				 MG_PLL_BIAS_VREF_RDAC(4) |
+				 MG_PLL_BIAS_IREFTRIM(iref_trim);
+
+	if (refclk_khz == 38400) {
+		pll_state->mg_pll_tdc_coldst_bias_mask = MG_PLL_TDC_COLDST_COLDSTART;
+		pll_state->mg_pll_bias_mask = 0;
+	} else {
+		pll_state->mg_pll_tdc_coldst_bias_mask = -1U;
+		pll_state->mg_pll_bias_mask = -1U;
 	}
 
+	pll_state->mg_pll_tdc_coldst_bias &= pll_state->mg_pll_tdc_coldst_bias_mask;
+	pll_state->mg_pll_bias &= pll_state->mg_pll_bias_mask;
+
 	return true;
 }
 
@@ -2897,6 +2905,7 @@ static i915_reg_t icl_pll_id_to_enable_reg(enum intel_dpll_id id)
 	switch (id) {
 	default:
 		MISSING_CASE(id);
+		/* fall through */
 	case DPLL_ID_ICL_DPLL0:
 	case DPLL_ID_ICL_DPLL1:
 		return CNL_DPLL_ENABLE(id);
@@ -2939,18 +2948,41 @@ static bool icl_pll_get_hw_state(struct drm_i915_private *dev_priv,
 	case DPLL_ID_ICL_MGPLL4:
 		port = icl_mg_pll_id_to_port(id);
 		hw_state->mg_refclkin_ctl = I915_READ(MG_REFCLKIN_CTL(port));
+		hw_state->mg_refclkin_ctl &= MG_REFCLKIN_CTL_OD_2_MUX_MASK;
+
 		hw_state->mg_clktop2_coreclkctl1 =
 			I915_READ(MG_CLKTOP2_CORECLKCTL1(port));
+		hw_state->mg_clktop2_coreclkctl1 &=
+			MG_CLKTOP2_CORECLKCTL1_A_DIVRATIO_MASK;
+
 		hw_state->mg_clktop2_hsclkctl =
 			I915_READ(MG_CLKTOP2_HSCLKCTL(port));
+		hw_state->mg_clktop2_hsclkctl &=
+			MG_CLKTOP2_HSCLKCTL_TLINEDRV_CLKSEL_MASK |
+			MG_CLKTOP2_HSCLKCTL_CORE_INPUTSEL_MASK |
+			MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO_MASK |
+			MG_CLKTOP2_HSCLKCTL_DSDIV_RATIO_MASK;
+
 		hw_state->mg_pll_div0 = I915_READ(MG_PLL_DIV0(port));
 		hw_state->mg_pll_div1 = I915_READ(MG_PLL_DIV1(port));
 		hw_state->mg_pll_lf = I915_READ(MG_PLL_LF(port));
 		hw_state->mg_pll_frac_lock = I915_READ(MG_PLL_FRAC_LOCK(port));
 		hw_state->mg_pll_ssc = I915_READ(MG_PLL_SSC(port));
+
 		hw_state->mg_pll_bias = I915_READ(MG_PLL_BIAS(port));
 		hw_state->mg_pll_tdc_coldst_bias =
 			I915_READ(MG_PLL_TDC_COLDST_BIAS(port));
+
+		if (dev_priv->cdclk.hw.ref == 38400) {
+			hw_state->mg_pll_tdc_coldst_bias_mask = MG_PLL_TDC_COLDST_COLDSTART;
+			hw_state->mg_pll_bias_mask = 0;
+		} else {
+			hw_state->mg_pll_tdc_coldst_bias_mask = -1U;
+			hw_state->mg_pll_bias_mask = -1U;
+		}
+
+		hw_state->mg_pll_tdc_coldst_bias &= hw_state->mg_pll_tdc_coldst_bias_mask;
+		hw_state->mg_pll_bias &= hw_state->mg_pll_bias_mask;
 		break;
 	default:
 		MISSING_CASE(id);
@@ -2978,19 +3010,48 @@ static void icl_mg_pll_write(struct drm_i915_private *dev_priv,
 {
 	struct intel_dpll_hw_state *hw_state = &pll->state.hw_state;
 	enum port port = icl_mg_pll_id_to_port(pll->info->id);
+	u32 val;
+
+	/*
+	 * Some of the following registers have reserved fields, so program
+	 * these with RMW based on a mask. The mask can be fixed or generated
+	 * during the calc/readout phase if the mask depends on some other HW
+	 * state like refclk, see icl_calc_mg_pll_state().
+	 */
+	val = I915_READ(MG_REFCLKIN_CTL(port));
+	val &= ~MG_REFCLKIN_CTL_OD_2_MUX_MASK;
+	val |= hw_state->mg_refclkin_ctl;
+	I915_WRITE(MG_REFCLKIN_CTL(port), val);
+
+	val = I915_READ(MG_CLKTOP2_CORECLKCTL1(port));
+	val &= ~MG_CLKTOP2_CORECLKCTL1_A_DIVRATIO_MASK;
+	val |= hw_state->mg_clktop2_coreclkctl1;
+	I915_WRITE(MG_CLKTOP2_CORECLKCTL1(port), val);
+
+	val = I915_READ(MG_CLKTOP2_HSCLKCTL(port));
+	val &= ~(MG_CLKTOP2_HSCLKCTL_TLINEDRV_CLKSEL_MASK |
+		 MG_CLKTOP2_HSCLKCTL_CORE_INPUTSEL_MASK |
+		 MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO_MASK |
+		 MG_CLKTOP2_HSCLKCTL_DSDIV_RATIO_MASK);
+	val |= hw_state->mg_clktop2_hsclkctl;
+	I915_WRITE(MG_CLKTOP2_HSCLKCTL(port), val);
 
-	I915_WRITE(MG_REFCLKIN_CTL(port), hw_state->mg_refclkin_ctl);
-	I915_WRITE(MG_CLKTOP2_CORECLKCTL1(port),
-		   hw_state->mg_clktop2_coreclkctl1);
-	I915_WRITE(MG_CLKTOP2_HSCLKCTL(port), hw_state->mg_clktop2_hsclkctl);
 	I915_WRITE(MG_PLL_DIV0(port), hw_state->mg_pll_div0);
 	I915_WRITE(MG_PLL_DIV1(port), hw_state->mg_pll_div1);
 	I915_WRITE(MG_PLL_LF(port), hw_state->mg_pll_lf);
 	I915_WRITE(MG_PLL_FRAC_LOCK(port), hw_state->mg_pll_frac_lock);
 	I915_WRITE(MG_PLL_SSC(port), hw_state->mg_pll_ssc);
-	I915_WRITE(MG_PLL_BIAS(port), hw_state->mg_pll_bias);
-	I915_WRITE(MG_PLL_TDC_COLDST_BIAS(port),
-		   hw_state->mg_pll_tdc_coldst_bias);
+
+	val = I915_READ(MG_PLL_BIAS(port));
+	val &= ~hw_state->mg_pll_bias_mask;
+	val |= hw_state->mg_pll_bias;
+	I915_WRITE(MG_PLL_BIAS(port), val);
+
+	val = I915_READ(MG_PLL_TDC_COLDST_BIAS(port));
+	val &= ~hw_state->mg_pll_tdc_coldst_bias_mask;
+	val |= hw_state->mg_pll_tdc_coldst_bias;
+	I915_WRITE(MG_PLL_TDC_COLDST_BIAS(port), val);
+
 	POSTING_READ(MG_PLL_TDC_COLDST_BIAS(port));
 }
 
diff --git a/drivers/gpu/drm/i915/intel_dpll_mgr.h b/drivers/gpu/drm/i915/intel_dpll_mgr.h
index ba925c7ee482..7e522cf4f13f 100644
--- a/drivers/gpu/drm/i915/intel_dpll_mgr.h
+++ b/drivers/gpu/drm/i915/intel_dpll_mgr.h
@@ -180,6 +180,8 @@ struct intel_dpll_hw_state {
 	uint32_t mg_pll_ssc;
 	uint32_t mg_pll_bias;
 	uint32_t mg_pll_tdc_coldst_bias;
+	uint32_t mg_pll_bias_mask;
+	uint32_t mg_pll_tdc_coldst_bias_mask;
 };
 
 /**
diff --git a/drivers/gpu/drm/i915/intel_drv.h b/drivers/gpu/drm/i915/intel_drv.h
index b41515bb9a15..5f63e1a9c25b 100644
--- a/drivers/gpu/drm/i915/intel_drv.h
+++ b/drivers/gpu/drm/i915/intel_drv.h
@@ -39,6 +39,7 @@
 #include <drm/drm_dp_mst_helper.h>
 #include <drm/drm_rect.h>
 #include <drm/drm_atomic.h>
+#include <media/cec-notifier.h>
 
 /**
  * __wait_for - magic wait macro
@@ -254,7 +255,8 @@ struct intel_encoder {
 			   struct intel_crtc_state *pipe_config);
 	/* Returns a mask of power domains that need to be referenced as part
 	 * of the hardware state readout code. */
-	u64 (*get_power_domains)(struct intel_encoder *encoder);
+	u64 (*get_power_domains)(struct intel_encoder *encoder,
+				 struct intel_crtc_state *crtc_state);
 	/*
 	 * Called during system suspend after all pending requests for the
 	 * encoder are flushed (for example for DP AUX transactions) and
@@ -303,6 +305,8 @@ struct intel_panel {
 	} backlight;
 };
 
+struct intel_digital_port;
+
 /*
  * This structure serves as a translation layer between the generic HDCP code
  * and the bus-specific code. What that means is that HDCP over HDMI differs
@@ -481,6 +485,8 @@ struct intel_atomic_state {
 	 */
 	bool skip_intermediate_wm;
 
+	bool rps_interactive;
+
 	/* Gen9+ only */
 	struct skl_ddb_values wm_results;
 
@@ -1011,6 +1017,7 @@ struct intel_hdmi {
 	bool has_audio;
 	bool rgb_quant_range_selectable;
 	struct intel_connector *attached_connector;
+	struct cec_notifier *cec_notifier;
 };
 
 struct intel_dp_mst_encoder;
@@ -1133,7 +1140,6 @@ struct intel_dp {
 	 * register with to kick off an AUX transaction.
 	 */
 	uint32_t (*get_aux_send_ctl)(struct intel_dp *dp,
-				     bool has_aux_irq,
 				     int send_bytes,
 				     uint32_t aux_clock_divider);
 
@@ -1246,22 +1252,29 @@ intel_attached_encoder(struct drm_connector *connector)
 	return to_intel_connector(connector)->encoder;
 }
 
-static inline struct intel_digital_port *
-enc_to_dig_port(struct drm_encoder *encoder)
+static inline bool intel_encoder_is_dig_port(struct intel_encoder *encoder)
 {
-	struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
-
-	switch (intel_encoder->type) {
+	switch (encoder->type) {
 	case INTEL_OUTPUT_DDI:
-		WARN_ON(!HAS_DDI(to_i915(encoder->dev)));
 	case INTEL_OUTPUT_DP:
 	case INTEL_OUTPUT_EDP:
 	case INTEL_OUTPUT_HDMI:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static inline struct intel_digital_port *
+enc_to_dig_port(struct drm_encoder *encoder)
+{
+	struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
+
+	if (intel_encoder_is_dig_port(intel_encoder))
 		return container_of(encoder, struct intel_digital_port,
 				    base.base);
-	default:
+	else
 		return NULL;
-	}
 }
 
 static inline struct intel_dp_mst_encoder *
@@ -1275,6 +1288,20 @@ static inline struct intel_dp *enc_to_intel_dp(struct drm_encoder *encoder)
 	return &enc_to_dig_port(encoder)->dp;
 }
 
+static inline bool intel_encoder_is_dp(struct intel_encoder *encoder)
+{
+	switch (encoder->type) {
+	case INTEL_OUTPUT_DP:
+	case INTEL_OUTPUT_EDP:
+		return true;
+	case INTEL_OUTPUT_DDI:
+		/* Skip pure HDMI/DVI DDI encoders */
+		return i915_mmio_reg_valid(enc_to_intel_dp(&encoder->base)->output_reg);
+	default:
+		return false;
+	}
+}
+
 static inline struct intel_digital_port *
 dp_to_dig_port(struct intel_dp *intel_dp)
 {
@@ -1331,9 +1358,6 @@ void intel_check_cpu_fifo_underruns(struct drm_i915_private *dev_priv);
 void intel_check_pch_fifo_underruns(struct drm_i915_private *dev_priv);
 
 /* i915_irq.c */
-bool gen11_reset_one_iir(struct drm_i915_private * const i915,
-			 const unsigned int bank,
-			 const unsigned int bit);
 void gen5_enable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask);
 void gen5_disable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask);
 void gen6_mask_pm_irq(struct drm_i915_private *dev_priv, u32 mask);
@@ -1384,8 +1408,7 @@ void hsw_fdi_link_train(struct intel_crtc *crtc,
 void intel_ddi_init(struct drm_i915_private *dev_priv, enum port port);
 bool intel_ddi_get_hw_state(struct intel_encoder *encoder, enum pipe *pipe);
 void intel_ddi_enable_transcoder_func(const struct intel_crtc_state *crtc_state);
-void intel_ddi_disable_transcoder_func(struct drm_i915_private *dev_priv,
-				       enum transcoder cpu_transcoder);
+void intel_ddi_disable_transcoder_func(const struct intel_crtc_state *crtc_state);
 void intel_ddi_enable_pipe_clock(const struct intel_crtc_state *crtc_state);
 void intel_ddi_disable_pipe_clock(const  struct intel_crtc_state *crtc_state);
 void intel_ddi_set_pipe_settings(const struct intel_crtc_state *crtc_state);
@@ -1664,8 +1687,6 @@ void intel_dp_sink_dpms(struct intel_dp *intel_dp, int mode);
 void intel_dp_encoder_reset(struct drm_encoder *encoder);
 void intel_dp_encoder_suspend(struct intel_encoder *intel_encoder);
 void intel_dp_encoder_destroy(struct drm_encoder *encoder);
-int intel_dp_sink_crc(struct intel_dp *intel_dp,
-		      struct intel_crtc_state *crtc_state, u8 *crc);
 bool intel_dp_compute_config(struct intel_encoder *encoder,
 			     struct intel_crtc_state *pipe_config,
 			     struct drm_connector_state *conn_state);
@@ -1679,8 +1700,8 @@ void intel_edp_backlight_off(const struct drm_connector_state *conn_state);
 void intel_edp_panel_vdd_on(struct intel_dp *intel_dp);
 void intel_edp_panel_on(struct intel_dp *intel_dp);
 void intel_edp_panel_off(struct intel_dp *intel_dp);
-void intel_dp_mst_suspend(struct drm_device *dev);
-void intel_dp_mst_resume(struct drm_device *dev);
+void intel_dp_mst_suspend(struct drm_i915_private *dev_priv);
+void intel_dp_mst_resume(struct drm_i915_private *dev_priv);
 int intel_dp_max_link_rate(struct intel_dp *intel_dp);
 int intel_dp_max_lane_count(struct intel_dp *intel_dp);
 int intel_dp_rate_select(struct intel_dp *intel_dp, int rate);
@@ -1730,8 +1751,8 @@ int intel_dp_aux_init_backlight_funcs(struct intel_connector *intel_connector);
 /* intel_dp_mst.c */
 int intel_dp_mst_encoder_init(struct intel_digital_port *intel_dig_port, int conn_id);
 void intel_dp_mst_encoder_cleanup(struct intel_digital_port *intel_dig_port);
-/* intel_dsi.c */
-void intel_dsi_init(struct drm_i915_private *dev_priv);
+/* vlv_dsi.c */
+void vlv_dsi_init(struct drm_i915_private *dev_priv);
 
 /* intel_dsi_dcs_backlight.c */
 int intel_dsi_dcs_init_backlight_funcs(struct intel_connector *intel_connector);
@@ -1873,7 +1894,6 @@ void intel_panel_enable_backlight(const struct intel_crtc_state *crtc_state,
 				  const struct drm_connector_state *conn_state);
 void intel_panel_disable_backlight(const struct drm_connector_state *old_conn_state);
 void intel_panel_destroy_backlight(struct drm_connector *connector);
-enum drm_connector_status intel_panel_detect(struct drm_i915_private *dev_priv);
 extern struct drm_display_mode *intel_find_panel_downclock(
 				struct drm_i915_private *dev_priv,
 				struct drm_display_mode *fixed_mode,
@@ -1921,6 +1941,8 @@ void intel_psr_compute_config(struct intel_dp *intel_dp,
 			      struct intel_crtc_state *crtc_state);
 void intel_psr_irq_control(struct drm_i915_private *dev_priv, bool debug);
 void intel_psr_irq_handler(struct drm_i915_private *dev_priv, u32 psr_iir);
+void intel_psr_short_pulse(struct intel_dp *intel_dp);
+int intel_psr_wait_for_idle(const struct intel_crtc_state *new_crtc_state);
 
 /* intel_runtime_pm.c */
 int intel_power_domains_init(struct drm_i915_private *);
@@ -2149,7 +2171,6 @@ void lspcon_resume(struct intel_lspcon *lspcon);
 void lspcon_wait_pcon_mode(struct intel_lspcon *lspcon);
 
 /* intel_pipe_crc.c */
-int intel_pipe_crc_create(struct drm_minor *minor);
 #ifdef CONFIG_DEBUG_FS
 int intel_crtc_set_crc_source(struct drm_crtc *crtc, const char *source_name);
 int intel_crtc_verify_crc_source(struct drm_crtc *crtc,
@@ -2170,5 +2191,4 @@ static inline void intel_crtc_enable_pipe_crc(struct intel_crtc *crtc)
 {
 }
 #endif
-extern const struct file_operations i915_display_crc_ctl_fops;
 #endif /* __INTEL_DRV_H__ */
diff --git a/drivers/gpu/drm/i915/intel_dsi.h b/drivers/gpu/drm/i915/intel_dsi.h
index 7afeb9580f41..ad7c1cb32983 100644
--- a/drivers/gpu/drm/i915/intel_dsi.h
+++ b/drivers/gpu/drm/i915/intel_dsi.h
@@ -129,21 +129,29 @@ static inline struct intel_dsi *enc_to_intel_dsi(struct drm_encoder *encoder)
 	return container_of(encoder, struct intel_dsi, base.base);
 }
 
-/* intel_dsi.c */
-void wait_for_dsi_fifo_empty(struct intel_dsi *intel_dsi, enum port port);
+/* vlv_dsi.c */
+void vlv_dsi_wait_for_fifo_empty(struct intel_dsi *intel_dsi, enum port port);
 enum mipi_dsi_pixel_format pixel_format_from_register_bits(u32 fmt);
 
-/* intel_dsi_pll.c */
-bool intel_dsi_pll_is_enabled(struct drm_i915_private *dev_priv);
-int intel_compute_dsi_pll(struct intel_encoder *encoder,
-			  struct intel_crtc_state *config);
-void intel_enable_dsi_pll(struct intel_encoder *encoder,
-			  const struct intel_crtc_state *config);
-void intel_disable_dsi_pll(struct intel_encoder *encoder);
-u32 intel_dsi_get_pclk(struct intel_encoder *encoder, int pipe_bpp,
-		       struct intel_crtc_state *config);
-void intel_dsi_reset_clocks(struct intel_encoder *encoder,
-			    enum port port);
+/* vlv_dsi_pll.c */
+int vlv_dsi_pll_compute(struct intel_encoder *encoder,
+			struct intel_crtc_state *config);
+void vlv_dsi_pll_enable(struct intel_encoder *encoder,
+			const struct intel_crtc_state *config);
+void vlv_dsi_pll_disable(struct intel_encoder *encoder);
+u32 vlv_dsi_get_pclk(struct intel_encoder *encoder, int pipe_bpp,
+		     struct intel_crtc_state *config);
+void vlv_dsi_reset_clocks(struct intel_encoder *encoder, enum port port);
+
+bool bxt_dsi_pll_is_enabled(struct drm_i915_private *dev_priv);
+int bxt_dsi_pll_compute(struct intel_encoder *encoder,
+			struct intel_crtc_state *config);
+void bxt_dsi_pll_enable(struct intel_encoder *encoder,
+			const struct intel_crtc_state *config);
+void bxt_dsi_pll_disable(struct intel_encoder *encoder);
+u32 bxt_dsi_get_pclk(struct intel_encoder *encoder, int pipe_bpp,
+		     struct intel_crtc_state *config);
+void bxt_dsi_reset_clocks(struct intel_encoder *encoder, enum port port);
 
 /* intel_dsi_vbt.c */
 bool intel_dsi_vbt_init(struct intel_dsi *intel_dsi, u16 panel_id);
diff --git a/drivers/gpu/drm/i915/intel_dsi_vbt.c b/drivers/gpu/drm/i915/intel_dsi_vbt.c
index 4d6ffa7b3e7b..ac83d6b89ae0 100644
--- a/drivers/gpu/drm/i915/intel_dsi_vbt.c
+++ b/drivers/gpu/drm/i915/intel_dsi_vbt.c
@@ -181,7 +181,7 @@ static const u8 *mipi_exec_send_packet(struct intel_dsi *intel_dsi,
 		break;
 	}
 
-	wait_for_dsi_fifo_empty(intel_dsi, port);
+	vlv_dsi_wait_for_fifo_empty(intel_dsi, port);
 
 out:
 	data += len;
diff --git a/drivers/gpu/drm/i915/intel_engine_cs.c b/drivers/gpu/drm/i915/intel_engine_cs.c
index 32bf3a408d46..2d1952849d69 100644
--- a/drivers/gpu/drm/i915/intel_engine_cs.c
+++ b/drivers/gpu/drm/i915/intel_engine_cs.c
@@ -25,7 +25,6 @@
 #include <drm/drm_print.h>
 
 #include "i915_drv.h"
-#include "i915_vgpu.h"
 #include "intel_ringbuffer.h"
 #include "intel_lrc.h"
 
@@ -230,6 +229,7 @@ __intel_engine_context_size(struct drm_i915_private *dev_priv, u8 class)
 		break;
 	default:
 		MISSING_CASE(class);
+		/* fall through */
 	case VIDEO_DECODE_CLASS:
 	case VIDEO_ENHANCEMENT_CLASS:
 	case COPY_ENGINE_CLASS:
@@ -302,6 +302,8 @@ intel_engine_setup(struct drm_i915_private *dev_priv,
 							   engine->class);
 	if (WARN_ON(engine->context_size > BIT(20)))
 		engine->context_size = 0;
+	if (engine->context_size)
+		DRIVER_CAPS(dev_priv)->has_logical_contexts = true;
 
 	/* Nothing to do here, execute in order of dependencies */
 	engine->schedule = NULL;
@@ -456,28 +458,16 @@ static void intel_engine_init_batch_pool(struct intel_engine_cs *engine)
 	i915_gem_batch_pool_init(&engine->batch_pool, engine);
 }
 
-static bool csb_force_mmio(struct drm_i915_private *i915)
-{
-	/* Older GVT emulation depends upon intercepting CSB mmio */
-	if (intel_vgpu_active(i915) && !intel_vgpu_has_hwsp_emulation(i915))
-		return true;
-
-	return false;
-}
-
 static void intel_engine_init_execlist(struct intel_engine_cs *engine)
 {
 	struct intel_engine_execlists * const execlists = &engine->execlists;
 
-	execlists->csb_use_mmio = csb_force_mmio(engine->i915);
-
 	execlists->port_mask = 1;
 	BUILD_BUG_ON_NOT_POWER_OF_2(execlists_num_ports(execlists));
 	GEM_BUG_ON(execlists_num_ports(execlists) > EXECLIST_MAX_PORTS);
 
 	execlists->queue_priority = INT_MIN;
-	execlists->queue = RB_ROOT;
-	execlists->first = NULL;
+	execlists->queue = RB_ROOT_CACHED;
 }
 
 /**
@@ -492,6 +482,7 @@ static void intel_engine_init_execlist(struct intel_engine_cs *engine)
 void intel_engine_setup_common(struct intel_engine_cs *engine)
 {
 	i915_timeline_init(engine->i915, &engine->timeline, engine->name);
+	lockdep_set_subclass(&engine->timeline.lock, TIMELINE_ENGINE);
 
 	intel_engine_init_execlist(engine);
 	intel_engine_init_hangcheck(engine);
@@ -998,19 +989,23 @@ bool intel_engine_is_idle(struct intel_engine_cs *engine)
 
 	/* Waiting to drain ELSP? */
 	if (READ_ONCE(engine->execlists.active)) {
-		struct intel_engine_execlists *execlists = &engine->execlists;
-
-		if (tasklet_trylock(&execlists->tasklet)) {
-			execlists->tasklet.func(execlists->tasklet.data);
-			tasklet_unlock(&execlists->tasklet);
+		struct tasklet_struct *t = &engine->execlists.tasklet;
+
+		local_bh_disable();
+		if (tasklet_trylock(t)) {
+			/* Must wait for any GPU reset in progress. */
+			if (__tasklet_is_enabled(t))
+				t->func(t->data);
+			tasklet_unlock(t);
 		}
+		local_bh_enable();
 
-		if (READ_ONCE(execlists->active))
+		if (READ_ONCE(engine->execlists.active))
 			return false;
 	}
 
 	/* ELSP is empty, but there are ready requests? E.g. after reset */
-	if (READ_ONCE(engine->execlists.first))
+	if (!RB_EMPTY_ROOT(&engine->execlists.queue.rb_root))
 		return false;
 
 	/* Ring stopped? */
@@ -1363,12 +1358,10 @@ static void intel_engine_print_registers(const struct intel_engine_cs *engine,
 		ptr = I915_READ(RING_CONTEXT_STATUS_PTR(engine));
 		read = GEN8_CSB_READ_PTR(ptr);
 		write = GEN8_CSB_WRITE_PTR(ptr);
-		drm_printf(m, "\tExeclist CSB read %d [%d cached], write %d [%d from hws], interrupt posted? %s, tasklet queued? %s (%s)\n",
+		drm_printf(m, "\tExeclist CSB read %d [%d cached], write %d [%d from hws], tasklet queued? %s (%s)\n",
 			   read, execlists->csb_head,
 			   write,
 			   intel_read_status_page(engine, intel_hws_csb_write_index(engine->i915)),
-			   yesno(test_bit(ENGINE_IRQ_EXECLIST,
-					  &engine->irq_posted)),
 			   yesno(test_bit(TASKLET_STATE_SCHED,
 					  &engine->execlists.tasklet.state)),
 			   enableddisabled(!atomic_read(&engine->execlists.tasklet.count)));
@@ -1548,7 +1541,7 @@ void intel_engine_dump(struct intel_engine_cs *engine,
 	last = NULL;
 	count = 0;
 	drm_printf(m, "\t\tQueue priority: %d\n", execlists->queue_priority);
-	for (rb = execlists->first; rb; rb = rb_next(rb)) {
+	for (rb = rb_first_cached(&execlists->queue); rb; rb = rb_next(rb)) {
 		struct i915_priolist *p =
 			rb_entry(rb, typeof(*p), node);
 
@@ -1580,11 +1573,9 @@ void intel_engine_dump(struct intel_engine_cs *engine,
 	spin_unlock(&b->rb_lock);
 	local_irq_restore(flags);
 
-	drm_printf(m, "IRQ? 0x%lx (breadcrumbs? %s) (execlists? %s)\n",
+	drm_printf(m, "IRQ? 0x%lx (breadcrumbs? %s)\n",
 		   engine->irq_posted,
 		   yesno(test_bit(ENGINE_IRQ_BREADCRUMB,
-				  &engine->irq_posted)),
-		   yesno(test_bit(ENGINE_IRQ_EXECLIST,
 				  &engine->irq_posted)));
 
 	drm_printf(m, "HWSP:\n");
@@ -1633,8 +1624,8 @@ int intel_enable_engine_stats(struct intel_engine_cs *engine)
 	if (!intel_engine_supports_stats(engine))
 		return -ENODEV;
 
-	tasklet_disable(&execlists->tasklet);
-	write_seqlock_irqsave(&engine->stats.lock, flags);
+	spin_lock_irqsave(&engine->timeline.lock, flags);
+	write_seqlock(&engine->stats.lock);
 
 	if (unlikely(engine->stats.enabled == ~0)) {
 		err = -EBUSY;
@@ -1658,8 +1649,8 @@ int intel_enable_engine_stats(struct intel_engine_cs *engine)
 	}
 
 unlock:
-	write_sequnlock_irqrestore(&engine->stats.lock, flags);
-	tasklet_enable(&execlists->tasklet);
+	write_sequnlock(&engine->stats.lock);
+	spin_unlock_irqrestore(&engine->timeline.lock, flags);
 
 	return err;
 }
diff --git a/drivers/gpu/drm/i915/intel_fbc.c b/drivers/gpu/drm/i915/intel_fbc.c
index b431b6733cc1..01d1d2088f04 100644
--- a/drivers/gpu/drm/i915/intel_fbc.c
+++ b/drivers/gpu/drm/i915/intel_fbc.c
@@ -399,89 +399,6 @@ bool intel_fbc_is_active(struct drm_i915_private *dev_priv)
 	return dev_priv->fbc.active;
 }
 
-static void intel_fbc_work_fn(struct work_struct *__work)
-{
-	struct drm_i915_private *dev_priv =
-		container_of(__work, struct drm_i915_private, fbc.work.work);
-	struct intel_fbc *fbc = &dev_priv->fbc;
-	struct intel_fbc_work *work = &fbc->work;
-	struct intel_crtc *crtc = fbc->crtc;
-	struct drm_vblank_crtc *vblank = &dev_priv->drm.vblank[crtc->pipe];
-
-	if (drm_crtc_vblank_get(&crtc->base)) {
-		/* CRTC is now off, leave FBC deactivated */
-		mutex_lock(&fbc->lock);
-		work->scheduled = false;
-		mutex_unlock(&fbc->lock);
-		return;
-	}
-
-retry:
-	/* Delay the actual enabling to let pageflipping cease and the
-	 * display to settle before starting the compression. Note that
-	 * this delay also serves a second purpose: it allows for a
-	 * vblank to pass after disabling the FBC before we attempt
-	 * to modify the control registers.
-	 *
-	 * WaFbcWaitForVBlankBeforeEnable:ilk,snb
-	 *
-	 * It is also worth mentioning that since work->scheduled_vblank can be
-	 * updated multiple times by the other threads, hitting the timeout is
-	 * not an error condition. We'll just end up hitting the "goto retry"
-	 * case below.
-	 */
-	wait_event_timeout(vblank->queue,
-		drm_crtc_vblank_count(&crtc->base) != work->scheduled_vblank,
-		msecs_to_jiffies(50));
-
-	mutex_lock(&fbc->lock);
-
-	/* Were we cancelled? */
-	if (!work->scheduled)
-		goto out;
-
-	/* Were we delayed again while this function was sleeping? */
-	if (drm_crtc_vblank_count(&crtc->base) == work->scheduled_vblank) {
-		mutex_unlock(&fbc->lock);
-		goto retry;
-	}
-
-	intel_fbc_hw_activate(dev_priv);
-
-	work->scheduled = false;
-
-out:
-	mutex_unlock(&fbc->lock);
-	drm_crtc_vblank_put(&crtc->base);
-}
-
-static void intel_fbc_schedule_activation(struct intel_crtc *crtc)
-{
-	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
-	struct intel_fbc *fbc = &dev_priv->fbc;
-	struct intel_fbc_work *work = &fbc->work;
-
-	WARN_ON(!mutex_is_locked(&fbc->lock));
-	if (WARN_ON(!fbc->enabled))
-		return;
-
-	if (drm_crtc_vblank_get(&crtc->base)) {
-		DRM_ERROR("vblank not available for FBC on pipe %c\n",
-			  pipe_name(crtc->pipe));
-		return;
-	}
-
-	/* It is useless to call intel_fbc_cancel_work() or cancel_work() in
-	 * this function since we're not releasing fbc.lock, so it won't have an
-	 * opportunity to grab it to discover that it was cancelled. So we just
-	 * update the expected jiffy count. */
-	work->scheduled = true;
-	work->scheduled_vblank = drm_crtc_vblank_count(&crtc->base);
-	drm_crtc_vblank_put(&crtc->base);
-
-	schedule_work(&work->work);
-}
-
 static void intel_fbc_deactivate(struct drm_i915_private *dev_priv,
 				 const char *reason)
 {
@@ -489,11 +406,6 @@ static void intel_fbc_deactivate(struct drm_i915_private *dev_priv,
 
 	WARN_ON(!mutex_is_locked(&fbc->lock));
 
-	/* Calling cancel_work() here won't help due to the fact that the work
-	 * function grabs fbc->lock. Just set scheduled to false so the work
-	 * function can know it was cancelled. */
-	fbc->work.scheduled = false;
-
 	if (fbc->active)
 		intel_fbc_hw_deactivate(dev_priv);
 
@@ -924,13 +836,6 @@ static void intel_fbc_get_reg_params(struct intel_crtc *crtc,
 						32 * fbc->threshold) * 8;
 }
 
-static bool intel_fbc_reg_params_equal(struct intel_fbc_reg_params *params1,
-				       struct intel_fbc_reg_params *params2)
-{
-	/* We can use this since intel_fbc_get_reg_params() does a memset. */
-	return memcmp(params1, params2, sizeof(*params1)) == 0;
-}
-
 void intel_fbc_pre_update(struct intel_crtc *crtc,
 			  struct intel_crtc_state *crtc_state,
 			  struct intel_plane_state *plane_state)
@@ -953,6 +858,7 @@ void intel_fbc_pre_update(struct intel_crtc *crtc,
 		goto unlock;
 
 	intel_fbc_update_state_cache(crtc, crtc_state, plane_state);
+	fbc->flip_pending = true;
 
 deactivate:
 	intel_fbc_deactivate(dev_priv, reason);
@@ -988,13 +894,15 @@ static void __intel_fbc_post_update(struct intel_crtc *crtc)
 {
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	struct intel_fbc *fbc = &dev_priv->fbc;
-	struct intel_fbc_reg_params old_params;
 
 	WARN_ON(!mutex_is_locked(&fbc->lock));
 
 	if (!fbc->enabled || fbc->crtc != crtc)
 		return;
 
+	fbc->flip_pending = false;
+	WARN_ON(fbc->active);
+
 	if (!i915_modparams.enable_fbc) {
 		intel_fbc_deactivate(dev_priv, "disabled at runtime per module param");
 		__intel_fbc_disable(dev_priv);
@@ -1002,25 +910,16 @@ static void __intel_fbc_post_update(struct intel_crtc *crtc)
 		return;
 	}
 
-	if (!intel_fbc_can_activate(crtc)) {
-		WARN_ON(fbc->active);
-		return;
-	}
-
-	old_params = fbc->params;
 	intel_fbc_get_reg_params(crtc, &fbc->params);
 
-	/* If the scanout has not changed, don't modify the FBC settings.
-	 * Note that we make the fundamental assumption that the fb->obj
-	 * cannot be unpinned (and have its GTT offset and fence revoked)
-	 * without first being decoupled from the scanout and FBC disabled.
-	 */
-	if (fbc->active &&
-	    intel_fbc_reg_params_equal(&old_params, &fbc->params))
+	if (!intel_fbc_can_activate(crtc))
 		return;
 
-	intel_fbc_deactivate(dev_priv, "FBC enabled (active or scheduled)");
-	intel_fbc_schedule_activation(crtc);
+	if (!fbc->busy_bits) {
+		intel_fbc_deactivate(dev_priv, "FBC enabled (active or scheduled)");
+		intel_fbc_hw_activate(dev_priv);
+	} else
+		intel_fbc_deactivate(dev_priv, "frontbuffer write");
 }
 
 void intel_fbc_post_update(struct intel_crtc *crtc)
@@ -1085,7 +984,7 @@ void intel_fbc_flush(struct drm_i915_private *dev_priv,
 	    (frontbuffer_bits & intel_fbc_get_frontbuffer_bit(fbc))) {
 		if (fbc->active)
 			intel_fbc_recompress(dev_priv);
-		else
+		else if (!fbc->flip_pending)
 			__intel_fbc_post_update(fbc->crtc);
 	}
 
@@ -1225,8 +1124,6 @@ void intel_fbc_disable(struct intel_crtc *crtc)
 	if (fbc->crtc == crtc)
 		__intel_fbc_disable(dev_priv);
 	mutex_unlock(&fbc->lock);
-
-	cancel_work_sync(&fbc->work.work);
 }
 
 /**
@@ -1248,8 +1145,6 @@ void intel_fbc_global_disable(struct drm_i915_private *dev_priv)
 		__intel_fbc_disable(dev_priv);
 	}
 	mutex_unlock(&fbc->lock);
-
-	cancel_work_sync(&fbc->work.work);
 }
 
 static void intel_fbc_underrun_work_fn(struct work_struct *work)
@@ -1400,12 +1295,10 @@ void intel_fbc_init(struct drm_i915_private *dev_priv)
 {
 	struct intel_fbc *fbc = &dev_priv->fbc;
 
-	INIT_WORK(&fbc->work.work, intel_fbc_work_fn);
 	INIT_WORK(&fbc->underrun_work, intel_fbc_underrun_work_fn);
 	mutex_init(&fbc->lock);
 	fbc->enabled = false;
 	fbc->active = false;
-	fbc->work.scheduled = false;
 
 	if (need_fbc_vtd_wa(dev_priv))
 		mkwrite_device_info(dev_priv)->has_fbc = false;
diff --git a/drivers/gpu/drm/i915/intel_guc.c b/drivers/gpu/drm/i915/intel_guc.c
index 1aff30b0870c..560c7406ae40 100644
--- a/drivers/gpu/drm/i915/intel_guc.c
+++ b/drivers/gpu/drm/i915/intel_guc.c
@@ -27,6 +27,8 @@
 #include "intel_guc_submission.h"
 #include "i915_drv.h"
 
+static void guc_init_ggtt_pin_bias(struct intel_guc *guc);
+
 static void gen8_guc_raise_irq(struct intel_guc *guc)
 {
 	struct drm_i915_private *dev_priv = guc_to_i915(guc);
@@ -73,7 +75,7 @@ void intel_guc_init_early(struct intel_guc *guc)
 	guc->notify = gen8_guc_raise_irq;
 }
 
-int intel_guc_init_wq(struct intel_guc *guc)
+static int guc_init_wq(struct intel_guc *guc)
 {
 	struct drm_i915_private *dev_priv = guc_to_i915(guc);
 
@@ -124,7 +126,7 @@ int intel_guc_init_wq(struct intel_guc *guc)
 	return 0;
 }
 
-void intel_guc_fini_wq(struct intel_guc *guc)
+static void guc_fini_wq(struct intel_guc *guc)
 {
 	struct drm_i915_private *dev_priv = guc_to_i915(guc);
 
@@ -135,6 +137,28 @@ void intel_guc_fini_wq(struct intel_guc *guc)
 	destroy_workqueue(guc->log.relay.flush_wq);
 }
 
+int intel_guc_init_misc(struct intel_guc *guc)
+{
+	struct drm_i915_private *i915 = guc_to_i915(guc);
+	int ret;
+
+	guc_init_ggtt_pin_bias(guc);
+
+	ret = guc_init_wq(guc);
+	if (ret)
+		return ret;
+
+	intel_uc_fw_fetch(i915, &guc->fw);
+
+	return 0;
+}
+
+void intel_guc_fini_misc(struct intel_guc *guc)
+{
+	intel_uc_fw_fini(&guc->fw);
+	guc_fini_wq(guc);
+}
+
 static int guc_shared_data_create(struct intel_guc *guc)
 {
 	struct i915_vma *vma;
@@ -169,7 +193,7 @@ int intel_guc_init(struct intel_guc *guc)
 
 	ret = guc_shared_data_create(guc);
 	if (ret)
-		return ret;
+		goto err_fetch;
 	GEM_BUG_ON(!guc->shared_data);
 
 	ret = intel_guc_log_create(&guc->log);
@@ -190,6 +214,8 @@ err_log:
 	intel_guc_log_destroy(&guc->log);
 err_shared:
 	guc_shared_data_destroy(guc);
+err_fetch:
+	intel_uc_fw_fini(&guc->fw);
 	return ret;
 }
 
@@ -201,12 +227,17 @@ void intel_guc_fini(struct intel_guc *guc)
 	intel_guc_ads_destroy(guc);
 	intel_guc_log_destroy(&guc->log);
 	guc_shared_data_destroy(guc);
+	intel_uc_fw_fini(&guc->fw);
 }
 
 static u32 guc_ctl_debug_flags(struct intel_guc *guc)
 {
 	u32 level = intel_guc_log_get_level(&guc->log);
-	u32 flags = 0;
+	u32 flags;
+	u32 ads;
+
+	ads = intel_guc_ggtt_offset(guc, guc->ads_vma) >> PAGE_SHIFT;
+	flags = ads << GUC_ADS_ADDR_SHIFT | GUC_ADS_ENABLED;
 
 	if (!GUC_LOG_LEVEL_IS_ENABLED(level))
 		flags |= GUC_LOG_DEFAULT_DISABLED;
@@ -217,13 +248,6 @@ static u32 guc_ctl_debug_flags(struct intel_guc *guc)
 		flags |= GUC_LOG_LEVEL_TO_VERBOSITY(level) <<
 			 GUC_LOG_VERBOSITY_SHIFT;
 
-	if (USES_GUC_SUBMISSION(guc_to_i915(guc))) {
-		u32 ads = intel_guc_ggtt_offset(guc, guc->ads_vma)
-			>> PAGE_SHIFT;
-
-		flags |= ads << GUC_ADS_ADDR_SHIFT | GUC_ADS_ENABLED;
-	}
-
 	return flags;
 }
 
@@ -327,6 +351,9 @@ void intel_guc_init_params(struct intel_guc *guc)
 	params[GUC_CTL_DEBUG] = guc_ctl_debug_flags(guc);
 	params[GUC_CTL_CTXINFO] = guc_ctl_ctxinfo_flags(guc);
 
+	for (i = 0; i < GUC_CTL_MAX_DWORDS; i++)
+		DRM_DEBUG_DRIVER("param[%2d] = %#x\n", i, params[i]);
+
 	/*
 	 * All SOFT_SCRATCH registers are in FORCEWAKE_BLITTER domain and
 	 * they are power context saved so it's ok to release forcewake
@@ -439,11 +466,13 @@ void intel_guc_to_host_event_handler_mmio(struct intel_guc *guc)
 	 * could happen that GuC sets the bit for 2nd interrupt but Host
 	 * clears out the bit on handling the 1st interrupt.
 	 */
+	disable_rpm_wakeref_asserts(dev_priv);
 	spin_lock(&guc->irq_lock);
 	val = I915_READ(SOFT_SCRATCH(15));
 	msg = val & guc->msg_enabled_mask;
 	I915_WRITE(SOFT_SCRATCH(15), val & ~msg);
 	spin_unlock(&guc->irq_lock);
+	enable_rpm_wakeref_asserts(dev_priv);
 
 	intel_guc_to_host_process_recv_msg(guc, msg);
 }
@@ -585,13 +614,13 @@ int intel_guc_resume(struct intel_guc *guc)
  */
 
 /**
- * intel_guc_init_ggtt_pin_bias() - Initialize the GuC ggtt_pin_bias value.
+ * guc_init_ggtt_pin_bias() - Initialize the GuC ggtt_pin_bias value.
  * @guc: intel_guc structure.
  *
  * This function will calculate and initialize the ggtt_pin_bias value based on
  * overall WOPCM size and GuC WOPCM size.
  */
-void intel_guc_init_ggtt_pin_bias(struct intel_guc *guc)
+static void guc_init_ggtt_pin_bias(struct intel_guc *guc)
 {
 	struct drm_i915_private *i915 = guc_to_i915(guc);
 
diff --git a/drivers/gpu/drm/i915/intel_guc.h b/drivers/gpu/drm/i915/intel_guc.h
index f1265e122d30..4121928a495e 100644
--- a/drivers/gpu/drm/i915/intel_guc.h
+++ b/drivers/gpu/drm/i915/intel_guc.h
@@ -151,11 +151,10 @@ static inline u32 intel_guc_ggtt_offset(struct intel_guc *guc,
 void intel_guc_init_early(struct intel_guc *guc);
 void intel_guc_init_send_regs(struct intel_guc *guc);
 void intel_guc_init_params(struct intel_guc *guc);
-void intel_guc_init_ggtt_pin_bias(struct intel_guc *guc);
-int intel_guc_init_wq(struct intel_guc *guc);
-void intel_guc_fini_wq(struct intel_guc *guc);
+int intel_guc_init_misc(struct intel_guc *guc);
 int intel_guc_init(struct intel_guc *guc);
 void intel_guc_fini(struct intel_guc *guc);
+void intel_guc_fini_misc(struct intel_guc *guc);
 int intel_guc_send_nop(struct intel_guc *guc, const u32 *action, u32 len,
 		       u32 *response_buf, u32 response_buf_size);
 int intel_guc_send_mmio(struct intel_guc *guc, const u32 *action, u32 len,
diff --git a/drivers/gpu/drm/i915/intel_guc_submission.c b/drivers/gpu/drm/i915/intel_guc_submission.c
index f3945258fe1b..4aa5e6463e7b 100644
--- a/drivers/gpu/drm/i915/intel_guc_submission.c
+++ b/drivers/gpu/drm/i915/intel_guc_submission.c
@@ -628,13 +628,14 @@ static void complete_preempt_context(struct intel_engine_cs *engine)
 
 	GEM_BUG_ON(!execlists_is_active(execlists, EXECLISTS_ACTIVE_PREEMPT));
 
+	if (inject_preempt_hang(execlists))
+		return;
+
 	execlists_cancel_port_requests(execlists);
 	execlists_unwind_incomplete_requests(execlists);
 
 	wait_for_guc_preempt_report(engine);
 	intel_write_status_page(engine, I915_GEM_HWS_PREEMPT_INDEX, 0);
-
-	execlists_clear_active(execlists, EXECLISTS_ACTIVE_PREEMPT);
 }
 
 /**
@@ -695,9 +696,6 @@ static bool __guc_dequeue(struct intel_engine_cs *engine)
 
 	lockdep_assert_held(&engine->timeline.lock);
 
-	rb = execlists->first;
-	GEM_BUG_ON(rb_first(&execlists->queue) != rb);
-
 	if (port_isset(port)) {
 		if (intel_engine_has_preemption(engine)) {
 			struct guc_preempt_work *preempt_work =
@@ -719,7 +717,7 @@ static bool __guc_dequeue(struct intel_engine_cs *engine)
 	}
 	GEM_BUG_ON(port_isset(port));
 
-	while (rb) {
+	while ((rb = rb_first_cached(&execlists->queue))) {
 		struct i915_priolist *p = to_priolist(rb);
 		struct i915_request *rq, *rn;
 
@@ -744,15 +742,13 @@ static bool __guc_dequeue(struct intel_engine_cs *engine)
 			submit = true;
 		}
 
-		rb = rb_next(rb);
-		rb_erase(&p->node, &execlists->queue);
+		rb_erase_cached(&p->node, &execlists->queue);
 		INIT_LIST_HEAD(&p->requests);
 		if (p->priority != I915_PRIORITY_NORMAL)
 			kmem_cache_free(engine->i915->priorities, p);
 	}
 done:
 	execlists->queue_priority = rb ? to_priolist(rb)->priority : INT_MIN;
-	execlists->first = rb;
 	if (submit)
 		port_assign(port, last);
 	if (last)
@@ -761,7 +757,8 @@ done:
 	/* We must always keep the beast fed if we have work piled up */
 	GEM_BUG_ON(port_isset(execlists->port) &&
 		   !execlists_is_active(execlists, EXECLISTS_ACTIVE_USER));
-	GEM_BUG_ON(execlists->first && !port_isset(execlists->port));
+	GEM_BUG_ON(rb_first_cached(&execlists->queue) &&
+		   !port_isset(execlists->port));
 
 	return submit;
 }
@@ -914,8 +911,12 @@ static void guc_clients_doorbell_fini(struct intel_guc *guc)
 		__update_doorbell_desc(guc->preempt_client,
 				       GUC_DOORBELL_INVALID);
 	}
-	__destroy_doorbell(guc->execbuf_client);
-	__update_doorbell_desc(guc->execbuf_client, GUC_DOORBELL_INVALID);
+
+	if (guc->execbuf_client) {
+		__destroy_doorbell(guc->execbuf_client);
+		__update_doorbell_desc(guc->execbuf_client,
+				       GUC_DOORBELL_INVALID);
+	}
 }
 
 /**
@@ -1128,7 +1129,8 @@ static void guc_clients_destroy(struct intel_guc *guc)
 		guc_client_free(client);
 
 	client = fetch_and_zero(&guc->execbuf_client);
-	guc_client_free(client);
+	if (client)
+		guc_client_free(client);
 }
 
 /*
@@ -1183,7 +1185,8 @@ void intel_guc_submission_fini(struct intel_guc *guc)
 	guc_clients_destroy(guc);
 	WARN_ON(!guc_verify_doorbells(guc));
 
-	guc_stage_desc_pool_destroy(guc);
+	if (guc->stage_desc_pool)
+		guc_stage_desc_pool_destroy(guc);
 }
 
 static void guc_interrupts_capture(struct drm_i915_private *dev_priv)
@@ -1266,6 +1269,31 @@ static void guc_submission_unpark(struct intel_engine_cs *engine)
 	intel_engine_pin_breadcrumbs_irq(engine);
 }
 
+static void guc_set_default_submission(struct intel_engine_cs *engine)
+{
+	/*
+	 * We inherit a bunch of functions from execlists that we'd like
+	 * to keep using:
+	 *
+	 *    engine->submit_request = execlists_submit_request;
+	 *    engine->cancel_requests = execlists_cancel_requests;
+	 *    engine->schedule = execlists_schedule;
+	 *
+	 * But we need to override the actual submission backend in order
+	 * to talk to the GuC.
+	 */
+	intel_execlists_set_default_submission(engine);
+
+	engine->execlists.tasklet.func = guc_submission_tasklet;
+
+	engine->park = guc_submission_park;
+	engine->unpark = guc_submission_unpark;
+
+	engine->reset.prepare = guc_reset_prepare;
+
+	engine->flags &= ~I915_ENGINE_SUPPORTS_STATS;
+}
+
 int intel_guc_submission_enable(struct intel_guc *guc)
 {
 	struct drm_i915_private *dev_priv = guc_to_i915(guc);
@@ -1304,17 +1332,8 @@ int intel_guc_submission_enable(struct intel_guc *guc)
 	guc_interrupts_capture(dev_priv);
 
 	for_each_engine(engine, dev_priv, id) {
-		struct intel_engine_execlists * const execlists =
-			&engine->execlists;
-
-		execlists->tasklet.func = guc_submission_tasklet;
-
-		engine->reset.prepare = guc_reset_prepare;
-
-		engine->park = guc_submission_park;
-		engine->unpark = guc_submission_unpark;
-
-		engine->flags &= ~I915_ENGINE_SUPPORTS_STATS;
+		engine->set_default_submission = guc_set_default_submission;
+		engine->set_default_submission(engine);
 	}
 
 	return 0;
@@ -1328,9 +1347,6 @@ void intel_guc_submission_disable(struct intel_guc *guc)
 
 	guc_interrupts_release(dev_priv);
 	guc_clients_doorbell_fini(guc);
-
-	/* Revert back to manual ELSP submission */
-	intel_engines_reset_default_submission(dev_priv);
 }
 
 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
diff --git a/drivers/gpu/drm/i915/intel_gvt.c b/drivers/gpu/drm/i915/intel_gvt.c
index a6291f60545b..c22b3e18a0f5 100644
--- a/drivers/gpu/drm/i915/intel_gvt.c
+++ b/drivers/gpu/drm/i915/intel_gvt.c
@@ -92,6 +92,9 @@ int intel_gvt_init(struct drm_i915_private *dev_priv)
 {
 	int ret;
 
+	if (i915_inject_load_failure())
+		return -ENODEV;
+
 	if (!i915_modparams.enable_gvt) {
 		DRM_DEBUG_DRIVER("GVT-g is disabled by kernel params\n");
 		return 0;
diff --git a/drivers/gpu/drm/i915/intel_hdmi.c b/drivers/gpu/drm/i915/intel_hdmi.c
index 8363fbd18ee8..a9076402dcb0 100644
--- a/drivers/gpu/drm/i915/intel_hdmi.c
+++ b/drivers/gpu/drm/i915/intel_hdmi.c
@@ -1899,6 +1899,8 @@ intel_hdmi_set_edid(struct drm_connector *connector)
 		connected = true;
 	}
 
+	cec_notifier_set_phys_addr_from_edid(intel_hdmi->cec_notifier, edid);
+
 	return connected;
 }
 
@@ -1907,6 +1909,7 @@ intel_hdmi_detect(struct drm_connector *connector, bool force)
 {
 	enum drm_connector_status status;
 	struct drm_i915_private *dev_priv = to_i915(connector->dev);
+	struct intel_hdmi *intel_hdmi = intel_attached_hdmi(connector);
 
 	DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n",
 		      connector->base.id, connector->name);
@@ -1922,6 +1925,9 @@ intel_hdmi_detect(struct drm_connector *connector, bool force)
 
 	intel_display_power_put(dev_priv, POWER_DOMAIN_GMBUS);
 
+	if (status != connector_status_connected)
+		cec_notifier_phys_addr_invalidate(intel_hdmi->cec_notifier);
+
 	return status;
 }
 
@@ -2062,6 +2068,8 @@ static void chv_hdmi_pre_enable(struct intel_encoder *encoder,
 
 static void intel_hdmi_destroy(struct drm_connector *connector)
 {
+	if (intel_attached_hdmi(connector)->cec_notifier)
+		cec_notifier_put(intel_attached_hdmi(connector)->cec_notifier);
 	kfree(to_intel_connector(connector)->detect_edid);
 	drm_connector_cleanup(connector);
 	kfree(connector);
@@ -2382,6 +2390,11 @@ void intel_hdmi_init_connector(struct intel_digital_port *intel_dig_port,
 		u32 temp = I915_READ(PEG_BAND_GAP_DATA);
 		I915_WRITE(PEG_BAND_GAP_DATA, (temp & ~0xf) | 0xd);
 	}
+
+	intel_hdmi->cec_notifier = cec_notifier_get_conn(dev->dev,
+							 port_identifier(port));
+	if (!intel_hdmi->cec_notifier)
+		DRM_DEBUG_KMS("CEC notifier get failed\n");
 }
 
 void intel_hdmi_init(struct drm_i915_private *dev_priv,
diff --git a/drivers/gpu/drm/i915/intel_hotplug.c b/drivers/gpu/drm/i915/intel_hotplug.c
index 43aa92beff2a..648a13c6043c 100644
--- a/drivers/gpu/drm/i915/intel_hotplug.c
+++ b/drivers/gpu/drm/i915/intel_hotplug.c
@@ -77,37 +77,6 @@
  */
 
 /**
- * intel_hpd_port - return port hard associated with certain pin.
- * @dev_priv: private driver data pointer
- * @pin: the hpd pin to get associated port
- *
- * Return port that is associatade with @pin and PORT_NONE if no port is
- * hard associated with that @pin.
- */
-enum port intel_hpd_pin_to_port(struct drm_i915_private *dev_priv,
-				enum hpd_pin pin)
-{
-	switch (pin) {
-	case HPD_PORT_A:
-		return PORT_A;
-	case HPD_PORT_B:
-		return PORT_B;
-	case HPD_PORT_C:
-		return PORT_C;
-	case HPD_PORT_D:
-		return PORT_D;
-	case HPD_PORT_E:
-		if (IS_CNL_WITH_PORT_F(dev_priv))
-			return PORT_F;
-		return PORT_E;
-	case HPD_PORT_F:
-		return PORT_F;
-	default:
-		return PORT_NONE; /* no port for this pin */
-	}
-}
-
-/**
  * intel_hpd_pin_default - return default pin associated with certain port.
  * @dev_priv: private driver data pointer
  * @port: the hpd port to get associated pin
@@ -241,25 +210,25 @@ static void intel_hpd_irq_storm_reenable_work(struct work_struct *work)
 		container_of(work, typeof(*dev_priv),
 			     hotplug.reenable_work.work);
 	struct drm_device *dev = &dev_priv->drm;
-	int i;
+	enum hpd_pin pin;
 
 	intel_runtime_pm_get(dev_priv);
 
 	spin_lock_irq(&dev_priv->irq_lock);
-	for_each_hpd_pin(i) {
+	for_each_hpd_pin(pin) {
 		struct drm_connector *connector;
 		struct drm_connector_list_iter conn_iter;
 
-		if (dev_priv->hotplug.stats[i].state != HPD_DISABLED)
+		if (dev_priv->hotplug.stats[pin].state != HPD_DISABLED)
 			continue;
 
-		dev_priv->hotplug.stats[i].state = HPD_ENABLED;
+		dev_priv->hotplug.stats[pin].state = HPD_ENABLED;
 
 		drm_connector_list_iter_begin(dev, &conn_iter);
 		drm_for_each_connector_iter(connector, &conn_iter) {
 			struct intel_connector *intel_connector = to_intel_connector(connector);
 
-			if (intel_connector->encoder->hpd_pin == i) {
+			if (intel_connector->encoder->hpd_pin == pin) {
 				if (connector->polled != intel_connector->polled)
 					DRM_DEBUG_DRIVER("Reenabling HPD on connector %s\n",
 							 connector->name);
@@ -301,13 +270,18 @@ bool intel_encoder_hotplug(struct intel_encoder *encoder,
 	return true;
 }
 
+static bool intel_encoder_has_hpd_pulse(struct intel_encoder *encoder)
+{
+	return intel_encoder_is_dig_port(encoder) &&
+		enc_to_dig_port(&encoder->base)->hpd_pulse != NULL;
+}
+
 static void i915_digport_work_func(struct work_struct *work)
 {
 	struct drm_i915_private *dev_priv =
 		container_of(work, struct drm_i915_private, hotplug.dig_port_work);
 	u32 long_port_mask, short_port_mask;
-	struct intel_digital_port *intel_dig_port;
-	int i;
+	struct intel_encoder *encoder;
 	u32 old_bits = 0;
 
 	spin_lock_irq(&dev_priv->irq_lock);
@@ -317,27 +291,27 @@ static void i915_digport_work_func(struct work_struct *work)
 	dev_priv->hotplug.short_port_mask = 0;
 	spin_unlock_irq(&dev_priv->irq_lock);
 
-	for (i = 0; i < I915_MAX_PORTS; i++) {
-		bool valid = false;
-		bool long_hpd = false;
-		intel_dig_port = dev_priv->hotplug.irq_port[i];
-		if (!intel_dig_port || !intel_dig_port->hpd_pulse)
+	for_each_intel_encoder(&dev_priv->drm, encoder) {
+		struct intel_digital_port *dig_port;
+		enum port port = encoder->port;
+		bool long_hpd, short_hpd;
+		enum irqreturn ret;
+
+		if (!intel_encoder_has_hpd_pulse(encoder))
 			continue;
 
-		if (long_port_mask & (1 << i))  {
-			valid = true;
-			long_hpd = true;
-		} else if (short_port_mask & (1 << i))
-			valid = true;
+		long_hpd = long_port_mask & BIT(port);
+		short_hpd = short_port_mask & BIT(port);
 
-		if (valid) {
-			enum irqreturn ret;
+		if (!long_hpd && !short_hpd)
+			continue;
 
-			ret = intel_dig_port->hpd_pulse(intel_dig_port, long_hpd);
-			if (ret == IRQ_NONE) {
-				/* fall back to old school hpd */
-				old_bits |= (1 << intel_dig_port->base.hpd_pin);
-			}
+		dig_port = enc_to_dig_port(&encoder->base);
+
+		ret = dig_port->hpd_pulse(dig_port, long_hpd);
+		if (ret == IRQ_NONE) {
+			/* fall back to old school hpd */
+			old_bits |= BIT(encoder->hpd_pin);
 		}
 	}
 
@@ -418,26 +392,24 @@ static void i915_hotplug_work_func(struct work_struct *work)
 void intel_hpd_irq_handler(struct drm_i915_private *dev_priv,
 			   u32 pin_mask, u32 long_mask)
 {
-	int i;
-	enum port port;
+	struct intel_encoder *encoder;
 	bool storm_detected = false;
 	bool queue_dig = false, queue_hp = false;
-	bool is_dig_port;
 
 	if (!pin_mask)
 		return;
 
 	spin_lock(&dev_priv->irq_lock);
-	for_each_hpd_pin(i) {
-		if (!(BIT(i) & pin_mask))
-			continue;
+	for_each_intel_encoder(&dev_priv->drm, encoder) {
+		enum hpd_pin pin = encoder->hpd_pin;
+		bool has_hpd_pulse = intel_encoder_has_hpd_pulse(encoder);
 
-		port = intel_hpd_pin_to_port(dev_priv, i);
-		is_dig_port = port != PORT_NONE &&
-			dev_priv->hotplug.irq_port[port];
+		if (!(BIT(pin) & pin_mask))
+			continue;
 
-		if (is_dig_port) {
-			bool long_hpd = long_mask & BIT(i);
+		if (has_hpd_pulse) {
+			bool long_hpd = long_mask & BIT(pin);
+			enum port port = encoder->port;
 
 			DRM_DEBUG_DRIVER("digital hpd port %c - %s\n", port_name(port),
 					 long_hpd ? "long" : "short");
@@ -455,7 +427,7 @@ void intel_hpd_irq_handler(struct drm_i915_private *dev_priv,
 			}
 		}
 
-		if (dev_priv->hotplug.stats[i].state == HPD_DISABLED) {
+		if (dev_priv->hotplug.stats[pin].state == HPD_DISABLED) {
 			/*
 			 * On GMCH platforms the interrupt mask bits only
 			 * prevent irq generation, not the setting of the
@@ -463,20 +435,20 @@ void intel_hpd_irq_handler(struct drm_i915_private *dev_priv,
 			 * interrupts on saner platforms.
 			 */
 			WARN_ONCE(!HAS_GMCH_DISPLAY(dev_priv),
-				  "Received HPD interrupt on pin %d although disabled\n", i);
+				  "Received HPD interrupt on pin %d although disabled\n", pin);
 			continue;
 		}
 
-		if (dev_priv->hotplug.stats[i].state != HPD_ENABLED)
+		if (dev_priv->hotplug.stats[pin].state != HPD_ENABLED)
 			continue;
 
-		if (!is_dig_port) {
-			dev_priv->hotplug.event_bits |= BIT(i);
+		if (!has_hpd_pulse) {
+			dev_priv->hotplug.event_bits |= BIT(pin);
 			queue_hp = true;
 		}
 
-		if (intel_hpd_irq_storm_detect(dev_priv, i)) {
-			dev_priv->hotplug.event_bits &= ~BIT(i);
+		if (intel_hpd_irq_storm_detect(dev_priv, pin)) {
+			dev_priv->hotplug.event_bits &= ~BIT(pin);
 			storm_detected = true;
 		}
 	}
diff --git a/drivers/gpu/drm/i915/intel_huc.c b/drivers/gpu/drm/i915/intel_huc.c
index 291285277403..ffcad5fad6a7 100644
--- a/drivers/gpu/drm/i915/intel_huc.c
+++ b/drivers/gpu/drm/i915/intel_huc.c
@@ -32,6 +32,14 @@ void intel_huc_init_early(struct intel_huc *huc)
 	intel_huc_fw_init_early(huc);
 }
 
+int intel_huc_init_misc(struct intel_huc *huc)
+{
+	struct drm_i915_private *i915 = huc_to_i915(huc);
+
+	intel_uc_fw_fetch(i915, &huc->fw);
+	return 0;
+}
+
 /**
  * intel_huc_auth() - Authenticate HuC uCode
  * @huc: intel_huc structure
diff --git a/drivers/gpu/drm/i915/intel_huc.h b/drivers/gpu/drm/i915/intel_huc.h
index aa854907abac..7e41d870b509 100644
--- a/drivers/gpu/drm/i915/intel_huc.h
+++ b/drivers/gpu/drm/i915/intel_huc.h
@@ -36,9 +36,15 @@ struct intel_huc {
 };
 
 void intel_huc_init_early(struct intel_huc *huc);
+int intel_huc_init_misc(struct intel_huc *huc);
 int intel_huc_auth(struct intel_huc *huc);
 int intel_huc_check_status(struct intel_huc *huc);
 
+static inline void intel_huc_fini_misc(struct intel_huc *huc)
+{
+	intel_uc_fw_fini(&huc->fw);
+}
+
 static inline int intel_huc_sanitize(struct intel_huc *huc)
 {
 	intel_uc_fw_sanitize(&huc->fw);
diff --git a/drivers/gpu/drm/i915/intel_i2c.c b/drivers/gpu/drm/i915/intel_i2c.c
index 97606c1be70d..bef32b7c248e 100644
--- a/drivers/gpu/drm/i915/intel_i2c.c
+++ b/drivers/gpu/drm/i915/intel_i2c.c
@@ -361,15 +361,39 @@ gmbus_wait_idle(struct drm_i915_private *dev_priv)
 	return ret;
 }
 
+static inline
+unsigned int gmbus_max_xfer_size(struct drm_i915_private *dev_priv)
+{
+	return INTEL_GEN(dev_priv) >= 9 ? GEN9_GMBUS_BYTE_COUNT_MAX :
+	       GMBUS_BYTE_COUNT_MAX;
+}
+
 static int
 gmbus_xfer_read_chunk(struct drm_i915_private *dev_priv,
 		      unsigned short addr, u8 *buf, unsigned int len,
-		      u32 gmbus1_index)
+		      u32 gmbus0_reg, u32 gmbus1_index)
 {
+	unsigned int size = len;
+	bool burst_read = len > gmbus_max_xfer_size(dev_priv);
+	bool extra_byte_added = false;
+
+	if (burst_read) {
+		/*
+		 * As per HW Spec, for 512Bytes need to read extra Byte and
+		 * Ignore the extra byte read.
+		 */
+		if (len == 512) {
+			extra_byte_added = true;
+			len++;
+		}
+		size = len % 256 + 256;
+		I915_WRITE_FW(GMBUS0, gmbus0_reg | GMBUS_BYTE_CNT_OVERRIDE);
+	}
+
 	I915_WRITE_FW(GMBUS1,
 		      gmbus1_index |
 		      GMBUS_CYCLE_WAIT |
-		      (len << GMBUS_BYTE_COUNT_SHIFT) |
+		      (size << GMBUS_BYTE_COUNT_SHIFT) |
 		      (addr << GMBUS_SLAVE_ADDR_SHIFT) |
 		      GMBUS_SLAVE_READ | GMBUS_SW_RDY);
 	while (len) {
@@ -382,17 +406,34 @@ gmbus_xfer_read_chunk(struct drm_i915_private *dev_priv,
 
 		val = I915_READ_FW(GMBUS3);
 		do {
+			if (extra_byte_added && len == 1)
+				break;
+
 			*buf++ = val & 0xff;
 			val >>= 8;
 		} while (--len && ++loop < 4);
+
+		if (burst_read && len == size - 4)
+			/* Reset the override bit */
+			I915_WRITE_FW(GMBUS0, gmbus0_reg);
 	}
 
 	return 0;
 }
 
+/*
+ * HW spec says that 512Bytes in Burst read need special treatment.
+ * But it doesn't talk about other multiple of 256Bytes. And couldn't locate
+ * an I2C slave, which supports such a lengthy burst read too for experiments.
+ *
+ * So until things get clarified on HW support, to avoid the burst read length
+ * in fold of 256Bytes except 512, max burst read length is fixed at 767Bytes.
+ */
+#define INTEL_GMBUS_BURST_READ_MAX_LEN		767U
+
 static int
 gmbus_xfer_read(struct drm_i915_private *dev_priv, struct i2c_msg *msg,
-		u32 gmbus1_index)
+		u32 gmbus0_reg, u32 gmbus1_index)
 {
 	u8 *buf = msg->buf;
 	unsigned int rx_size = msg->len;
@@ -400,10 +441,13 @@ gmbus_xfer_read(struct drm_i915_private *dev_priv, struct i2c_msg *msg,
 	int ret;
 
 	do {
-		len = min(rx_size, GMBUS_BYTE_COUNT_MAX);
+		if (HAS_GMBUS_BURST_READ(dev_priv))
+			len = min(rx_size, INTEL_GMBUS_BURST_READ_MAX_LEN);
+		else
+			len = min(rx_size, gmbus_max_xfer_size(dev_priv));
 
-		ret = gmbus_xfer_read_chunk(dev_priv, msg->addr,
-					    buf, len, gmbus1_index);
+		ret = gmbus_xfer_read_chunk(dev_priv, msg->addr, buf, len,
+					    gmbus0_reg, gmbus1_index);
 		if (ret)
 			return ret;
 
@@ -462,7 +506,7 @@ gmbus_xfer_write(struct drm_i915_private *dev_priv, struct i2c_msg *msg,
 	int ret;
 
 	do {
-		len = min(tx_size, GMBUS_BYTE_COUNT_MAX);
+		len = min(tx_size, gmbus_max_xfer_size(dev_priv));
 
 		ret = gmbus_xfer_write_chunk(dev_priv, msg->addr, buf, len,
 					     gmbus1_index);
@@ -491,7 +535,8 @@ gmbus_is_index_xfer(struct i2c_msg *msgs, int i, int num)
 }
 
 static int
-gmbus_index_xfer(struct drm_i915_private *dev_priv, struct i2c_msg *msgs)
+gmbus_index_xfer(struct drm_i915_private *dev_priv, struct i2c_msg *msgs,
+		 u32 gmbus0_reg)
 {
 	u32 gmbus1_index = 0;
 	u32 gmbus5 = 0;
@@ -509,7 +554,8 @@ gmbus_index_xfer(struct drm_i915_private *dev_priv, struct i2c_msg *msgs)
 		I915_WRITE_FW(GMBUS5, gmbus5);
 
 	if (msgs[1].flags & I2C_M_RD)
-		ret = gmbus_xfer_read(dev_priv, &msgs[1], gmbus1_index);
+		ret = gmbus_xfer_read(dev_priv, &msgs[1], gmbus0_reg,
+				      gmbus1_index);
 	else
 		ret = gmbus_xfer_write(dev_priv, &msgs[1], gmbus1_index);
 
@@ -544,10 +590,12 @@ retry:
 	for (; i < num; i += inc) {
 		inc = 1;
 		if (gmbus_is_index_xfer(msgs, i, num)) {
-			ret = gmbus_index_xfer(dev_priv, &msgs[i]);
+			ret = gmbus_index_xfer(dev_priv, &msgs[i],
+					       gmbus0_source | bus->reg0);
 			inc = 2; /* an index transmission is two msgs */
 		} else if (msgs[i].flags & I2C_M_RD) {
-			ret = gmbus_xfer_read(dev_priv, &msgs[i], 0);
+			ret = gmbus_xfer_read(dev_priv, &msgs[i],
+					      gmbus0_source | bus->reg0, 0);
 		} else {
 			ret = gmbus_xfer_write(dev_priv, &msgs[i], 0);
 		}
diff --git a/drivers/gpu/drm/i915/intel_lpe_audio.c b/drivers/gpu/drm/i915/intel_lpe_audio.c
index 6269750e2b54..cdf19553ffac 100644
--- a/drivers/gpu/drm/i915/intel_lpe_audio.c
+++ b/drivers/gpu/drm/i915/intel_lpe_audio.c
@@ -62,6 +62,7 @@
 
 #include <linux/acpi.h>
 #include <linux/device.h>
+#include <linux/irq.h>
 #include <linux/pci.h>
 #include <linux/pm_runtime.h>
 
@@ -126,9 +127,7 @@ lpe_audio_platdev_create(struct drm_i915_private *dev_priv)
 		return platdev;
 	}
 
-	pm_runtime_forbid(&platdev->dev);
-	pm_runtime_set_active(&platdev->dev);
-	pm_runtime_enable(&platdev->dev);
+	pm_runtime_no_callbacks(&platdev->dev);
 
 	return platdev;
 }
diff --git a/drivers/gpu/drm/i915/intel_lrc.c b/drivers/gpu/drm/i915/intel_lrc.c
index 33bc914c2ef5..174479232e94 100644
--- a/drivers/gpu/drm/i915/intel_lrc.c
+++ b/drivers/gpu/drm/i915/intel_lrc.c
@@ -137,6 +137,7 @@
 #include <drm/i915_drm.h>
 #include "i915_drv.h"
 #include "i915_gem_render_state.h"
+#include "i915_vgpu.h"
 #include "intel_lrc_reg.h"
 #include "intel_mocs.h"
 #include "intel_workarounds.h"
@@ -272,7 +273,7 @@ lookup_priolist(struct intel_engine_cs *engine, int prio)
 find_priolist:
 	/* most positive priority is scheduled first, equal priorities fifo */
 	rb = NULL;
-	parent = &execlists->queue.rb_node;
+	parent = &execlists->queue.rb_root.rb_node;
 	while (*parent) {
 		rb = *parent;
 		p = to_priolist(rb);
@@ -310,10 +311,7 @@ find_priolist:
 	p->priority = prio;
 	INIT_LIST_HEAD(&p->requests);
 	rb_link_node(&p->node, rb, parent);
-	rb_insert_color(&p->node, &execlists->queue);
-
-	if (first)
-		execlists->first = &p->node;
+	rb_insert_color_cached(&p->node, &execlists->queue, first);
 
 	return p;
 }
@@ -455,6 +453,16 @@ static void execlists_submit_ports(struct intel_engine_cs *engine)
 	unsigned int n;
 
 	/*
+	 * We can skip acquiring intel_runtime_pm_get() here as it was taken
+	 * on our behalf by the request (see i915_gem_mark_busy()) and it will
+	 * not be relinquished until the device is idle (see
+	 * i915_gem_idle_work_handler()). As a precaution, we make sure
+	 * that all ELSP are drained i.e. we have processed the CSB,
+	 * before allowing ourselves to idle and calling intel_runtime_pm_put().
+	 */
+	GEM_BUG_ON(!engine->i915->gt.awake);
+
+	/*
 	 * ELSQ note: the submit queue is not cleared after being submitted
 	 * to the HW so we need to make sure we always clean it up. This is
 	 * currently ensured by the fact that we always write the same number
@@ -561,13 +569,16 @@ static void complete_preempt_context(struct intel_engine_execlists *execlists)
 {
 	GEM_BUG_ON(!execlists_is_active(execlists, EXECLISTS_ACTIVE_PREEMPT));
 
-	execlists_cancel_port_requests(execlists);
-	execlists_unwind_incomplete_requests(execlists);
+	if (inject_preempt_hang(execlists))
+		return;
 
-	execlists_clear_active(execlists, EXECLISTS_ACTIVE_PREEMPT);
+	execlists_cancel_port_requests(execlists);
+	__unwind_incomplete_requests(container_of(execlists,
+						  struct intel_engine_cs,
+						  execlists));
 }
 
-static bool __execlists_dequeue(struct intel_engine_cs *engine)
+static void execlists_dequeue(struct intel_engine_cs *engine)
 {
 	struct intel_engine_execlists * const execlists = &engine->execlists;
 	struct execlist_port *port = execlists->port;
@@ -577,9 +588,8 @@ static bool __execlists_dequeue(struct intel_engine_cs *engine)
 	struct rb_node *rb;
 	bool submit = false;
 
-	lockdep_assert_held(&engine->timeline.lock);
-
-	/* Hardware submission is through 2 ports. Conceptually each port
+	/*
+	 * Hardware submission is through 2 ports. Conceptually each port
 	 * has a (RING_START, RING_HEAD, RING_TAIL) tuple. RING_START is
 	 * static for a context, and unique to each, so we only execute
 	 * requests belonging to a single context from each ring. RING_HEAD
@@ -600,9 +610,6 @@ static bool __execlists_dequeue(struct intel_engine_cs *engine)
 	 * and context switches) submission.
 	 */
 
-	rb = execlists->first;
-	GEM_BUG_ON(rb_first(&execlists->queue) != rb);
-
 	if (last) {
 		/*
 		 * Don't resubmit or switch until all outstanding
@@ -622,11 +629,11 @@ static bool __execlists_dequeue(struct intel_engine_cs *engine)
 		 * the HW to indicate that it has had a chance to respond.
 		 */
 		if (!execlists_is_active(execlists, EXECLISTS_ACTIVE_HWACK))
-			return false;
+			return;
 
 		if (need_preempt(engine, last, execlists->queue_priority)) {
 			inject_preempt_context(engine);
-			return false;
+			return;
 		}
 
 		/*
@@ -651,7 +658,7 @@ static bool __execlists_dequeue(struct intel_engine_cs *engine)
 		 * priorities of the ports haven't been switch.
 		 */
 		if (port_count(&port[1]))
-			return false;
+			return;
 
 		/*
 		 * WaIdleLiteRestore:bdw,skl
@@ -664,7 +671,7 @@ static bool __execlists_dequeue(struct intel_engine_cs *engine)
 		last->tail = last->wa_tail;
 	}
 
-	while (rb) {
+	while ((rb = rb_first_cached(&execlists->queue))) {
 		struct i915_priolist *p = to_priolist(rb);
 		struct i915_request *rq, *rn;
 
@@ -723,8 +730,7 @@ static bool __execlists_dequeue(struct intel_engine_cs *engine)
 			submit = true;
 		}
 
-		rb = rb_next(rb);
-		rb_erase(&p->node, &execlists->queue);
+		rb_erase_cached(&p->node, &execlists->queue);
 		INIT_LIST_HEAD(&p->requests);
 		if (p->priority != I915_PRIORITY_NORMAL)
 			kmem_cache_free(engine->i915->priorities, p);
@@ -750,35 +756,23 @@ done:
 	execlists->queue_priority =
 		port != execlists->port ? rq_prio(last) : INT_MIN;
 
-	execlists->first = rb;
-	if (submit)
+	if (submit) {
 		port_assign(port, last);
+		execlists_submit_ports(engine);
+	}
 
 	/* We must always keep the beast fed if we have work piled up */
-	GEM_BUG_ON(execlists->first && !port_isset(execlists->port));
+	GEM_BUG_ON(rb_first_cached(&execlists->queue) &&
+		   !port_isset(execlists->port));
 
 	/* Re-evaluate the executing context setup after each preemptive kick */
 	if (last)
 		execlists_user_begin(execlists, execlists->port);
 
-	return submit;
-}
-
-static void execlists_dequeue(struct intel_engine_cs *engine)
-{
-	struct intel_engine_execlists * const execlists = &engine->execlists;
-	unsigned long flags;
-	bool submit;
-
-	spin_lock_irqsave(&engine->timeline.lock, flags);
-	submit = __execlists_dequeue(engine);
-	spin_unlock_irqrestore(&engine->timeline.lock, flags);
-
-	if (submit)
-		execlists_submit_ports(engine);
-
-	GEM_BUG_ON(port_isset(execlists->port) &&
-		   !execlists_is_active(execlists, EXECLISTS_ACTIVE_USER));
+	/* If the engine is now idle, so should be the flag; and vice versa. */
+	GEM_BUG_ON(execlists_is_active(&engine->execlists,
+				       EXECLISTS_ACTIVE_USER) ==
+		   !port_isset(engine->execlists.port));
 }
 
 void
@@ -809,82 +803,27 @@ execlists_cancel_port_requests(struct intel_engine_execlists * const execlists)
 		port++;
 	}
 
-	execlists_user_end(execlists);
+	execlists_clear_all_active(execlists);
 }
 
-static void clear_gtiir(struct intel_engine_cs *engine)
+static void reset_csb_pointers(struct intel_engine_execlists *execlists)
 {
-	struct drm_i915_private *dev_priv = engine->i915;
-	int i;
-
 	/*
-	 * Clear any pending interrupt state.
-	 *
-	 * We do it twice out of paranoia that some of the IIR are
-	 * double buffered, and so if we only reset it once there may
-	 * still be an interrupt pending.
+	 * After a reset, the HW starts writing into CSB entry [0]. We
+	 * therefore have to set our HEAD pointer back one entry so that
+	 * the *first* entry we check is entry 0. To complicate this further,
+	 * as we don't wait for the first interrupt after reset, we have to
+	 * fake the HW write to point back to the last entry so that our
+	 * inline comparison of our cached head position against the last HW
+	 * write works even before the first interrupt.
 	 */
-	if (INTEL_GEN(dev_priv) >= 11) {
-		static const struct {
-			u8 bank;
-			u8 bit;
-		} gen11_gtiir[] = {
-			[RCS] = {0, GEN11_RCS0},
-			[BCS] = {0, GEN11_BCS},
-			[_VCS(0)] = {1, GEN11_VCS(0)},
-			[_VCS(1)] = {1, GEN11_VCS(1)},
-			[_VCS(2)] = {1, GEN11_VCS(2)},
-			[_VCS(3)] = {1, GEN11_VCS(3)},
-			[_VECS(0)] = {1, GEN11_VECS(0)},
-			[_VECS(1)] = {1, GEN11_VECS(1)},
-		};
-		unsigned long irqflags;
-
-		GEM_BUG_ON(engine->id >= ARRAY_SIZE(gen11_gtiir));
-
-		spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
-		for (i = 0; i < 2; i++) {
-			gen11_reset_one_iir(dev_priv,
-					    gen11_gtiir[engine->id].bank,
-					    gen11_gtiir[engine->id].bit);
-		}
-		spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
-	} else {
-		static const u8 gtiir[] = {
-			[RCS]  = 0,
-			[BCS]  = 0,
-			[VCS]  = 1,
-			[VCS2] = 1,
-			[VECS] = 3,
-		};
-
-		GEM_BUG_ON(engine->id >= ARRAY_SIZE(gtiir));
-
-		for (i = 0; i < 2; i++) {
-			I915_WRITE(GEN8_GT_IIR(gtiir[engine->id]),
-				   engine->irq_keep_mask);
-			POSTING_READ(GEN8_GT_IIR(gtiir[engine->id]));
-		}
-		GEM_BUG_ON(I915_READ(GEN8_GT_IIR(gtiir[engine->id])) &
-			   engine->irq_keep_mask);
-	}
+	execlists->csb_head = execlists->csb_write_reset;
+	WRITE_ONCE(*execlists->csb_write, execlists->csb_write_reset);
 }
 
-static void reset_irq(struct intel_engine_cs *engine)
+static void nop_submission_tasklet(unsigned long data)
 {
-	/* Mark all CS interrupts as complete */
-	smp_store_mb(engine->execlists.active, 0);
-	synchronize_hardirq(engine->i915->drm.irq);
-
-	clear_gtiir(engine);
-
-	/*
-	 * The port is checked prior to scheduling a tasklet, but
-	 * just in case we have suspended the tasklet to do the
-	 * wedging make sure that when it wakes, it decides there
-	 * is no work to do by clearing the irq_posted bit.
-	 */
-	clear_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted);
+	/* The driver is wedged; don't process any more events. */
 }
 
 static void execlists_cancel_requests(struct intel_engine_cs *engine)
@@ -911,13 +850,11 @@ static void execlists_cancel_requests(struct intel_engine_cs *engine)
 	 * submission's irq state, we also wish to remind ourselves that
 	 * it is irq state.)
 	 */
-	local_irq_save(flags);
+	spin_lock_irqsave(&engine->timeline.lock, flags);
 
 	/* Cancel the requests on the HW and clear the ELSP tracker. */
 	execlists_cancel_port_requests(execlists);
-	reset_irq(engine);
-
-	spin_lock(&engine->timeline.lock);
+	execlists_user_end(execlists);
 
 	/* Mark all executing requests as skipped. */
 	list_for_each_entry(rq, &engine->timeline.requests, link) {
@@ -927,8 +864,7 @@ static void execlists_cancel_requests(struct intel_engine_cs *engine)
 	}
 
 	/* Flush the queued requests to the timeline list (for retiring). */
-	rb = execlists->first;
-	while (rb) {
+	while ((rb = rb_first_cached(&execlists->queue))) {
 		struct i915_priolist *p = to_priolist(rb);
 
 		list_for_each_entry_safe(rq, rn, &p->requests, sched.link) {
@@ -938,8 +874,7 @@ static void execlists_cancel_requests(struct intel_engine_cs *engine)
 			__i915_request_submit(rq);
 		}
 
-		rb = rb_next(rb);
-		rb_erase(&p->node, &execlists->queue);
+		rb_erase_cached(&p->node, &execlists->queue);
 		INIT_LIST_HEAD(&p->requests);
 		if (p->priority != I915_PRIORITY_NORMAL)
 			kmem_cache_free(engine->i915->priorities, p);
@@ -948,183 +883,179 @@ static void execlists_cancel_requests(struct intel_engine_cs *engine)
 	/* Remaining _unready_ requests will be nop'ed when submitted */
 
 	execlists->queue_priority = INT_MIN;
-	execlists->queue = RB_ROOT;
-	execlists->first = NULL;
+	execlists->queue = RB_ROOT_CACHED;
 	GEM_BUG_ON(port_isset(execlists->port));
 
-	spin_unlock(&engine->timeline.lock);
+	GEM_BUG_ON(__tasklet_is_enabled(&execlists->tasklet));
+	execlists->tasklet.func = nop_submission_tasklet;
 
-	local_irq_restore(flags);
+	spin_unlock_irqrestore(&engine->timeline.lock, flags);
+}
+
+static inline bool
+reset_in_progress(const struct intel_engine_execlists *execlists)
+{
+	return unlikely(!__tasklet_is_enabled(&execlists->tasklet));
 }
 
 static void process_csb(struct intel_engine_cs *engine)
 {
 	struct intel_engine_execlists * const execlists = &engine->execlists;
 	struct execlist_port *port = execlists->port;
-	struct drm_i915_private *i915 = engine->i915;
-	bool fw = false;
+	const u32 * const buf = execlists->csb_status;
+	u8 head, tail;
+
+	/*
+	 * Note that csb_write, csb_status may be either in HWSP or mmio.
+	 * When reading from the csb_write mmio register, we have to be
+	 * careful to only use the GEN8_CSB_WRITE_PTR portion, which is
+	 * the low 4bits. As it happens we know the next 4bits are always
+	 * zero and so we can simply masked off the low u8 of the register
+	 * and treat it identically to reading from the HWSP (without having
+	 * to use explicit shifting and masking, and probably bifurcating
+	 * the code to handle the legacy mmio read).
+	 */
+	head = execlists->csb_head;
+	tail = READ_ONCE(*execlists->csb_write);
+	GEM_TRACE("%s cs-irq head=%d, tail=%d\n", engine->name, head, tail);
+	if (unlikely(head == tail))
+		return;
+
+	/*
+	 * Hopefully paired with a wmb() in HW!
+	 *
+	 * We must complete the read of the write pointer before any reads
+	 * from the CSB, so that we do not see stale values. Without an rmb
+	 * (lfence) the HW may speculatively perform the CSB[] reads *before*
+	 * we perform the READ_ONCE(*csb_write).
+	 */
+	rmb();
 
 	do {
-		/* The HWSP contains a (cacheable) mirror of the CSB */
-		const u32 *buf =
-			&engine->status_page.page_addr[I915_HWS_CSB_BUF0_INDEX];
-		unsigned int head, tail;
+		struct i915_request *rq;
+		unsigned int status;
+		unsigned int count;
 
-		/* Clear before reading to catch new interrupts */
-		clear_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted);
-		smp_mb__after_atomic();
+		if (++head == GEN8_CSB_ENTRIES)
+			head = 0;
 
-		if (unlikely(execlists->csb_use_mmio)) {
-			if (!fw) {
-				intel_uncore_forcewake_get(i915, execlists->fw_domains);
-				fw = true;
-			}
+		/*
+		 * We are flying near dragons again.
+		 *
+		 * We hold a reference to the request in execlist_port[]
+		 * but no more than that. We are operating in softirq
+		 * context and so cannot hold any mutex or sleep. That
+		 * prevents us stopping the requests we are processing
+		 * in port[] from being retired simultaneously (the
+		 * breadcrumb will be complete before we see the
+		 * context-switch). As we only hold the reference to the
+		 * request, any pointer chasing underneath the request
+		 * is subject to a potential use-after-free. Thus we
+		 * store all of the bookkeeping within port[] as
+		 * required, and avoid using unguarded pointers beneath
+		 * request itself. The same applies to the atomic
+		 * status notifier.
+		 */
 
-			buf = (u32 * __force)
-				(i915->regs + i915_mmio_reg_offset(RING_CONTEXT_STATUS_BUF_LO(engine, 0)));
+		GEM_TRACE("%s csb[%d]: status=0x%08x:0x%08x, active=0x%x\n",
+			  engine->name, head,
+			  buf[2 * head + 0], buf[2 * head + 1],
+			  execlists->active);
+
+		status = buf[2 * head];
+		if (status & (GEN8_CTX_STATUS_IDLE_ACTIVE |
+			      GEN8_CTX_STATUS_PREEMPTED))
+			execlists_set_active(execlists,
+					     EXECLISTS_ACTIVE_HWACK);
+		if (status & GEN8_CTX_STATUS_ACTIVE_IDLE)
+			execlists_clear_active(execlists,
+					       EXECLISTS_ACTIVE_HWACK);
+
+		if (!(status & GEN8_CTX_STATUS_COMPLETED_MASK))
+			continue;
 
-			head = readl(i915->regs + i915_mmio_reg_offset(RING_CONTEXT_STATUS_PTR(engine)));
-			tail = GEN8_CSB_WRITE_PTR(head);
-			head = GEN8_CSB_READ_PTR(head);
-			execlists->csb_head = head;
-		} else {
-			const int write_idx =
-				intel_hws_csb_write_index(i915) -
-				I915_HWS_CSB_BUF0_INDEX;
+		/* We should never get a COMPLETED | IDLE_ACTIVE! */
+		GEM_BUG_ON(status & GEN8_CTX_STATUS_IDLE_ACTIVE);
 
-			head = execlists->csb_head;
-			tail = READ_ONCE(buf[write_idx]);
-			rmb(); /* Hopefully paired with a wmb() in HW */
+		if (status & GEN8_CTX_STATUS_COMPLETE &&
+		    buf[2*head + 1] == execlists->preempt_complete_status) {
+			GEM_TRACE("%s preempt-idle\n", engine->name);
+			complete_preempt_context(execlists);
+			continue;
 		}
-		GEM_TRACE("%s cs-irq head=%d [%d%s], tail=%d [%d%s]\n",
-			  engine->name,
-			  head, GEN8_CSB_READ_PTR(readl(i915->regs + i915_mmio_reg_offset(RING_CONTEXT_STATUS_PTR(engine)))), fw ? "" : "?",
-			  tail, GEN8_CSB_WRITE_PTR(readl(i915->regs + i915_mmio_reg_offset(RING_CONTEXT_STATUS_PTR(engine)))), fw ? "" : "?");
 
-		while (head != tail) {
-			struct i915_request *rq;
-			unsigned int status;
-			unsigned int count;
+		if (status & GEN8_CTX_STATUS_PREEMPTED &&
+		    execlists_is_active(execlists,
+					EXECLISTS_ACTIVE_PREEMPT))
+			continue;
 
-			if (++head == GEN8_CSB_ENTRIES)
-				head = 0;
+		GEM_BUG_ON(!execlists_is_active(execlists,
+						EXECLISTS_ACTIVE_USER));
 
+		rq = port_unpack(port, &count);
+		GEM_TRACE("%s out[0]: ctx=%d.%d, global=%d (fence %llx:%d) (current %d), prio=%d\n",
+			  engine->name,
+			  port->context_id, count,
+			  rq ? rq->global_seqno : 0,
+			  rq ? rq->fence.context : 0,
+			  rq ? rq->fence.seqno : 0,
+			  intel_engine_get_seqno(engine),
+			  rq ? rq_prio(rq) : 0);
+
+		/* Check the context/desc id for this event matches */
+		GEM_DEBUG_BUG_ON(buf[2 * head + 1] != port->context_id);
+
+		GEM_BUG_ON(count == 0);
+		if (--count == 0) {
 			/*
-			 * We are flying near dragons again.
-			 *
-			 * We hold a reference to the request in execlist_port[]
-			 * but no more than that. We are operating in softirq
-			 * context and so cannot hold any mutex or sleep. That
-			 * prevents us stopping the requests we are processing
-			 * in port[] from being retired simultaneously (the
-			 * breadcrumb will be complete before we see the
-			 * context-switch). As we only hold the reference to the
-			 * request, any pointer chasing underneath the request
-			 * is subject to a potential use-after-free. Thus we
-			 * store all of the bookkeeping within port[] as
-			 * required, and avoid using unguarded pointers beneath
-			 * request itself. The same applies to the atomic
-			 * status notifier.
+			 * On the final event corresponding to the
+			 * submission of this context, we expect either
+			 * an element-switch event or a completion
+			 * event (and on completion, the active-idle
+			 * marker). No more preemptions, lite-restore
+			 * or otherwise.
 			 */
+			GEM_BUG_ON(status & GEN8_CTX_STATUS_PREEMPTED);
+			GEM_BUG_ON(port_isset(&port[1]) &&
+				   !(status & GEN8_CTX_STATUS_ELEMENT_SWITCH));
+			GEM_BUG_ON(!port_isset(&port[1]) &&
+				   !(status & GEN8_CTX_STATUS_ACTIVE_IDLE));
 
-			status = READ_ONCE(buf[2 * head]); /* maybe mmio! */
-			GEM_TRACE("%s csb[%d]: status=0x%08x:0x%08x, active=0x%x\n",
-				  engine->name, head,
-				  status, buf[2*head + 1],
-				  execlists->active);
-
-			if (status & (GEN8_CTX_STATUS_IDLE_ACTIVE |
-				      GEN8_CTX_STATUS_PREEMPTED))
-				execlists_set_active(execlists,
-						     EXECLISTS_ACTIVE_HWACK);
-			if (status & GEN8_CTX_STATUS_ACTIVE_IDLE)
-				execlists_clear_active(execlists,
-						       EXECLISTS_ACTIVE_HWACK);
-
-			if (!(status & GEN8_CTX_STATUS_COMPLETED_MASK))
-				continue;
-
-			/* We should never get a COMPLETED | IDLE_ACTIVE! */
-			GEM_BUG_ON(status & GEN8_CTX_STATUS_IDLE_ACTIVE);
-
-			if (status & GEN8_CTX_STATUS_COMPLETE &&
-			    buf[2*head + 1] == execlists->preempt_complete_status) {
-				GEM_TRACE("%s preempt-idle\n", engine->name);
-				complete_preempt_context(execlists);
-				continue;
-			}
-
-			if (status & GEN8_CTX_STATUS_PREEMPTED &&
-			    execlists_is_active(execlists,
-						EXECLISTS_ACTIVE_PREEMPT))
-				continue;
-
-			GEM_BUG_ON(!execlists_is_active(execlists,
-							EXECLISTS_ACTIVE_USER));
-
-			rq = port_unpack(port, &count);
-			GEM_TRACE("%s out[0]: ctx=%d.%d, global=%d (fence %llx:%d) (current %d), prio=%d\n",
-				  engine->name,
-				  port->context_id, count,
-				  rq ? rq->global_seqno : 0,
-				  rq ? rq->fence.context : 0,
-				  rq ? rq->fence.seqno : 0,
-				  intel_engine_get_seqno(engine),
-				  rq ? rq_prio(rq) : 0);
+			/*
+			 * We rely on the hardware being strongly
+			 * ordered, that the breadcrumb write is
+			 * coherent (visible from the CPU) before the
+			 * user interrupt and CSB is processed.
+			 */
+			GEM_BUG_ON(!i915_request_completed(rq));
 
-			/* Check the context/desc id for this event matches */
-			GEM_DEBUG_BUG_ON(buf[2 * head + 1] != port->context_id);
+			execlists_context_schedule_out(rq,
+						       INTEL_CONTEXT_SCHEDULE_OUT);
+			i915_request_put(rq);
 
-			GEM_BUG_ON(count == 0);
-			if (--count == 0) {
-				/*
-				 * On the final event corresponding to the
-				 * submission of this context, we expect either
-				 * an element-switch event or a completion
-				 * event (and on completion, the active-idle
-				 * marker). No more preemptions, lite-restore
-				 * or otherwise.
-				 */
-				GEM_BUG_ON(status & GEN8_CTX_STATUS_PREEMPTED);
-				GEM_BUG_ON(port_isset(&port[1]) &&
-					   !(status & GEN8_CTX_STATUS_ELEMENT_SWITCH));
-				GEM_BUG_ON(!port_isset(&port[1]) &&
-					   !(status & GEN8_CTX_STATUS_ACTIVE_IDLE));
+			GEM_TRACE("%s completed ctx=%d\n",
+				  engine->name, port->context_id);
 
-				/*
-				 * We rely on the hardware being strongly
-				 * ordered, that the breadcrumb write is
-				 * coherent (visible from the CPU) before the
-				 * user interrupt and CSB is processed.
-				 */
-				GEM_BUG_ON(!i915_request_completed(rq));
-
-				execlists_context_schedule_out(rq,
-							       INTEL_CONTEXT_SCHEDULE_OUT);
-				i915_request_put(rq);
-
-				GEM_TRACE("%s completed ctx=%d\n",
-					  engine->name, port->context_id);
-
-				port = execlists_port_complete(execlists, port);
-				if (port_isset(port))
-					execlists_user_begin(execlists, port);
-				else
-					execlists_user_end(execlists);
-			} else {
-				port_set(port, port_pack(rq, count));
-			}
+			port = execlists_port_complete(execlists, port);
+			if (port_isset(port))
+				execlists_user_begin(execlists, port);
+			else
+				execlists_user_end(execlists);
+		} else {
+			port_set(port, port_pack(rq, count));
 		}
+	} while (head != tail);
 
-		if (head != execlists->csb_head) {
-			execlists->csb_head = head;
-			writel(_MASKED_FIELD(GEN8_CSB_READ_PTR_MASK, head << 8),
-			       i915->regs + i915_mmio_reg_offset(RING_CONTEXT_STATUS_PTR(engine)));
-		}
-	} while (test_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted));
+	execlists->csb_head = head;
+}
 
-	if (unlikely(fw))
-		intel_uncore_forcewake_put(i915, execlists->fw_domains);
+static void __execlists_submission_tasklet(struct intel_engine_cs *const engine)
+{
+	lockdep_assert_held(&engine->timeline.lock);
+
+	process_csb(engine);
+	if (!execlists_is_active(&engine->execlists, EXECLISTS_ACTIVE_PREEMPT))
+		execlists_dequeue(engine);
 }
 
 /*
@@ -1134,38 +1065,16 @@ static void process_csb(struct intel_engine_cs *engine)
 static void execlists_submission_tasklet(unsigned long data)
 {
 	struct intel_engine_cs * const engine = (struct intel_engine_cs *)data;
+	unsigned long flags;
 
-	GEM_TRACE("%s awake?=%d, active=%x, irq-posted?=%d\n",
+	GEM_TRACE("%s awake?=%d, active=%x\n",
 		  engine->name,
 		  engine->i915->gt.awake,
-		  engine->execlists.active,
-		  test_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted));
-
-	/*
-	 * We can skip acquiring intel_runtime_pm_get() here as it was taken
-	 * on our behalf by the request (see i915_gem_mark_busy()) and it will
-	 * not be relinquished until the device is idle (see
-	 * i915_gem_idle_work_handler()). As a precaution, we make sure
-	 * that all ELSP are drained i.e. we have processed the CSB,
-	 * before allowing ourselves to idle and calling intel_runtime_pm_put().
-	 */
-	GEM_BUG_ON(!engine->i915->gt.awake);
-
-	/*
-	 * Prefer doing test_and_clear_bit() as a two stage operation to avoid
-	 * imposing the cost of a locked atomic transaction when submitting a
-	 * new request (outside of the context-switch interrupt).
-	 */
-	if (test_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted))
-		process_csb(engine);
+		  engine->execlists.active);
 
-	if (!execlists_is_active(&engine->execlists, EXECLISTS_ACTIVE_PREEMPT))
-		execlists_dequeue(engine);
-
-	/* If the engine is now idle, so should be the flag; and vice versa. */
-	GEM_BUG_ON(execlists_is_active(&engine->execlists,
-				       EXECLISTS_ACTIVE_USER) ==
-		   !port_isset(engine->execlists.port));
+	spin_lock_irqsave(&engine->timeline.lock, flags);
+	__execlists_submission_tasklet(engine);
+	spin_unlock_irqrestore(&engine->timeline.lock, flags);
 }
 
 static void queue_request(struct intel_engine_cs *engine,
@@ -1176,16 +1085,30 @@ static void queue_request(struct intel_engine_cs *engine,
 		      &lookup_priolist(engine, prio)->requests);
 }
 
-static void __submit_queue(struct intel_engine_cs *engine, int prio)
+static void __update_queue(struct intel_engine_cs *engine, int prio)
 {
 	engine->execlists.queue_priority = prio;
-	tasklet_hi_schedule(&engine->execlists.tasklet);
+}
+
+static void __submit_queue_imm(struct intel_engine_cs *engine)
+{
+	struct intel_engine_execlists * const execlists = &engine->execlists;
+
+	if (reset_in_progress(execlists))
+		return; /* defer until we restart the engine following reset */
+
+	if (execlists->tasklet.func == execlists_submission_tasklet)
+		__execlists_submission_tasklet(engine);
+	else
+		tasklet_hi_schedule(&execlists->tasklet);
 }
 
 static void submit_queue(struct intel_engine_cs *engine, int prio)
 {
-	if (prio > engine->execlists.queue_priority)
-		__submit_queue(engine, prio);
+	if (prio > engine->execlists.queue_priority) {
+		__update_queue(engine, prio);
+		__submit_queue_imm(engine);
+	}
 }
 
 static void execlists_submit_request(struct i915_request *request)
@@ -1197,11 +1120,12 @@ static void execlists_submit_request(struct i915_request *request)
 	spin_lock_irqsave(&engine->timeline.lock, flags);
 
 	queue_request(engine, &request->sched, rq_prio(request));
-	submit_queue(engine, rq_prio(request));
 
-	GEM_BUG_ON(!engine->execlists.first);
+	GEM_BUG_ON(RB_EMPTY_ROOT(&engine->execlists.queue.rb_root));
 	GEM_BUG_ON(list_empty(&request->sched.link));
 
+	submit_queue(engine, rq_prio(request));
+
 	spin_unlock_irqrestore(&engine->timeline.lock, flags);
 }
 
@@ -1328,8 +1252,11 @@ static void execlists_schedule(struct i915_request *request,
 		}
 
 		if (prio > engine->execlists.queue_priority &&
-		    i915_sw_fence_done(&sched_to_request(node)->submit))
-			__submit_queue(engine, prio);
+		    i915_sw_fence_done(&sched_to_request(node)->submit)) {
+			/* defer submission until after all of our updates */
+			__update_queue(engine, prio);
+			tasklet_hi_schedule(&engine->execlists.tasklet);
+		}
 	}
 
 	spin_unlock_irq(&engine->timeline.lock);
@@ -1337,11 +1264,15 @@ static void execlists_schedule(struct i915_request *request,
 
 static void execlists_context_destroy(struct intel_context *ce)
 {
-	GEM_BUG_ON(!ce->state);
 	GEM_BUG_ON(ce->pin_count);
 
+	if (!ce->state)
+		return;
+
 	intel_ring_free(ce->ring);
-	__i915_gem_object_release_unless_active(ce->state->obj);
+
+	GEM_BUG_ON(i915_gem_object_is_active(ce->state->obj));
+	i915_gem_object_put(ce->state->obj);
 }
 
 static void execlists_context_unpin(struct intel_context *ce)
@@ -1851,7 +1782,6 @@ static int gen8_init_common_ring(struct intel_engine_cs *engine)
 		return ret;
 
 	intel_engine_reset_breadcrumbs(engine);
-	intel_engine_init_hangcheck(engine);
 
 	if (GEM_SHOW_DEBUG() && unexpected_starting_state(engine)) {
 		struct drm_printer p = drm_debug_printer(__func__);
@@ -1906,6 +1836,7 @@ execlists_reset_prepare(struct intel_engine_cs *engine)
 {
 	struct intel_engine_execlists * const execlists = &engine->execlists;
 	struct i915_request *request, *active;
+	unsigned long flags;
 
 	GEM_TRACE("%s\n", engine->name);
 
@@ -1920,6 +1851,8 @@ execlists_reset_prepare(struct intel_engine_cs *engine)
 	 */
 	__tasklet_disable_sync_once(&execlists->tasklet);
 
+	spin_lock_irqsave(&engine->timeline.lock, flags);
+
 	/*
 	 * We want to flush the pending context switches, having disabled
 	 * the tasklet above, we can assume exclusive access to the execlists.
@@ -1927,8 +1860,7 @@ execlists_reset_prepare(struct intel_engine_cs *engine)
 	 * and avoid blaming an innocent request if the stall was due to the
 	 * preemption itself.
 	 */
-	if (test_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted))
-		process_csb(engine);
+	process_csb(engine);
 
 	/*
 	 * The last active request can then be no later than the last request
@@ -1938,15 +1870,12 @@ execlists_reset_prepare(struct intel_engine_cs *engine)
 	active = NULL;
 	request = port_request(execlists->port);
 	if (request) {
-		unsigned long flags;
-
 		/*
 		 * Prevent the breadcrumb from advancing before we decide
 		 * which request is currently active.
 		 */
 		intel_engine_stop_cs(engine);
 
-		spin_lock_irqsave(&engine->timeline.lock, flags);
 		list_for_each_entry_from_reverse(request,
 						 &engine->timeline.requests,
 						 link) {
@@ -1956,9 +1885,10 @@ execlists_reset_prepare(struct intel_engine_cs *engine)
 
 			active = request;
 		}
-		spin_unlock_irqrestore(&engine->timeline.lock, flags);
 	}
 
+	spin_unlock_irqrestore(&engine->timeline.lock, flags);
+
 	return active;
 }
 
@@ -1973,8 +1903,7 @@ static void execlists_reset(struct intel_engine_cs *engine,
 		  engine->name, request ? request->global_seqno : 0,
 		  intel_engine_get_seqno(engine));
 
-	/* See execlists_cancel_requests() for the irq/spinlock split. */
-	local_irq_save(flags);
+	spin_lock_irqsave(&engine->timeline.lock, flags);
 
 	/*
 	 * Catch up with any missed context-switch interrupts.
@@ -1986,17 +1915,14 @@ static void execlists_reset(struct intel_engine_cs *engine,
 	 * requests were completed.
 	 */
 	execlists_cancel_port_requests(execlists);
-	reset_irq(engine);
 
 	/* Push back any incomplete requests for replay after the reset. */
-	spin_lock(&engine->timeline.lock);
 	__unwind_incomplete_requests(engine);
-	spin_unlock(&engine->timeline.lock);
 
 	/* Following the reset, we need to reload the CSB read/write pointers */
-	engine->execlists.csb_head = GEN8_CSB_ENTRIES - 1;
+	reset_csb_pointers(&engine->execlists);
 
-	local_irq_restore(flags);
+	spin_unlock_irqrestore(&engine->timeline.lock, flags);
 
 	/*
 	 * If the request was innocent, we leave the request in the ELSP
@@ -2046,7 +1972,7 @@ static void execlists_reset_finish(struct intel_engine_cs *engine)
 	struct intel_engine_execlists * const execlists = &engine->execlists;
 
 	/* After a GPU reset, we may have requests to replay */
-	if (execlists->first)
+	if (!RB_EMPTY_ROOT(&execlists->queue.rb_root))
 		tasklet_schedule(&execlists->tasklet);
 
 	/*
@@ -2366,7 +2292,7 @@ void intel_logical_ring_cleanup(struct intel_engine_cs *engine)
 	kfree(engine);
 }
 
-static void execlists_set_default_submission(struct intel_engine_cs *engine)
+void intel_execlists_set_default_submission(struct intel_engine_cs *engine)
 {
 	engine->submit_request = execlists_submit_request;
 	engine->cancel_requests = execlists_cancel_requests;
@@ -2406,7 +2332,7 @@ logical_ring_default_vfuncs(struct intel_engine_cs *engine)
 	engine->emit_breadcrumb = gen8_emit_breadcrumb;
 	engine->emit_breadcrumb_sz = gen8_emit_breadcrumb_sz;
 
-	engine->set_default_submission = execlists_set_default_submission;
+	engine->set_default_submission = intel_execlists_set_default_submission;
 
 	if (INTEL_GEN(engine->i915) < 11) {
 		engine->irq_enable = gen8_logical_ring_enable_irq;
@@ -2446,28 +2372,11 @@ logical_ring_default_irqs(struct intel_engine_cs *engine)
 static void
 logical_ring_setup(struct intel_engine_cs *engine)
 {
-	struct drm_i915_private *dev_priv = engine->i915;
-	enum forcewake_domains fw_domains;
-
 	intel_engine_setup_common(engine);
 
 	/* Intentionally left blank. */
 	engine->buffer = NULL;
 
-	fw_domains = intel_uncore_forcewake_for_reg(dev_priv,
-						    RING_ELSP(engine),
-						    FW_REG_WRITE);
-
-	fw_domains |= intel_uncore_forcewake_for_reg(dev_priv,
-						     RING_CONTEXT_STATUS_PTR(engine),
-						     FW_REG_READ | FW_REG_WRITE);
-
-	fw_domains |= intel_uncore_forcewake_for_reg(dev_priv,
-						     RING_CONTEXT_STATUS_BUF_BASE(engine),
-						     FW_REG_READ);
-
-	engine->execlists.fw_domains = fw_domains;
-
 	tasklet_init(&engine->execlists.tasklet,
 		     execlists_submission_tasklet, (unsigned long)engine);
 
@@ -2475,34 +2384,60 @@ logical_ring_setup(struct intel_engine_cs *engine)
 	logical_ring_default_irqs(engine);
 }
 
+static bool csb_force_mmio(struct drm_i915_private *i915)
+{
+	/* Older GVT emulation depends upon intercepting CSB mmio */
+	return intel_vgpu_active(i915) && !intel_vgpu_has_hwsp_emulation(i915);
+}
+
 static int logical_ring_init(struct intel_engine_cs *engine)
 {
+	struct drm_i915_private *i915 = engine->i915;
+	struct intel_engine_execlists * const execlists = &engine->execlists;
 	int ret;
 
 	ret = intel_engine_init_common(engine);
 	if (ret)
 		goto error;
 
-	if (HAS_LOGICAL_RING_ELSQ(engine->i915)) {
-		engine->execlists.submit_reg = engine->i915->regs +
+	if (HAS_LOGICAL_RING_ELSQ(i915)) {
+		execlists->submit_reg = i915->regs +
 			i915_mmio_reg_offset(RING_EXECLIST_SQ_CONTENTS(engine));
-		engine->execlists.ctrl_reg = engine->i915->regs +
+		execlists->ctrl_reg = i915->regs +
 			i915_mmio_reg_offset(RING_EXECLIST_CONTROL(engine));
 	} else {
-		engine->execlists.submit_reg = engine->i915->regs +
+		execlists->submit_reg = i915->regs +
 			i915_mmio_reg_offset(RING_ELSP(engine));
 	}
 
-	engine->execlists.preempt_complete_status = ~0u;
-	if (engine->i915->preempt_context) {
+	execlists->preempt_complete_status = ~0u;
+	if (i915->preempt_context) {
 		struct intel_context *ce =
-			to_intel_context(engine->i915->preempt_context, engine);
+			to_intel_context(i915->preempt_context, engine);
 
-		engine->execlists.preempt_complete_status =
+		execlists->preempt_complete_status =
 			upper_32_bits(ce->lrc_desc);
 	}
 
-	engine->execlists.csb_head = GEN8_CSB_ENTRIES - 1;
+	execlists->csb_read =
+		i915->regs + i915_mmio_reg_offset(RING_CONTEXT_STATUS_PTR(engine));
+	if (csb_force_mmio(i915)) {
+		execlists->csb_status = (u32 __force *)
+			(i915->regs + i915_mmio_reg_offset(RING_CONTEXT_STATUS_BUF_LO(engine, 0)));
+
+		execlists->csb_write = (u32 __force *)execlists->csb_read;
+		execlists->csb_write_reset =
+			_MASKED_FIELD(GEN8_CSB_WRITE_PTR_MASK,
+				      GEN8_CSB_ENTRIES - 1);
+	} else {
+		execlists->csb_status =
+			&engine->status_page.page_addr[I915_HWS_CSB_BUF0_INDEX];
+
+		execlists->csb_write =
+			&engine->status_page.page_addr[intel_hws_csb_write_index(i915)];
+		execlists->csb_write_reset = GEN8_CSB_ENTRIES - 1;
+	}
+	reset_csb_pointers(execlists);
 
 	return 0;
 
diff --git a/drivers/gpu/drm/i915/intel_lrc.h b/drivers/gpu/drm/i915/intel_lrc.h
index 1593194e930c..4dfb78e3ec7e 100644
--- a/drivers/gpu/drm/i915/intel_lrc.h
+++ b/drivers/gpu/drm/i915/intel_lrc.h
@@ -104,4 +104,6 @@ struct i915_gem_context;
 
 void intel_lr_context_resume(struct drm_i915_private *dev_priv);
 
+void intel_execlists_set_default_submission(struct intel_engine_cs *engine);
+
 #endif /* _INTEL_LRC_H_ */
diff --git a/drivers/gpu/drm/i915/intel_lvds.c b/drivers/gpu/drm/i915/intel_lvds.c
index ca55b0a82ba6..f9f3b0885ba5 100644
--- a/drivers/gpu/drm/i915/intel_lvds.c
+++ b/drivers/gpu/drm/i915/intel_lvds.c
@@ -44,8 +44,6 @@
 /* Private structure for the integrated LVDS support */
 struct intel_lvds_connector {
 	struct intel_connector base;
-
-	struct notifier_block lid_notifier;
 };
 
 struct intel_lvds_pps {
@@ -452,26 +450,9 @@ static bool intel_lvds_compute_config(struct intel_encoder *intel_encoder,
 	return true;
 }
 
-/*
- * Detect the LVDS connection.
- *
- * Since LVDS doesn't have hotlug, we use the lid as a proxy.  Open means
- * connected and closed means disconnected.  We also send hotplug events as
- * needed, using lid status notification from the input layer.
- */
 static enum drm_connector_status
 intel_lvds_detect(struct drm_connector *connector, bool force)
 {
-	struct drm_i915_private *dev_priv = to_i915(connector->dev);
-	enum drm_connector_status status;
-
-	DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n",
-		      connector->base.id, connector->name);
-
-	status = intel_panel_detect(dev_priv);
-	if (status != connector_status_unknown)
-		return status;
-
 	return connector_status_connected;
 }
 
@@ -496,117 +477,6 @@ static int intel_lvds_get_modes(struct drm_connector *connector)
 	return 1;
 }
 
-static int intel_no_modeset_on_lid_dmi_callback(const struct dmi_system_id *id)
-{
-	DRM_INFO("Skipping forced modeset for %s\n", id->ident);
-	return 1;
-}
-
-/* The GPU hangs up on these systems if modeset is performed on LID open */
-static const struct dmi_system_id intel_no_modeset_on_lid[] = {
-	{
-		.callback = intel_no_modeset_on_lid_dmi_callback,
-		.ident = "Toshiba Tecra A11",
-		.matches = {
-			DMI_MATCH(DMI_SYS_VENDOR, "TOSHIBA"),
-			DMI_MATCH(DMI_PRODUCT_NAME, "TECRA A11"),
-		},
-	},
-
-	{ }	/* terminating entry */
-};
-
-/*
- * Lid events. Note the use of 'modeset':
- *  - we set it to MODESET_ON_LID_OPEN on lid close,
- *    and set it to MODESET_DONE on open
- *  - we use it as a "only once" bit (ie we ignore
- *    duplicate events where it was already properly set)
- *  - the suspend/resume paths will set it to
- *    MODESET_SUSPENDED and ignore the lid open event,
- *    because they restore the mode ("lid open").
- */
-static int intel_lid_notify(struct notifier_block *nb, unsigned long val,
-			    void *unused)
-{
-	struct intel_lvds_connector *lvds_connector =
-		container_of(nb, struct intel_lvds_connector, lid_notifier);
-	struct drm_connector *connector = &lvds_connector->base.base;
-	struct drm_device *dev = connector->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-
-	if (dev->switch_power_state != DRM_SWITCH_POWER_ON)
-		return NOTIFY_OK;
-
-	mutex_lock(&dev_priv->modeset_restore_lock);
-	if (dev_priv->modeset_restore == MODESET_SUSPENDED)
-		goto exit;
-	/*
-	 * check and update the status of LVDS connector after receiving
-	 * the LID nofication event.
-	 */
-	connector->status = connector->funcs->detect(connector, false);
-
-	/* Don't force modeset on machines where it causes a GPU lockup */
-	if (dmi_check_system(intel_no_modeset_on_lid))
-		goto exit;
-	if (!acpi_lid_open()) {
-		/* do modeset on next lid open event */
-		dev_priv->modeset_restore = MODESET_ON_LID_OPEN;
-		goto exit;
-	}
-
-	if (dev_priv->modeset_restore == MODESET_DONE)
-		goto exit;
-
-	/*
-	 * Some old platform's BIOS love to wreak havoc while the lid is closed.
-	 * We try to detect this here and undo any damage. The split for PCH
-	 * platforms is rather conservative and a bit arbitrary expect that on
-	 * those platforms VGA disabling requires actual legacy VGA I/O access,
-	 * and as part of the cleanup in the hw state restore we also redisable
-	 * the vga plane.
-	 */
-	if (!HAS_PCH_SPLIT(dev_priv))
-		intel_display_resume(dev);
-
-	dev_priv->modeset_restore = MODESET_DONE;
-
-exit:
-	mutex_unlock(&dev_priv->modeset_restore_lock);
-	return NOTIFY_OK;
-}
-
-static int
-intel_lvds_connector_register(struct drm_connector *connector)
-{
-	struct intel_lvds_connector *lvds = to_lvds_connector(connector);
-	int ret;
-
-	ret = intel_connector_register(connector);
-	if (ret)
-		return ret;
-
-	lvds->lid_notifier.notifier_call = intel_lid_notify;
-	if (acpi_lid_notifier_register(&lvds->lid_notifier)) {
-		DRM_DEBUG_KMS("lid notifier registration failed\n");
-		lvds->lid_notifier.notifier_call = NULL;
-	}
-
-	return 0;
-}
-
-static void
-intel_lvds_connector_unregister(struct drm_connector *connector)
-{
-	struct intel_lvds_connector *lvds = to_lvds_connector(connector);
-
-	if (lvds->lid_notifier.notifier_call)
-		acpi_lid_notifier_unregister(&lvds->lid_notifier);
-
-	intel_connector_unregister(connector);
-}
-
 /**
  * intel_lvds_destroy - unregister and free LVDS structures
  * @connector: connector to free
@@ -639,8 +509,8 @@ static const struct drm_connector_funcs intel_lvds_connector_funcs = {
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.atomic_get_property = intel_digital_connector_atomic_get_property,
 	.atomic_set_property = intel_digital_connector_atomic_set_property,
-	.late_register = intel_lvds_connector_register,
-	.early_unregister = intel_lvds_connector_unregister,
+	.late_register = intel_connector_register,
+	.early_unregister = intel_connector_unregister,
 	.destroy = intel_lvds_destroy,
 	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
 	.atomic_duplicate_state = intel_digital_connector_duplicate_state,
@@ -1114,8 +984,6 @@ void intel_lvds_init(struct drm_i915_private *dev_priv)
 	 * 2) check for VBT data
 	 * 3) check to see if LVDS is already on
 	 *    if none of the above, no panel
-	 * 4) make sure lid is open
-	 *    if closed, act like it's not there for now
 	 */
 
 	/*
diff --git a/drivers/gpu/drm/i915/intel_panel.c b/drivers/gpu/drm/i915/intel_panel.c
index 14b827ec5427..4a9f139e7b73 100644
--- a/drivers/gpu/drm/i915/intel_panel.c
+++ b/drivers/gpu/drm/i915/intel_panel.c
@@ -375,26 +375,6 @@ out:
 	pipe_config->gmch_pfit.lvds_border_bits = border;
 }
 
-enum drm_connector_status
-intel_panel_detect(struct drm_i915_private *dev_priv)
-{
-	/* Assume that the BIOS does not lie through the OpRegion... */
-	if (!i915_modparams.panel_ignore_lid && dev_priv->opregion.lid_state) {
-		return *dev_priv->opregion.lid_state & 0x1 ?
-			connector_status_connected :
-			connector_status_disconnected;
-	}
-
-	switch (i915_modparams.panel_ignore_lid) {
-	case -2:
-		return connector_status_connected;
-	case -1:
-		return connector_status_disconnected;
-	default:
-		return connector_status_unknown;
-	}
-}
-
 /**
  * scale - scale values from one range to another
  * @source_val: value in range [@source_min..@source_max]
diff --git a/drivers/gpu/drm/i915/intel_pipe_crc.c b/drivers/gpu/drm/i915/intel_pipe_crc.c
index 27d560f7a817..f3c9010e332a 100644
--- a/drivers/gpu/drm/i915/intel_pipe_crc.c
+++ b/drivers/gpu/drm/i915/intel_pipe_crc.c
@@ -30,160 +30,6 @@
 #include <linux/debugfs.h>
 #include "intel_drv.h"
 
-struct pipe_crc_info {
-	const char *name;
-	struct drm_i915_private *dev_priv;
-	enum pipe pipe;
-};
-
-static int i915_pipe_crc_open(struct inode *inode, struct file *filep)
-{
-	struct pipe_crc_info *info = inode->i_private;
-	struct drm_i915_private *dev_priv = info->dev_priv;
-	struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[info->pipe];
-
-	if (info->pipe >= INTEL_INFO(dev_priv)->num_pipes)
-		return -ENODEV;
-
-	spin_lock_irq(&pipe_crc->lock);
-
-	if (pipe_crc->opened) {
-		spin_unlock_irq(&pipe_crc->lock);
-		return -EBUSY; /* already open */
-	}
-
-	pipe_crc->opened = true;
-	filep->private_data = inode->i_private;
-
-	spin_unlock_irq(&pipe_crc->lock);
-
-	return 0;
-}
-
-static int i915_pipe_crc_release(struct inode *inode, struct file *filep)
-{
-	struct pipe_crc_info *info = inode->i_private;
-	struct drm_i915_private *dev_priv = info->dev_priv;
-	struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[info->pipe];
-
-	spin_lock_irq(&pipe_crc->lock);
-	pipe_crc->opened = false;
-	spin_unlock_irq(&pipe_crc->lock);
-
-	return 0;
-}
-
-/* (6 fields, 8 chars each, space separated (5) + '\n') */
-#define PIPE_CRC_LINE_LEN	(6 * 8 + 5 + 1)
-/* account for \'0' */
-#define PIPE_CRC_BUFFER_LEN	(PIPE_CRC_LINE_LEN + 1)
-
-static int pipe_crc_data_count(struct intel_pipe_crc *pipe_crc)
-{
-	lockdep_assert_held(&pipe_crc->lock);
-	return CIRC_CNT(pipe_crc->head, pipe_crc->tail,
-			INTEL_PIPE_CRC_ENTRIES_NR);
-}
-
-static ssize_t
-i915_pipe_crc_read(struct file *filep, char __user *user_buf, size_t count,
-		   loff_t *pos)
-{
-	struct pipe_crc_info *info = filep->private_data;
-	struct drm_i915_private *dev_priv = info->dev_priv;
-	struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[info->pipe];
-	char buf[PIPE_CRC_BUFFER_LEN];
-	int n_entries;
-	ssize_t bytes_read;
-
-	/*
-	 * Don't allow user space to provide buffers not big enough to hold
-	 * a line of data.
-	 */
-	if (count < PIPE_CRC_LINE_LEN)
-		return -EINVAL;
-
-	if (pipe_crc->source == INTEL_PIPE_CRC_SOURCE_NONE)
-		return 0;
-
-	/* nothing to read */
-	spin_lock_irq(&pipe_crc->lock);
-	while (pipe_crc_data_count(pipe_crc) == 0) {
-		int ret;
-
-		if (filep->f_flags & O_NONBLOCK) {
-			spin_unlock_irq(&pipe_crc->lock);
-			return -EAGAIN;
-		}
-
-		ret = wait_event_interruptible_lock_irq(pipe_crc->wq,
-				pipe_crc_data_count(pipe_crc), pipe_crc->lock);
-		if (ret) {
-			spin_unlock_irq(&pipe_crc->lock);
-			return ret;
-		}
-	}
-
-	/* We now have one or more entries to read */
-	n_entries = count / PIPE_CRC_LINE_LEN;
-
-	bytes_read = 0;
-	while (n_entries > 0) {
-		struct intel_pipe_crc_entry *entry =
-			&pipe_crc->entries[pipe_crc->tail];
-
-		if (CIRC_CNT(pipe_crc->head, pipe_crc->tail,
-			     INTEL_PIPE_CRC_ENTRIES_NR) < 1)
-			break;
-
-		BUILD_BUG_ON_NOT_POWER_OF_2(INTEL_PIPE_CRC_ENTRIES_NR);
-		pipe_crc->tail = (pipe_crc->tail + 1) &
-				 (INTEL_PIPE_CRC_ENTRIES_NR - 1);
-
-		bytes_read += snprintf(buf, PIPE_CRC_BUFFER_LEN,
-				       "%8u %8x %8x %8x %8x %8x\n",
-				       entry->frame, entry->crc[0],
-				       entry->crc[1], entry->crc[2],
-				       entry->crc[3], entry->crc[4]);
-
-		spin_unlock_irq(&pipe_crc->lock);
-
-		if (copy_to_user(user_buf, buf, PIPE_CRC_LINE_LEN))
-			return -EFAULT;
-
-		user_buf += PIPE_CRC_LINE_LEN;
-		n_entries--;
-
-		spin_lock_irq(&pipe_crc->lock);
-	}
-
-	spin_unlock_irq(&pipe_crc->lock);
-
-	return bytes_read;
-}
-
-static const struct file_operations i915_pipe_crc_fops = {
-	.owner = THIS_MODULE,
-	.open = i915_pipe_crc_open,
-	.read = i915_pipe_crc_read,
-	.release = i915_pipe_crc_release,
-};
-
-static struct pipe_crc_info i915_pipe_crc_data[I915_MAX_PIPES] = {
-	{
-		.name = "i915_pipe_A_crc",
-		.pipe = PIPE_A,
-	},
-	{
-		.name = "i915_pipe_B_crc",
-		.pipe = PIPE_B,
-	},
-	{
-		.name = "i915_pipe_C_crc",
-		.pipe = PIPE_C,
-	},
-};
-
 static const char * const pipe_crc_sources[] = {
 	"none",
 	"plane1",
@@ -197,29 +43,6 @@ static const char * const pipe_crc_sources[] = {
 	"auto",
 };
 
-static const char *pipe_crc_source_name(enum intel_pipe_crc_source source)
-{
-	BUILD_BUG_ON(ARRAY_SIZE(pipe_crc_sources) != INTEL_PIPE_CRC_SOURCE_MAX);
-	return pipe_crc_sources[source];
-}
-
-static int display_crc_ctl_show(struct seq_file *m, void *data)
-{
-	struct drm_i915_private *dev_priv = m->private;
-	enum pipe pipe;
-
-	for_each_pipe(dev_priv, pipe)
-		seq_printf(m, "%c %s\n", pipe_name(pipe),
-			   pipe_crc_source_name(dev_priv->pipe_crc[pipe].source));
-
-	return 0;
-}
-
-static int display_crc_ctl_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, display_crc_ctl_show, inode->i_private);
-}
-
 static int i8xx_pipe_crc_ctl_reg(enum intel_pipe_crc_source *source,
 				 uint32_t *val)
 {
@@ -616,177 +439,6 @@ static int get_new_crc_ctl_reg(struct drm_i915_private *dev_priv,
 		return ivb_pipe_crc_ctl_reg(dev_priv, pipe, source, val, set_wa);
 }
 
-static int pipe_crc_set_source(struct drm_i915_private *dev_priv,
-			       enum pipe pipe,
-			       enum intel_pipe_crc_source source)
-{
-	struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe];
-	enum intel_display_power_domain power_domain;
-	u32 val = 0; /* shut up gcc */
-	int ret;
-
-	if (pipe_crc->source == source)
-		return 0;
-
-	/* forbid changing the source without going back to 'none' */
-	if (pipe_crc->source && source)
-		return -EINVAL;
-
-	power_domain = POWER_DOMAIN_PIPE(pipe);
-	if (!intel_display_power_get_if_enabled(dev_priv, power_domain)) {
-		DRM_DEBUG_KMS("Trying to capture CRC while pipe is off\n");
-		return -EIO;
-	}
-
-	ret = get_new_crc_ctl_reg(dev_priv, pipe, &source, &val, true);
-	if (ret != 0)
-		goto out;
-
-	/* none -> real source transition */
-	if (source) {
-		struct intel_pipe_crc_entry *entries;
-
-		DRM_DEBUG_DRIVER("collecting CRCs for pipe %c, %s\n",
-				 pipe_name(pipe), pipe_crc_source_name(source));
-
-		entries = kcalloc(INTEL_PIPE_CRC_ENTRIES_NR,
-				  sizeof(pipe_crc->entries[0]),
-				  GFP_KERNEL);
-		if (!entries) {
-			ret = -ENOMEM;
-			goto out;
-		}
-
-		spin_lock_irq(&pipe_crc->lock);
-		kfree(pipe_crc->entries);
-		pipe_crc->entries = entries;
-		pipe_crc->head = 0;
-		pipe_crc->tail = 0;
-		spin_unlock_irq(&pipe_crc->lock);
-	}
-
-	pipe_crc->source = source;
-
-	I915_WRITE(PIPE_CRC_CTL(pipe), val);
-	POSTING_READ(PIPE_CRC_CTL(pipe));
-
-	/* real source -> none transition */
-	if (!source) {
-		struct intel_pipe_crc_entry *entries;
-		struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv,
-								  pipe);
-
-		DRM_DEBUG_DRIVER("stopping CRCs for pipe %c\n",
-				 pipe_name(pipe));
-
-		drm_modeset_lock(&crtc->base.mutex, NULL);
-		if (crtc->base.state->active)
-			intel_wait_for_vblank(dev_priv, pipe);
-		drm_modeset_unlock(&crtc->base.mutex);
-
-		spin_lock_irq(&pipe_crc->lock);
-		entries = pipe_crc->entries;
-		pipe_crc->entries = NULL;
-		pipe_crc->head = 0;
-		pipe_crc->tail = 0;
-		spin_unlock_irq(&pipe_crc->lock);
-
-		kfree(entries);
-
-		if (IS_G4X(dev_priv))
-			g4x_undo_pipe_scramble_reset(dev_priv, pipe);
-		else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
-			vlv_undo_pipe_scramble_reset(dev_priv, pipe);
-		else if ((IS_HASWELL(dev_priv) ||
-			  IS_BROADWELL(dev_priv)) && pipe == PIPE_A)
-			hsw_pipe_A_crc_wa(dev_priv, false);
-	}
-
-	ret = 0;
-
-out:
-	intel_display_power_put(dev_priv, power_domain);
-
-	return ret;
-}
-
-/*
- * Parse pipe CRC command strings:
- *   command: wsp* object wsp+ name wsp+ source wsp*
- *   object: 'pipe'
- *   name: (A | B | C)
- *   source: (none | plane1 | plane2 | pf)
- *   wsp: (#0x20 | #0x9 | #0xA)+
- *
- * eg.:
- *  "pipe A plane1"  ->  Start CRC computations on plane1 of pipe A
- *  "pipe A none"    ->  Stop CRC
- */
-static int display_crc_ctl_tokenize(char *buf, char *words[], int max_words)
-{
-	int n_words = 0;
-
-	while (*buf) {
-		char *end;
-
-		/* skip leading white space */
-		buf = skip_spaces(buf);
-		if (!*buf)
-			break;	/* end of buffer */
-
-		/* find end of word */
-		for (end = buf; *end && !isspace(*end); end++)
-			;
-
-		if (n_words == max_words) {
-			DRM_DEBUG_DRIVER("too many words, allowed <= %d\n",
-					 max_words);
-			return -EINVAL;	/* ran out of words[] before bytes */
-		}
-
-		if (*end)
-			*end++ = '\0';
-		words[n_words++] = buf;
-		buf = end;
-	}
-
-	return n_words;
-}
-
-enum intel_pipe_crc_object {
-	PIPE_CRC_OBJECT_PIPE,
-};
-
-static const char * const pipe_crc_objects[] = {
-	"pipe",
-};
-
-static int
-display_crc_ctl_parse_object(const char *buf, enum intel_pipe_crc_object *o)
-{
-	int i;
-
-	i = match_string(pipe_crc_objects, ARRAY_SIZE(pipe_crc_objects), buf);
-	if (i < 0)
-		return i;
-
-	*o = i;
-	return 0;
-}
-
-static int display_crc_ctl_parse_pipe(struct drm_i915_private *dev_priv,
-				      const char *buf, enum pipe *pipe)
-{
-	const char name = buf[0];
-
-	if (name < 'A' || name >= pipe_name(INTEL_INFO(dev_priv)->num_pipes))
-		return -EINVAL;
-
-	*pipe = name - 'A';
-
-	return 0;
-}
-
 static int
 display_crc_ctl_parse_source(const char *buf, enum intel_pipe_crc_source *s)
 {
@@ -805,81 +457,6 @@ display_crc_ctl_parse_source(const char *buf, enum intel_pipe_crc_source *s)
 	return 0;
 }
 
-static int display_crc_ctl_parse(struct drm_i915_private *dev_priv,
-				 char *buf, size_t len)
-{
-#define N_WORDS 3
-	int n_words;
-	char *words[N_WORDS];
-	enum pipe pipe;
-	enum intel_pipe_crc_object object;
-	enum intel_pipe_crc_source source;
-
-	n_words = display_crc_ctl_tokenize(buf, words, N_WORDS);
-	if (n_words != N_WORDS) {
-		DRM_DEBUG_DRIVER("tokenize failed, a command is %d words\n",
-				 N_WORDS);
-		return -EINVAL;
-	}
-
-	if (display_crc_ctl_parse_object(words[0], &object) < 0) {
-		DRM_DEBUG_DRIVER("unknown object %s\n", words[0]);
-		return -EINVAL;
-	}
-
-	if (display_crc_ctl_parse_pipe(dev_priv, words[1], &pipe) < 0) {
-		DRM_DEBUG_DRIVER("unknown pipe %s\n", words[1]);
-		return -EINVAL;
-	}
-
-	if (display_crc_ctl_parse_source(words[2], &source) < 0) {
-		DRM_DEBUG_DRIVER("unknown source %s\n", words[2]);
-		return -EINVAL;
-	}
-
-	return pipe_crc_set_source(dev_priv, pipe, source);
-}
-
-static ssize_t display_crc_ctl_write(struct file *file, const char __user *ubuf,
-				     size_t len, loff_t *offp)
-{
-	struct seq_file *m = file->private_data;
-	struct drm_i915_private *dev_priv = m->private;
-	char *tmpbuf;
-	int ret;
-
-	if (len == 0)
-		return 0;
-
-	if (len > PAGE_SIZE - 1) {
-		DRM_DEBUG_DRIVER("expected <%lu bytes into pipe crc control\n",
-				 PAGE_SIZE);
-		return -E2BIG;
-	}
-
-	tmpbuf = memdup_user_nul(ubuf, len);
-	if (IS_ERR(tmpbuf))
-		return PTR_ERR(tmpbuf);
-
-	ret = display_crc_ctl_parse(dev_priv, tmpbuf, len);
-
-	kfree(tmpbuf);
-	if (ret < 0)
-		return ret;
-
-	*offp += len;
-	return len;
-}
-
-const struct file_operations i915_display_crc_ctl_fops = {
-	.owner = THIS_MODULE,
-	.open = display_crc_ctl_open,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = single_release,
-	.write = display_crc_ctl_write
-};
-
 void intel_display_crc_init(struct drm_i915_private *dev_priv)
 {
 	enum pipe pipe;
@@ -887,30 +464,8 @@ void intel_display_crc_init(struct drm_i915_private *dev_priv)
 	for_each_pipe(dev_priv, pipe) {
 		struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe];
 
-		pipe_crc->opened = false;
 		spin_lock_init(&pipe_crc->lock);
-		init_waitqueue_head(&pipe_crc->wq);
-	}
-}
-
-int intel_pipe_crc_create(struct drm_minor *minor)
-{
-	struct drm_i915_private *dev_priv = to_i915(minor->dev);
-	struct dentry *ent;
-	int i;
-
-	for (i = 0; i < ARRAY_SIZE(i915_pipe_crc_data); i++) {
-		struct pipe_crc_info *info = &i915_pipe_crc_data[i];
-
-		info->dev_priv = dev_priv;
-		ent = debugfs_create_file(info->name, S_IRUGO,
-					  minor->debugfs_root, info,
-					  &i915_pipe_crc_fops);
-		if (!ent)
-			return -ENOMEM;
 	}
-
-	return 0;
 }
 
 static int i8xx_crc_source_valid(struct drm_i915_private *dev_priv,
diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c
index 53aaaa3e6886..43ae9de12ba3 100644
--- a/drivers/gpu/drm/i915/intel_pm.c
+++ b/drivers/gpu/drm/i915/intel_pm.c
@@ -6264,42 +6264,15 @@ static u32 intel_rps_limits(struct drm_i915_private *dev_priv, u8 val)
 	return limits;
 }
 
-static void gen6_set_rps_thresholds(struct drm_i915_private *dev_priv, u8 val)
+static void rps_set_power(struct drm_i915_private *dev_priv, int new_power)
 {
 	struct intel_rps *rps = &dev_priv->gt_pm.rps;
-	int new_power;
 	u32 threshold_up = 0, threshold_down = 0; /* in % */
 	u32 ei_up = 0, ei_down = 0;
 
-	new_power = rps->power;
-	switch (rps->power) {
-	case LOW_POWER:
-		if (val > rps->efficient_freq + 1 &&
-		    val > rps->cur_freq)
-			new_power = BETWEEN;
-		break;
-
-	case BETWEEN:
-		if (val <= rps->efficient_freq &&
-		    val < rps->cur_freq)
-			new_power = LOW_POWER;
-		else if (val >= rps->rp0_freq &&
-			 val > rps->cur_freq)
-			new_power = HIGH_POWER;
-		break;
+	lockdep_assert_held(&rps->power.mutex);
 
-	case HIGH_POWER:
-		if (val < (rps->rp1_freq + rps->rp0_freq) >> 1 &&
-		    val < rps->cur_freq)
-			new_power = BETWEEN;
-		break;
-	}
-	/* Max/min bins are special */
-	if (val <= rps->min_freq_softlimit)
-		new_power = LOW_POWER;
-	if (val >= rps->max_freq_softlimit)
-		new_power = HIGH_POWER;
-	if (new_power == rps->power)
+	if (new_power == rps->power.mode)
 		return;
 
 	/* Note the units here are not exactly 1us, but 1280ns. */
@@ -6362,12 +6335,71 @@ static void gen6_set_rps_thresholds(struct drm_i915_private *dev_priv, u8 val)
 		   GEN6_RP_DOWN_IDLE_AVG);
 
 skip_hw_write:
-	rps->power = new_power;
-	rps->up_threshold = threshold_up;
-	rps->down_threshold = threshold_down;
+	rps->power.mode = new_power;
+	rps->power.up_threshold = threshold_up;
+	rps->power.down_threshold = threshold_down;
+}
+
+static void gen6_set_rps_thresholds(struct drm_i915_private *dev_priv, u8 val)
+{
+	struct intel_rps *rps = &dev_priv->gt_pm.rps;
+	int new_power;
+
+	new_power = rps->power.mode;
+	switch (rps->power.mode) {
+	case LOW_POWER:
+		if (val > rps->efficient_freq + 1 &&
+		    val > rps->cur_freq)
+			new_power = BETWEEN;
+		break;
+
+	case BETWEEN:
+		if (val <= rps->efficient_freq &&
+		    val < rps->cur_freq)
+			new_power = LOW_POWER;
+		else if (val >= rps->rp0_freq &&
+			 val > rps->cur_freq)
+			new_power = HIGH_POWER;
+		break;
+
+	case HIGH_POWER:
+		if (val < (rps->rp1_freq + rps->rp0_freq) >> 1 &&
+		    val < rps->cur_freq)
+			new_power = BETWEEN;
+		break;
+	}
+	/* Max/min bins are special */
+	if (val <= rps->min_freq_softlimit)
+		new_power = LOW_POWER;
+	if (val >= rps->max_freq_softlimit)
+		new_power = HIGH_POWER;
+
+	mutex_lock(&rps->power.mutex);
+	if (rps->power.interactive)
+		new_power = HIGH_POWER;
+	rps_set_power(dev_priv, new_power);
+	mutex_unlock(&rps->power.mutex);
 	rps->last_adj = 0;
 }
 
+void intel_rps_mark_interactive(struct drm_i915_private *i915, bool interactive)
+{
+	struct intel_rps *rps = &i915->gt_pm.rps;
+
+	if (INTEL_GEN(i915) < 6)
+		return;
+
+	mutex_lock(&rps->power.mutex);
+	if (interactive) {
+		if (!rps->power.interactive++ && READ_ONCE(i915->gt.awake))
+			rps_set_power(i915, HIGH_POWER);
+	} else {
+		GEM_BUG_ON(!rps->power.interactive);
+		rps->power.interactive--;
+	}
+	mutex_unlock(&rps->power.mutex);
+}
+
 static u32 gen6_rps_pm_mask(struct drm_i915_private *dev_priv, u8 val)
 {
 	struct intel_rps *rps = &dev_priv->gt_pm.rps;
@@ -6780,7 +6812,7 @@ static void reset_rps(struct drm_i915_private *dev_priv,
 	u8 freq = rps->cur_freq;
 
 	/* force a reset */
-	rps->power = -1;
+	rps->power.mode = -1;
 	rps->cur_freq = -1;
 
 	if (set(dev_priv, freq))
@@ -7347,11 +7379,11 @@ out:
 
 static void valleyview_cleanup_pctx(struct drm_i915_private *dev_priv)
 {
-	if (WARN_ON(!dev_priv->vlv_pctx))
-		return;
+	struct drm_i915_gem_object *pctx;
 
-	i915_gem_object_put(dev_priv->vlv_pctx);
-	dev_priv->vlv_pctx = NULL;
+	pctx = fetch_and_zero(&dev_priv->vlv_pctx);
+	if (pctx)
+		i915_gem_object_put(pctx);
 }
 
 static void vlv_init_gpll_ref_freq(struct drm_i915_private *dev_priv)
@@ -9604,6 +9636,7 @@ int intel_freq_opcode(struct drm_i915_private *dev_priv, int val)
 void intel_pm_setup(struct drm_i915_private *dev_priv)
 {
 	mutex_init(&dev_priv->pcu_lock);
+	mutex_init(&dev_priv->gt_pm.rps.power.mutex);
 
 	atomic_set(&dev_priv->gt_pm.rps.num_waiters, 0);
 
diff --git a/drivers/gpu/drm/i915/intel_psr.c b/drivers/gpu/drm/i915/intel_psr.c
index d4cd19fea148..4bd5768731ee 100644
--- a/drivers/gpu/drm/i915/intel_psr.c
+++ b/drivers/gpu/drm/i915/intel_psr.c
@@ -56,43 +56,6 @@
 #include "intel_drv.h"
 #include "i915_drv.h"
 
-static inline enum intel_display_power_domain
-psr_aux_domain(struct intel_dp *intel_dp)
-{
-	/* CNL HW requires corresponding AUX IOs to be powered up for PSR.
-	 * However, for non-A AUX ports the corresponding non-EDP transcoders
-	 * would have already enabled power well 2 and DC_OFF. This means we can
-	 * acquire a wider POWER_DOMAIN_AUX_{B,C,D,F} reference instead of a
-	 * specific AUX_IO reference without powering up any extra wells.
-	 * Note that PSR is enabled only on Port A even though this function
-	 * returns the correct domain for other ports too.
-	 */
-	return intel_dp->aux_ch == AUX_CH_A ? POWER_DOMAIN_AUX_IO_A :
-					      intel_dp->aux_power_domain;
-}
-
-static void psr_aux_io_power_get(struct intel_dp *intel_dp)
-{
-	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-	struct drm_i915_private *dev_priv = to_i915(intel_dig_port->base.base.dev);
-
-	if (INTEL_GEN(dev_priv) < 10)
-		return;
-
-	intel_display_power_get(dev_priv, psr_aux_domain(intel_dp));
-}
-
-static void psr_aux_io_power_put(struct intel_dp *intel_dp)
-{
-	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-	struct drm_i915_private *dev_priv = to_i915(intel_dig_port->base.base.dev);
-
-	if (INTEL_GEN(dev_priv) < 10)
-		return;
-
-	intel_display_power_put(dev_priv, psr_aux_domain(intel_dp));
-}
-
 void intel_psr_irq_control(struct drm_i915_private *dev_priv, bool debug)
 {
 	u32 debug_mask, mask;
@@ -278,8 +241,8 @@ void intel_psr_init_dpcd(struct intel_dp *intel_dp)
 	}
 }
 
-static void hsw_psr_setup_vsc(struct intel_dp *intel_dp,
-			      const struct intel_crtc_state *crtc_state)
+static void intel_psr_setup_vsc(struct intel_dp *intel_dp,
+				const struct intel_crtc_state *crtc_state)
 {
 	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
 	struct drm_i915_private *dev_priv = to_i915(intel_dig_port->base.base.dev);
@@ -336,7 +299,7 @@ static void hsw_psr_setup_aux(struct intel_dp *intel_dp)
 	aux_clock_divider = intel_dp->get_aux_clock_divider(intel_dp, 0);
 
 	/* Start with bits set for DDI_AUX_CTL register */
-	aux_ctl = intel_dp->get_aux_send_ctl(intel_dp, 0, sizeof(aux_msg),
+	aux_ctl = intel_dp->get_aux_send_ctl(intel_dp, sizeof(aux_msg),
 					     aux_clock_divider);
 
 	/* Select only valid bits for SRD_AUX_CTL */
@@ -344,7 +307,7 @@ static void hsw_psr_setup_aux(struct intel_dp *intel_dp)
 	I915_WRITE(EDP_PSR_AUX_CTL, aux_ctl);
 }
 
-static void hsw_psr_enable_sink(struct intel_dp *intel_dp)
+static void intel_psr_enable_sink(struct intel_dp *intel_dp)
 {
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
 	struct drm_device *dev = dig_port->base.base.dev;
@@ -360,6 +323,8 @@ static void hsw_psr_enable_sink(struct intel_dp *intel_dp)
 
 	if (dev_priv->psr.link_standby)
 		dpcd_val |= DP_PSR_MAIN_LINK_ACTIVE;
+	if (!dev_priv->psr.psr2_enabled && INTEL_GEN(dev_priv) >= 8)
+		dpcd_val |= DP_PSR_CRC_VERIFICATION;
 	drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_EN_CFG, dpcd_val);
 
 	drm_dp_dpcd_writeb(&intel_dp->aux, DP_SET_POWER, DP_SET_POWER_D0);
@@ -415,6 +380,9 @@ static void hsw_activate_psr1(struct intel_dp *intel_dp)
 	else
 		val |= EDP_PSR_TP1_TP2_SEL;
 
+	if (INTEL_GEN(dev_priv) >= 8)
+		val |= EDP_PSR_CRC_ENABLE;
+
 	val |= I915_READ(EDP_PSR_CTL) & EDP_PSR_RESTORE_PSR_ACTIVE_CTX_MASK;
 	I915_WRITE(EDP_PSR_CTL, val);
 }
@@ -456,24 +424,6 @@ static void hsw_activate_psr2(struct intel_dp *intel_dp)
 	I915_WRITE(EDP_PSR2_CTL, val);
 }
 
-static void hsw_psr_activate(struct intel_dp *intel_dp)
-{
-	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
-	struct drm_device *dev = dig_port->base.base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-
-	/* On HSW+ after we enable PSR on source it will activate it
-	 * as soon as it match configure idle_frame count. So
-	 * we just actually enable it here on activation time.
-	 */
-
-	/* psr1 and psr2 are mutually exclusive.*/
-	if (dev_priv->psr.psr2_enabled)
-		hsw_activate_psr2(intel_dp);
-	else
-		hsw_activate_psr1(intel_dp);
-}
-
 static bool intel_psr2_config_valid(struct intel_dp *intel_dp,
 				    struct intel_crtc_state *crtc_state)
 {
@@ -576,27 +526,29 @@ static void intel_psr_activate(struct intel_dp *intel_dp)
 	struct drm_device *dev = intel_dig_port->base.base.dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
 
-	if (dev_priv->psr.psr2_enabled)
+	if (INTEL_GEN(dev_priv) >= 9)
 		WARN_ON(I915_READ(EDP_PSR2_CTL) & EDP_PSR2_ENABLE);
-	else
-		WARN_ON(I915_READ(EDP_PSR_CTL) & EDP_PSR_ENABLE);
+	WARN_ON(I915_READ(EDP_PSR_CTL) & EDP_PSR_ENABLE);
 	WARN_ON(dev_priv->psr.active);
 	lockdep_assert_held(&dev_priv->psr.lock);
 
-	dev_priv->psr.activate(intel_dp);
+	/* psr1 and psr2 are mutually exclusive.*/
+	if (dev_priv->psr.psr2_enabled)
+		hsw_activate_psr2(intel_dp);
+	else
+		hsw_activate_psr1(intel_dp);
+
 	dev_priv->psr.active = true;
 }
 
-static void hsw_psr_enable_source(struct intel_dp *intel_dp,
-				  const struct intel_crtc_state *crtc_state)
+static void intel_psr_enable_source(struct intel_dp *intel_dp,
+				    const struct intel_crtc_state *crtc_state)
 {
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
 	struct drm_device *dev = dig_port->base.base.dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
 
-	psr_aux_io_power_get(intel_dp);
-
 	/* Only HSW and BDW have PSR AUX registers that need to be setup. SKL+
 	 * use hardcoded values PSR AUX transactions
 	 */
@@ -632,7 +584,8 @@ static void hsw_psr_enable_source(struct intel_dp *intel_dp,
 			   EDP_PSR_DEBUG_MASK_MEMUP |
 			   EDP_PSR_DEBUG_MASK_HPD |
 			   EDP_PSR_DEBUG_MASK_LPSP |
-			   EDP_PSR_DEBUG_MASK_DISP_REG_WRITE);
+			   EDP_PSR_DEBUG_MASK_DISP_REG_WRITE |
+			   EDP_PSR_DEBUG_MASK_MAX_SLEEP);
 	}
 }
 
@@ -666,9 +619,9 @@ void intel_psr_enable(struct intel_dp *intel_dp,
 	dev_priv->psr.psr2_enabled = crtc_state->has_psr2;
 	dev_priv->psr.busy_frontbuffer_bits = 0;
 
-	dev_priv->psr.setup_vsc(intel_dp, crtc_state);
-	dev_priv->psr.enable_sink(intel_dp);
-	dev_priv->psr.enable_source(intel_dp, crtc_state);
+	intel_psr_setup_vsc(intel_dp, crtc_state);
+	intel_psr_enable_sink(intel_dp);
+	intel_psr_enable_source(intel_dp, crtc_state);
 	dev_priv->psr.enabled = intel_dp;
 
 	intel_psr_activate(intel_dp);
@@ -677,8 +630,8 @@ unlock:
 	mutex_unlock(&dev_priv->psr.lock);
 }
 
-static void hsw_psr_disable(struct intel_dp *intel_dp,
-			    const struct intel_crtc_state *old_crtc_state)
+static void
+intel_psr_disable_source(struct intel_dp *intel_dp)
 {
 	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
 	struct drm_device *dev = intel_dig_port->base.base.dev;
@@ -717,8 +670,25 @@ static void hsw_psr_disable(struct intel_dp *intel_dp,
 		else
 			WARN_ON(I915_READ(EDP_PSR_CTL) & EDP_PSR_ENABLE);
 	}
+}
+
+static void intel_psr_disable_locked(struct intel_dp *intel_dp)
+{
+	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
+	struct drm_device *dev = intel_dig_port->base.base.dev;
+	struct drm_i915_private *dev_priv = to_i915(dev);
+
+	lockdep_assert_held(&dev_priv->psr.lock);
 
-	psr_aux_io_power_put(intel_dp);
+	if (!dev_priv->psr.enabled)
+		return;
+
+	intel_psr_disable_source(intel_dp);
+
+	/* Disable PSR on Sink */
+	drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_EN_CFG, 0);
+
+	dev_priv->psr.enabled = NULL;
 }
 
 /**
@@ -742,22 +712,49 @@ void intel_psr_disable(struct intel_dp *intel_dp,
 		return;
 
 	mutex_lock(&dev_priv->psr.lock);
-	if (!dev_priv->psr.enabled) {
-		mutex_unlock(&dev_priv->psr.lock);
-		return;
-	}
+	intel_psr_disable_locked(intel_dp);
+	mutex_unlock(&dev_priv->psr.lock);
+	cancel_work_sync(&dev_priv->psr.work);
+}
 
-	dev_priv->psr.disable_source(intel_dp, old_crtc_state);
+int intel_psr_wait_for_idle(const struct intel_crtc_state *new_crtc_state)
+{
+	struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->base.crtc);
+	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
+	i915_reg_t reg;
+	u32 mask;
 
-	/* Disable PSR on Sink */
-	drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_EN_CFG, 0);
+	if (!new_crtc_state->has_psr)
+		return 0;
 
-	dev_priv->psr.enabled = NULL;
-	mutex_unlock(&dev_priv->psr.lock);
-	cancel_work_sync(&dev_priv->psr.work);
+	/*
+	 * The sole user right now is intel_pipe_update_start(),
+	 * which won't race with psr_enable/disable, which is
+	 * where psr2_enabled is written to. So, we don't need
+	 * to acquire the psr.lock. More importantly, we want the
+	 * latency inside intel_pipe_update_start() to be as low
+	 * as possible, so no need to acquire psr.lock when it is
+	 * not needed and will induce latencies in the atomic
+	 * update path.
+	 */
+	if (dev_priv->psr.psr2_enabled) {
+		reg = EDP_PSR2_STATUS;
+		mask = EDP_PSR2_STATUS_STATE_MASK;
+	} else {
+		reg = EDP_PSR_STATUS;
+		mask = EDP_PSR_STATUS_STATE_MASK;
+	}
+
+	/*
+	 * Max time for PSR to idle = Inverse of the refresh rate +
+	 * 6 ms of exit training time + 1.5 ms of aux channel
+	 * handshake. 50 msec is defesive enough to cover everything.
+	 */
+	return intel_wait_for_register(dev_priv, reg, mask,
+				       EDP_PSR_STATUS_STATE_IDLE, 50);
 }
 
-static bool psr_wait_for_idle(struct drm_i915_private *dev_priv)
+static bool __psr_wait_for_idle_locked(struct drm_i915_private *dev_priv)
 {
 	struct intel_dp *intel_dp;
 	i915_reg_t reg;
@@ -803,7 +800,7 @@ static void intel_psr_work(struct work_struct *work)
 	 * PSR might take some time to get fully disabled
 	 * and be ready for re-enable.
 	 */
-	if (!psr_wait_for_idle(dev_priv))
+	if (!__psr_wait_for_idle_locked(dev_priv))
 		goto unlock;
 
 	/*
@@ -811,7 +808,7 @@ static void intel_psr_work(struct work_struct *work)
 	 * recheck. Since psr_flush first clears this and then reschedules we
 	 * won't ever miss a flush when bailing out here.
 	 */
-	if (dev_priv->psr.busy_frontbuffer_bits)
+	if (dev_priv->psr.busy_frontbuffer_bits || dev_priv->psr.active)
 		goto unlock;
 
 	intel_psr_activate(dev_priv->psr.enabled);
@@ -974,23 +971,60 @@ void intel_psr_init(struct drm_i915_private *dev_priv)
 		/* For new platforms let's respect VBT back again */
 		dev_priv->psr.link_standby = dev_priv->vbt.psr.full_link;
 
-	/* Override link_standby x link_off defaults */
-	if (i915_modparams.enable_psr == 2 && !dev_priv->psr.link_standby) {
-		DRM_DEBUG_KMS("PSR: Forcing link standby\n");
-		dev_priv->psr.link_standby = true;
-	}
-	if (i915_modparams.enable_psr == 3 && dev_priv->psr.link_standby) {
-		DRM_DEBUG_KMS("PSR: Forcing main link off\n");
-		dev_priv->psr.link_standby = false;
-	}
-
 	INIT_WORK(&dev_priv->psr.work, intel_psr_work);
 	mutex_init(&dev_priv->psr.lock);
+}
 
-	dev_priv->psr.enable_source = hsw_psr_enable_source;
-	dev_priv->psr.disable_source = hsw_psr_disable;
-	dev_priv->psr.enable_sink = hsw_psr_enable_sink;
-	dev_priv->psr.activate = hsw_psr_activate;
-	dev_priv->psr.setup_vsc = hsw_psr_setup_vsc;
+void intel_psr_short_pulse(struct intel_dp *intel_dp)
+{
+	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
+	struct drm_device *dev = intel_dig_port->base.base.dev;
+	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct i915_psr *psr = &dev_priv->psr;
+	u8 val;
+	const u8 errors = DP_PSR_RFB_STORAGE_ERROR |
+			  DP_PSR_VSC_SDP_UNCORRECTABLE_ERROR |
+			  DP_PSR_LINK_CRC_ERROR;
+
+	if (!CAN_PSR(dev_priv) || !intel_dp_is_edp(intel_dp))
+		return;
+
+	mutex_lock(&psr->lock);
+
+	if (psr->enabled != intel_dp)
+		goto exit;
+
+	if (drm_dp_dpcd_readb(&intel_dp->aux, DP_PSR_STATUS, &val) != 1) {
+		DRM_ERROR("PSR_STATUS dpcd read failed\n");
+		goto exit;
+	}
+
+	if ((val & DP_PSR_SINK_STATE_MASK) == DP_PSR_SINK_INTERNAL_ERROR) {
+		DRM_DEBUG_KMS("PSR sink internal error, disabling PSR\n");
+		intel_psr_disable_locked(intel_dp);
+	}
+
+	if (drm_dp_dpcd_readb(&intel_dp->aux, DP_PSR_ERROR_STATUS, &val) != 1) {
+		DRM_ERROR("PSR_ERROR_STATUS dpcd read failed\n");
+		goto exit;
+	}
 
+	if (val & DP_PSR_RFB_STORAGE_ERROR)
+		DRM_DEBUG_KMS("PSR RFB storage error, disabling PSR\n");
+	if (val & DP_PSR_VSC_SDP_UNCORRECTABLE_ERROR)
+		DRM_DEBUG_KMS("PSR VSC SDP uncorrectable error, disabling PSR\n");
+	if (val & DP_PSR_LINK_CRC_ERROR)
+		DRM_ERROR("PSR Link CRC error, disabling PSR\n");
+
+	if (val & ~errors)
+		DRM_ERROR("PSR_ERROR_STATUS unhandled errors %x\n",
+			  val & ~errors);
+	if (val & errors)
+		intel_psr_disable_locked(intel_dp);
+	/* clear status register */
+	drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_ERROR_STATUS, val);
+
+	/* TODO: handle PSR2 errors */
+exit:
+	mutex_unlock(&psr->lock);
 }
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c
index e0448eff12bd..6a8f27d0a742 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.c
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.c
@@ -387,8 +387,18 @@ static void intel_ring_setup_status_page(struct intel_engine_cs *engine)
 		mmio = RING_HWS_PGA(engine->mmio_base);
 	}
 
-	if (INTEL_GEN(dev_priv) >= 6)
-		I915_WRITE(RING_HWSTAM(engine->mmio_base), 0xffffffff);
+	if (INTEL_GEN(dev_priv) >= 6) {
+		u32 mask = ~0u;
+
+		/*
+		 * Keep the render interrupt unmasked as this papers over
+		 * lost interrupts following a reset.
+		 */
+		if (engine->id == RCS)
+			mask &= ~BIT(0);
+
+		I915_WRITE(RING_HWSTAM(engine->mmio_base), mask);
+	}
 
 	I915_WRITE(mmio, engine->status_page.ggtt_offset);
 	POSTING_READ(mmio);
@@ -524,8 +534,6 @@ static int init_ring_common(struct intel_engine_cs *engine)
 		goto out;
 	}
 
-	intel_engine_init_hangcheck(engine);
-
 	if (INTEL_GEN(dev_priv) > 2)
 		I915_WRITE_MODE(engine, _MASKED_BIT_DISABLE(STOP_RING));
 
@@ -1089,6 +1097,7 @@ void intel_ring_unpin(struct intel_ring *ring)
 static struct i915_vma *
 intel_ring_create_vma(struct drm_i915_private *dev_priv, int size)
 {
+	struct i915_address_space *vm = &dev_priv->ggtt.vm;
 	struct drm_i915_gem_object *obj;
 	struct i915_vma *vma;
 
@@ -1098,10 +1107,14 @@ intel_ring_create_vma(struct drm_i915_private *dev_priv, int size)
 	if (IS_ERR(obj))
 		return ERR_CAST(obj);
 
-	/* mark ring buffers as read-only from GPU side by default */
-	obj->gt_ro = 1;
+	/*
+	 * Mark ring buffers as read-only from GPU side (so no stray overwrites)
+	 * if supported by the platform's GGTT.
+	 */
+	if (vm->has_read_only)
+		i915_gem_object_set_readonly(obj);
 
-	vma = i915_vma_instance(obj, &dev_priv->ggtt.vm, NULL);
+	vma = i915_vma_instance(obj, vm, NULL);
 	if (IS_ERR(vma))
 		goto err;
 
@@ -1169,8 +1182,11 @@ static void intel_ring_context_destroy(struct intel_context *ce)
 {
 	GEM_BUG_ON(ce->pin_count);
 
-	if (ce->state)
-		__i915_gem_object_release_unless_active(ce->state->obj);
+	if (!ce->state)
+		return;
+
+	GEM_BUG_ON(i915_gem_object_is_active(ce->state->obj));
+	i915_gem_object_put(ce->state->obj);
 }
 
 static int __context_pin_ppgtt(struct i915_gem_context *ctx)
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.h b/drivers/gpu/drm/i915/intel_ringbuffer.h
index a0bc7a8222b4..f5ffa6d31e82 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.h
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.h
@@ -193,6 +193,11 @@ struct i915_priolist {
 	int priority;
 };
 
+struct st_preempt_hang {
+	struct completion completion;
+	bool inject_hang;
+};
+
 /**
  * struct intel_engine_execlists - execlist submission queue and port state
  *
@@ -292,32 +297,49 @@ struct intel_engine_execlists {
 	/**
 	 * @queue: queue of requests, in priority lists
 	 */
-	struct rb_root queue;
+	struct rb_root_cached queue;
 
 	/**
-	 * @first: leftmost level in priority @queue
+	 * @csb_read: control register for Context Switch buffer
+	 *
+	 * Note this register is always in mmio.
 	 */
-	struct rb_node *first;
+	u32 __iomem *csb_read;
 
 	/**
-	 * @fw_domains: forcewake domains for irq tasklet
+	 * @csb_write: control register for Context Switch buffer
+	 *
+	 * Note this register may be either mmio or HWSP shadow.
 	 */
-	unsigned int fw_domains;
+	u32 *csb_write;
 
 	/**
-	 * @csb_head: context status buffer head
+	 * @csb_status: status array for Context Switch buffer
+	 *
+	 * Note these register may be either mmio or HWSP shadow.
+	 */
+	u32 *csb_status;
+
+	/**
+	 * @preempt_complete_status: expected CSB upon completing preemption
 	 */
-	unsigned int csb_head;
+	u32 preempt_complete_status;
 
 	/**
-	 * @csb_use_mmio: access csb through mmio, instead of hwsp
+	 * @csb_write_reset: reset value for CSB write pointer
+	 *
+	 * As the CSB write pointer maybe either in HWSP or as a field
+	 * inside an mmio register, we want to reprogram it slightly
+	 * differently to avoid later confusion.
 	 */
-	bool csb_use_mmio;
+	u32 csb_write_reset;
 
 	/**
-	 * @preempt_complete_status: expected CSB upon completing preemption
+	 * @csb_head: context status buffer head
 	 */
-	u32 preempt_complete_status;
+	u8 csb_head;
+
+	I915_SELFTEST_DECLARE(struct st_preempt_hang preempt_hang;)
 };
 
 #define INTEL_ENGINE_CS_MAX_NAME 8
@@ -345,10 +367,8 @@ struct intel_engine_cs {
 	struct drm_i915_gem_object *default_state;
 	void *pinned_default_state;
 
-	atomic_t irq_count;
 	unsigned long irq_posted;
 #define ENGINE_IRQ_BREADCRUMB 0
-#define ENGINE_IRQ_EXECLIST 1
 
 	/* Rather than have every client wait upon all user interrupts,
 	 * with the herd waking after every interrupt and each doing the
@@ -380,6 +400,7 @@ struct intel_engine_cs {
 
 		unsigned int hangcheck_interrupts;
 		unsigned int irq_enabled;
+		unsigned int irq_count;
 
 		bool irq_armed : 1;
 		I915_SELFTEST_DECLARE(bool mock : 1);
@@ -669,6 +690,12 @@ execlists_clear_active(struct intel_engine_execlists *execlists,
 	__clear_bit(bit, (unsigned long *)&execlists->active);
 }
 
+static inline void
+execlists_clear_all_active(struct intel_engine_execlists *execlists)
+{
+	execlists->active = 0;
+}
+
 static inline bool
 execlists_is_active(const struct intel_engine_execlists *execlists,
 		    unsigned int bit)
@@ -928,11 +955,10 @@ static inline u32 intel_hws_preempt_done_address(struct intel_engine_cs *engine)
 /* intel_breadcrumbs.c -- user interrupt bottom-half for waiters */
 int intel_engine_init_breadcrumbs(struct intel_engine_cs *engine);
 
-static inline void intel_wait_init(struct intel_wait *wait,
-				   struct i915_request *rq)
+static inline void intel_wait_init(struct intel_wait *wait)
 {
 	wait->tsk = current;
-	wait->request = rq;
+	wait->request = NULL;
 }
 
 static inline void intel_wait_init_for_seqno(struct intel_wait *wait, u32 seqno)
@@ -1136,4 +1162,24 @@ void intel_disable_engine_stats(struct intel_engine_cs *engine);
 
 ktime_t intel_engine_get_busy_time(struct intel_engine_cs *engine);
 
+#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
+
+static inline bool inject_preempt_hang(struct intel_engine_execlists *execlists)
+{
+	if (!execlists->preempt_hang.inject_hang)
+		return false;
+
+	complete(&execlists->preempt_hang.completion);
+	return true;
+}
+
+#else
+
+static inline bool inject_preempt_hang(struct intel_engine_execlists *execlists)
+{
+	return false;
+}
+
+#endif
+
 #endif /* _INTEL_RINGBUFFER_H_ */
diff --git a/drivers/gpu/drm/i915/intel_runtime_pm.c b/drivers/gpu/drm/i915/intel_runtime_pm.c
index de3a81034f77..6b5aa3b074ec 100644
--- a/drivers/gpu/drm/i915/intel_runtime_pm.c
+++ b/drivers/gpu/drm/i915/intel_runtime_pm.c
@@ -134,6 +134,14 @@ intel_display_power_domain_str(enum intel_display_power_domain domain)
 		return "AUX_F";
 	case POWER_DOMAIN_AUX_IO_A:
 		return "AUX_IO_A";
+	case POWER_DOMAIN_AUX_TBT1:
+		return "AUX_TBT1";
+	case POWER_DOMAIN_AUX_TBT2:
+		return "AUX_TBT2";
+	case POWER_DOMAIN_AUX_TBT3:
+		return "AUX_TBT3";
+	case POWER_DOMAIN_AUX_TBT4:
+		return "AUX_TBT4";
 	case POWER_DOMAIN_GMBUS:
 		return "GMBUS";
 	case POWER_DOMAIN_INIT:
@@ -384,7 +392,8 @@ static void hsw_power_well_enable(struct drm_i915_private *dev_priv,
 	u32 val;
 
 	if (wait_fuses) {
-		pg = SKL_PW_TO_PG(id);
+		pg = INTEL_GEN(dev_priv) >= 11 ? ICL_PW_TO_PG(id) :
+						 SKL_PW_TO_PG(id);
 		/*
 		 * For PW1 we have to wait both for the PW0/PG0 fuse state
 		 * before enabling the power well and PW1/PG1's own fuse
@@ -430,6 +439,43 @@ static void hsw_power_well_disable(struct drm_i915_private *dev_priv,
 	hsw_wait_for_power_well_disable(dev_priv, power_well);
 }
 
+#define ICL_AUX_PW_TO_PORT(pw)	((pw) - ICL_DISP_PW_AUX_A)
+
+static void
+icl_combo_phy_aux_power_well_enable(struct drm_i915_private *dev_priv,
+				    struct i915_power_well *power_well)
+{
+	enum i915_power_well_id id = power_well->id;
+	enum port port = ICL_AUX_PW_TO_PORT(id);
+	u32 val;
+
+	val = I915_READ(HSW_PWR_WELL_CTL_DRIVER(id));
+	I915_WRITE(HSW_PWR_WELL_CTL_DRIVER(id), val | HSW_PWR_WELL_CTL_REQ(id));
+
+	val = I915_READ(ICL_PORT_CL_DW12(port));
+	I915_WRITE(ICL_PORT_CL_DW12(port), val | ICL_LANE_ENABLE_AUX);
+
+	hsw_wait_for_power_well_enable(dev_priv, power_well);
+}
+
+static void
+icl_combo_phy_aux_power_well_disable(struct drm_i915_private *dev_priv,
+				     struct i915_power_well *power_well)
+{
+	enum i915_power_well_id id = power_well->id;
+	enum port port = ICL_AUX_PW_TO_PORT(id);
+	u32 val;
+
+	val = I915_READ(ICL_PORT_CL_DW12(port));
+	I915_WRITE(ICL_PORT_CL_DW12(port), val & ~ICL_LANE_ENABLE_AUX);
+
+	val = I915_READ(HSW_PWR_WELL_CTL_DRIVER(id));
+	I915_WRITE(HSW_PWR_WELL_CTL_DRIVER(id),
+		   val & ~HSW_PWR_WELL_CTL_REQ(id));
+
+	hsw_wait_for_power_well_disable(dev_priv, power_well);
+}
+
 /*
  * We should only use the power well if we explicitly asked the hardware to
  * enable it, so check if it's enabled and also check if we've requested it to
@@ -1824,6 +1870,7 @@ void intel_display_power_put(struct drm_i915_private *dev_priv,
 	BIT_ULL(POWER_DOMAIN_INIT))
 #define GLK_DISPLAY_AUX_A_POWER_DOMAINS (		\
 	BIT_ULL(POWER_DOMAIN_AUX_A) |		\
+	BIT_ULL(POWER_DOMAIN_AUX_IO_A) |		\
 	BIT_ULL(POWER_DOMAIN_INIT))
 #define GLK_DISPLAY_AUX_B_POWER_DOMAINS (		\
 	BIT_ULL(POWER_DOMAIN_AUX_B) |		\
@@ -1896,6 +1943,105 @@ void intel_display_power_put(struct drm_i915_private *dev_priv,
 	BIT_ULL(POWER_DOMAIN_AUX_A) |			\
 	BIT_ULL(POWER_DOMAIN_INIT))
 
+/*
+ * ICL PW_0/PG_0 domains (HW/DMC control):
+ * - PCI
+ * - clocks except port PLL
+ * - central power except FBC
+ * - shared functions except pipe interrupts, pipe MBUS, DBUF registers
+ * ICL PW_1/PG_1 domains (HW/DMC control):
+ * - DBUF function
+ * - PIPE_A and its planes, except VGA
+ * - transcoder EDP + PSR
+ * - transcoder DSI
+ * - DDI_A
+ * - FBC
+ */
+#define ICL_PW_4_POWER_DOMAINS (			\
+	BIT_ULL(POWER_DOMAIN_PIPE_C) |			\
+	BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |	\
+	BIT_ULL(POWER_DOMAIN_INIT))
+	/* VDSC/joining */
+#define ICL_PW_3_POWER_DOMAINS (			\
+	ICL_PW_4_POWER_DOMAINS |			\
+	BIT_ULL(POWER_DOMAIN_PIPE_B) |			\
+	BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |		\
+	BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |		\
+	BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |		\
+	BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |	\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |	\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_B_IO) |		\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |	\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_C_IO) |		\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |	\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_D_IO) |		\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_E_LANES) |	\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_E_IO) |		\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_F_LANES) |	\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_F_IO) |		\
+	BIT_ULL(POWER_DOMAIN_AUX_B) |			\
+	BIT_ULL(POWER_DOMAIN_AUX_C) |			\
+	BIT_ULL(POWER_DOMAIN_AUX_D) |			\
+	BIT_ULL(POWER_DOMAIN_AUX_E) |			\
+	BIT_ULL(POWER_DOMAIN_AUX_F) |			\
+	BIT_ULL(POWER_DOMAIN_AUX_TBT1) |		\
+	BIT_ULL(POWER_DOMAIN_AUX_TBT2) |		\
+	BIT_ULL(POWER_DOMAIN_AUX_TBT3) |		\
+	BIT_ULL(POWER_DOMAIN_AUX_TBT4) |		\
+	BIT_ULL(POWER_DOMAIN_VGA) |			\
+	BIT_ULL(POWER_DOMAIN_AUDIO) |			\
+	BIT_ULL(POWER_DOMAIN_INIT))
+	/*
+	 * - transcoder WD
+	 * - KVMR (HW control)
+	 */
+#define ICL_PW_2_POWER_DOMAINS (			\
+	ICL_PW_3_POWER_DOMAINS |			\
+	BIT_ULL(POWER_DOMAIN_INIT))
+	/*
+	 * - eDP/DSI VDSC
+	 * - KVMR (HW control)
+	 */
+#define ICL_DISPLAY_DC_OFF_POWER_DOMAINS (		\
+	ICL_PW_2_POWER_DOMAINS |			\
+	BIT_ULL(POWER_DOMAIN_MODESET) |			\
+	BIT_ULL(POWER_DOMAIN_AUX_A) |			\
+	BIT_ULL(POWER_DOMAIN_INIT))
+
+#define ICL_DDI_IO_A_POWER_DOMAINS (			\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_A_IO))
+#define ICL_DDI_IO_B_POWER_DOMAINS (			\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_B_IO))
+#define ICL_DDI_IO_C_POWER_DOMAINS (			\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_C_IO))
+#define ICL_DDI_IO_D_POWER_DOMAINS (			\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_D_IO))
+#define ICL_DDI_IO_E_POWER_DOMAINS (			\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_E_IO))
+#define ICL_DDI_IO_F_POWER_DOMAINS (			\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_F_IO))
+
+#define ICL_AUX_A_IO_POWER_DOMAINS (			\
+	BIT_ULL(POWER_DOMAIN_AUX_A))
+#define ICL_AUX_B_IO_POWER_DOMAINS (			\
+	BIT_ULL(POWER_DOMAIN_AUX_B))
+#define ICL_AUX_C_IO_POWER_DOMAINS (			\
+	BIT_ULL(POWER_DOMAIN_AUX_C))
+#define ICL_AUX_D_IO_POWER_DOMAINS (			\
+	BIT_ULL(POWER_DOMAIN_AUX_D))
+#define ICL_AUX_E_IO_POWER_DOMAINS (			\
+	BIT_ULL(POWER_DOMAIN_AUX_E))
+#define ICL_AUX_F_IO_POWER_DOMAINS (			\
+	BIT_ULL(POWER_DOMAIN_AUX_F))
+#define ICL_AUX_TBT1_IO_POWER_DOMAINS (			\
+	BIT_ULL(POWER_DOMAIN_AUX_TBT1))
+#define ICL_AUX_TBT2_IO_POWER_DOMAINS (			\
+	BIT_ULL(POWER_DOMAIN_AUX_TBT2))
+#define ICL_AUX_TBT3_IO_POWER_DOMAINS (			\
+	BIT_ULL(POWER_DOMAIN_AUX_TBT3))
+#define ICL_AUX_TBT4_IO_POWER_DOMAINS (			\
+	BIT_ULL(POWER_DOMAIN_AUX_TBT4))
+
 static const struct i915_power_well_ops i9xx_always_on_power_well_ops = {
 	.sync_hw = i9xx_power_well_sync_hw_noop,
 	.enable = i9xx_always_on_power_well_noop,
@@ -2453,6 +2599,157 @@ static struct i915_power_well cnl_power_wells[] = {
 	},
 };
 
+static const struct i915_power_well_ops icl_combo_phy_aux_power_well_ops = {
+	.sync_hw = hsw_power_well_sync_hw,
+	.enable = icl_combo_phy_aux_power_well_enable,
+	.disable = icl_combo_phy_aux_power_well_disable,
+	.is_enabled = hsw_power_well_enabled,
+};
+
+static struct i915_power_well icl_power_wells[] = {
+	{
+		.name = "always-on",
+		.always_on = 1,
+		.domains = POWER_DOMAIN_MASK,
+		.ops = &i9xx_always_on_power_well_ops,
+		.id = I915_DISP_PW_ALWAYS_ON,
+	},
+	{
+		.name = "power well 1",
+		/* Handled by the DMC firmware */
+		.domains = 0,
+		.ops = &hsw_power_well_ops,
+		.id = ICL_DISP_PW_1,
+		.hsw.has_fuses = true,
+	},
+	{
+		.name = "power well 2",
+		.domains = ICL_PW_2_POWER_DOMAINS,
+		.ops = &hsw_power_well_ops,
+		.id = ICL_DISP_PW_2,
+		.hsw.has_fuses = true,
+	},
+	{
+		.name = "DC off",
+		.domains = ICL_DISPLAY_DC_OFF_POWER_DOMAINS,
+		.ops = &gen9_dc_off_power_well_ops,
+		.id = SKL_DISP_PW_DC_OFF,
+	},
+	{
+		.name = "power well 3",
+		.domains = ICL_PW_3_POWER_DOMAINS,
+		.ops = &hsw_power_well_ops,
+		.id = ICL_DISP_PW_3,
+		.hsw.irq_pipe_mask = BIT(PIPE_B),
+		.hsw.has_vga = true,
+		.hsw.has_fuses = true,
+	},
+	{
+		.name = "DDI A IO",
+		.domains = ICL_DDI_IO_A_POWER_DOMAINS,
+		.ops = &hsw_power_well_ops,
+		.id = ICL_DISP_PW_DDI_A,
+	},
+	{
+		.name = "DDI B IO",
+		.domains = ICL_DDI_IO_B_POWER_DOMAINS,
+		.ops = &hsw_power_well_ops,
+		.id = ICL_DISP_PW_DDI_B,
+	},
+	{
+		.name = "DDI C IO",
+		.domains = ICL_DDI_IO_C_POWER_DOMAINS,
+		.ops = &hsw_power_well_ops,
+		.id = ICL_DISP_PW_DDI_C,
+	},
+	{
+		.name = "DDI D IO",
+		.domains = ICL_DDI_IO_D_POWER_DOMAINS,
+		.ops = &hsw_power_well_ops,
+		.id = ICL_DISP_PW_DDI_D,
+	},
+	{
+		.name = "DDI E IO",
+		.domains = ICL_DDI_IO_E_POWER_DOMAINS,
+		.ops = &hsw_power_well_ops,
+		.id = ICL_DISP_PW_DDI_E,
+	},
+	{
+		.name = "DDI F IO",
+		.domains = ICL_DDI_IO_F_POWER_DOMAINS,
+		.ops = &hsw_power_well_ops,
+		.id = ICL_DISP_PW_DDI_F,
+	},
+	{
+		.name = "AUX A",
+		.domains = ICL_AUX_A_IO_POWER_DOMAINS,
+		.ops = &icl_combo_phy_aux_power_well_ops,
+		.id = ICL_DISP_PW_AUX_A,
+	},
+	{
+		.name = "AUX B",
+		.domains = ICL_AUX_B_IO_POWER_DOMAINS,
+		.ops = &icl_combo_phy_aux_power_well_ops,
+		.id = ICL_DISP_PW_AUX_B,
+	},
+	{
+		.name = "AUX C",
+		.domains = ICL_AUX_C_IO_POWER_DOMAINS,
+		.ops = &hsw_power_well_ops,
+		.id = ICL_DISP_PW_AUX_C,
+	},
+	{
+		.name = "AUX D",
+		.domains = ICL_AUX_D_IO_POWER_DOMAINS,
+		.ops = &hsw_power_well_ops,
+		.id = ICL_DISP_PW_AUX_D,
+	},
+	{
+		.name = "AUX E",
+		.domains = ICL_AUX_E_IO_POWER_DOMAINS,
+		.ops = &hsw_power_well_ops,
+		.id = ICL_DISP_PW_AUX_E,
+	},
+	{
+		.name = "AUX F",
+		.domains = ICL_AUX_F_IO_POWER_DOMAINS,
+		.ops = &hsw_power_well_ops,
+		.id = ICL_DISP_PW_AUX_F,
+	},
+	{
+		.name = "AUX TBT1",
+		.domains = ICL_AUX_TBT1_IO_POWER_DOMAINS,
+		.ops = &hsw_power_well_ops,
+		.id = ICL_DISP_PW_AUX_TBT1,
+	},
+	{
+		.name = "AUX TBT2",
+		.domains = ICL_AUX_TBT2_IO_POWER_DOMAINS,
+		.ops = &hsw_power_well_ops,
+		.id = ICL_DISP_PW_AUX_TBT2,
+	},
+	{
+		.name = "AUX TBT3",
+		.domains = ICL_AUX_TBT3_IO_POWER_DOMAINS,
+		.ops = &hsw_power_well_ops,
+		.id = ICL_DISP_PW_AUX_TBT3,
+	},
+	{
+		.name = "AUX TBT4",
+		.domains = ICL_AUX_TBT4_IO_POWER_DOMAINS,
+		.ops = &hsw_power_well_ops,
+		.id = ICL_DISP_PW_AUX_TBT4,
+	},
+	{
+		.name = "power well 4",
+		.domains = ICL_PW_4_POWER_DOMAINS,
+		.ops = &hsw_power_well_ops,
+		.id = ICL_DISP_PW_4,
+		.hsw.has_fuses = true,
+		.hsw.irq_pipe_mask = BIT(PIPE_C),
+	},
+};
+
 static int
 sanitize_disable_power_well_option(const struct drm_i915_private *dev_priv,
 				   int disable_power_well)
@@ -2470,7 +2767,7 @@ static uint32_t get_allowed_dc_mask(const struct drm_i915_private *dev_priv,
 	int requested_dc;
 	int max_dc;
 
-	if (IS_GEN9_BC(dev_priv) || IS_CANNONLAKE(dev_priv)) {
+	if (IS_GEN9_BC(dev_priv) || INTEL_INFO(dev_priv)->gen >= 10) {
 		max_dc = 2;
 		mask = 0;
 	} else if (IS_GEN9_LP(dev_priv)) {
@@ -2558,7 +2855,9 @@ int intel_power_domains_init(struct drm_i915_private *dev_priv)
 	 * The enabling order will be from lower to higher indexed wells,
 	 * the disabling order is reversed.
 	 */
-	if (IS_HASWELL(dev_priv)) {
+	if (IS_ICELAKE(dev_priv)) {
+		set_power_wells(power_domains, icl_power_wells);
+	} else if (IS_HASWELL(dev_priv)) {
 		set_power_wells(power_domains, hsw_power_wells);
 	} else if (IS_BROADWELL(dev_priv)) {
 		set_power_wells(power_domains, bdw_power_wells);
@@ -2913,6 +3212,7 @@ static void cnl_set_procmon_ref_values(struct drm_i915_private *dev_priv,
 	switch (val & (PROCESS_INFO_MASK | VOLTAGE_INFO_MASK)) {
 	default:
 		MISSING_CASE(val);
+		/* fall through */
 	case VOLTAGE_INFO_0_85V | PROCESS_INFO_DOT_0:
 		procmon = &cnl_procmon_values[PROCMON_0_85V_DOT_0];
 		break;
@@ -3025,6 +3325,8 @@ static void cnl_display_core_uninit(struct drm_i915_private *dev_priv)
 static void icl_display_core_init(struct drm_i915_private *dev_priv,
 				  bool resume)
 {
+	struct i915_power_domains *power_domains = &dev_priv->power_domains;
+	struct i915_power_well *well;
 	enum port port;
 	u32 val;
 
@@ -3053,8 +3355,14 @@ static void icl_display_core_init(struct drm_i915_private *dev_priv,
 		I915_WRITE(ICL_PORT_CL_DW5(port), val);
 	}
 
-	/* 4. Enable power well 1 (PG1) and aux IO power. */
-	/* FIXME: ICL power wells code not here yet. */
+	/*
+	 * 4. Enable Power Well 1 (PG1).
+	 *    The AUX IO power wells will be enabled on demand.
+	 */
+	mutex_lock(&power_domains->lock);
+	well = lookup_power_well(dev_priv, ICL_DISP_PW_1);
+	intel_power_well_enable(dev_priv, well);
+	mutex_unlock(&power_domains->lock);
 
 	/* 5. Enable CDCLK. */
 	icl_init_cdclk(dev_priv);
@@ -3072,6 +3380,8 @@ static void icl_display_core_init(struct drm_i915_private *dev_priv,
 
 static void icl_display_core_uninit(struct drm_i915_private *dev_priv)
 {
+	struct i915_power_domains *power_domains = &dev_priv->power_domains;
+	struct i915_power_well *well;
 	enum port port;
 	u32 val;
 
@@ -3085,8 +3395,15 @@ static void icl_display_core_uninit(struct drm_i915_private *dev_priv)
 	/* 3. Disable CD clock */
 	icl_uninit_cdclk(dev_priv);
 
-	/* 4. Disable Power Well 1 (PG1) and Aux IO Power */
-	/* FIXME: ICL power wells code not here yet. */
+	/*
+	 * 4. Disable Power Well 1 (PG1).
+	 *    The AUX IO power wells are toggled on demand, so they are already
+	 *    disabled at this point.
+	 */
+	mutex_lock(&power_domains->lock);
+	well = lookup_power_well(dev_priv, ICL_DISP_PW_1);
+	intel_power_well_disable(dev_priv, well);
+	mutex_unlock(&power_domains->lock);
 
 	/* 5. Disable Comp */
 	for (port = PORT_A; port <= PORT_B; port++) {
diff --git a/drivers/gpu/drm/i915/intel_sdvo.c b/drivers/gpu/drm/i915/intel_sdvo.c
index a88611a73b03..812fe7b06f87 100644
--- a/drivers/gpu/drm/i915/intel_sdvo.c
+++ b/drivers/gpu/drm/i915/intel_sdvo.c
@@ -1340,6 +1340,7 @@ static void intel_sdvo_pre_enable(struct intel_encoder *intel_encoder,
 	switch (crtc_state->pixel_multiplier) {
 	default:
 		WARN(1, "unknown pixel multiplier specified\n");
+		/* fall through */
 	case 1: rate = SDVO_CLOCK_RATE_MULT_1X; break;
 	case 2: rate = SDVO_CLOCK_RATE_MULT_2X; break;
 	case 4: rate = SDVO_CLOCK_RATE_MULT_4X; break;
@@ -1400,10 +1401,7 @@ static bool intel_sdvo_connector_get_hw_state(struct intel_connector *connector)
 
 	intel_sdvo_get_active_outputs(intel_sdvo, &active_outputs);
 
-	if (active_outputs & intel_sdvo_connector->output_flag)
-		return true;
-	else
-		return false;
+	return active_outputs & intel_sdvo_connector->output_flag;
 }
 
 bool intel_sdvo_port_enabled(struct drm_i915_private *dev_priv,
@@ -2316,14 +2314,19 @@ intel_sdvo_guess_ddc_bus(struct intel_sdvo *sdvo)
 	switch (sdvo->controlled_output) {
 	case SDVO_OUTPUT_LVDS1:
 		mask |= SDVO_OUTPUT_LVDS1;
+		/* fall through */
 	case SDVO_OUTPUT_LVDS0:
 		mask |= SDVO_OUTPUT_LVDS0;
+		/* fall through */
 	case SDVO_OUTPUT_TMDS1:
 		mask |= SDVO_OUTPUT_TMDS1;
+		/* fall through */
 	case SDVO_OUTPUT_TMDS0:
 		mask |= SDVO_OUTPUT_TMDS0;
+		/* fall through */
 	case SDVO_OUTPUT_RGB1:
 		mask |= SDVO_OUTPUT_RGB1;
+		/* fall through */
 	case SDVO_OUTPUT_RGB0:
 		mask |= SDVO_OUTPUT_RGB0;
 		break;
diff --git a/drivers/gpu/drm/i915/intel_sprite.c b/drivers/gpu/drm/i915/intel_sprite.c
index 1bb7bc3f84fe..f7026e887fa9 100644
--- a/drivers/gpu/drm/i915/intel_sprite.c
+++ b/drivers/gpu/drm/i915/intel_sprite.c
@@ -93,13 +93,21 @@ void intel_pipe_update_start(const struct intel_crtc_state *new_crtc_state)
 						      VBLANK_EVASION_TIME_US);
 	max = vblank_start - 1;
 
-	local_irq_disable();
-
 	if (min <= 0 || max <= 0)
-		return;
+		goto irq_disable;
 
 	if (WARN_ON(drm_crtc_vblank_get(&crtc->base)))
-		return;
+		goto irq_disable;
+
+	/*
+	 * Wait for psr to idle out after enabling the VBL interrupts
+	 * VBL interrupts will start the PSR exit and prevent a PSR
+	 * re-entry as well.
+	 */
+	if (intel_psr_wait_for_idle(new_crtc_state))
+		DRM_ERROR("PSR idle timed out, atomic update may fail\n");
+
+	local_irq_disable();
 
 	crtc->debug.min_vbl = min;
 	crtc->debug.max_vbl = max;
@@ -157,6 +165,10 @@ void intel_pipe_update_start(const struct intel_crtc_state *new_crtc_state)
 	crtc->debug.start_vbl_count = intel_crtc_get_vblank_counter(crtc);
 
 	trace_i915_pipe_update_vblank_evaded(crtc);
+	return;
+
+irq_disable:
+	local_irq_disable();
 }
 
 /**
diff --git a/drivers/gpu/drm/i915/intel_tv.c b/drivers/gpu/drm/i915/intel_tv.c
index 24dc368fdaa1..b5b04cb892e9 100644
--- a/drivers/gpu/drm/i915/intel_tv.c
+++ b/drivers/gpu/drm/i915/intel_tv.c
@@ -1347,8 +1347,7 @@ intel_tv_get_modes(struct drm_connector *connector)
 		mode_ptr = drm_mode_create(connector->dev);
 		if (!mode_ptr)
 			continue;
-		strncpy(mode_ptr->name, input->name, DRM_DISPLAY_MODE_LEN);
-		mode_ptr->name[DRM_DISPLAY_MODE_LEN - 1] = '\0';
+		strlcpy(mode_ptr->name, input->name, DRM_DISPLAY_MODE_LEN);
 
 		mode_ptr->hdisplay = hactive_s;
 		mode_ptr->hsync_start = hactive_s + 1;
diff --git a/drivers/gpu/drm/i915/intel_uc.c b/drivers/gpu/drm/i915/intel_uc.c
index 94e8863bd97c..7c95697e1a35 100644
--- a/drivers/gpu/drm/i915/intel_uc.c
+++ b/drivers/gpu/drm/i915/intel_uc.c
@@ -171,24 +171,11 @@ void intel_uc_init_early(struct drm_i915_private *i915)
 	intel_huc_init_early(huc);
 
 	sanitize_options_early(i915);
-
-	if (USES_GUC(i915))
-		intel_uc_fw_fetch(i915, &guc->fw);
-
-	if (USES_HUC(i915))
-		intel_uc_fw_fetch(i915, &huc->fw);
 }
 
 void intel_uc_cleanup_early(struct drm_i915_private *i915)
 {
 	struct intel_guc *guc = &i915->guc;
-	struct intel_huc *huc = &i915->huc;
-
-	if (USES_HUC(i915))
-		intel_uc_fw_fini(&huc->fw);
-
-	if (USES_GUC(i915))
-		intel_uc_fw_fini(&guc->fw);
 
 	guc_free_load_err_log(guc);
 }
@@ -252,28 +239,41 @@ static void guc_disable_communication(struct intel_guc *guc)
 int intel_uc_init_misc(struct drm_i915_private *i915)
 {
 	struct intel_guc *guc = &i915->guc;
+	struct intel_huc *huc = &i915->huc;
 	int ret;
 
 	if (!USES_GUC(i915))
 		return 0;
 
-	intel_guc_init_ggtt_pin_bias(guc);
-
-	ret = intel_guc_init_wq(guc);
+	ret = intel_guc_init_misc(guc);
 	if (ret)
 		return ret;
 
+	if (USES_HUC(i915)) {
+		ret = intel_huc_init_misc(huc);
+		if (ret)
+			goto err_guc;
+	}
+
 	return 0;
+
+err_guc:
+	intel_guc_fini_misc(guc);
+	return ret;
 }
 
 void intel_uc_fini_misc(struct drm_i915_private *i915)
 {
 	struct intel_guc *guc = &i915->guc;
+	struct intel_huc *huc = &i915->huc;
 
 	if (!USES_GUC(i915))
 		return;
 
-	intel_guc_fini_wq(guc);
+	if (USES_HUC(i915))
+		intel_huc_fini_misc(huc);
+
+	intel_guc_fini_misc(guc);
 }
 
 int intel_uc_init(struct drm_i915_private *i915)
diff --git a/drivers/gpu/drm/i915/intel_uncore.c b/drivers/gpu/drm/i915/intel_uncore.c
index b892ca8396e8..50b39aa4ffb8 100644
--- a/drivers/gpu/drm/i915/intel_uncore.c
+++ b/drivers/gpu/drm/i915/intel_uncore.c
@@ -359,8 +359,8 @@ intel_uncore_fw_release_timer(struct hrtimer *timer)
 }
 
 /* Note callers must have acquired the PUNIT->PMIC bus, before calling this. */
-static void intel_uncore_forcewake_reset(struct drm_i915_private *dev_priv,
-					 bool restore)
+static unsigned int
+intel_uncore_forcewake_reset(struct drm_i915_private *dev_priv)
 {
 	unsigned long irqflags;
 	struct intel_uncore_forcewake_domain *domain;
@@ -412,20 +412,11 @@ static void intel_uncore_forcewake_reset(struct drm_i915_private *dev_priv,
 		dev_priv->uncore.funcs.force_wake_put(dev_priv, fw);
 
 	fw_domains_reset(dev_priv, dev_priv->uncore.fw_domains);
-
-	if (restore) { /* If reset with a user forcewake, try to restore */
-		if (fw)
-			dev_priv->uncore.funcs.force_wake_get(dev_priv, fw);
-
-		if (IS_GEN6(dev_priv) || IS_GEN7(dev_priv))
-			dev_priv->uncore.fifo_count =
-				fifo_free_entries(dev_priv);
-	}
-
-	if (!restore)
-		assert_forcewakes_inactive(dev_priv);
+	assert_forcewakes_inactive(dev_priv);
 
 	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
+
+	return fw; /* track the lost user forcewake domains */
 }
 
 static u64 gen9_edram_size(struct drm_i915_private *dev_priv)
@@ -534,7 +525,7 @@ check_for_unclaimed_mmio(struct drm_i915_private *dev_priv)
 }
 
 static void __intel_uncore_early_sanitize(struct drm_i915_private *dev_priv,
-					  bool restore_forcewake)
+					  unsigned int restore_forcewake)
 {
 	/* clear out unclaimed reg detection bit */
 	if (check_for_unclaimed_mmio(dev_priv))
@@ -549,7 +540,17 @@ static void __intel_uncore_early_sanitize(struct drm_i915_private *dev_priv,
 	}
 
 	iosf_mbi_punit_acquire();
-	intel_uncore_forcewake_reset(dev_priv, restore_forcewake);
+	intel_uncore_forcewake_reset(dev_priv);
+	if (restore_forcewake) {
+		spin_lock_irq(&dev_priv->uncore.lock);
+		dev_priv->uncore.funcs.force_wake_get(dev_priv,
+						      restore_forcewake);
+
+		if (IS_GEN6(dev_priv) || IS_GEN7(dev_priv))
+			dev_priv->uncore.fifo_count =
+				fifo_free_entries(dev_priv);
+		spin_unlock_irq(&dev_priv->uncore.lock);
+	}
 	iosf_mbi_punit_release();
 }
 
@@ -558,13 +559,18 @@ void intel_uncore_suspend(struct drm_i915_private *dev_priv)
 	iosf_mbi_punit_acquire();
 	iosf_mbi_unregister_pmic_bus_access_notifier_unlocked(
 		&dev_priv->uncore.pmic_bus_access_nb);
-	intel_uncore_forcewake_reset(dev_priv, false);
+	dev_priv->uncore.fw_domains_saved =
+		intel_uncore_forcewake_reset(dev_priv);
 	iosf_mbi_punit_release();
 }
 
 void intel_uncore_resume_early(struct drm_i915_private *dev_priv)
 {
-	__intel_uncore_early_sanitize(dev_priv, true);
+	unsigned int restore_forcewake;
+
+	restore_forcewake = fetch_and_zero(&dev_priv->uncore.fw_domains_saved);
+	__intel_uncore_early_sanitize(dev_priv, restore_forcewake);
+
 	iosf_mbi_register_pmic_bus_access_notifier(
 		&dev_priv->uncore.pmic_bus_access_nb);
 	i915_check_and_clear_faults(dev_priv);
@@ -1545,7 +1551,7 @@ void intel_uncore_init(struct drm_i915_private *dev_priv)
 
 	intel_uncore_edram_detect(dev_priv);
 	intel_uncore_fw_domains_init(dev_priv);
-	__intel_uncore_early_sanitize(dev_priv, false);
+	__intel_uncore_early_sanitize(dev_priv, 0);
 
 	dev_priv->uncore.unclaimed_mmio_check = 1;
 	dev_priv->uncore.pmic_bus_access_nb.notifier_call =
@@ -1632,7 +1638,7 @@ void intel_uncore_fini(struct drm_i915_private *dev_priv)
 	iosf_mbi_punit_acquire();
 	iosf_mbi_unregister_pmic_bus_access_notifier_unlocked(
 		&dev_priv->uncore.pmic_bus_access_nb);
-	intel_uncore_forcewake_reset(dev_priv, false);
+	intel_uncore_forcewake_reset(dev_priv);
 	iosf_mbi_punit_release();
 }
 
diff --git a/drivers/gpu/drm/i915/intel_uncore.h b/drivers/gpu/drm/i915/intel_uncore.h
index 2fbe93178fb2..e5e157d288de 100644
--- a/drivers/gpu/drm/i915/intel_uncore.h
+++ b/drivers/gpu/drm/i915/intel_uncore.h
@@ -104,6 +104,7 @@ struct intel_uncore {
 
 	enum forcewake_domains fw_domains;
 	enum forcewake_domains fw_domains_active;
+	enum forcewake_domains fw_domains_saved; /* user domains saved for S3 */
 
 	u32 fw_set;
 	u32 fw_clear;
diff --git a/drivers/gpu/drm/i915/intel_workarounds.c b/drivers/gpu/drm/i915/intel_workarounds.c
index f8bb32e974f6..4bcdeaf8d98f 100644
--- a/drivers/gpu/drm/i915/intel_workarounds.c
+++ b/drivers/gpu/drm/i915/intel_workarounds.c
@@ -508,9 +508,6 @@ static int icl_ctx_workarounds_init(struct drm_i915_private *dev_priv)
 		WA_SET_BIT_MASKED(GEN11_COMMON_SLICE_CHICKEN3,
 				  GEN11_BLEND_EMB_FIX_DISABLE_IN_RCC);
 
-	/* WaEnableFloatBlendOptimization:icl */
-	WA_SET_BIT_MASKED(GEN10_CACHE_MODE_SS, FLOAT_BLEND_OPTIMIZATION_ENABLE);
-
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/i915/selftests/huge_pages.c b/drivers/gpu/drm/i915/selftests/huge_pages.c
index fbe4324116d7..7efb326badcd 100644
--- a/drivers/gpu/drm/i915/selftests/huge_pages.c
+++ b/drivers/gpu/drm/i915/selftests/huge_pages.c
@@ -570,6 +570,7 @@ static int igt_mock_ppgtt_misaligned_dma(void *arg)
 		i915_vma_close(vma);
 
 		i915_gem_object_unpin_pages(obj);
+		__i915_gem_object_put_pages(obj, I915_MM_NORMAL);
 		i915_gem_object_put(obj);
 	}
 
@@ -597,6 +598,7 @@ static void close_object_list(struct list_head *objects,
 
 		list_del(&obj->st_link);
 		i915_gem_object_unpin_pages(obj);
+		__i915_gem_object_put_pages(obj, I915_MM_NORMAL);
 		i915_gem_object_put(obj);
 	}
 }
@@ -866,6 +868,7 @@ static int igt_mock_ppgtt_64K(void *arg)
 			i915_vma_close(vma);
 
 			i915_gem_object_unpin_pages(obj);
+			__i915_gem_object_put_pages(obj, I915_MM_NORMAL);
 			i915_gem_object_put(obj);
 		}
 	}
@@ -919,12 +922,12 @@ gpu_write_dw(struct i915_vma *vma, u64 offset, u32 val)
 			*cmd++ = val;
 		} else if (gen >= 4) {
 			*cmd++ = MI_STORE_DWORD_IMM_GEN4 |
-				(gen < 6 ? 1 << 22 : 0);
+				(gen < 6 ? MI_USE_GGTT : 0);
 			*cmd++ = 0;
 			*cmd++ = offset;
 			*cmd++ = val;
 		} else {
-			*cmd++ = MI_STORE_DWORD_IMM | 1 << 22;
+			*cmd++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL;
 			*cmd++ = offset;
 			*cmd++ = val;
 		}
@@ -985,7 +988,10 @@ static int gpu_write(struct i915_vma *vma,
 		goto err_request;
 	}
 
-	i915_vma_move_to_active(batch, rq, 0);
+	err = i915_vma_move_to_active(batch, rq, 0);
+	if (err)
+		goto err_request;
+
 	i915_gem_object_set_active_reference(batch->obj);
 	i915_vma_unpin(batch);
 	i915_vma_close(batch);
@@ -996,11 +1002,9 @@ static int gpu_write(struct i915_vma *vma,
 	if (err)
 		goto err_request;
 
-	i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
-
-	reservation_object_lock(vma->resv, NULL);
-	reservation_object_add_excl_fence(vma->resv, &rq->fence);
-	reservation_object_unlock(vma->resv);
+	err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
+	if (err)
+		i915_request_skip(rq, err);
 
 err_request:
 	i915_request_add(rq);
@@ -1264,6 +1268,7 @@ static int igt_ppgtt_exhaust_huge(void *arg)
 			}
 
 			i915_gem_object_unpin_pages(obj);
+			__i915_gem_object_put_pages(obj, I915_MM_NORMAL);
 			i915_gem_object_put(obj);
 		}
 	}
@@ -1325,6 +1330,7 @@ static int igt_ppgtt_internal_huge(void *arg)
 		}
 
 		i915_gem_object_unpin_pages(obj);
+		__i915_gem_object_put_pages(obj, I915_MM_NORMAL);
 		i915_gem_object_put(obj);
 	}
 
@@ -1393,6 +1399,7 @@ static int igt_ppgtt_gemfs_huge(void *arg)
 		}
 
 		i915_gem_object_unpin_pages(obj);
+		__i915_gem_object_put_pages(obj, I915_MM_NORMAL);
 		i915_gem_object_put(obj);
 	}
 
@@ -1694,7 +1701,7 @@ int i915_gem_huge_page_mock_selftests(void)
 	dma_coerce_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(39));
 
 	mutex_lock(&dev_priv->drm.struct_mutex);
-	ppgtt = i915_ppgtt_create(dev_priv, ERR_PTR(-ENODEV), "mock");
+	ppgtt = i915_ppgtt_create(dev_priv, ERR_PTR(-ENODEV));
 	if (IS_ERR(ppgtt)) {
 		err = PTR_ERR(ppgtt);
 		goto out_unlock;
@@ -1724,7 +1731,7 @@ out_unlock:
 
 	i915_modparams.enable_ppgtt = saved_ppgtt;
 
-	drm_dev_unref(&dev_priv->drm);
+	drm_dev_put(&dev_priv->drm);
 
 	return err;
 }
@@ -1748,6 +1755,9 @@ int i915_gem_huge_page_live_selftests(struct drm_i915_private *dev_priv)
 		return 0;
 	}
 
+	if (i915_terminally_wedged(&dev_priv->gpu_error))
+		return 0;
+
 	file = mock_file(dev_priv);
 	if (IS_ERR(file))
 		return PTR_ERR(file);
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_coherency.c b/drivers/gpu/drm/i915/selftests/i915_gem_coherency.c
index a4900091ae3d..3a095c37c120 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_coherency.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_coherency.c
@@ -42,11 +42,21 @@ static int cpu_set(struct drm_i915_gem_object *obj,
 
 	page = i915_gem_object_get_page(obj, offset >> PAGE_SHIFT);
 	map = kmap_atomic(page);
-	if (needs_clflush & CLFLUSH_BEFORE)
+
+	if (needs_clflush & CLFLUSH_BEFORE) {
+		mb();
 		clflush(map+offset_in_page(offset) / sizeof(*map));
+		mb();
+	}
+
 	map[offset_in_page(offset) / sizeof(*map)] = v;
-	if (needs_clflush & CLFLUSH_AFTER)
+
+	if (needs_clflush & CLFLUSH_AFTER) {
+		mb();
 		clflush(map+offset_in_page(offset) / sizeof(*map));
+		mb();
+	}
+
 	kunmap_atomic(map);
 
 	i915_gem_obj_finish_shmem_access(obj);
@@ -68,8 +78,13 @@ static int cpu_get(struct drm_i915_gem_object *obj,
 
 	page = i915_gem_object_get_page(obj, offset >> PAGE_SHIFT);
 	map = kmap_atomic(page);
-	if (needs_clflush & CLFLUSH_BEFORE)
+
+	if (needs_clflush & CLFLUSH_BEFORE) {
+		mb();
 		clflush(map+offset_in_page(offset) / sizeof(*map));
+		mb();
+	}
+
 	*v = map[offset_in_page(offset) / sizeof(*map)];
 	kunmap_atomic(map);
 
@@ -210,28 +225,24 @@ static int gpu_set(struct drm_i915_gem_object *obj,
 		*cs++ = upper_32_bits(i915_ggtt_offset(vma) + offset);
 		*cs++ = v;
 	} else if (INTEL_GEN(i915) >= 4) {
-		*cs++ = MI_STORE_DWORD_IMM_GEN4 | 1 << 22;
+		*cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
 		*cs++ = 0;
 		*cs++ = i915_ggtt_offset(vma) + offset;
 		*cs++ = v;
 	} else {
-		*cs++ = MI_STORE_DWORD_IMM | 1 << 22;
+		*cs++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL;
 		*cs++ = i915_ggtt_offset(vma) + offset;
 		*cs++ = v;
 		*cs++ = MI_NOOP;
 	}
 	intel_ring_advance(rq, cs);
 
-	i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
+	err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
 	i915_vma_unpin(vma);
 
-	reservation_object_lock(obj->resv, NULL);
-	reservation_object_add_excl_fence(obj->resv, &rq->fence);
-	reservation_object_unlock(obj->resv);
-
 	i915_request_add(rq);
 
-	return 0;
+	return err;
 }
 
 static bool always_valid(struct drm_i915_private *i915)
@@ -239,8 +250,16 @@ static bool always_valid(struct drm_i915_private *i915)
 	return true;
 }
 
+static bool needs_fence_registers(struct drm_i915_private *i915)
+{
+	return !i915_terminally_wedged(&i915->gpu_error);
+}
+
 static bool needs_mi_store_dword(struct drm_i915_private *i915)
 {
+	if (i915_terminally_wedged(&i915->gpu_error))
+		return false;
+
 	return intel_engine_can_store_dword(i915->engine[RCS]);
 }
 
@@ -251,7 +270,7 @@ static const struct igt_coherency_mode {
 	bool (*valid)(struct drm_i915_private *i915);
 } igt_coherency_mode[] = {
 	{ "cpu", cpu_set, cpu_get, always_valid },
-	{ "gtt", gtt_set, gtt_get, always_valid },
+	{ "gtt", gtt_set, gtt_get, needs_fence_registers },
 	{ "wc", wc_set, wc_get, always_valid },
 	{ "gpu", gpu_set, NULL, needs_mi_store_dword },
 	{ },
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_context.c b/drivers/gpu/drm/i915/selftests/i915_gem_context.c
index 90c3c36173ba..1c92560d35da 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_context.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_context.c
@@ -23,6 +23,7 @@
  */
 
 #include "../i915_selftest.h"
+#include "i915_random.h"
 #include "igt_flush_test.h"
 
 #include "mock_drm.h"
@@ -63,12 +64,12 @@ gpu_fill_dw(struct i915_vma *vma, u64 offset, unsigned long count, u32 value)
 			*cmd++ = value;
 		} else if (gen >= 4) {
 			*cmd++ = MI_STORE_DWORD_IMM_GEN4 |
-				(gen < 6 ? 1 << 22 : 0);
+				(gen < 6 ? MI_USE_GGTT : 0);
 			*cmd++ = 0;
 			*cmd++ = offset;
 			*cmd++ = value;
 		} else {
-			*cmd++ = MI_STORE_DWORD_IMM | 1 << 22;
+			*cmd++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL;
 			*cmd++ = offset;
 			*cmd++ = value;
 		}
@@ -170,22 +171,26 @@ static int gpu_fill(struct drm_i915_gem_object *obj,
 	if (err)
 		goto err_request;
 
-	i915_vma_move_to_active(batch, rq, 0);
+	err = i915_vma_move_to_active(batch, rq, 0);
+	if (err)
+		goto skip_request;
+
+	err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
+	if (err)
+		goto skip_request;
+
 	i915_gem_object_set_active_reference(batch->obj);
 	i915_vma_unpin(batch);
 	i915_vma_close(batch);
 
-	i915_vma_move_to_active(vma, rq, 0);
 	i915_vma_unpin(vma);
 
-	reservation_object_lock(obj->resv, NULL);
-	reservation_object_add_excl_fence(obj->resv, &rq->fence);
-	reservation_object_unlock(obj->resv);
-
 	i915_request_add(rq);
 
 	return 0;
 
+skip_request:
+	i915_request_skip(rq, err);
 err_request:
 	i915_request_add(rq);
 err_batch:
@@ -248,9 +253,9 @@ static int cpu_check(struct drm_i915_gem_object *obj, unsigned int max)
 		}
 
 		for (; m < DW_PER_PAGE; m++) {
-			if (map[m] != 0xdeadbeef) {
+			if (map[m] != STACK_MAGIC) {
 				pr_err("Invalid value at page %d, offset %d: found %x expected %x\n",
-				       n, m, map[m], 0xdeadbeef);
+				       n, m, map[m], STACK_MAGIC);
 				err = -EINVAL;
 				goto out_unmap;
 			}
@@ -306,7 +311,7 @@ create_test_object(struct i915_gem_context *ctx,
 	if (err)
 		return ERR_PTR(err);
 
-	err = cpu_fill(obj, 0xdeadbeef);
+	err = cpu_fill(obj, STACK_MAGIC);
 	if (err) {
 		pr_err("Failed to fill object with cpu, err=%d\n",
 		       err);
@@ -336,11 +341,15 @@ static int igt_ctx_exec(void *arg)
 	bool first_shared_gtt = true;
 	int err = -ENODEV;
 
-	/* Create a few different contexts (with different mm) and write
+	/*
+	 * Create a few different contexts (with different mm) and write
 	 * through each ctx/mm using the GPU making sure those writes end
 	 * up in the expected pages of our obj.
 	 */
 
+	if (!DRIVER_CAPS(i915)->has_logical_contexts)
+		return 0;
+
 	file = mock_file(i915);
 	if (IS_ERR(file))
 		return PTR_ERR(file);
@@ -367,6 +376,9 @@ static int igt_ctx_exec(void *arg)
 		}
 
 		for_each_engine(engine, i915, id) {
+			if (!engine->context_size)
+				continue; /* No logical context support in HW */
+
 			if (!intel_engine_can_store_dword(engine))
 				continue;
 
@@ -421,6 +433,111 @@ out_unlock:
 	return err;
 }
 
+static int igt_ctx_readonly(void *arg)
+{
+	struct drm_i915_private *i915 = arg;
+	struct drm_i915_gem_object *obj = NULL;
+	struct drm_file *file;
+	I915_RND_STATE(prng);
+	IGT_TIMEOUT(end_time);
+	LIST_HEAD(objects);
+	struct i915_gem_context *ctx;
+	struct i915_hw_ppgtt *ppgtt;
+	unsigned long ndwords, dw;
+	int err = -ENODEV;
+
+	/*
+	 * Create a few read-only objects (with the occasional writable object)
+	 * and try to write into these object checking that the GPU discards
+	 * any write to a read-only object.
+	 */
+
+	file = mock_file(i915);
+	if (IS_ERR(file))
+		return PTR_ERR(file);
+
+	mutex_lock(&i915->drm.struct_mutex);
+
+	ctx = i915_gem_create_context(i915, file->driver_priv);
+	if (IS_ERR(ctx)) {
+		err = PTR_ERR(ctx);
+		goto out_unlock;
+	}
+
+	ppgtt = ctx->ppgtt ?: i915->mm.aliasing_ppgtt;
+	if (!ppgtt || !ppgtt->vm.has_read_only) {
+		err = 0;
+		goto out_unlock;
+	}
+
+	ndwords = 0;
+	dw = 0;
+	while (!time_after(jiffies, end_time)) {
+		struct intel_engine_cs *engine;
+		unsigned int id;
+
+		for_each_engine(engine, i915, id) {
+			if (!intel_engine_can_store_dword(engine))
+				continue;
+
+			if (!obj) {
+				obj = create_test_object(ctx, file, &objects);
+				if (IS_ERR(obj)) {
+					err = PTR_ERR(obj);
+					goto out_unlock;
+				}
+
+				if (prandom_u32_state(&prng) & 1)
+					i915_gem_object_set_readonly(obj);
+			}
+
+			intel_runtime_pm_get(i915);
+			err = gpu_fill(obj, ctx, engine, dw);
+			intel_runtime_pm_put(i915);
+			if (err) {
+				pr_err("Failed to fill dword %lu [%lu/%lu] with gpu (%s) in ctx %u [full-ppgtt? %s], err=%d\n",
+				       ndwords, dw, max_dwords(obj),
+				       engine->name, ctx->hw_id,
+				       yesno(!!ctx->ppgtt), err);
+				goto out_unlock;
+			}
+
+			if (++dw == max_dwords(obj)) {
+				obj = NULL;
+				dw = 0;
+			}
+			ndwords++;
+		}
+	}
+	pr_info("Submitted %lu dwords (across %u engines)\n",
+		ndwords, INTEL_INFO(i915)->num_rings);
+
+	dw = 0;
+	list_for_each_entry(obj, &objects, st_link) {
+		unsigned int rem =
+			min_t(unsigned int, ndwords - dw, max_dwords(obj));
+		unsigned int num_writes;
+
+		num_writes = rem;
+		if (i915_gem_object_is_readonly(obj))
+			num_writes = 0;
+
+		err = cpu_check(obj, num_writes);
+		if (err)
+			break;
+
+		dw += rem;
+	}
+
+out_unlock:
+	if (igt_flush_test(i915, I915_WAIT_LOCKED))
+		err = -EIO;
+	mutex_unlock(&i915->drm.struct_mutex);
+
+	mock_file_free(i915, file);
+	return err;
+}
+
 static __maybe_unused const char *
 __engine_name(struct drm_i915_private *i915, unsigned int engines)
 {
@@ -467,7 +584,9 @@ static int __igt_switch_to_kernel_context(struct drm_i915_private *i915,
 		}
 	}
 
-	err = i915_gem_wait_for_idle(i915, I915_WAIT_LOCKED);
+	err = i915_gem_wait_for_idle(i915,
+				     I915_WAIT_LOCKED,
+				     MAX_SCHEDULE_TIMEOUT);
 	if (err)
 		return err;
 
@@ -586,7 +705,7 @@ int i915_gem_context_mock_selftests(void)
 
 	err = i915_subtests(tests, i915);
 
-	drm_dev_unref(&i915->drm);
+	drm_dev_put(&i915->drm);
 	return err;
 }
 
@@ -595,10 +714,14 @@ int i915_gem_context_live_selftests(struct drm_i915_private *dev_priv)
 	static const struct i915_subtest tests[] = {
 		SUBTEST(igt_switch_to_kernel_context),
 		SUBTEST(igt_ctx_exec),
+		SUBTEST(igt_ctx_readonly),
 	};
 	bool fake_alias = false;
 	int err;
 
+	if (i915_terminally_wedged(&dev_priv->gpu_error))
+		return 0;
+
 	/* Install a fake aliasing gtt for exercise */
 	if (USES_PPGTT(dev_priv) && !dev_priv->mm.aliasing_ppgtt) {
 		mutex_lock(&dev_priv->drm.struct_mutex);
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_dmabuf.c b/drivers/gpu/drm/i915/selftests/i915_gem_dmabuf.c
index 89dc25a5a53b..a7055b12e53c 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_dmabuf.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_dmabuf.c
@@ -389,7 +389,7 @@ int i915_gem_dmabuf_mock_selftests(void)
 
 	err = i915_subtests(tests, i915);
 
-	drm_dev_unref(&i915->drm);
+	drm_dev_put(&i915->drm);
 	return err;
 }
 
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_evict.c b/drivers/gpu/drm/i915/selftests/i915_gem_evict.c
index 2dc72a984d45..128ad1cf0647 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_evict.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_evict.c
@@ -490,7 +490,7 @@ int i915_gem_evict_mock_selftests(void)
 	err = i915_subtests(tests, i915);
 	mutex_unlock(&i915->drm.struct_mutex);
 
-	drm_dev_unref(&i915->drm);
+	drm_dev_put(&i915->drm);
 	return err;
 }
 
@@ -500,5 +500,8 @@ int i915_gem_evict_live_selftests(struct drm_i915_private *i915)
 		SUBTEST(igt_evict_contexts),
 	};
 
+	if (i915_terminally_wedged(&i915->gpu_error))
+		return 0;
+
 	return i915_subtests(tests, i915);
 }
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c b/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c
index a4060238bef0..8e2e269db97e 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c
@@ -32,6 +32,20 @@
 #include "mock_drm.h"
 #include "mock_gem_device.h"
 
+static void cleanup_freed_objects(struct drm_i915_private *i915)
+{
+	/*
+	 * As we may hold onto the struct_mutex for inordinate lengths of
+	 * time, the NMI khungtaskd detector may fire for the free objects
+	 * worker.
+	 */
+	mutex_unlock(&i915->drm.struct_mutex);
+
+	i915_gem_drain_freed_objects(i915);
+
+	mutex_lock(&i915->drm.struct_mutex);
+}
+
 static void fake_free_pages(struct drm_i915_gem_object *obj,
 			    struct sg_table *pages)
 {
@@ -134,7 +148,7 @@ static int igt_ppgtt_alloc(void *arg)
 {
 	struct drm_i915_private *dev_priv = arg;
 	struct i915_hw_ppgtt *ppgtt;
-	u64 size, last;
+	u64 size, last, limit;
 	int err = 0;
 
 	/* Allocate a ppggt and try to fill the entire range */
@@ -142,20 +156,25 @@ static int igt_ppgtt_alloc(void *arg)
 	if (!USES_PPGTT(dev_priv))
 		return 0;
 
-	mutex_lock(&dev_priv->drm.struct_mutex);
 	ppgtt = __hw_ppgtt_create(dev_priv);
-	if (IS_ERR(ppgtt)) {
-		err = PTR_ERR(ppgtt);
-		goto err_unlock;
-	}
+	if (IS_ERR(ppgtt))
+		return PTR_ERR(ppgtt);
 
 	if (!ppgtt->vm.allocate_va_range)
 		goto err_ppgtt_cleanup;
 
+	/*
+	 * While we only allocate the page tables here and so we could
+	 * address a much larger GTT than we could actually fit into
+	 * RAM, a practical limit is the amount of physical pages in the system.
+	 * This should ensure that we do not run into the oomkiller during
+	 * the test and take down the machine wilfully.
+	 */
+	limit = totalram_pages << PAGE_SHIFT;
+	limit = min(ppgtt->vm.total, limit);
+
 	/* Check we can allocate the entire range */
-	for (size = 4096;
-	     size <= ppgtt->vm.total;
-	     size <<= 2) {
+	for (size = 4096; size <= limit; size <<= 2) {
 		err = ppgtt->vm.allocate_va_range(&ppgtt->vm, 0, size);
 		if (err) {
 			if (err == -ENOMEM) {
@@ -166,13 +185,13 @@ static int igt_ppgtt_alloc(void *arg)
 			goto err_ppgtt_cleanup;
 		}
 
+		cond_resched();
+
 		ppgtt->vm.clear_range(&ppgtt->vm, 0, size);
 	}
 
 	/* Check we can incrementally allocate the entire range */
-	for (last = 0, size = 4096;
-	     size <= ppgtt->vm.total;
-	     last = size, size <<= 2) {
+	for (last = 0, size = 4096; size <= limit; last = size, size <<= 2) {
 		err = ppgtt->vm.allocate_va_range(&ppgtt->vm,
 						  last, size - last);
 		if (err) {
@@ -183,12 +202,13 @@ static int igt_ppgtt_alloc(void *arg)
 			}
 			goto err_ppgtt_cleanup;
 		}
+
+		cond_resched();
 	}
 
 err_ppgtt_cleanup:
-	ppgtt->vm.cleanup(&ppgtt->vm);
-	kfree(ppgtt);
-err_unlock:
+	mutex_lock(&dev_priv->drm.struct_mutex);
+	i915_ppgtt_put(ppgtt);
 	mutex_unlock(&dev_priv->drm.struct_mutex);
 	return err;
 }
@@ -291,6 +311,8 @@ static int lowlevel_hole(struct drm_i915_private *i915,
 		i915_gem_object_put(obj);
 
 		kfree(order);
+
+		cleanup_freed_objects(i915);
 	}
 
 	return 0;
@@ -519,6 +541,7 @@ static int fill_hole(struct drm_i915_private *i915,
 		}
 
 		close_object_list(&objects, vm);
+		cleanup_freed_objects(i915);
 	}
 
 	return 0;
@@ -605,6 +628,8 @@ err_put:
 		i915_gem_object_put(obj);
 		if (err)
 			return err;
+
+		cleanup_freed_objects(i915);
 	}
 
 	return 0;
@@ -789,6 +814,8 @@ err_obj:
 		kfree(order);
 		if (err)
 			return err;
+
+		cleanup_freed_objects(i915);
 	}
 
 	return 0;
@@ -857,6 +884,7 @@ static int __shrink_hole(struct drm_i915_private *i915,
 	}
 
 	close_object_list(&objects, vm);
+	cleanup_freed_objects(i915);
 	return err;
 }
 
@@ -949,6 +977,7 @@ static int shrink_boom(struct drm_i915_private *i915,
 		i915_gem_object_put(explode);
 
 		memset(&vm->fault_attr, 0, sizeof(vm->fault_attr));
+		cleanup_freed_objects(i915);
 	}
 
 	return 0;
@@ -980,7 +1009,7 @@ static int exercise_ppgtt(struct drm_i915_private *dev_priv,
 		return PTR_ERR(file);
 
 	mutex_lock(&dev_priv->drm.struct_mutex);
-	ppgtt = i915_ppgtt_create(dev_priv, file->driver_priv, "mock");
+	ppgtt = i915_ppgtt_create(dev_priv, file->driver_priv);
 	if (IS_ERR(ppgtt)) {
 		err = PTR_ERR(ppgtt);
 		goto out_unlock;
@@ -1215,6 +1244,7 @@ static int exercise_mock(struct drm_i915_private *i915,
 				     u64 hole_start, u64 hole_end,
 				     unsigned long end_time))
 {
+	const u64 limit = totalram_pages << PAGE_SHIFT;
 	struct i915_gem_context *ctx;
 	struct i915_hw_ppgtt *ppgtt;
 	IGT_TIMEOUT(end_time);
@@ -1227,7 +1257,7 @@ static int exercise_mock(struct drm_i915_private *i915,
 	ppgtt = ctx->ppgtt;
 	GEM_BUG_ON(!ppgtt);
 
-	err = func(i915, &ppgtt->vm, 0, ppgtt->vm.total, end_time);
+	err = func(i915, &ppgtt->vm, 0, min(ppgtt->vm.total, limit), end_time);
 
 	mock_context_close(ctx);
 	return err;
@@ -1644,7 +1674,7 @@ int i915_gem_gtt_mock_selftests(void)
 	err = i915_subtests(tests, i915);
 	mutex_unlock(&i915->drm.struct_mutex);
 
-	drm_dev_unref(&i915->drm);
+	drm_dev_put(&i915->drm);
 	return err;
 }
 
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_object.c b/drivers/gpu/drm/i915/selftests/i915_gem_object.c
index 2b2dde94526f..ba4f322d56b8 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_object.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_object.c
@@ -169,9 +169,16 @@ static u64 tiled_offset(const struct tile *tile, u64 v)
 		v += y * tile->width;
 		v += div64_u64_rem(x, tile->width, &x) << tile->size;
 		v += x;
-	} else {
+	} else if (tile->width == 128) {
 		const unsigned int ytile_span = 16;
-		const unsigned int ytile_height = 32 * ytile_span;
+		const unsigned int ytile_height = 512;
+
+		v += y * ytile_span;
+		v += div64_u64_rem(x, ytile_span, &x) * ytile_height;
+		v += x;
+	} else {
+		const unsigned int ytile_span = 32;
+		const unsigned int ytile_height = 256;
 
 		v += y * ytile_span;
 		v += div64_u64_rem(x, ytile_span, &x) * ytile_height;
@@ -288,6 +295,8 @@ static int check_partial_mapping(struct drm_i915_gem_object *obj,
 		kunmap(p);
 		if (err)
 			return err;
+
+		i915_vma_destroy(vma);
 	}
 
 	return 0;
@@ -347,6 +356,14 @@ static int igt_partial_tiling(void *arg)
 		unsigned int pitch;
 		struct tile tile;
 
+		if (i915->quirks & QUIRK_PIN_SWIZZLED_PAGES)
+			/*
+			 * The swizzling pattern is actually unknown as it
+			 * varies based on physical address of each page.
+			 * See i915_gem_detect_bit_6_swizzle().
+			 */
+			break;
+
 		tile.tiling = tiling;
 		switch (tiling) {
 		case I915_TILING_X:
@@ -357,7 +374,8 @@ static int igt_partial_tiling(void *arg)
 			break;
 		}
 
-		if (tile.swizzle == I915_BIT_6_SWIZZLE_UNKNOWN ||
+		GEM_BUG_ON(tile.swizzle == I915_BIT_6_SWIZZLE_UNKNOWN);
+		if (tile.swizzle == I915_BIT_6_SWIZZLE_9_17 ||
 		    tile.swizzle == I915_BIT_6_SWIZZLE_9_10_17)
 			continue;
 
@@ -454,12 +472,14 @@ static int make_obj_busy(struct drm_i915_gem_object *obj)
 		return PTR_ERR(rq);
 	}
 
-	i915_vma_move_to_active(vma, rq, 0);
+	err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
+
 	i915_request_add(rq);
 
-	i915_gem_object_set_active_reference(obj);
+	__i915_gem_object_release_unless_active(obj);
 	i915_vma_unpin(vma);
-	return 0;
+
+	return err;
 }
 
 static bool assert_mmap_offset(struct drm_i915_private *i915,
@@ -479,6 +499,19 @@ static bool assert_mmap_offset(struct drm_i915_private *i915,
 	return err == expected;
 }
 
+static void disable_retire_worker(struct drm_i915_private *i915)
+{
+	mutex_lock(&i915->drm.struct_mutex);
+	if (!i915->gt.active_requests++) {
+		intel_runtime_pm_get(i915);
+		i915_gem_unpark(i915);
+		intel_runtime_pm_put(i915);
+	}
+	mutex_unlock(&i915->drm.struct_mutex);
+	cancel_delayed_work_sync(&i915->gt.retire_work);
+	cancel_delayed_work_sync(&i915->gt.idle_work);
+}
+
 static int igt_mmap_offset_exhaustion(void *arg)
 {
 	struct drm_i915_private *i915 = arg;
@@ -488,6 +521,10 @@ static int igt_mmap_offset_exhaustion(void *arg)
 	u64 hole_start, hole_end;
 	int loop, err;
 
+	/* Disable background reaper */
+	disable_retire_worker(i915);
+	GEM_BUG_ON(!i915->gt.awake);
+
 	/* Trim the device mmap space to only a page */
 	memset(&resv, 0, sizeof(resv));
 	drm_mm_for_each_hole(hole, mm, hole_start, hole_end) {
@@ -496,7 +533,7 @@ static int igt_mmap_offset_exhaustion(void *arg)
 		err = drm_mm_reserve_node(mm, &resv);
 		if (err) {
 			pr_err("Failed to trim VMA manager, err=%d\n", err);
-			return err;
+			goto out_park;
 		}
 		break;
 	}
@@ -538,6 +575,9 @@ static int igt_mmap_offset_exhaustion(void *arg)
 
 	/* Now fill with busy dead objects that we expect to reap */
 	for (loop = 0; loop < 3; loop++) {
+		if (i915_terminally_wedged(&i915->gpu_error))
+			break;
+
 		obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
 		if (IS_ERR(obj)) {
 			err = PTR_ERR(obj);
@@ -554,6 +594,7 @@ static int igt_mmap_offset_exhaustion(void *arg)
 			goto err_obj;
 		}
 
+		/* NB we rely on the _active_ reference to access obj now */
 		GEM_BUG_ON(!i915_gem_object_is_active(obj));
 		err = i915_gem_object_create_mmap_offset(obj);
 		if (err) {
@@ -565,6 +606,13 @@ static int igt_mmap_offset_exhaustion(void *arg)
 
 out:
 	drm_mm_remove_node(&resv);
+out_park:
+	mutex_lock(&i915->drm.struct_mutex);
+	if (--i915->gt.active_requests)
+		queue_delayed_work(i915->wq, &i915->gt.retire_work, 0);
+	else
+		queue_delayed_work(i915->wq, &i915->gt.idle_work, 0);
+	mutex_unlock(&i915->drm.struct_mutex);
 	return err;
 err_obj:
 	i915_gem_object_put(obj);
@@ -586,7 +634,7 @@ int i915_gem_object_mock_selftests(void)
 
 	err = i915_subtests(tests, i915);
 
-	drm_dev_unref(&i915->drm);
+	drm_dev_put(&i915->drm);
 	return err;
 }
 
diff --git a/drivers/gpu/drm/i915/selftests/i915_request.c b/drivers/gpu/drm/i915/selftests/i915_request.c
index 63cd9486cc13..c4aac6141e04 100644
--- a/drivers/gpu/drm/i915/selftests/i915_request.c
+++ b/drivers/gpu/drm/i915/selftests/i915_request.c
@@ -262,7 +262,7 @@ int i915_request_mock_selftests(void)
 		return -ENOMEM;
 
 	err = i915_subtests(tests, i915);
-	drm_dev_unref(&i915->drm);
+	drm_dev_put(&i915->drm);
 
 	return err;
 }
@@ -286,7 +286,9 @@ static int begin_live_test(struct live_test *t,
 	t->func = func;
 	t->name = name;
 
-	err = i915_gem_wait_for_idle(i915, I915_WAIT_LOCKED);
+	err = i915_gem_wait_for_idle(i915,
+				     I915_WAIT_LOCKED,
+				     MAX_SCHEDULE_TIMEOUT);
 	if (err) {
 		pr_err("%s(%s): failed to idle before, with err=%d!",
 		       func, name, err);
@@ -594,11 +596,8 @@ static struct i915_vma *recursive_batch(struct drm_i915_private *i915)
 	} else if (gen >= 6) {
 		*cmd++ = MI_BATCH_BUFFER_START | 1 << 8;
 		*cmd++ = lower_32_bits(vma->node.start);
-	} else if (gen >= 4) {
-		*cmd++ = MI_BATCH_BUFFER_START | MI_BATCH_GTT;
-		*cmd++ = lower_32_bits(vma->node.start);
 	} else {
-		*cmd++ = MI_BATCH_BUFFER_START | MI_BATCH_GTT | 1;
+		*cmd++ = MI_BATCH_BUFFER_START | MI_BATCH_GTT;
 		*cmd++ = lower_32_bits(vma->node.start);
 	}
 	*cmd++ = MI_BATCH_BUFFER_END; /* terminate early in case of error */
@@ -678,7 +677,9 @@ static int live_all_engines(void *arg)
 			i915_gem_object_set_active_reference(batch->obj);
 		}
 
-		i915_vma_move_to_active(batch, request[id], 0);
+		err = i915_vma_move_to_active(batch, request[id], 0);
+		GEM_BUG_ON(err);
+
 		i915_request_get(request[id]);
 		i915_request_add(request[id]);
 	}
@@ -788,7 +789,9 @@ static int live_sequential_engines(void *arg)
 		GEM_BUG_ON(err);
 		request[id]->batch = batch;
 
-		i915_vma_move_to_active(batch, request[id], 0);
+		err = i915_vma_move_to_active(batch, request[id], 0);
+		GEM_BUG_ON(err);
+
 		i915_gem_object_set_active_reference(batch->obj);
 		i915_vma_get(batch);
 
@@ -862,5 +865,9 @@ int i915_request_live_selftests(struct drm_i915_private *i915)
 		SUBTEST(live_sequential_engines),
 		SUBTEST(live_empty_request),
 	};
+
+	if (i915_terminally_wedged(&i915->gpu_error))
+		return 0;
+
 	return i915_subtests(tests, i915);
 }
diff --git a/drivers/gpu/drm/i915/selftests/i915_selftest.c b/drivers/gpu/drm/i915/selftests/i915_selftest.c
index addc5a599c4a..86c54ea37f48 100644
--- a/drivers/gpu/drm/i915/selftests/i915_selftest.c
+++ b/drivers/gpu/drm/i915/selftests/i915_selftest.c
@@ -210,6 +210,8 @@ int __i915_subtests(const char *caller,
 			return -EINTR;
 
 		pr_debug(DRIVER_NAME ": Running %s/%s\n", caller, st->name);
+		GEM_TRACE("Running %s/%s\n", caller, st->name);
+
 		err = st->func(data);
 		if (err && err != -EINTR) {
 			pr_err(DRIVER_NAME "/%s: %s failed with error %d\n",
diff --git a/drivers/gpu/drm/i915/selftests/i915_vma.c b/drivers/gpu/drm/i915/selftests/i915_vma.c
index 8400a8cc5cf2..ffa74290e054 100644
--- a/drivers/gpu/drm/i915/selftests/i915_vma.c
+++ b/drivers/gpu/drm/i915/selftests/i915_vma.c
@@ -733,7 +733,7 @@ int i915_vma_mock_selftests(void)
 	err = i915_subtests(tests, i915);
 	mutex_unlock(&i915->drm.struct_mutex);
 
-	drm_dev_unref(&i915->drm);
+	drm_dev_put(&i915->drm);
 	return err;
 }
 
diff --git a/drivers/gpu/drm/i915/selftests/igt_flush_test.c b/drivers/gpu/drm/i915/selftests/igt_flush_test.c
index 0d06f559243f..af66e3d4e23a 100644
--- a/drivers/gpu/drm/i915/selftests/igt_flush_test.c
+++ b/drivers/gpu/drm/i915/selftests/igt_flush_test.c
@@ -9,52 +9,8 @@
 #include "../i915_selftest.h"
 #include "igt_flush_test.h"
 
-struct wedge_me {
-	struct delayed_work work;
-	struct drm_i915_private *i915;
-	const void *symbol;
-};
-
-static void wedge_me(struct work_struct *work)
-{
-	struct wedge_me *w = container_of(work, typeof(*w), work.work);
-
-	pr_err("%pS timed out, cancelling all further testing.\n", w->symbol);
-
-	GEM_TRACE("%pS timed out.\n", w->symbol);
-	GEM_TRACE_DUMP();
-
-	i915_gem_set_wedged(w->i915);
-}
-
-static void __init_wedge(struct wedge_me *w,
-			 struct drm_i915_private *i915,
-			 long timeout,
-			 const void *symbol)
-{
-	w->i915 = i915;
-	w->symbol = symbol;
-
-	INIT_DELAYED_WORK_ONSTACK(&w->work, wedge_me);
-	schedule_delayed_work(&w->work, timeout);
-}
-
-static void __fini_wedge(struct wedge_me *w)
-{
-	cancel_delayed_work_sync(&w->work);
-	destroy_delayed_work_on_stack(&w->work);
-	w->i915 = NULL;
-}
-
-#define wedge_on_timeout(W, DEV, TIMEOUT)				\
-	for (__init_wedge((W), (DEV), (TIMEOUT), __builtin_return_address(0)); \
-	     (W)->i915;							\
-	     __fini_wedge((W)))
-
 int igt_flush_test(struct drm_i915_private *i915, unsigned int flags)
 {
-	struct wedge_me w;
-
 	cond_resched();
 
 	if (flags & I915_WAIT_LOCKED &&
@@ -63,8 +19,15 @@ int igt_flush_test(struct drm_i915_private *i915, unsigned int flags)
 		i915_gem_set_wedged(i915);
 	}
 
-	wedge_on_timeout(&w, i915, HZ)
-		i915_gem_wait_for_idle(i915, flags);
+	if (i915_gem_wait_for_idle(i915, flags, HZ / 5) == -ETIME) {
+		pr_err("%pS timed out, cancelling all further testing.\n",
+		       __builtin_return_address(0));
+
+		GEM_TRACE("%pS timed out.\n", __builtin_return_address(0));
+		GEM_TRACE_DUMP();
+
+		i915_gem_set_wedged(i915);
+	}
 
 	return i915_terminally_wedged(&i915->gpu_error) ? -EIO : 0;
 }
diff --git a/drivers/gpu/drm/i915/selftests/igt_wedge_me.h b/drivers/gpu/drm/i915/selftests/igt_wedge_me.h
new file mode 100644
index 000000000000..08e5ff11bbd9
--- /dev/null
+++ b/drivers/gpu/drm/i915/selftests/igt_wedge_me.h
@@ -0,0 +1,58 @@
+/*
+ * SPDX-License-Identifier: MIT
+ *
+ * Copyright © 2018 Intel Corporation
+ */
+
+#ifndef IGT_WEDGE_ME_H
+#define IGT_WEDGE_ME_H
+
+#include <linux/workqueue.h>
+
+#include "../i915_gem.h"
+
+struct drm_i915_private;
+
+struct igt_wedge_me {
+	struct delayed_work work;
+	struct drm_i915_private *i915;
+	const char *name;
+};
+
+static void __igt_wedge_me(struct work_struct *work)
+{
+	struct igt_wedge_me *w = container_of(work, typeof(*w), work.work);
+
+	pr_err("%s timed out, cancelling test.\n", w->name);
+
+	GEM_TRACE("%s timed out.\n", w->name);
+	GEM_TRACE_DUMP();
+
+	i915_gem_set_wedged(w->i915);
+}
+
+static void __igt_init_wedge(struct igt_wedge_me *w,
+			     struct drm_i915_private *i915,
+			     long timeout,
+			     const char *name)
+{
+	w->i915 = i915;
+	w->name = name;
+
+	INIT_DELAYED_WORK_ONSTACK(&w->work, __igt_wedge_me);
+	schedule_delayed_work(&w->work, timeout);
+}
+
+static void __igt_fini_wedge(struct igt_wedge_me *w)
+{
+	cancel_delayed_work_sync(&w->work);
+	destroy_delayed_work_on_stack(&w->work);
+	w->i915 = NULL;
+}
+
+#define igt_wedge_on_timeout(W, DEV, TIMEOUT)				\
+	for (__igt_init_wedge((W), (DEV), (TIMEOUT), __func__);		\
+	     (W)->i915;							\
+	     __igt_fini_wedge((W)))
+
+#endif /* IGT_WEDGE_ME_H */
diff --git a/drivers/gpu/drm/i915/selftests/intel_breadcrumbs.c b/drivers/gpu/drm/i915/selftests/intel_breadcrumbs.c
index d6926e7820e5..f03b407fdbe2 100644
--- a/drivers/gpu/drm/i915/selftests/intel_breadcrumbs.c
+++ b/drivers/gpu/drm/i915/selftests/intel_breadcrumbs.c
@@ -464,7 +464,7 @@ int intel_breadcrumbs_mock_selftests(void)
 		return -ENOMEM;
 
 	err = i915_subtests(tests, i915->engine[RCS]);
-	drm_dev_unref(&i915->drm);
+	drm_dev_put(&i915->drm);
 
 	return err;
 }
diff --git a/drivers/gpu/drm/i915/selftests/intel_guc.c b/drivers/gpu/drm/i915/selftests/intel_guc.c
index fb74e2cf8a0a..407c98fb9170 100644
--- a/drivers/gpu/drm/i915/selftests/intel_guc.c
+++ b/drivers/gpu/drm/i915/selftests/intel_guc.c
@@ -196,19 +196,23 @@ static int igt_guc_clients(void *args)
 	}
 
 	unreserve_doorbell(guc->execbuf_client);
-	err = guc_clients_doorbell_init(guc);
+
+	__create_doorbell(guc->execbuf_client);
+	err = __guc_allocate_doorbell(guc, guc->execbuf_client->stage_id);
 	if (err != -EIO) {
 		pr_err("unexpected (err = %d)", err);
-		goto out;
+		goto out_db;
 	}
 
 	if (!available_dbs(guc, guc->execbuf_client->priority)) {
 		pr_err("doorbell not available when it should\n");
 		err = -EIO;
-		goto out;
+		goto out_db;
 	}
 
+out_db:
 	/* clean after test */
+	__destroy_doorbell(guc->execbuf_client);
 	err = reserve_doorbell(guc->execbuf_client);
 	if (err) {
 		pr_err("failed to reserve back the doorbell back\n");
diff --git a/drivers/gpu/drm/i915/selftests/intel_hangcheck.c b/drivers/gpu/drm/i915/selftests/intel_hangcheck.c
index fe7d3190ebfe..65d66cdedd26 100644
--- a/drivers/gpu/drm/i915/selftests/intel_hangcheck.c
+++ b/drivers/gpu/drm/i915/selftests/intel_hangcheck.c
@@ -27,6 +27,7 @@
 #include "../i915_selftest.h"
 #include "i915_random.h"
 #include "igt_flush_test.h"
+#include "igt_wedge_me.h"
 
 #include "mock_context.h"
 #include "mock_drm.h"
@@ -130,13 +131,19 @@ static int emit_recurse_batch(struct hang *h,
 	if (err)
 		goto unpin_vma;
 
-	i915_vma_move_to_active(vma, rq, 0);
+	err = i915_vma_move_to_active(vma, rq, 0);
+	if (err)
+		goto unpin_hws;
+
 	if (!i915_gem_object_has_active_reference(vma->obj)) {
 		i915_gem_object_get(vma->obj);
 		i915_gem_object_set_active_reference(vma->obj);
 	}
 
-	i915_vma_move_to_active(hws, rq, 0);
+	err = i915_vma_move_to_active(hws, rq, 0);
+	if (err)
+		goto unpin_hws;
+
 	if (!i915_gem_object_has_active_reference(hws->obj)) {
 		i915_gem_object_get(hws->obj);
 		i915_gem_object_set_active_reference(hws->obj);
@@ -171,7 +178,7 @@ static int emit_recurse_batch(struct hang *h,
 		*batch++ = MI_BATCH_BUFFER_START | 1 << 8;
 		*batch++ = lower_32_bits(vma->node.start);
 	} else if (INTEL_GEN(i915) >= 4) {
-		*batch++ = MI_STORE_DWORD_IMM_GEN4 | 1 << 22;
+		*batch++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
 		*batch++ = 0;
 		*batch++ = lower_32_bits(hws_address(hws, rq));
 		*batch++ = rq->fence.seqno;
@@ -184,7 +191,7 @@ static int emit_recurse_batch(struct hang *h,
 		*batch++ = MI_BATCH_BUFFER_START | 2 << 6;
 		*batch++ = lower_32_bits(vma->node.start);
 	} else {
-		*batch++ = MI_STORE_DWORD_IMM;
+		*batch++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL;
 		*batch++ = lower_32_bits(hws_address(hws, rq));
 		*batch++ = rq->fence.seqno;
 		*batch++ = MI_ARB_CHECK;
@@ -193,7 +200,7 @@ static int emit_recurse_batch(struct hang *h,
 		batch += 1024 / sizeof(*batch);
 
 		*batch++ = MI_ARB_CHECK;
-		*batch++ = MI_BATCH_BUFFER_START | 2 << 6 | 1;
+		*batch++ = MI_BATCH_BUFFER_START | 2 << 6;
 		*batch++ = lower_32_bits(vma->node.start);
 	}
 	*batch++ = MI_BATCH_BUFFER_END; /* not reached */
@@ -205,6 +212,7 @@ static int emit_recurse_batch(struct hang *h,
 
 	err = rq->engine->emit_bb_start(rq, vma->node.start, PAGE_SIZE, flags);
 
+unpin_hws:
 	i915_vma_unpin(hws);
 unpin_vma:
 	i915_vma_unpin(vma);
@@ -914,7 +922,7 @@ static u32 fake_hangcheck(struct i915_request *rq, u32 mask)
 	return reset_count;
 }
 
-static int igt_wait_reset(void *arg)
+static int igt_reset_wait(void *arg)
 {
 	struct drm_i915_private *i915 = arg;
 	struct i915_request *rq;
@@ -988,6 +996,170 @@ unlock:
 	return err;
 }
 
+struct evict_vma {
+	struct completion completion;
+	struct i915_vma *vma;
+};
+
+static int evict_vma(void *data)
+{
+	struct evict_vma *arg = data;
+	struct i915_address_space *vm = arg->vma->vm;
+	struct drm_i915_private *i915 = vm->i915;
+	struct drm_mm_node evict = arg->vma->node;
+	int err;
+
+	complete(&arg->completion);
+
+	mutex_lock(&i915->drm.struct_mutex);
+	err = i915_gem_evict_for_node(vm, &evict, 0);
+	mutex_unlock(&i915->drm.struct_mutex);
+
+	return err;
+}
+
+static int __igt_reset_evict_vma(struct drm_i915_private *i915,
+				 struct i915_address_space *vm)
+{
+	struct drm_i915_gem_object *obj;
+	struct task_struct *tsk = NULL;
+	struct i915_request *rq;
+	struct evict_vma arg;
+	struct hang h;
+	int err;
+
+	if (!intel_engine_can_store_dword(i915->engine[RCS]))
+		return 0;
+
+	/* Check that we can recover an unbind stuck on a hanging request */
+
+	global_reset_lock(i915);
+
+	mutex_lock(&i915->drm.struct_mutex);
+	err = hang_init(&h, i915);
+	if (err)
+		goto unlock;
+
+	obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
+	if (IS_ERR(obj)) {
+		err = PTR_ERR(obj);
+		goto fini;
+	}
+
+	arg.vma = i915_vma_instance(obj, vm, NULL);
+	if (IS_ERR(arg.vma)) {
+		err = PTR_ERR(arg.vma);
+		goto out_obj;
+	}
+
+	rq = hang_create_request(&h, i915->engine[RCS]);
+	if (IS_ERR(rq)) {
+		err = PTR_ERR(rq);
+		goto out_obj;
+	}
+
+	err = i915_vma_pin(arg.vma, 0, 0,
+			   i915_vma_is_ggtt(arg.vma) ? PIN_GLOBAL : PIN_USER);
+	if (err)
+		goto out_obj;
+
+	err = i915_vma_move_to_active(arg.vma, rq, EXEC_OBJECT_WRITE);
+	i915_vma_unpin(arg.vma);
+
+	i915_request_get(rq);
+	i915_request_add(rq);
+	if (err)
+		goto out_rq;
+
+	mutex_unlock(&i915->drm.struct_mutex);
+
+	if (!wait_until_running(&h, rq)) {
+		struct drm_printer p = drm_info_printer(i915->drm.dev);
+
+		pr_err("%s: Failed to start request %x, at %x\n",
+		       __func__, rq->fence.seqno, hws_seqno(&h, rq));
+		intel_engine_dump(rq->engine, &p, "%s\n", rq->engine->name);
+
+		i915_gem_set_wedged(i915);
+		goto out_reset;
+	}
+
+	init_completion(&arg.completion);
+
+	tsk = kthread_run(evict_vma, &arg, "igt/evict_vma");
+	if (IS_ERR(tsk)) {
+		err = PTR_ERR(tsk);
+		tsk = NULL;
+		goto out_reset;
+	}
+
+	wait_for_completion(&arg.completion);
+
+	if (wait_for(waitqueue_active(&rq->execute), 10)) {
+		struct drm_printer p = drm_info_printer(i915->drm.dev);
+
+		pr_err("igt/evict_vma kthread did not wait\n");
+		intel_engine_dump(rq->engine, &p, "%s\n", rq->engine->name);
+
+		i915_gem_set_wedged(i915);
+		goto out_reset;
+	}
+
+out_reset:
+	fake_hangcheck(rq, intel_engine_flag(rq->engine));
+
+	if (tsk) {
+		struct igt_wedge_me w;
+
+		/* The reset, even indirectly, should take less than 10ms. */
+		igt_wedge_on_timeout(&w, i915, HZ / 10 /* 100ms timeout*/)
+			err = kthread_stop(tsk);
+	}
+
+	mutex_lock(&i915->drm.struct_mutex);
+out_rq:
+	i915_request_put(rq);
+out_obj:
+	i915_gem_object_put(obj);
+fini:
+	hang_fini(&h);
+unlock:
+	mutex_unlock(&i915->drm.struct_mutex);
+	global_reset_unlock(i915);
+
+	if (i915_terminally_wedged(&i915->gpu_error))
+		return -EIO;
+
+	return err;
+}
+
+static int igt_reset_evict_ggtt(void *arg)
+{
+	struct drm_i915_private *i915 = arg;
+
+	return __igt_reset_evict_vma(i915, &i915->ggtt.vm);
+}
+
+static int igt_reset_evict_ppgtt(void *arg)
+{
+	struct drm_i915_private *i915 = arg;
+	struct i915_gem_context *ctx;
+	int err;
+
+	mutex_lock(&i915->drm.struct_mutex);
+	ctx = kernel_context(i915);
+	mutex_unlock(&i915->drm.struct_mutex);
+	if (IS_ERR(ctx))
+		return PTR_ERR(ctx);
+
+	err = 0;
+	if (ctx->ppgtt) /* aliasing == global gtt locking, covered above */
+		err = __igt_reset_evict_vma(i915, &ctx->ppgtt->vm);
+
+	kernel_context_close(ctx);
+	return err;
+}
+
 static int wait_for_others(struct drm_i915_private *i915,
 			   struct intel_engine_cs *exclude)
 {
@@ -1233,8 +1405,10 @@ int intel_hangcheck_live_selftests(struct drm_i915_private *i915)
 		SUBTEST(igt_reset_idle_engine),
 		SUBTEST(igt_reset_active_engine),
 		SUBTEST(igt_reset_engines),
-		SUBTEST(igt_wait_reset),
 		SUBTEST(igt_reset_queue),
+		SUBTEST(igt_reset_wait),
+		SUBTEST(igt_reset_evict_ggtt),
+		SUBTEST(igt_reset_evict_ppgtt),
 		SUBTEST(igt_handle_error),
 	};
 	bool saved_hangcheck;
@@ -1243,6 +1417,9 @@ int intel_hangcheck_live_selftests(struct drm_i915_private *i915)
 	if (!intel_has_gpu_reset(i915))
 		return 0;
 
+	if (i915_terminally_wedged(&i915->gpu_error))
+		return -EIO; /* we're long past hope of a successful reset */
+
 	intel_runtime_pm_get(i915);
 	saved_hangcheck = fetch_and_zero(&i915_modparams.enable_hangcheck);
 
diff --git a/drivers/gpu/drm/i915/selftests/intel_lrc.c b/drivers/gpu/drm/i915/selftests/intel_lrc.c
index ea27c7cfbf96..582566faef09 100644
--- a/drivers/gpu/drm/i915/selftests/intel_lrc.c
+++ b/drivers/gpu/drm/i915/selftests/intel_lrc.c
@@ -104,13 +104,19 @@ static int emit_recurse_batch(struct spinner *spin,
 	if (err)
 		goto unpin_vma;
 
-	i915_vma_move_to_active(vma, rq, 0);
+	err = i915_vma_move_to_active(vma, rq, 0);
+	if (err)
+		goto unpin_hws;
+
 	if (!i915_gem_object_has_active_reference(vma->obj)) {
 		i915_gem_object_get(vma->obj);
 		i915_gem_object_set_active_reference(vma->obj);
 	}
 
-	i915_vma_move_to_active(hws, rq, 0);
+	err = i915_vma_move_to_active(hws, rq, 0);
+	if (err)
+		goto unpin_hws;
+
 	if (!i915_gem_object_has_active_reference(hws->obj)) {
 		i915_gem_object_get(hws->obj);
 		i915_gem_object_set_active_reference(hws->obj);
@@ -134,6 +140,7 @@ static int emit_recurse_batch(struct spinner *spin,
 
 	err = rq->engine->emit_bb_start(rq, vma->node.start, PAGE_SIZE, 0);
 
+unpin_hws:
 	i915_vma_unpin(hws);
 unpin_vma:
 	i915_vma_unpin(vma);
@@ -444,16 +451,134 @@ err_wedged:
 	goto err_ctx_lo;
 }
 
+static int live_preempt_hang(void *arg)
+{
+	struct drm_i915_private *i915 = arg;
+	struct i915_gem_context *ctx_hi, *ctx_lo;
+	struct spinner spin_hi, spin_lo;
+	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
+	int err = -ENOMEM;
+
+	if (!HAS_LOGICAL_RING_PREEMPTION(i915))
+		return 0;
+
+	if (!intel_has_reset_engine(i915))
+		return 0;
+
+	mutex_lock(&i915->drm.struct_mutex);
+
+	if (spinner_init(&spin_hi, i915))
+		goto err_unlock;
+
+	if (spinner_init(&spin_lo, i915))
+		goto err_spin_hi;
+
+	ctx_hi = kernel_context(i915);
+	if (!ctx_hi)
+		goto err_spin_lo;
+	ctx_hi->sched.priority = I915_CONTEXT_MAX_USER_PRIORITY;
+
+	ctx_lo = kernel_context(i915);
+	if (!ctx_lo)
+		goto err_ctx_hi;
+	ctx_lo->sched.priority = I915_CONTEXT_MIN_USER_PRIORITY;
+
+	for_each_engine(engine, i915, id) {
+		struct i915_request *rq;
+
+		if (!intel_engine_has_preemption(engine))
+			continue;
+
+		rq = spinner_create_request(&spin_lo, ctx_lo, engine,
+					    MI_ARB_CHECK);
+		if (IS_ERR(rq)) {
+			err = PTR_ERR(rq);
+			goto err_ctx_lo;
+		}
+
+		i915_request_add(rq);
+		if (!wait_for_spinner(&spin_lo, rq)) {
+			GEM_TRACE("lo spinner failed to start\n");
+			GEM_TRACE_DUMP();
+			i915_gem_set_wedged(i915);
+			err = -EIO;
+			goto err_ctx_lo;
+		}
+
+		rq = spinner_create_request(&spin_hi, ctx_hi, engine,
+					    MI_ARB_CHECK);
+		if (IS_ERR(rq)) {
+			spinner_end(&spin_lo);
+			err = PTR_ERR(rq);
+			goto err_ctx_lo;
+		}
+
+		init_completion(&engine->execlists.preempt_hang.completion);
+		engine->execlists.preempt_hang.inject_hang = true;
+
+		i915_request_add(rq);
+
+		if (!wait_for_completion_timeout(&engine->execlists.preempt_hang.completion,
+						 HZ / 10)) {
+			pr_err("Preemption did not occur within timeout!");
+			GEM_TRACE_DUMP();
+			i915_gem_set_wedged(i915);
+			err = -EIO;
+			goto err_ctx_lo;
+		}
+
+		set_bit(I915_RESET_ENGINE + id, &i915->gpu_error.flags);
+		i915_reset_engine(engine, NULL);
+		clear_bit(I915_RESET_ENGINE + id, &i915->gpu_error.flags);
+
+		engine->execlists.preempt_hang.inject_hang = false;
+
+		if (!wait_for_spinner(&spin_hi, rq)) {
+			GEM_TRACE("hi spinner failed to start\n");
+			GEM_TRACE_DUMP();
+			i915_gem_set_wedged(i915);
+			err = -EIO;
+			goto err_ctx_lo;
+		}
+
+		spinner_end(&spin_hi);
+		spinner_end(&spin_lo);
+		if (igt_flush_test(i915, I915_WAIT_LOCKED)) {
+			err = -EIO;
+			goto err_ctx_lo;
+		}
+	}
+
+	err = 0;
+err_ctx_lo:
+	kernel_context_close(ctx_lo);
+err_ctx_hi:
+	kernel_context_close(ctx_hi);
+err_spin_lo:
+	spinner_fini(&spin_lo);
+err_spin_hi:
+	spinner_fini(&spin_hi);
+err_unlock:
+	igt_flush_test(i915, I915_WAIT_LOCKED);
+	mutex_unlock(&i915->drm.struct_mutex);
+	return err;
+}
+
 int intel_execlists_live_selftests(struct drm_i915_private *i915)
 {
 	static const struct i915_subtest tests[] = {
 		SUBTEST(live_sanitycheck),
 		SUBTEST(live_preempt),
 		SUBTEST(live_late_preempt),
+		SUBTEST(live_preempt_hang),
 	};
 
 	if (!HAS_EXECLISTS(i915))
 		return 0;
 
+	if (i915_terminally_wedged(&i915->gpu_error))
+		return 0;
+
 	return i915_subtests(tests, i915);
 }
diff --git a/drivers/gpu/drm/i915/selftests/intel_uncore.c b/drivers/gpu/drm/i915/selftests/intel_uncore.c
index 47bc5b2ddb56..81d9d31042a9 100644
--- a/drivers/gpu/drm/i915/selftests/intel_uncore.c
+++ b/drivers/gpu/drm/i915/selftests/intel_uncore.c
@@ -160,7 +160,7 @@ static int intel_uncore_check_forcewake_domains(struct drm_i915_private *dev_pri
 		i915_reg_t reg = { offset };
 
 		iosf_mbi_punit_acquire();
-		intel_uncore_forcewake_reset(dev_priv, false);
+		intel_uncore_forcewake_reset(dev_priv);
 		iosf_mbi_punit_release();
 
 		check_for_unclaimed_mmio(dev_priv);
diff --git a/drivers/gpu/drm/i915/selftests/intel_workarounds.c b/drivers/gpu/drm/i915/selftests/intel_workarounds.c
index e1ea2d2bedd2..0d39b3bf0c0d 100644
--- a/drivers/gpu/drm/i915/selftests/intel_workarounds.c
+++ b/drivers/gpu/drm/i915/selftests/intel_workarounds.c
@@ -6,6 +6,7 @@
 
 #include "../i915_selftest.h"
 
+#include "igt_wedge_me.h"
 #include "mock_context.h"
 
 static struct drm_i915_gem_object *
@@ -49,6 +50,10 @@ read_nonprivs(struct i915_gem_context *ctx, struct intel_engine_cs *engine)
 		goto err_pin;
 	}
 
+	err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
+	if (err)
+		goto err_req;
+
 	srm = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT;
 	if (INTEL_GEN(ctx->i915) >= 8)
 		srm++;
@@ -67,11 +72,6 @@ read_nonprivs(struct i915_gem_context *ctx, struct intel_engine_cs *engine)
 	}
 	intel_ring_advance(rq, cs);
 
-	i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
-	reservation_object_lock(vma->resv, NULL);
-	reservation_object_add_excl_fence(vma->resv, &rq->fence);
-	reservation_object_unlock(vma->resv);
-
 	i915_gem_object_get(result);
 	i915_gem_object_set_active_reference(result);
 
@@ -112,6 +112,7 @@ static int check_whitelist(const struct whitelist *w,
 			   struct intel_engine_cs *engine)
 {
 	struct drm_i915_gem_object *results;
+	struct igt_wedge_me wedge;
 	u32 *vaddr;
 	int err;
 	int i;
@@ -120,7 +121,11 @@ static int check_whitelist(const struct whitelist *w,
 	if (IS_ERR(results))
 		return PTR_ERR(results);
 
-	err = i915_gem_object_set_to_cpu_domain(results, false);
+	err = 0;
+	igt_wedge_on_timeout(&wedge, ctx->i915, HZ / 5) /* a safety net! */
+		err = i915_gem_object_set_to_cpu_domain(results, false);
+	if (i915_terminally_wedged(&ctx->i915->gpu_error))
+		err = -EIO;
 	if (err)
 		goto out_put;
 
@@ -283,6 +288,9 @@ int intel_workarounds_live_selftests(struct drm_i915_private *i915)
 	};
 	int err;
 
+	if (i915_terminally_wedged(&i915->gpu_error))
+		return 0;
+
 	mutex_lock(&i915->drm.struct_mutex);
 	err = i915_subtests(tests, i915);
 	mutex_unlock(&i915->drm.struct_mutex);
diff --git a/drivers/gpu/drm/i915/selftests/mock_engine.c b/drivers/gpu/drm/i915/selftests/mock_engine.c
index c2a0451336cf..22a73da45ad5 100644
--- a/drivers/gpu/drm/i915/selftests/mock_engine.c
+++ b/drivers/gpu/drm/i915/selftests/mock_engine.c
@@ -200,6 +200,8 @@ struct intel_engine_cs *mock_engine(struct drm_i915_private *i915,
 	engine->base.submit_request = mock_submit_request;
 
 	i915_timeline_init(i915, &engine->base.timeline, engine->base.name);
+	lockdep_set_subclass(&engine->base.timeline.lock, TIMELINE_ENGINE);
+
 	intel_engine_init_breadcrumbs(&engine->base);
 	engine->base.breadcrumbs.mock = true; /* prevent touching HW for irqs */
 
diff --git a/drivers/gpu/drm/i915/selftests/mock_gem_device.c b/drivers/gpu/drm/i915/selftests/mock_gem_device.c
index c97075c5ccaf..43ed8b28aeaa 100644
--- a/drivers/gpu/drm/i915/selftests/mock_gem_device.c
+++ b/drivers/gpu/drm/i915/selftests/mock_gem_device.c
@@ -157,7 +157,8 @@ struct drm_i915_private *mock_gem_device(void)
 	dev_pm_domain_set(&pdev->dev, &pm_domain);
 	pm_runtime_enable(&pdev->dev);
 	pm_runtime_dont_use_autosuspend(&pdev->dev);
-	WARN_ON(pm_runtime_get_sync(&pdev->dev));
+	if (pm_runtime_enabled(&pdev->dev))
+		WARN_ON(pm_runtime_get_sync(&pdev->dev));
 
 	i915 = (struct drm_i915_private *)(pdev + 1);
 	pci_set_drvdata(pdev, i915);
diff --git a/drivers/gpu/drm/i915/selftests/mock_gtt.c b/drivers/gpu/drm/i915/selftests/mock_gtt.c
index 6a7f4da7b523..a140ea5c3a7c 100644
--- a/drivers/gpu/drm/i915/selftests/mock_gtt.c
+++ b/drivers/gpu/drm/i915/selftests/mock_gtt.c
@@ -70,12 +70,7 @@ mock_ppgtt(struct drm_i915_private *i915,
 	ppgtt->vm.total = round_down(U64_MAX, PAGE_SIZE);
 	ppgtt->vm.file = ERR_PTR(-ENODEV);
 
-	INIT_LIST_HEAD(&ppgtt->vm.active_list);
-	INIT_LIST_HEAD(&ppgtt->vm.inactive_list);
-	INIT_LIST_HEAD(&ppgtt->vm.unbound_list);
-
-	INIT_LIST_HEAD(&ppgtt->vm.global_link);
-	drm_mm_init(&ppgtt->vm.mm, 0, ppgtt->vm.total);
+	i915_address_space_init(&ppgtt->vm, i915);
 
 	ppgtt->vm.clear_range = nop_clear_range;
 	ppgtt->vm.insert_page = mock_insert_page;
@@ -106,8 +101,6 @@ void mock_init_ggtt(struct drm_i915_private *i915)
 {
 	struct i915_ggtt *ggtt = &i915->ggtt;
 
-	INIT_LIST_HEAD(&i915->vm_list);
-
 	ggtt->vm.i915 = i915;
 
 	ggtt->gmadr = (struct resource) DEFINE_RES_MEM(0, 2048 * PAGE_SIZE);
@@ -124,7 +117,7 @@ void mock_init_ggtt(struct drm_i915_private *i915)
 	ggtt->vm.vma_ops.set_pages   = ggtt_set_pages;
 	ggtt->vm.vma_ops.clear_pages = clear_pages;
 
-	i915_address_space_init(&ggtt->vm, i915, "global");
+	i915_address_space_init(&ggtt->vm, i915);
 }
 
 void mock_fini_ggtt(struct drm_i915_private *i915)
diff --git a/drivers/gpu/drm/i915/intel_dsi.c b/drivers/gpu/drm/i915/vlv_dsi.c
index 3b7acb5a70b3..435a2c35ee8c 100644
--- a/drivers/gpu/drm/i915/intel_dsi.c
+++ b/drivers/gpu/drm/i915/vlv_dsi.c
@@ -69,7 +69,7 @@ enum mipi_dsi_pixel_format pixel_format_from_register_bits(u32 fmt)
 	}
 }
 
-void wait_for_dsi_fifo_empty(struct intel_dsi *intel_dsi, enum port port)
+void vlv_dsi_wait_for_fifo_empty(struct intel_dsi *intel_dsi, enum port port)
 {
 	struct drm_encoder *encoder = &intel_dsi->base.base;
 	struct drm_device *dev = encoder->dev;
@@ -342,11 +342,15 @@ static bool intel_dsi_compute_config(struct intel_encoder *encoder,
 			pipe_config->cpu_transcoder = TRANSCODER_DSI_C;
 		else
 			pipe_config->cpu_transcoder = TRANSCODER_DSI_A;
-	}
 
-	ret = intel_compute_dsi_pll(encoder, pipe_config);
-	if (ret)
-		return false;
+		ret = bxt_dsi_pll_compute(encoder, pipe_config);
+		if (ret)
+			return false;
+	} else {
+		ret = vlv_dsi_pll_compute(encoder, pipe_config);
+		if (ret)
+			return false;
+	}
 
 	pipe_config->clock_set = true;
 
@@ -546,12 +550,12 @@ static void intel_dsi_device_ready(struct intel_encoder *encoder)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 
-	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
-		vlv_dsi_device_ready(encoder);
-	else if (IS_BROXTON(dev_priv))
-		bxt_dsi_device_ready(encoder);
-	else if (IS_GEMINILAKE(dev_priv))
+	if (IS_GEMINILAKE(dev_priv))
 		glk_dsi_device_ready(encoder);
+	else if (IS_GEN9_LP(dev_priv))
+		bxt_dsi_device_ready(encoder);
+	else
+		vlv_dsi_device_ready(encoder);
 }
 
 static void glk_dsi_enter_low_power_mode(struct intel_encoder *encoder)
@@ -810,8 +814,13 @@ static void intel_dsi_pre_enable(struct intel_encoder *encoder,
 	 * The BIOS may leave the PLL in a wonky state where it doesn't
 	 * lock. It needs to be fully powered down to fix it.
 	 */
-	intel_disable_dsi_pll(encoder);
-	intel_enable_dsi_pll(encoder, pipe_config);
+	if (IS_GEN9_LP(dev_priv)) {
+		bxt_dsi_pll_disable(encoder);
+		bxt_dsi_pll_enable(encoder, pipe_config);
+	} else {
+		vlv_dsi_pll_disable(encoder);
+		vlv_dsi_pll_enable(encoder, pipe_config);
+	}
 
 	if (IS_BROXTON(dev_priv)) {
 		/* Add MIPI IO reset programming for modeset */
@@ -929,11 +938,10 @@ static void intel_dsi_clear_device_ready(struct intel_encoder *encoder)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 
-	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv) ||
-	    IS_BROXTON(dev_priv))
-		vlv_dsi_clear_device_ready(encoder);
-	else if (IS_GEMINILAKE(dev_priv))
+	if (IS_GEMINILAKE(dev_priv))
 		glk_dsi_clear_device_ready(encoder);
+	else
+		vlv_dsi_clear_device_ready(encoder);
 }
 
 static void intel_dsi_post_disable(struct intel_encoder *encoder,
@@ -949,7 +957,7 @@ static void intel_dsi_post_disable(struct intel_encoder *encoder,
 
 	if (is_vid_mode(intel_dsi)) {
 		for_each_dsi_port(port, intel_dsi->ports)
-			wait_for_dsi_fifo_empty(intel_dsi, port);
+			vlv_dsi_wait_for_fifo_empty(intel_dsi, port);
 
 		intel_dsi_port_disable(encoder);
 		usleep_range(2000, 5000);
@@ -979,11 +987,13 @@ static void intel_dsi_post_disable(struct intel_encoder *encoder,
 				val & ~MIPIO_RST_CTRL);
 	}
 
-	intel_disable_dsi_pll(encoder);
-
-	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
+	if (IS_GEN9_LP(dev_priv)) {
+		bxt_dsi_pll_disable(encoder);
+	} else {
 		u32 val;
 
+		vlv_dsi_pll_disable(encoder);
+
 		val = I915_READ(DSPCLK_GATE_D);
 		val &= ~DPOUNIT_CLOCK_GATE_DISABLE;
 		I915_WRITE(DSPCLK_GATE_D, val);
@@ -1024,7 +1034,7 @@ static bool intel_dsi_get_hw_state(struct intel_encoder *encoder,
 	 * configuration, otherwise accessing DSI registers will hang the
 	 * machine. See BSpec North Display Engine registers/MIPI[BXT].
 	 */
-	if (IS_GEN9_LP(dev_priv) && !intel_dsi_pll_is_enabled(dev_priv))
+	if (IS_GEN9_LP(dev_priv) && !bxt_dsi_pll_is_enabled(dev_priv))
 		goto out_put_power;
 
 	/* XXX: this only works for one DSI output */
@@ -1247,16 +1257,19 @@ static void intel_dsi_get_config(struct intel_encoder *encoder,
 
 	pipe_config->output_types |= BIT(INTEL_OUTPUT_DSI);
 
-	if (IS_GEN9_LP(dev_priv))
+	if (IS_GEN9_LP(dev_priv)) {
 		bxt_dsi_get_pipe_config(encoder, pipe_config);
+		pclk = bxt_dsi_get_pclk(encoder, pipe_config->pipe_bpp,
+					pipe_config);
+	} else {
+		pclk = vlv_dsi_get_pclk(encoder, pipe_config->pipe_bpp,
+					pipe_config);
+	}
 
-	pclk = intel_dsi_get_pclk(encoder, pipe_config->pipe_bpp,
-				  pipe_config);
-	if (!pclk)
-		return;
-
-	pipe_config->base.adjusted_mode.crtc_clock = pclk;
-	pipe_config->port_clock = pclk;
+	if (pclk) {
+		pipe_config->base.adjusted_mode.crtc_clock = pclk;
+		pipe_config->port_clock = pclk;
+	}
 }
 
 static enum drm_mode_status
@@ -1585,20 +1598,24 @@ static void intel_dsi_unprepare(struct intel_encoder *encoder)
 	enum port port;
 	u32 val;
 
-	if (!IS_GEMINILAKE(dev_priv)) {
-		for_each_dsi_port(port, intel_dsi->ports) {
-			/* Panel commands can be sent when clock is in LP11 */
-			I915_WRITE(MIPI_DEVICE_READY(port), 0x0);
+	if (IS_GEMINILAKE(dev_priv))
+		return;
 
-			intel_dsi_reset_clocks(encoder, port);
-			I915_WRITE(MIPI_EOT_DISABLE(port), CLOCKSTOP);
+	for_each_dsi_port(port, intel_dsi->ports) {
+		/* Panel commands can be sent when clock is in LP11 */
+		I915_WRITE(MIPI_DEVICE_READY(port), 0x0);
 
-			val = I915_READ(MIPI_DSI_FUNC_PRG(port));
-			val &= ~VID_MODE_FORMAT_MASK;
-			I915_WRITE(MIPI_DSI_FUNC_PRG(port), val);
+		if (IS_GEN9_LP(dev_priv))
+			bxt_dsi_reset_clocks(encoder, port);
+		else
+			vlv_dsi_reset_clocks(encoder, port);
+		I915_WRITE(MIPI_EOT_DISABLE(port), CLOCKSTOP);
 
-			I915_WRITE(MIPI_DEVICE_READY(port), 0x1);
-		}
+		val = I915_READ(MIPI_DSI_FUNC_PRG(port));
+		val &= ~VID_MODE_FORMAT_MASK;
+		I915_WRITE(MIPI_DSI_FUNC_PRG(port), val);
+
+		I915_WRITE(MIPI_DEVICE_READY(port), 0x1);
 	}
 }
 
@@ -1713,7 +1730,7 @@ static void intel_dsi_add_properties(struct intel_connector *connector)
 	}
 }
 
-void intel_dsi_init(struct drm_i915_private *dev_priv)
+void vlv_dsi_init(struct drm_i915_private *dev_priv)
 {
 	struct drm_device *dev = &dev_priv->drm;
 	struct intel_dsi *intel_dsi;
@@ -1730,14 +1747,10 @@ void intel_dsi_init(struct drm_i915_private *dev_priv)
 	if (!intel_bios_is_dsi_present(dev_priv, &port))
 		return;
 
-	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
-		dev_priv->mipi_mmio_base = VLV_MIPI_BASE;
-	} else if (IS_GEN9_LP(dev_priv)) {
+	if (IS_GEN9_LP(dev_priv))
 		dev_priv->mipi_mmio_base = BXT_MIPI_BASE;
-	} else {
-		DRM_ERROR("Unsupported Mipi device to reg base");
-		return;
-	}
+	else
+		dev_priv->mipi_mmio_base = VLV_MIPI_BASE;
 
 	intel_dsi = kzalloc(sizeof(*intel_dsi), GFP_KERNEL);
 	if (!intel_dsi)
diff --git a/drivers/gpu/drm/i915/intel_dsi_pll.c b/drivers/gpu/drm/i915/vlv_dsi_pll.c
index 2ff2ee7f3b78..a132a8037ecc 100644
--- a/drivers/gpu/drm/i915/intel_dsi_pll.c
+++ b/drivers/gpu/drm/i915/vlv_dsi_pll.c
@@ -111,8 +111,8 @@ static int dsi_calc_mnp(struct drm_i915_private *dev_priv,
  * XXX: The muxing and gating is hard coded for now. Need to add support for
  * sharing PLLs with two DSI outputs.
  */
-static int vlv_compute_dsi_pll(struct intel_encoder *encoder,
-			       struct intel_crtc_state *config)
+int vlv_dsi_pll_compute(struct intel_encoder *encoder,
+			struct intel_crtc_state *config)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
@@ -142,8 +142,8 @@ static int vlv_compute_dsi_pll(struct intel_encoder *encoder,
 	return 0;
 }
 
-static void vlv_enable_dsi_pll(struct intel_encoder *encoder,
-			       const struct intel_crtc_state *config)
+void vlv_dsi_pll_enable(struct intel_encoder *encoder,
+			const struct intel_crtc_state *config)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 
@@ -175,7 +175,7 @@ static void vlv_enable_dsi_pll(struct intel_encoder *encoder,
 	DRM_DEBUG_KMS("DSI PLL locked\n");
 }
 
-static void vlv_disable_dsi_pll(struct intel_encoder *encoder)
+void vlv_dsi_pll_disable(struct intel_encoder *encoder)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	u32 tmp;
@@ -192,7 +192,7 @@ static void vlv_disable_dsi_pll(struct intel_encoder *encoder)
 	mutex_unlock(&dev_priv->sb_lock);
 }
 
-static bool bxt_dsi_pll_is_enabled(struct drm_i915_private *dev_priv)
+bool bxt_dsi_pll_is_enabled(struct drm_i915_private *dev_priv)
 {
 	bool enabled;
 	u32 val;
@@ -229,7 +229,7 @@ static bool bxt_dsi_pll_is_enabled(struct drm_i915_private *dev_priv)
 	return enabled;
 }
 
-static void bxt_disable_dsi_pll(struct intel_encoder *encoder)
+void bxt_dsi_pll_disable(struct intel_encoder *encoder)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	u32 val;
@@ -261,8 +261,8 @@ static void assert_bpp_mismatch(enum mipi_dsi_pixel_format fmt, int pipe_bpp)
 	     bpp, pipe_bpp);
 }
 
-static u32 vlv_dsi_get_pclk(struct intel_encoder *encoder, int pipe_bpp,
-			    struct intel_crtc_state *config)
+u32 vlv_dsi_get_pclk(struct intel_encoder *encoder, int pipe_bpp,
+		     struct intel_crtc_state *config)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
@@ -327,8 +327,8 @@ static u32 vlv_dsi_get_pclk(struct intel_encoder *encoder, int pipe_bpp,
 	return pclk;
 }
 
-static u32 bxt_dsi_get_pclk(struct intel_encoder *encoder, int pipe_bpp,
-			    struct intel_crtc_state *config)
+u32 bxt_dsi_get_pclk(struct intel_encoder *encoder, int pipe_bpp,
+		     struct intel_crtc_state *config)
 {
 	u32 pclk;
 	u32 dsi_clk;
@@ -357,16 +357,7 @@ static u32 bxt_dsi_get_pclk(struct intel_encoder *encoder, int pipe_bpp,
 	return pclk;
 }
 
-u32 intel_dsi_get_pclk(struct intel_encoder *encoder, int pipe_bpp,
-		       struct intel_crtc_state *config)
-{
-	if (IS_GEN9_LP(to_i915(encoder->base.dev)))
-		return bxt_dsi_get_pclk(encoder, pipe_bpp, config);
-	else
-		return vlv_dsi_get_pclk(encoder, pipe_bpp, config);
-}
-
-static void vlv_dsi_reset_clocks(struct intel_encoder *encoder, enum port port)
+void vlv_dsi_reset_clocks(struct intel_encoder *encoder, enum port port)
 {
 	u32 temp;
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
@@ -480,8 +471,8 @@ static void bxt_dsi_program_clocks(struct drm_device *dev, enum port port,
 	I915_WRITE(BXT_MIPI_CLOCK_CTL, tmp);
 }
 
-static int gen9lp_compute_dsi_pll(struct intel_encoder *encoder,
-			       struct intel_crtc_state *config)
+int bxt_dsi_pll_compute(struct intel_encoder *encoder,
+			struct intel_crtc_state *config)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
@@ -528,8 +519,8 @@ static int gen9lp_compute_dsi_pll(struct intel_encoder *encoder,
 	return 0;
 }
 
-static void gen9lp_enable_dsi_pll(struct intel_encoder *encoder,
-			       const struct intel_crtc_state *config)
+void bxt_dsi_pll_enable(struct intel_encoder *encoder,
+			const struct intel_crtc_state *config)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
@@ -568,52 +559,7 @@ static void gen9lp_enable_dsi_pll(struct intel_encoder *encoder,
 	DRM_DEBUG_KMS("DSI PLL locked\n");
 }
 
-bool intel_dsi_pll_is_enabled(struct drm_i915_private *dev_priv)
-{
-	if (IS_GEN9_LP(dev_priv))
-		return bxt_dsi_pll_is_enabled(dev_priv);
-
-	MISSING_CASE(INTEL_DEVID(dev_priv));
-
-	return false;
-}
-
-int intel_compute_dsi_pll(struct intel_encoder *encoder,
-			  struct intel_crtc_state *config)
-{
-	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-
-	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
-		return vlv_compute_dsi_pll(encoder, config);
-	else if (IS_GEN9_LP(dev_priv))
-		return gen9lp_compute_dsi_pll(encoder, config);
-
-	return -ENODEV;
-}
-
-void intel_enable_dsi_pll(struct intel_encoder *encoder,
-			  const struct intel_crtc_state *config)
-{
-	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-
-	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
-		vlv_enable_dsi_pll(encoder, config);
-	else if (IS_GEN9_LP(dev_priv))
-		gen9lp_enable_dsi_pll(encoder, config);
-}
-
-void intel_disable_dsi_pll(struct intel_encoder *encoder)
-{
-	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-
-	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
-		vlv_disable_dsi_pll(encoder);
-	else if (IS_GEN9_LP(dev_priv))
-		bxt_disable_dsi_pll(encoder);
-}
-
-static void gen9lp_dsi_reset_clocks(struct intel_encoder *encoder,
-				    enum port port)
+void bxt_dsi_reset_clocks(struct intel_encoder *encoder, enum port port)
 {
 	u32 tmp;
 	struct drm_device *dev = encoder->base.dev;
@@ -638,13 +584,3 @@ static void gen9lp_dsi_reset_clocks(struct intel_encoder *encoder,
 	}
 	I915_WRITE(MIPI_EOT_DISABLE(port), CLOCKSTOP);
 }
-
-void intel_dsi_reset_clocks(struct intel_encoder *encoder, enum port port)
-{
-	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-
-	if (IS_GEN9_LP(dev_priv))
-		gen9lp_dsi_reset_clocks(encoder, port);
-	else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
-		vlv_dsi_reset_clocks(encoder, port);
-}
diff --git a/drivers/gpu/drm/imx/imx-drm-core.c b/drivers/gpu/drm/imx/imx-drm-core.c
index 1d053bbefc02..5ea0c82f9957 100644
--- a/drivers/gpu/drm/imx/imx-drm-core.c
+++ b/drivers/gpu/drm/imx/imx-drm-core.c
@@ -35,12 +35,6 @@
 
 #define MAX_CRTC	4
 
-struct imx_drm_device {
-	struct drm_device			*drm;
-	unsigned int				pipes;
-	struct drm_atomic_state			*state;
-};
-
 #if IS_ENABLED(CONFIG_DRM_FBDEV_EMULATION)
 static int legacyfb_depth = 16;
 module_param(legacyfb_depth, int, 0444);
@@ -219,22 +213,12 @@ static int compare_of(struct device *dev, void *data)
 static int imx_drm_bind(struct device *dev)
 {
 	struct drm_device *drm;
-	struct imx_drm_device *imxdrm;
 	int ret;
 
 	drm = drm_dev_alloc(&imx_drm_driver, dev);
 	if (IS_ERR(drm))
 		return PTR_ERR(drm);
 
-	imxdrm = devm_kzalloc(dev, sizeof(*imxdrm), GFP_KERNEL);
-	if (!imxdrm) {
-		ret = -ENOMEM;
-		goto err_unref;
-	}
-
-	imxdrm->drm = drm;
-	drm->dev_private = imxdrm;
-
 	/*
 	 * enable drm irq mode.
 	 * - with irq_enabled = true, we can use the vblank feature.
@@ -306,8 +290,7 @@ err_unbind:
 	component_unbind_all(drm->dev, drm);
 err_kms:
 	drm_mode_config_cleanup(drm);
-err_unref:
-	drm_dev_unref(drm);
+	drm_dev_put(drm);
 
 	return ret;
 }
@@ -327,7 +310,7 @@ static void imx_drm_unbind(struct device *dev)
 	component_unbind_all(drm->dev, drm);
 	dev_set_drvdata(dev, NULL);
 
-	drm_dev_unref(drm);
+	drm_dev_put(drm);
 }
 
 static const struct component_master_ops imx_drm_ops = {
@@ -355,37 +338,15 @@ static int imx_drm_platform_remove(struct platform_device *pdev)
 static int imx_drm_suspend(struct device *dev)
 {
 	struct drm_device *drm_dev = dev_get_drvdata(dev);
-	struct imx_drm_device *imxdrm;
-
-	/* The drm_dev is NULL before .load hook is called */
-	if (drm_dev == NULL)
-		return 0;
-
-	drm_kms_helper_poll_disable(drm_dev);
 
-	imxdrm = drm_dev->dev_private;
-	imxdrm->state = drm_atomic_helper_suspend(drm_dev);
-	if (IS_ERR(imxdrm->state)) {
-		drm_kms_helper_poll_enable(drm_dev);
-		return PTR_ERR(imxdrm->state);
-	}
-
-	return 0;
+	return drm_mode_config_helper_suspend(drm_dev);
 }
 
 static int imx_drm_resume(struct device *dev)
 {
 	struct drm_device *drm_dev = dev_get_drvdata(dev);
-	struct imx_drm_device *imx_drm;
 
-	if (drm_dev == NULL)
-		return 0;
-
-	imx_drm = drm_dev->dev_private;
-	drm_atomic_helper_resume(drm_dev, imx_drm->state);
-	drm_kms_helper_poll_enable(drm_dev);
-
-	return 0;
+	return drm_mode_config_helper_resume(drm_dev);
 }
 #endif
 
diff --git a/drivers/gpu/drm/imx/imx-drm.h b/drivers/gpu/drm/imx/imx-drm.h
index 15c2bec47a04..ab9c6f706eb3 100644
--- a/drivers/gpu/drm/imx/imx-drm.h
+++ b/drivers/gpu/drm/imx/imx-drm.h
@@ -10,7 +10,6 @@ struct drm_display_mode;
 struct drm_encoder;
 struct drm_framebuffer;
 struct drm_plane;
-struct imx_drm_crtc;
 struct platform_device;
 
 struct imx_crtc_state {
diff --git a/drivers/gpu/drm/imx/imx-ldb.c b/drivers/gpu/drm/imx/imx-ldb.c
index 7312beb6f1fc..3bd0f8a18e74 100644
--- a/drivers/gpu/drm/imx/imx-ldb.c
+++ b/drivers/gpu/drm/imx/imx-ldb.c
@@ -611,6 +611,9 @@ static int imx_ldb_bind(struct device *dev, struct device *master, void *data)
 		return PTR_ERR(imx_ldb->regmap);
 	}
 
+	/* disable LDB by resetting the control register to POR default */
+	regmap_write(imx_ldb->regmap, IOMUXC_GPR2, 0);
+
 	imx_ldb->dev = dev;
 
 	if (of_id)
@@ -651,14 +654,14 @@ static int imx_ldb_bind(struct device *dev, struct device *master, void *data)
 		if (ret || i < 0 || i > 1)
 			return -EINVAL;
 
+		if (!of_device_is_available(child))
+			continue;
+
 		if (dual && i > 0) {
 			dev_warn(dev, "dual-channel mode, ignoring second output\n");
 			continue;
 		}
 
-		if (!of_device_is_available(child))
-			continue;
-
 		channel = &imx_ldb->channel[i];
 		channel->ldb = imx_ldb;
 		channel->chno = i;
diff --git a/drivers/gpu/drm/imx/ipuv3-crtc.c b/drivers/gpu/drm/imx/ipuv3-crtc.c
index 21d002859ae0..7d4b710b837a 100644
--- a/drivers/gpu/drm/imx/ipuv3-crtc.c
+++ b/drivers/gpu/drm/imx/ipuv3-crtc.c
@@ -35,7 +35,6 @@
 struct ipu_crtc {
 	struct device		*dev;
 	struct drm_crtc		base;
-	struct imx_drm_crtc	*imx_crtc;
 
 	/* plane[0] is the full plane, plane[1] is the partial plane */
 	struct ipu_plane	*plane[2];
diff --git a/drivers/gpu/drm/mediatek/mtk_cec.c b/drivers/gpu/drm/mediatek/mtk_cec.c
index 7a3eb8c17ef9..5ce84d0dbf81 100644
--- a/drivers/gpu/drm/mediatek/mtk_cec.c
+++ b/drivers/gpu/drm/mediatek/mtk_cec.c
@@ -15,6 +15,7 @@
 #include <linux/delay.h>
 #include <linux/io.h>
 #include <linux/interrupt.h>
+#include <linux/mod_devicetable.h>
 #include <linux/platform_device.h>
 
 #include "mtk_cec.h"
diff --git a/drivers/gpu/drm/mediatek/mtk_drm_crtc.c b/drivers/gpu/drm/mediatek/mtk_drm_crtc.c
index 658b8dd45b83..2d6aa150a9ff 100644
--- a/drivers/gpu/drm/mediatek/mtk_drm_crtc.c
+++ b/drivers/gpu/drm/mediatek/mtk_drm_crtc.c
@@ -539,6 +539,9 @@ int mtk_drm_crtc_create(struct drm_device *drm_dev,
 	int ret;
 	int i;
 
+	if (!path)
+		return 0;
+
 	for (i = 0; i < path_len; i++) {
 		enum mtk_ddp_comp_id comp_id = path[i];
 		struct device_node *node;
diff --git a/drivers/gpu/drm/mediatek/mtk_drm_ddp.c b/drivers/gpu/drm/mediatek/mtk_drm_ddp.c
index 8130f3dab661..87e4191c250e 100644
--- a/drivers/gpu/drm/mediatek/mtk_drm_ddp.c
+++ b/drivers/gpu/drm/mediatek/mtk_drm_ddp.c
@@ -28,8 +28,12 @@
 #define DISP_REG_CONFIG_DISP_UFOE_MOUT_EN	0x050
 #define DISP_REG_CONFIG_DISP_COLOR0_SEL_IN	0x084
 #define DISP_REG_CONFIG_DISP_COLOR1_SEL_IN	0x088
+#define DISP_REG_CONFIG_DSIE_SEL_IN		0x0a4
+#define DISP_REG_CONFIG_DSIO_SEL_IN		0x0a8
 #define DISP_REG_CONFIG_DPI_SEL_IN		0x0ac
-#define DISP_REG_CONFIG_DISP_RDMA1_MOUT_EN	0x0c8
+#define DISP_REG_CONFIG_DISP_RDMA2_SOUT		0x0b8
+#define DISP_REG_CONFIG_DISP_RDMA0_SOUT_EN	0x0c4
+#define DISP_REG_CONFIG_DISP_RDMA1_SOUT_EN	0x0c8
 #define DISP_REG_CONFIG_MMSYS_CG_CON0		0x100
 
 #define DISP_REG_CONFIG_DISP_OVL_MOUT_EN	0x030
@@ -41,45 +45,89 @@
 #define DISP_REG_MUTEX_RST(n)	(0x28 + 0x20 * (n))
 #define DISP_REG_MUTEX_MOD(n)	(0x2c + 0x20 * (n))
 #define DISP_REG_MUTEX_SOF(n)	(0x30 + 0x20 * (n))
+#define DISP_REG_MUTEX_MOD2(n)	(0x34 + 0x20 * (n))
 
 #define INT_MUTEX				BIT(1)
 
-#define MT8173_MUTEX_MOD_DISP_OVL0		BIT(11)
-#define MT8173_MUTEX_MOD_DISP_OVL1		BIT(12)
-#define MT8173_MUTEX_MOD_DISP_RDMA0		BIT(13)
-#define MT8173_MUTEX_MOD_DISP_RDMA1		BIT(14)
-#define MT8173_MUTEX_MOD_DISP_RDMA2		BIT(15)
-#define MT8173_MUTEX_MOD_DISP_WDMA0		BIT(16)
-#define MT8173_MUTEX_MOD_DISP_WDMA1		BIT(17)
-#define MT8173_MUTEX_MOD_DISP_COLOR0		BIT(18)
-#define MT8173_MUTEX_MOD_DISP_COLOR1		BIT(19)
-#define MT8173_MUTEX_MOD_DISP_AAL		BIT(20)
-#define MT8173_MUTEX_MOD_DISP_GAMMA		BIT(21)
-#define MT8173_MUTEX_MOD_DISP_UFOE		BIT(22)
-#define MT8173_MUTEX_MOD_DISP_PWM0		BIT(23)
-#define MT8173_MUTEX_MOD_DISP_PWM1		BIT(24)
-#define MT8173_MUTEX_MOD_DISP_OD		BIT(25)
-
-#define MT2701_MUTEX_MOD_DISP_OVL		BIT(3)
-#define MT2701_MUTEX_MOD_DISP_WDMA		BIT(6)
-#define MT2701_MUTEX_MOD_DISP_COLOR		BIT(7)
-#define MT2701_MUTEX_MOD_DISP_BLS		BIT(9)
-#define MT2701_MUTEX_MOD_DISP_RDMA0		BIT(10)
-#define MT2701_MUTEX_MOD_DISP_RDMA1		BIT(12)
+#define MT8173_MUTEX_MOD_DISP_OVL0		11
+#define MT8173_MUTEX_MOD_DISP_OVL1		12
+#define MT8173_MUTEX_MOD_DISP_RDMA0		13
+#define MT8173_MUTEX_MOD_DISP_RDMA1		14
+#define MT8173_MUTEX_MOD_DISP_RDMA2		15
+#define MT8173_MUTEX_MOD_DISP_WDMA0		16
+#define MT8173_MUTEX_MOD_DISP_WDMA1		17
+#define MT8173_MUTEX_MOD_DISP_COLOR0		18
+#define MT8173_MUTEX_MOD_DISP_COLOR1		19
+#define MT8173_MUTEX_MOD_DISP_AAL		20
+#define MT8173_MUTEX_MOD_DISP_GAMMA		21
+#define MT8173_MUTEX_MOD_DISP_UFOE		22
+#define MT8173_MUTEX_MOD_DISP_PWM0		23
+#define MT8173_MUTEX_MOD_DISP_PWM1		24
+#define MT8173_MUTEX_MOD_DISP_OD		25
+
+#define MT2712_MUTEX_MOD_DISP_PWM2		10
+#define MT2712_MUTEX_MOD_DISP_OVL0		11
+#define MT2712_MUTEX_MOD_DISP_OVL1		12
+#define MT2712_MUTEX_MOD_DISP_RDMA0		13
+#define MT2712_MUTEX_MOD_DISP_RDMA1		14
+#define MT2712_MUTEX_MOD_DISP_RDMA2		15
+#define MT2712_MUTEX_MOD_DISP_WDMA0		16
+#define MT2712_MUTEX_MOD_DISP_WDMA1		17
+#define MT2712_MUTEX_MOD_DISP_COLOR0		18
+#define MT2712_MUTEX_MOD_DISP_COLOR1		19
+#define MT2712_MUTEX_MOD_DISP_AAL0		20
+#define MT2712_MUTEX_MOD_DISP_UFOE		22
+#define MT2712_MUTEX_MOD_DISP_PWM0		23
+#define MT2712_MUTEX_MOD_DISP_PWM1		24
+#define MT2712_MUTEX_MOD_DISP_OD0		25
+#define MT2712_MUTEX_MOD2_DISP_AAL1		33
+#define MT2712_MUTEX_MOD2_DISP_OD1		34
+
+#define MT2701_MUTEX_MOD_DISP_OVL		3
+#define MT2701_MUTEX_MOD_DISP_WDMA		6
+#define MT2701_MUTEX_MOD_DISP_COLOR		7
+#define MT2701_MUTEX_MOD_DISP_BLS		9
+#define MT2701_MUTEX_MOD_DISP_RDMA0		10
+#define MT2701_MUTEX_MOD_DISP_RDMA1		12
 
 #define MUTEX_SOF_SINGLE_MODE		0
 #define MUTEX_SOF_DSI0			1
 #define MUTEX_SOF_DSI1			2
 #define MUTEX_SOF_DPI0			3
+#define MUTEX_SOF_DPI1			4
+#define MUTEX_SOF_DSI2			5
+#define MUTEX_SOF_DSI3			6
 
 #define OVL0_MOUT_EN_COLOR0		0x1
 #define OD_MOUT_EN_RDMA0		0x1
+#define OD1_MOUT_EN_RDMA1		BIT(16)
 #define UFOE_MOUT_EN_DSI0		0x1
 #define COLOR0_SEL_IN_OVL0		0x1
 #define OVL1_MOUT_EN_COLOR1		0x1
 #define GAMMA_MOUT_EN_RDMA1		0x1
-#define RDMA1_MOUT_DPI0			0x2
+#define RDMA0_SOUT_DPI0			0x2
+#define RDMA0_SOUT_DSI2			0x4
+#define RDMA0_SOUT_DSI3			0x5
+#define RDMA1_SOUT_DPI0			0x2
+#define RDMA1_SOUT_DPI1			0x3
+#define RDMA1_SOUT_DSI1			0x1
+#define RDMA1_SOUT_DSI2			0x4
+#define RDMA1_SOUT_DSI3			0x5
+#define RDMA2_SOUT_DPI0			0x2
+#define RDMA2_SOUT_DPI1			0x3
+#define RDMA2_SOUT_DSI1			0x1
+#define RDMA2_SOUT_DSI2			0x4
+#define RDMA2_SOUT_DSI3			0x5
 #define DPI0_SEL_IN_RDMA1		0x1
+#define DPI0_SEL_IN_RDMA2		0x3
+#define DPI1_SEL_IN_RDMA1		(0x1 << 8)
+#define DPI1_SEL_IN_RDMA2		(0x3 << 8)
+#define DSI1_SEL_IN_RDMA1		0x1
+#define DSI1_SEL_IN_RDMA2		0x4
+#define DSI2_SEL_IN_RDMA1		(0x1 << 16)
+#define DSI2_SEL_IN_RDMA2		(0x4 << 16)
+#define DSI3_SEL_IN_RDMA1		(0x1 << 16)
+#define DSI3_SEL_IN_RDMA2		(0x4 << 16)
 #define COLOR1_SEL_IN_OVL1		0x1
 
 #define OVL_MOUT_EN_RDMA		0x1
@@ -108,12 +156,32 @@ static const unsigned int mt2701_mutex_mod[DDP_COMPONENT_ID_MAX] = {
 	[DDP_COMPONENT_WDMA0] = MT2701_MUTEX_MOD_DISP_WDMA,
 };
 
+static const unsigned int mt2712_mutex_mod[DDP_COMPONENT_ID_MAX] = {
+	[DDP_COMPONENT_AAL0] = MT2712_MUTEX_MOD_DISP_AAL0,
+	[DDP_COMPONENT_AAL1] = MT2712_MUTEX_MOD2_DISP_AAL1,
+	[DDP_COMPONENT_COLOR0] = MT2712_MUTEX_MOD_DISP_COLOR0,
+	[DDP_COMPONENT_COLOR1] = MT2712_MUTEX_MOD_DISP_COLOR1,
+	[DDP_COMPONENT_OD0] = MT2712_MUTEX_MOD_DISP_OD0,
+	[DDP_COMPONENT_OD1] = MT2712_MUTEX_MOD2_DISP_OD1,
+	[DDP_COMPONENT_OVL0] = MT2712_MUTEX_MOD_DISP_OVL0,
+	[DDP_COMPONENT_OVL1] = MT2712_MUTEX_MOD_DISP_OVL1,
+	[DDP_COMPONENT_PWM0] = MT2712_MUTEX_MOD_DISP_PWM0,
+	[DDP_COMPONENT_PWM1] = MT2712_MUTEX_MOD_DISP_PWM1,
+	[DDP_COMPONENT_PWM2] = MT2712_MUTEX_MOD_DISP_PWM2,
+	[DDP_COMPONENT_RDMA0] = MT2712_MUTEX_MOD_DISP_RDMA0,
+	[DDP_COMPONENT_RDMA1] = MT2712_MUTEX_MOD_DISP_RDMA1,
+	[DDP_COMPONENT_RDMA2] = MT2712_MUTEX_MOD_DISP_RDMA2,
+	[DDP_COMPONENT_UFOE] = MT2712_MUTEX_MOD_DISP_UFOE,
+	[DDP_COMPONENT_WDMA0] = MT2712_MUTEX_MOD_DISP_WDMA0,
+	[DDP_COMPONENT_WDMA1] = MT2712_MUTEX_MOD_DISP_WDMA1,
+};
+
 static const unsigned int mt8173_mutex_mod[DDP_COMPONENT_ID_MAX] = {
-	[DDP_COMPONENT_AAL] = MT8173_MUTEX_MOD_DISP_AAL,
+	[DDP_COMPONENT_AAL0] = MT8173_MUTEX_MOD_DISP_AAL,
 	[DDP_COMPONENT_COLOR0] = MT8173_MUTEX_MOD_DISP_COLOR0,
 	[DDP_COMPONENT_COLOR1] = MT8173_MUTEX_MOD_DISP_COLOR1,
 	[DDP_COMPONENT_GAMMA] = MT8173_MUTEX_MOD_DISP_GAMMA,
-	[DDP_COMPONENT_OD] = MT8173_MUTEX_MOD_DISP_OD,
+	[DDP_COMPONENT_OD0] = MT8173_MUTEX_MOD_DISP_OD,
 	[DDP_COMPONENT_OVL0] = MT8173_MUTEX_MOD_DISP_OVL0,
 	[DDP_COMPONENT_OVL1] = MT8173_MUTEX_MOD_DISP_OVL1,
 	[DDP_COMPONENT_PWM0] = MT8173_MUTEX_MOD_DISP_PWM0,
@@ -138,7 +206,7 @@ static unsigned int mtk_ddp_mout_en(enum mtk_ddp_comp_id cur,
 	} else if (cur == DDP_COMPONENT_OVL0 && next == DDP_COMPONENT_RDMA0) {
 		*addr = DISP_REG_CONFIG_DISP_OVL_MOUT_EN;
 		value = OVL_MOUT_EN_RDMA;
-	} else if (cur == DDP_COMPONENT_OD && next == DDP_COMPONENT_RDMA0) {
+	} else if (cur == DDP_COMPONENT_OD0 && next == DDP_COMPONENT_RDMA0) {
 		*addr = DISP_REG_CONFIG_DISP_OD_MOUT_EN;
 		value = OD_MOUT_EN_RDMA0;
 	} else if (cur == DDP_COMPONENT_UFOE && next == DDP_COMPONENT_DSI0) {
@@ -150,9 +218,48 @@ static unsigned int mtk_ddp_mout_en(enum mtk_ddp_comp_id cur,
 	} else if (cur == DDP_COMPONENT_GAMMA && next == DDP_COMPONENT_RDMA1) {
 		*addr = DISP_REG_CONFIG_DISP_GAMMA_MOUT_EN;
 		value = GAMMA_MOUT_EN_RDMA1;
+	} else if (cur == DDP_COMPONENT_OD1 && next == DDP_COMPONENT_RDMA1) {
+		*addr = DISP_REG_CONFIG_DISP_OD_MOUT_EN;
+		value = OD1_MOUT_EN_RDMA1;
+	} else if (cur == DDP_COMPONENT_RDMA0 && next == DDP_COMPONENT_DPI0) {
+		*addr = DISP_REG_CONFIG_DISP_RDMA0_SOUT_EN;
+		value = RDMA0_SOUT_DPI0;
+	} else if (cur == DDP_COMPONENT_RDMA0 && next == DDP_COMPONENT_DSI2) {
+		*addr = DISP_REG_CONFIG_DISP_RDMA0_SOUT_EN;
+		value = RDMA0_SOUT_DSI2;
+	} else if (cur == DDP_COMPONENT_RDMA0 && next == DDP_COMPONENT_DSI3) {
+		*addr = DISP_REG_CONFIG_DISP_RDMA0_SOUT_EN;
+		value = RDMA0_SOUT_DSI3;
+	} else if (cur == DDP_COMPONENT_RDMA1 && next == DDP_COMPONENT_DSI1) {
+		*addr = DISP_REG_CONFIG_DISP_RDMA1_SOUT_EN;
+		value = RDMA1_SOUT_DSI1;
+	} else if (cur == DDP_COMPONENT_RDMA1 && next == DDP_COMPONENT_DSI2) {
+		*addr = DISP_REG_CONFIG_DISP_RDMA1_SOUT_EN;
+		value = RDMA1_SOUT_DSI2;
+	} else if (cur == DDP_COMPONENT_RDMA1 && next == DDP_COMPONENT_DSI3) {
+		*addr = DISP_REG_CONFIG_DISP_RDMA1_SOUT_EN;
+		value = RDMA1_SOUT_DSI3;
 	} else if (cur == DDP_COMPONENT_RDMA1 && next == DDP_COMPONENT_DPI0) {
-		*addr = DISP_REG_CONFIG_DISP_RDMA1_MOUT_EN;
-		value = RDMA1_MOUT_DPI0;
+		*addr = DISP_REG_CONFIG_DISP_RDMA1_SOUT_EN;
+		value = RDMA1_SOUT_DPI0;
+	} else if (cur == DDP_COMPONENT_RDMA1 && next == DDP_COMPONENT_DPI1) {
+		*addr = DISP_REG_CONFIG_DISP_RDMA1_SOUT_EN;
+		value = RDMA1_SOUT_DPI1;
+	} else if (cur == DDP_COMPONENT_RDMA2 && next == DDP_COMPONENT_DPI0) {
+		*addr = DISP_REG_CONFIG_DISP_RDMA2_SOUT;
+		value = RDMA2_SOUT_DPI0;
+	} else if (cur == DDP_COMPONENT_RDMA2 && next == DDP_COMPONENT_DPI1) {
+		*addr = DISP_REG_CONFIG_DISP_RDMA2_SOUT;
+		value = RDMA2_SOUT_DPI1;
+	} else if (cur == DDP_COMPONENT_RDMA2 && next == DDP_COMPONENT_DSI1) {
+		*addr = DISP_REG_CONFIG_DISP_RDMA2_SOUT;
+		value = RDMA2_SOUT_DSI1;
+	} else if (cur == DDP_COMPONENT_RDMA2 && next == DDP_COMPONENT_DSI2) {
+		*addr = DISP_REG_CONFIG_DISP_RDMA2_SOUT;
+		value = RDMA2_SOUT_DSI2;
+	} else if (cur == DDP_COMPONENT_RDMA2 && next == DDP_COMPONENT_DSI3) {
+		*addr = DISP_REG_CONFIG_DISP_RDMA2_SOUT;
+		value = RDMA2_SOUT_DSI3;
 	} else {
 		value = 0;
 	}
@@ -172,6 +279,33 @@ static unsigned int mtk_ddp_sel_in(enum mtk_ddp_comp_id cur,
 	} else if (cur == DDP_COMPONENT_RDMA1 && next == DDP_COMPONENT_DPI0) {
 		*addr = DISP_REG_CONFIG_DPI_SEL_IN;
 		value = DPI0_SEL_IN_RDMA1;
+	} else if (cur == DDP_COMPONENT_RDMA1 && next == DDP_COMPONENT_DPI1) {
+		*addr = DISP_REG_CONFIG_DPI_SEL_IN;
+		value = DPI1_SEL_IN_RDMA1;
+	} else if (cur == DDP_COMPONENT_RDMA1 && next == DDP_COMPONENT_DSI1) {
+		*addr = DISP_REG_CONFIG_DSIO_SEL_IN;
+		value = DSI1_SEL_IN_RDMA1;
+	} else if (cur == DDP_COMPONENT_RDMA1 && next == DDP_COMPONENT_DSI2) {
+		*addr = DISP_REG_CONFIG_DSIE_SEL_IN;
+		value = DSI2_SEL_IN_RDMA1;
+	} else if (cur == DDP_COMPONENT_RDMA1 && next == DDP_COMPONENT_DSI3) {
+		*addr = DISP_REG_CONFIG_DSIO_SEL_IN;
+		value = DSI3_SEL_IN_RDMA1;
+	} else if (cur == DDP_COMPONENT_RDMA2 && next == DDP_COMPONENT_DPI0) {
+		*addr = DISP_REG_CONFIG_DPI_SEL_IN;
+		value = DPI0_SEL_IN_RDMA2;
+	} else if (cur == DDP_COMPONENT_RDMA2 && next == DDP_COMPONENT_DPI1) {
+		*addr = DISP_REG_CONFIG_DPI_SEL_IN;
+		value = DPI1_SEL_IN_RDMA2;
+	} else if (cur == DDP_COMPONENT_RDMA2 && next == DDP_COMPONENT_DSI1) {
+		*addr = DISP_REG_CONFIG_DSIE_SEL_IN;
+		value = DSI1_SEL_IN_RDMA2;
+	} else if (cur == DDP_COMPONENT_RDMA2 && next == DDP_COMPONENT_DSI2) {
+		*addr = DISP_REG_CONFIG_DSIE_SEL_IN;
+		value = DSI2_SEL_IN_RDMA2;
+	} else if (cur == DDP_COMPONENT_RDMA2 && next == DDP_COMPONENT_DSI3) {
+		*addr = DISP_REG_CONFIG_DSIE_SEL_IN;
+		value = DSI3_SEL_IN_RDMA2;
 	} else if (cur == DDP_COMPONENT_OVL1 && next == DDP_COMPONENT_COLOR1) {
 		*addr = DISP_REG_CONFIG_DISP_COLOR1_SEL_IN;
 		value = COLOR1_SEL_IN_OVL1;
@@ -278,6 +412,7 @@ void mtk_disp_mutex_add_comp(struct mtk_disp_mutex *mutex,
 	struct mtk_ddp *ddp = container_of(mutex, struct mtk_ddp,
 					   mutex[mutex->id]);
 	unsigned int reg;
+	unsigned int offset;
 
 	WARN_ON(&ddp->mutex[mutex->id] != mutex);
 
@@ -288,13 +423,30 @@ void mtk_disp_mutex_add_comp(struct mtk_disp_mutex *mutex,
 	case DDP_COMPONENT_DSI1:
 		reg = MUTEX_SOF_DSI0;
 		break;
+	case DDP_COMPONENT_DSI2:
+		reg = MUTEX_SOF_DSI2;
+		break;
+	case DDP_COMPONENT_DSI3:
+		reg = MUTEX_SOF_DSI3;
+		break;
 	case DDP_COMPONENT_DPI0:
 		reg = MUTEX_SOF_DPI0;
 		break;
+	case DDP_COMPONENT_DPI1:
+		reg = MUTEX_SOF_DPI1;
+		break;
 	default:
-		reg = readl_relaxed(ddp->regs + DISP_REG_MUTEX_MOD(mutex->id));
-		reg |= ddp->mutex_mod[id];
-		writel_relaxed(reg, ddp->regs + DISP_REG_MUTEX_MOD(mutex->id));
+		if (ddp->mutex_mod[id] < 32) {
+			offset = DISP_REG_MUTEX_MOD(mutex->id);
+			reg = readl_relaxed(ddp->regs + offset);
+			reg |= 1 << ddp->mutex_mod[id];
+			writel_relaxed(reg, ddp->regs + offset);
+		} else {
+			offset = DISP_REG_MUTEX_MOD2(mutex->id);
+			reg = readl_relaxed(ddp->regs + offset);
+			reg |= 1 << (ddp->mutex_mod[id] - 32);
+			writel_relaxed(reg, ddp->regs + offset);
+		}
 		return;
 	}
 
@@ -307,20 +459,32 @@ void mtk_disp_mutex_remove_comp(struct mtk_disp_mutex *mutex,
 	struct mtk_ddp *ddp = container_of(mutex, struct mtk_ddp,
 					   mutex[mutex->id]);
 	unsigned int reg;
+	unsigned int offset;
 
 	WARN_ON(&ddp->mutex[mutex->id] != mutex);
 
 	switch (id) {
 	case DDP_COMPONENT_DSI0:
 	case DDP_COMPONENT_DSI1:
+	case DDP_COMPONENT_DSI2:
+	case DDP_COMPONENT_DSI3:
 	case DDP_COMPONENT_DPI0:
+	case DDP_COMPONENT_DPI1:
 		writel_relaxed(MUTEX_SOF_SINGLE_MODE,
 			       ddp->regs + DISP_REG_MUTEX_SOF(mutex->id));
 		break;
 	default:
-		reg = readl_relaxed(ddp->regs + DISP_REG_MUTEX_MOD(mutex->id));
-		reg &= ~(ddp->mutex_mod[id]);
-		writel_relaxed(reg, ddp->regs + DISP_REG_MUTEX_MOD(mutex->id));
+		if (ddp->mutex_mod[id] < 32) {
+			offset = DISP_REG_MUTEX_MOD(mutex->id);
+			reg = readl_relaxed(ddp->regs + offset);
+			reg &= ~(1 << ddp->mutex_mod[id]);
+			writel_relaxed(reg, ddp->regs + offset);
+		} else {
+			offset = DISP_REG_MUTEX_MOD2(mutex->id);
+			reg = readl_relaxed(ddp->regs + offset);
+			reg &= ~(1 << (ddp->mutex_mod[id] - 32));
+			writel_relaxed(reg, ddp->regs + offset);
+		}
 		break;
 	}
 }
@@ -407,6 +571,7 @@ static int mtk_ddp_remove(struct platform_device *pdev)
 
 static const struct of_device_id ddp_driver_dt_match[] = {
 	{ .compatible = "mediatek,mt2701-disp-mutex", .data = mt2701_mutex_mod},
+	{ .compatible = "mediatek,mt2712-disp-mutex", .data = mt2712_mutex_mod},
 	{ .compatible = "mediatek,mt8173-disp-mutex", .data = mt8173_mutex_mod},
 	{},
 };
diff --git a/drivers/gpu/drm/mediatek/mtk_drm_ddp_comp.c b/drivers/gpu/drm/mediatek/mtk_drm_ddp_comp.c
index 4672317e3ad1..ff974d82a4a6 100644
--- a/drivers/gpu/drm/mediatek/mtk_drm_ddp_comp.c
+++ b/drivers/gpu/drm/mediatek/mtk_drm_ddp_comp.c
@@ -218,18 +218,25 @@ struct mtk_ddp_comp_match {
 };
 
 static const struct mtk_ddp_comp_match mtk_ddp_matches[DDP_COMPONENT_ID_MAX] = {
-	[DDP_COMPONENT_AAL]	= { MTK_DISP_AAL,	0, &ddp_aal },
+	[DDP_COMPONENT_AAL0]	= { MTK_DISP_AAL,	0, &ddp_aal },
+	[DDP_COMPONENT_AAL1]	= { MTK_DISP_AAL,	1, &ddp_aal },
 	[DDP_COMPONENT_BLS]	= { MTK_DISP_BLS,	0, NULL },
 	[DDP_COMPONENT_COLOR0]	= { MTK_DISP_COLOR,	0, NULL },
 	[DDP_COMPONENT_COLOR1]	= { MTK_DISP_COLOR,	1, NULL },
 	[DDP_COMPONENT_DPI0]	= { MTK_DPI,		0, NULL },
+	[DDP_COMPONENT_DPI1]	= { MTK_DPI,		1, NULL },
 	[DDP_COMPONENT_DSI0]	= { MTK_DSI,		0, NULL },
 	[DDP_COMPONENT_DSI1]	= { MTK_DSI,		1, NULL },
+	[DDP_COMPONENT_DSI2]	= { MTK_DSI,		2, NULL },
+	[DDP_COMPONENT_DSI3]	= { MTK_DSI,		3, NULL },
 	[DDP_COMPONENT_GAMMA]	= { MTK_DISP_GAMMA,	0, &ddp_gamma },
-	[DDP_COMPONENT_OD]	= { MTK_DISP_OD,	0, &ddp_od },
+	[DDP_COMPONENT_OD0]	= { MTK_DISP_OD,	0, &ddp_od },
+	[DDP_COMPONENT_OD1]	= { MTK_DISP_OD,	1, &ddp_od },
 	[DDP_COMPONENT_OVL0]	= { MTK_DISP_OVL,	0, NULL },
 	[DDP_COMPONENT_OVL1]	= { MTK_DISP_OVL,	1, NULL },
 	[DDP_COMPONENT_PWM0]	= { MTK_DISP_PWM,	0, NULL },
+	[DDP_COMPONENT_PWM1]	= { MTK_DISP_PWM,	1, NULL },
+	[DDP_COMPONENT_PWM2]	= { MTK_DISP_PWM,	2, NULL },
 	[DDP_COMPONENT_RDMA0]	= { MTK_DISP_RDMA,	0, NULL },
 	[DDP_COMPONENT_RDMA1]	= { MTK_DISP_RDMA,	1, NULL },
 	[DDP_COMPONENT_RDMA2]	= { MTK_DISP_RDMA,	2, NULL },
@@ -271,7 +278,11 @@ int mtk_ddp_comp_init(struct device *dev, struct device_node *node,
 
 	if (comp_id == DDP_COMPONENT_BLS ||
 	    comp_id == DDP_COMPONENT_DPI0 ||
+	    comp_id == DDP_COMPONENT_DPI1 ||
 	    comp_id == DDP_COMPONENT_DSI0 ||
+	    comp_id == DDP_COMPONENT_DSI1 ||
+	    comp_id == DDP_COMPONENT_DSI2 ||
+	    comp_id == DDP_COMPONENT_DSI3 ||
 	    comp_id == DDP_COMPONENT_PWM0) {
 		comp->regs = NULL;
 		comp->clk = NULL;
diff --git a/drivers/gpu/drm/mediatek/mtk_drm_ddp_comp.h b/drivers/gpu/drm/mediatek/mtk_drm_ddp_comp.h
index 0828cf8bf85c..7413ffeb3c9d 100644
--- a/drivers/gpu/drm/mediatek/mtk_drm_ddp_comp.h
+++ b/drivers/gpu/drm/mediatek/mtk_drm_ddp_comp.h
@@ -41,19 +41,25 @@ enum mtk_ddp_comp_type {
 };
 
 enum mtk_ddp_comp_id {
-	DDP_COMPONENT_AAL,
+	DDP_COMPONENT_AAL0,
+	DDP_COMPONENT_AAL1,
 	DDP_COMPONENT_BLS,
 	DDP_COMPONENT_COLOR0,
 	DDP_COMPONENT_COLOR1,
 	DDP_COMPONENT_DPI0,
+	DDP_COMPONENT_DPI1,
 	DDP_COMPONENT_DSI0,
 	DDP_COMPONENT_DSI1,
+	DDP_COMPONENT_DSI2,
+	DDP_COMPONENT_DSI3,
 	DDP_COMPONENT_GAMMA,
-	DDP_COMPONENT_OD,
+	DDP_COMPONENT_OD0,
+	DDP_COMPONENT_OD1,
 	DDP_COMPONENT_OVL0,
 	DDP_COMPONENT_OVL1,
 	DDP_COMPONENT_PWM0,
 	DDP_COMPONENT_PWM1,
+	DDP_COMPONENT_PWM2,
 	DDP_COMPONENT_RDMA0,
 	DDP_COMPONENT_RDMA1,
 	DDP_COMPONENT_RDMA2,
diff --git a/drivers/gpu/drm/mediatek/mtk_drm_drv.c b/drivers/gpu/drm/mediatek/mtk_drm_drv.c
index a2ca90fc403c..39721119713b 100644
--- a/drivers/gpu/drm/mediatek/mtk_drm_drv.c
+++ b/drivers/gpu/drm/mediatek/mtk_drm_drv.c
@@ -146,11 +146,37 @@ static const enum mtk_ddp_comp_id mt2701_mtk_ddp_ext[] = {
 	DDP_COMPONENT_DPI0,
 };
 
+static const enum mtk_ddp_comp_id mt2712_mtk_ddp_main[] = {
+	DDP_COMPONENT_OVL0,
+	DDP_COMPONENT_COLOR0,
+	DDP_COMPONENT_AAL0,
+	DDP_COMPONENT_OD0,
+	DDP_COMPONENT_RDMA0,
+	DDP_COMPONENT_DPI0,
+	DDP_COMPONENT_PWM0,
+};
+
+static const enum mtk_ddp_comp_id mt2712_mtk_ddp_ext[] = {
+	DDP_COMPONENT_OVL1,
+	DDP_COMPONENT_COLOR1,
+	DDP_COMPONENT_AAL1,
+	DDP_COMPONENT_OD1,
+	DDP_COMPONENT_RDMA1,
+	DDP_COMPONENT_DPI1,
+	DDP_COMPONENT_PWM1,
+};
+
+static const enum mtk_ddp_comp_id mt2712_mtk_ddp_third[] = {
+	DDP_COMPONENT_RDMA2,
+	DDP_COMPONENT_DSI3,
+	DDP_COMPONENT_PWM2,
+};
+
 static const enum mtk_ddp_comp_id mt8173_mtk_ddp_main[] = {
 	DDP_COMPONENT_OVL0,
 	DDP_COMPONENT_COLOR0,
-	DDP_COMPONENT_AAL,
-	DDP_COMPONENT_OD,
+	DDP_COMPONENT_AAL0,
+	DDP_COMPONENT_OD0,
 	DDP_COMPONENT_RDMA0,
 	DDP_COMPONENT_UFOE,
 	DDP_COMPONENT_DSI0,
@@ -173,6 +199,15 @@ static const struct mtk_mmsys_driver_data mt2701_mmsys_driver_data = {
 	.shadow_register = true,
 };
 
+static const struct mtk_mmsys_driver_data mt2712_mmsys_driver_data = {
+	.main_path = mt2712_mtk_ddp_main,
+	.main_len = ARRAY_SIZE(mt2712_mtk_ddp_main),
+	.ext_path = mt2712_mtk_ddp_ext,
+	.ext_len = ARRAY_SIZE(mt2712_mtk_ddp_ext),
+	.third_path = mt2712_mtk_ddp_third,
+	.third_len = ARRAY_SIZE(mt2712_mtk_ddp_third),
+};
+
 static const struct mtk_mmsys_driver_data mt8173_mmsys_driver_data = {
 	.main_path = mt8173_mtk_ddp_main,
 	.main_len = ARRAY_SIZE(mt8173_mtk_ddp_main),
@@ -232,6 +267,11 @@ static int mtk_drm_kms_init(struct drm_device *drm)
 	if (ret < 0)
 		goto err_component_unbind;
 
+	ret = mtk_drm_crtc_create(drm, private->data->third_path,
+				  private->data->third_len);
+	if (ret < 0)
+		goto err_component_unbind;
+
 	/* Use OVL device for all DMA memory allocations */
 	np = private->comp_node[private->data->main_path[0]] ?:
 	     private->comp_node[private->data->ext_path[0]];
@@ -360,24 +400,44 @@ static const struct component_master_ops mtk_drm_ops = {
 };
 
 static const struct of_device_id mtk_ddp_comp_dt_ids[] = {
-	{ .compatible = "mediatek,mt2701-disp-ovl",   .data = (void *)MTK_DISP_OVL },
-	{ .compatible = "mediatek,mt8173-disp-ovl",   .data = (void *)MTK_DISP_OVL },
-	{ .compatible = "mediatek,mt2701-disp-rdma",  .data = (void *)MTK_DISP_RDMA },
-	{ .compatible = "mediatek,mt8173-disp-rdma",  .data = (void *)MTK_DISP_RDMA },
-	{ .compatible = "mediatek,mt8173-disp-wdma",  .data = (void *)MTK_DISP_WDMA },
-	{ .compatible = "mediatek,mt2701-disp-color", .data = (void *)MTK_DISP_COLOR },
-	{ .compatible = "mediatek,mt8173-disp-color", .data = (void *)MTK_DISP_COLOR },
-	{ .compatible = "mediatek,mt8173-disp-aal",   .data = (void *)MTK_DISP_AAL},
-	{ .compatible = "mediatek,mt8173-disp-gamma", .data = (void *)MTK_DISP_GAMMA, },
-	{ .compatible = "mediatek,mt8173-disp-ufoe",  .data = (void *)MTK_DISP_UFOE },
-	{ .compatible = "mediatek,mt2701-dsi",	      .data = (void *)MTK_DSI },
-	{ .compatible = "mediatek,mt8173-dsi",        .data = (void *)MTK_DSI },
-	{ .compatible = "mediatek,mt8173-dpi",        .data = (void *)MTK_DPI },
-	{ .compatible = "mediatek,mt2701-disp-mutex", .data = (void *)MTK_DISP_MUTEX },
-	{ .compatible = "mediatek,mt8173-disp-mutex", .data = (void *)MTK_DISP_MUTEX },
-	{ .compatible = "mediatek,mt2701-disp-pwm",   .data = (void *)MTK_DISP_BLS },
-	{ .compatible = "mediatek,mt8173-disp-pwm",   .data = (void *)MTK_DISP_PWM },
-	{ .compatible = "mediatek,mt8173-disp-od",    .data = (void *)MTK_DISP_OD },
+	{ .compatible = "mediatek,mt2701-disp-ovl",
+	  .data = (void *)MTK_DISP_OVL },
+	{ .compatible = "mediatek,mt8173-disp-ovl",
+	  .data = (void *)MTK_DISP_OVL },
+	{ .compatible = "mediatek,mt2701-disp-rdma",
+	  .data = (void *)MTK_DISP_RDMA },
+	{ .compatible = "mediatek,mt8173-disp-rdma",
+	  .data = (void *)MTK_DISP_RDMA },
+	{ .compatible = "mediatek,mt8173-disp-wdma",
+	  .data = (void *)MTK_DISP_WDMA },
+	{ .compatible = "mediatek,mt2701-disp-color",
+	  .data = (void *)MTK_DISP_COLOR },
+	{ .compatible = "mediatek,mt8173-disp-color",
+	  .data = (void *)MTK_DISP_COLOR },
+	{ .compatible = "mediatek,mt8173-disp-aal",
+	  .data = (void *)MTK_DISP_AAL},
+	{ .compatible = "mediatek,mt8173-disp-gamma",
+	  .data = (void *)MTK_DISP_GAMMA, },
+	{ .compatible = "mediatek,mt8173-disp-ufoe",
+	  .data = (void *)MTK_DISP_UFOE },
+	{ .compatible = "mediatek,mt2701-dsi",
+	  .data = (void *)MTK_DSI },
+	{ .compatible = "mediatek,mt8173-dsi",
+	  .data = (void *)MTK_DSI },
+	{ .compatible = "mediatek,mt8173-dpi",
+	  .data = (void *)MTK_DPI },
+	{ .compatible = "mediatek,mt2701-disp-mutex",
+	  .data = (void *)MTK_DISP_MUTEX },
+	{ .compatible = "mediatek,mt2712-disp-mutex",
+	  .data = (void *)MTK_DISP_MUTEX },
+	{ .compatible = "mediatek,mt8173-disp-mutex",
+	  .data = (void *)MTK_DISP_MUTEX },
+	{ .compatible = "mediatek,mt2701-disp-pwm",
+	  .data = (void *)MTK_DISP_BLS },
+	{ .compatible = "mediatek,mt8173-disp-pwm",
+	  .data = (void *)MTK_DISP_PWM },
+	{ .compatible = "mediatek,mt8173-disp-od",
+	  .data = (void *)MTK_DISP_OD },
 	{ }
 };
 
@@ -552,6 +612,8 @@ static SIMPLE_DEV_PM_OPS(mtk_drm_pm_ops, mtk_drm_sys_suspend,
 static const struct of_device_id mtk_drm_of_ids[] = {
 	{ .compatible = "mediatek,mt2701-mmsys",
 	  .data = &mt2701_mmsys_driver_data},
+	{ .compatible = "mediatek,mt2712-mmsys",
+	  .data = &mt2712_mmsys_driver_data},
 	{ .compatible = "mediatek,mt8173-mmsys",
 	  .data = &mt8173_mmsys_driver_data},
 	{ }
diff --git a/drivers/gpu/drm/mediatek/mtk_drm_drv.h b/drivers/gpu/drm/mediatek/mtk_drm_drv.h
index c3378c452c0a..ecc00ca3221d 100644
--- a/drivers/gpu/drm/mediatek/mtk_drm_drv.h
+++ b/drivers/gpu/drm/mediatek/mtk_drm_drv.h
@@ -17,7 +17,7 @@
 #include <linux/io.h>
 #include "mtk_drm_ddp_comp.h"
 
-#define MAX_CRTC	2
+#define MAX_CRTC	3
 #define MAX_CONNECTOR	2
 
 struct device;
@@ -33,6 +33,9 @@ struct mtk_mmsys_driver_data {
 	unsigned int main_len;
 	const enum mtk_ddp_comp_id *ext_path;
 	unsigned int ext_len;
+	const enum mtk_ddp_comp_id *third_path;
+	unsigned int third_len;
+
 	bool shadow_register;
 };
 
diff --git a/drivers/gpu/drm/msm/Kconfig b/drivers/gpu/drm/msm/Kconfig
index 38cbde971b48..843a9d40c05e 100644
--- a/drivers/gpu/drm/msm/Kconfig
+++ b/drivers/gpu/drm/msm/Kconfig
@@ -12,6 +12,7 @@ config DRM_MSM
 	select SHMEM
 	select TMPFS
 	select QCOM_SCM
+	select WANT_DEV_COREDUMP
 	select SND_SOC_HDMI_CODEC if SND_SOC
 	select SYNC_FILE
 	select PM_OPP
diff --git a/drivers/gpu/drm/msm/Makefile b/drivers/gpu/drm/msm/Makefile
index cd40c050b2d7..261fa79d456d 100644
--- a/drivers/gpu/drm/msm/Makefile
+++ b/drivers/gpu/drm/msm/Makefile
@@ -1,5 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0
 ccflags-y := -Idrivers/gpu/drm/msm
+ccflags-y += -Idrivers/gpu/drm/msm/disp/dpu1
 ccflags-$(CONFIG_DRM_MSM_DSI) += -Idrivers/gpu/drm/msm/dsi
 
 msm-y := \
@@ -10,6 +11,9 @@ msm-y := \
 	adreno/a5xx_gpu.o \
 	adreno/a5xx_power.o \
 	adreno/a5xx_preempt.o \
+	adreno/a6xx_gpu.o \
+	adreno/a6xx_gmu.o \
+	adreno/a6xx_hfi.o \
 	hdmi/hdmi.o \
 	hdmi/hdmi_audio.o \
 	hdmi/hdmi_bridge.o \
@@ -45,6 +49,33 @@ msm-y := \
 	disp/mdp5/mdp5_mixer.o \
 	disp/mdp5/mdp5_plane.o \
 	disp/mdp5/mdp5_smp.o \
+	disp/dpu1/dpu_core_irq.o \
+	disp/dpu1/dpu_core_perf.o \
+	disp/dpu1/dpu_crtc.o \
+	disp/dpu1/dpu_encoder.o \
+	disp/dpu1/dpu_encoder_phys_cmd.o \
+	disp/dpu1/dpu_encoder_phys_vid.o \
+	disp/dpu1/dpu_formats.o \
+	disp/dpu1/dpu_hw_blk.o \
+	disp/dpu1/dpu_hw_catalog.o \
+	disp/dpu1/dpu_hw_cdm.o \
+	disp/dpu1/dpu_hw_ctl.o \
+	disp/dpu1/dpu_hw_interrupts.o \
+	disp/dpu1/dpu_hw_intf.o \
+	disp/dpu1/dpu_hw_lm.o \
+	disp/dpu1/dpu_hw_pingpong.o \
+	disp/dpu1/dpu_hw_sspp.o \
+	disp/dpu1/dpu_hw_top.o \
+	disp/dpu1/dpu_hw_util.o \
+	disp/dpu1/dpu_hw_vbif.o \
+	disp/dpu1/dpu_io_util.o \
+	disp/dpu1/dpu_irq.o \
+	disp/dpu1/dpu_kms.o \
+	disp/dpu1/dpu_mdss.o \
+	disp/dpu1/dpu_plane.o \
+	disp/dpu1/dpu_power_handle.o \
+	disp/dpu1/dpu_rm.o \
+	disp/dpu1/dpu_vbif.o \
 	msm_atomic.o \
 	msm_debugfs.o \
 	msm_drv.o \
@@ -62,7 +93,8 @@ msm-y := \
 	msm_ringbuffer.o \
 	msm_submitqueue.o
 
-msm-$(CONFIG_DEBUG_FS) += adreno/a5xx_debugfs.o
+msm-$(CONFIG_DEBUG_FS) += adreno/a5xx_debugfs.o \
+			  disp/dpu1/dpu_dbg.o
 
 msm-$(CONFIG_DRM_FBDEV_EMULATION) += msm_fbdev.o
 msm-$(CONFIG_COMMON_CLK) += disp/mdp4/mdp4_lvds_pll.o
diff --git a/drivers/gpu/drm/msm/adreno/a2xx.xml.h b/drivers/gpu/drm/msm/adreno/a2xx.xml.h
index 644374c7b3e0..4bff0a740c7d 100644
--- a/drivers/gpu/drm/msm/adreno/a2xx.xml.h
+++ b/drivers/gpu/drm/msm/adreno/a2xx.xml.h
@@ -8,17 +8,19 @@ http://github.com/freedreno/envytools/
 git clone https://github.com/freedreno/envytools.git
 
 The rules-ng-ng source files this header was generated from are:
-- /home/robclark/src/freedreno/envytools/rnndb/adreno.xml               (    431 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml  (   1572 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/a2xx.xml          (  37162 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_common.xml (  13324 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_pm4.xml    (  31866 bytes, from 2017-06-06 18:26:14)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/a3xx.xml          (  83840 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/a4xx.xml          ( 111898 bytes, from 2017-06-06 18:23:59)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/a5xx.xml          ( 139480 bytes, from 2017-06-16 12:44:39)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/ocmem.xml         (   1773 bytes, from 2017-05-17 13:21:27)
-
-Copyright (C) 2013-2017 by the following authors:
+- /home/robclark/src/envytools/rnndb/adreno.xml               (    501 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/freedreno_copyright.xml  (   1572 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/adreno/a2xx.xml          (  36805 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/adreno/adreno_common.xml (  13634 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/adreno/adreno_pm4.xml    (  42393 bytes, from 2018-08-06 18:45:45)
+- /home/robclark/src/envytools/rnndb/adreno/a3xx.xml          (  83840 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/adreno/a4xx.xml          ( 112086 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/adreno/a5xx.xml          ( 147240 bytes, from 2018-08-06 18:45:45)
+- /home/robclark/src/envytools/rnndb/adreno/a6xx.xml          ( 101627 bytes, from 2018-08-06 18:45:45)
+- /home/robclark/src/envytools/rnndb/adreno/a6xx_gmu.xml      (  10431 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/adreno/ocmem.xml         (   1773 bytes, from 2018-07-03 19:37:13)
+
+Copyright (C) 2013-2018 by the following authors:
 - Rob Clark <robdclark@gmail.com> (robclark)
 - Ilia Mirkin <imirkin@alum.mit.edu> (imirkin)
 
@@ -84,13 +86,12 @@ enum a2xx_sq_surfaceformat {
 	FMT_5_5_5_1 = 13,
 	FMT_8_8_8_8_A = 14,
 	FMT_4_4_4_4 = 15,
-	FMT_10_11_11 = 16,
-	FMT_11_11_10 = 17,
+	FMT_8_8_8 = 16,
 	FMT_DXT1 = 18,
 	FMT_DXT2_3 = 19,
 	FMT_DXT4_5 = 20,
+	FMT_10_10_10_2 = 21,
 	FMT_24_8 = 22,
-	FMT_24_8_FLOAT = 23,
 	FMT_16 = 24,
 	FMT_16_16 = 25,
 	FMT_16_16_16_16 = 26,
@@ -106,29 +107,23 @@ enum a2xx_sq_surfaceformat {
 	FMT_32_FLOAT = 36,
 	FMT_32_32_FLOAT = 37,
 	FMT_32_32_32_32_FLOAT = 38,
-	FMT_32_AS_8 = 39,
-	FMT_32_AS_8_8 = 40,
-	FMT_16_MPEG = 41,
-	FMT_16_16_MPEG = 42,
-	FMT_8_INTERLACED = 43,
-	FMT_32_AS_8_INTERLACED = 44,
-	FMT_32_AS_8_8_INTERLACED = 45,
-	FMT_16_INTERLACED = 46,
-	FMT_16_MPEG_INTERLACED = 47,
-	FMT_16_16_MPEG_INTERLACED = 48,
+	FMT_ATI_TC_RGB = 39,
+	FMT_ATI_TC_RGBA = 40,
+	FMT_ATI_TC_555_565_RGB = 41,
+	FMT_ATI_TC_555_565_RGBA = 42,
+	FMT_ATI_TC_RGBA_INTERP = 43,
+	FMT_ATI_TC_555_565_RGBA_INTERP = 44,
+	FMT_ETC1_RGBA_INTERP = 46,
+	FMT_ETC1_RGB = 47,
+	FMT_ETC1_RGBA = 48,
 	FMT_DXN = 49,
-	FMT_8_8_8_8_AS_16_16_16_16 = 50,
-	FMT_DXT1_AS_16_16_16_16 = 51,
-	FMT_DXT2_3_AS_16_16_16_16 = 52,
-	FMT_DXT4_5_AS_16_16_16_16 = 53,
+	FMT_2_3_3 = 51,
 	FMT_2_10_10_10_AS_16_16_16_16 = 54,
-	FMT_10_11_11_AS_16_16_16_16 = 55,
-	FMT_11_11_10_AS_16_16_16_16 = 56,
+	FMT_10_10_10_2_AS_16_16_16_16 = 55,
 	FMT_32_32_32_FLOAT = 57,
 	FMT_DXT3A = 58,
 	FMT_DXT5A = 59,
 	FMT_CTX1 = 60,
-	FMT_DXT3A_AS_1_1_1_1 = 61,
 };
 
 enum a2xx_sq_ps_vtx_mode {
diff --git a/drivers/gpu/drm/msm/adreno/a3xx.xml.h b/drivers/gpu/drm/msm/adreno/a3xx.xml.h
index 663a73216926..645a19aef399 100644
--- a/drivers/gpu/drm/msm/adreno/a3xx.xml.h
+++ b/drivers/gpu/drm/msm/adreno/a3xx.xml.h
@@ -8,17 +8,19 @@ http://github.com/freedreno/envytools/
 git clone https://github.com/freedreno/envytools.git
 
 The rules-ng-ng source files this header was generated from are:
-- /home/robclark/src/freedreno/envytools/rnndb/adreno.xml               (    431 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml  (   1572 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/a2xx.xml          (  37162 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_common.xml (  13324 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_pm4.xml    (  31866 bytes, from 2017-06-06 18:26:14)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/a3xx.xml          (  83840 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/a4xx.xml          ( 111898 bytes, from 2017-06-06 18:23:59)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/a5xx.xml          ( 139480 bytes, from 2017-06-16 12:44:39)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/ocmem.xml         (   1773 bytes, from 2017-05-17 13:21:27)
-
-Copyright (C) 2013-2017 by the following authors:
+- /home/robclark/src/envytools/rnndb/adreno.xml               (    501 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/freedreno_copyright.xml  (   1572 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/adreno/a2xx.xml          (  36805 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/adreno/adreno_common.xml (  13634 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/adreno/adreno_pm4.xml    (  42393 bytes, from 2018-08-06 18:45:45)
+- /home/robclark/src/envytools/rnndb/adreno/a3xx.xml          (  83840 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/adreno/a4xx.xml          ( 112086 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/adreno/a5xx.xml          ( 147240 bytes, from 2018-08-06 18:45:45)
+- /home/robclark/src/envytools/rnndb/adreno/a6xx.xml          ( 101627 bytes, from 2018-08-06 18:45:45)
+- /home/robclark/src/envytools/rnndb/adreno/a6xx_gmu.xml      (  10431 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/adreno/ocmem.xml         (   1773 bytes, from 2018-07-03 19:37:13)
+
+Copyright (C) 2013-2018 by the following authors:
 - Rob Clark <robdclark@gmail.com> (robclark)
 - Ilia Mirkin <imirkin@alum.mit.edu> (imirkin)
 
diff --git a/drivers/gpu/drm/msm/adreno/a3xx_gpu.c b/drivers/gpu/drm/msm/adreno/a3xx_gpu.c
index 3ebbeb3a9b68..669c2d4b070d 100644
--- a/drivers/gpu/drm/msm/adreno/a3xx_gpu.c
+++ b/drivers/gpu/drm/msm/adreno/a3xx_gpu.c
@@ -411,15 +411,6 @@ static const unsigned int a3xx_registers[] = {
 	~0   /* sentinel */
 };
 
-#ifdef CONFIG_DEBUG_FS
-static void a3xx_show(struct msm_gpu *gpu, struct seq_file *m)
-{
-	seq_printf(m, "status:   %08x\n",
-			gpu_read(gpu, REG_A3XX_RBBM_STATUS));
-	adreno_show(gpu, m);
-}
-#endif
-
 /* would be nice to not have to duplicate the _show() stuff with printk(): */
 static void a3xx_dump(struct msm_gpu *gpu)
 {
@@ -427,6 +418,21 @@ static void a3xx_dump(struct msm_gpu *gpu)
 			gpu_read(gpu, REG_A3XX_RBBM_STATUS));
 	adreno_dump(gpu);
 }
+
+static struct msm_gpu_state *a3xx_gpu_state_get(struct msm_gpu *gpu)
+{
+	struct msm_gpu_state *state = kzalloc(sizeof(*state), GFP_KERNEL);
+
+	if (!state)
+		return ERR_PTR(-ENOMEM);
+
+	adreno_gpu_state_get(gpu, state);
+
+	state->rbbm_status = gpu_read(gpu, REG_A3XX_RBBM_STATUS);
+
+	return state;
+}
+
 /* Register offset defines for A3XX */
 static const unsigned int a3xx_register_offsets[REG_ADRENO_REGISTER_MAX] = {
 	REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_BASE, REG_AXXX_CP_RB_BASE),
@@ -450,9 +456,11 @@ static const struct adreno_gpu_funcs funcs = {
 		.active_ring = adreno_active_ring,
 		.irq = a3xx_irq,
 		.destroy = a3xx_destroy,
-#ifdef CONFIG_DEBUG_FS
-		.show = a3xx_show,
+#if defined(CONFIG_DEBUG_FS) || defined(CONFIG_DEV_COREDUMP)
+		.show = adreno_show,
 #endif
+		.gpu_state_get = a3xx_gpu_state_get,
+		.gpu_state_put = adreno_gpu_state_put,
 	},
 };
 
diff --git a/drivers/gpu/drm/msm/adreno/a4xx.xml.h b/drivers/gpu/drm/msm/adreno/a4xx.xml.h
index 1a14f4a40b9c..19565e87aa7b 100644
--- a/drivers/gpu/drm/msm/adreno/a4xx.xml.h
+++ b/drivers/gpu/drm/msm/adreno/a4xx.xml.h
@@ -8,17 +8,19 @@ http://github.com/freedreno/envytools/
 git clone https://github.com/freedreno/envytools.git
 
 The rules-ng-ng source files this header was generated from are:
-- /home/robclark/src/freedreno/envytools/rnndb/adreno.xml               (    431 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml  (   1572 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/a2xx.xml          (  37162 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_common.xml (  13324 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_pm4.xml    (  31866 bytes, from 2017-06-06 18:26:14)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/a3xx.xml          (  83840 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/a4xx.xml          ( 111898 bytes, from 2017-06-06 18:23:59)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/a5xx.xml          ( 139480 bytes, from 2017-06-16 12:44:39)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/ocmem.xml         (   1773 bytes, from 2017-05-17 13:21:27)
-
-Copyright (C) 2013-2017 by the following authors:
+- /home/robclark/src/envytools/rnndb/adreno.xml               (    501 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/freedreno_copyright.xml  (   1572 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/adreno/a2xx.xml          (  36805 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/adreno/adreno_common.xml (  13634 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/adreno/adreno_pm4.xml    (  42393 bytes, from 2018-08-06 18:45:45)
+- /home/robclark/src/envytools/rnndb/adreno/a3xx.xml          (  83840 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/adreno/a4xx.xml          ( 112086 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/adreno/a5xx.xml          ( 147240 bytes, from 2018-08-06 18:45:45)
+- /home/robclark/src/envytools/rnndb/adreno/a6xx.xml          ( 101627 bytes, from 2018-08-06 18:45:45)
+- /home/robclark/src/envytools/rnndb/adreno/a6xx_gmu.xml      (  10431 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/adreno/ocmem.xml         (   1773 bytes, from 2018-07-03 19:37:13)
+
+Copyright (C) 2013-2018 by the following authors:
 - Rob Clark <robdclark@gmail.com> (robclark)
 - Ilia Mirkin <imirkin@alum.mit.edu> (imirkin)
 
@@ -263,12 +265,6 @@ enum a4xx_depth_format {
 	DEPTH4_32 = 3,
 };
 
-enum a4xx_tess_spacing {
-	EQUAL_SPACING = 0,
-	ODD_SPACING = 2,
-	EVEN_SPACING = 3,
-};
-
 enum a4xx_ccu_perfcounter_select {
 	CCU_BUSY_CYCLES = 0,
 	CCU_RB_DEPTH_RETURN_STALL = 2,
@@ -3544,12 +3540,13 @@ static inline uint32_t A4XX_HLSQ_VS_CONTROL_REG_CONSTLENGTH(uint32_t val)
 {
 	return ((val) << A4XX_HLSQ_VS_CONTROL_REG_CONSTLENGTH__SHIFT) & A4XX_HLSQ_VS_CONTROL_REG_CONSTLENGTH__MASK;
 }
-#define A4XX_HLSQ_VS_CONTROL_REG_CONSTOBJECTOFFSET__MASK	0x0000ff00
+#define A4XX_HLSQ_VS_CONTROL_REG_CONSTOBJECTOFFSET__MASK	0x00007f00
 #define A4XX_HLSQ_VS_CONTROL_REG_CONSTOBJECTOFFSET__SHIFT	8
 static inline uint32_t A4XX_HLSQ_VS_CONTROL_REG_CONSTOBJECTOFFSET(uint32_t val)
 {
 	return ((val) << A4XX_HLSQ_VS_CONTROL_REG_CONSTOBJECTOFFSET__SHIFT) & A4XX_HLSQ_VS_CONTROL_REG_CONSTOBJECTOFFSET__MASK;
 }
+#define A4XX_HLSQ_VS_CONTROL_REG_SSBO_ENABLE			0x00008000
 #define A4XX_HLSQ_VS_CONTROL_REG_ENABLED			0x00010000
 #define A4XX_HLSQ_VS_CONTROL_REG_SHADEROBJOFFSET__MASK		0x00fe0000
 #define A4XX_HLSQ_VS_CONTROL_REG_SHADEROBJOFFSET__SHIFT		17
@@ -3571,12 +3568,13 @@ static inline uint32_t A4XX_HLSQ_FS_CONTROL_REG_CONSTLENGTH(uint32_t val)
 {
 	return ((val) << A4XX_HLSQ_FS_CONTROL_REG_CONSTLENGTH__SHIFT) & A4XX_HLSQ_FS_CONTROL_REG_CONSTLENGTH__MASK;
 }
-#define A4XX_HLSQ_FS_CONTROL_REG_CONSTOBJECTOFFSET__MASK	0x0000ff00
+#define A4XX_HLSQ_FS_CONTROL_REG_CONSTOBJECTOFFSET__MASK	0x00007f00
 #define A4XX_HLSQ_FS_CONTROL_REG_CONSTOBJECTOFFSET__SHIFT	8
 static inline uint32_t A4XX_HLSQ_FS_CONTROL_REG_CONSTOBJECTOFFSET(uint32_t val)
 {
 	return ((val) << A4XX_HLSQ_FS_CONTROL_REG_CONSTOBJECTOFFSET__SHIFT) & A4XX_HLSQ_FS_CONTROL_REG_CONSTOBJECTOFFSET__MASK;
 }
+#define A4XX_HLSQ_FS_CONTROL_REG_SSBO_ENABLE			0x00008000
 #define A4XX_HLSQ_FS_CONTROL_REG_ENABLED			0x00010000
 #define A4XX_HLSQ_FS_CONTROL_REG_SHADEROBJOFFSET__MASK		0x00fe0000
 #define A4XX_HLSQ_FS_CONTROL_REG_SHADEROBJOFFSET__SHIFT		17
@@ -3598,12 +3596,13 @@ static inline uint32_t A4XX_HLSQ_HS_CONTROL_REG_CONSTLENGTH(uint32_t val)
 {
 	return ((val) << A4XX_HLSQ_HS_CONTROL_REG_CONSTLENGTH__SHIFT) & A4XX_HLSQ_HS_CONTROL_REG_CONSTLENGTH__MASK;
 }
-#define A4XX_HLSQ_HS_CONTROL_REG_CONSTOBJECTOFFSET__MASK	0x0000ff00
+#define A4XX_HLSQ_HS_CONTROL_REG_CONSTOBJECTOFFSET__MASK	0x00007f00
 #define A4XX_HLSQ_HS_CONTROL_REG_CONSTOBJECTOFFSET__SHIFT	8
 static inline uint32_t A4XX_HLSQ_HS_CONTROL_REG_CONSTOBJECTOFFSET(uint32_t val)
 {
 	return ((val) << A4XX_HLSQ_HS_CONTROL_REG_CONSTOBJECTOFFSET__SHIFT) & A4XX_HLSQ_HS_CONTROL_REG_CONSTOBJECTOFFSET__MASK;
 }
+#define A4XX_HLSQ_HS_CONTROL_REG_SSBO_ENABLE			0x00008000
 #define A4XX_HLSQ_HS_CONTROL_REG_ENABLED			0x00010000
 #define A4XX_HLSQ_HS_CONTROL_REG_SHADEROBJOFFSET__MASK		0x00fe0000
 #define A4XX_HLSQ_HS_CONTROL_REG_SHADEROBJOFFSET__SHIFT		17
@@ -3625,12 +3624,13 @@ static inline uint32_t A4XX_HLSQ_DS_CONTROL_REG_CONSTLENGTH(uint32_t val)
 {
 	return ((val) << A4XX_HLSQ_DS_CONTROL_REG_CONSTLENGTH__SHIFT) & A4XX_HLSQ_DS_CONTROL_REG_CONSTLENGTH__MASK;
 }
-#define A4XX_HLSQ_DS_CONTROL_REG_CONSTOBJECTOFFSET__MASK	0x0000ff00
+#define A4XX_HLSQ_DS_CONTROL_REG_CONSTOBJECTOFFSET__MASK	0x00007f00
 #define A4XX_HLSQ_DS_CONTROL_REG_CONSTOBJECTOFFSET__SHIFT	8
 static inline uint32_t A4XX_HLSQ_DS_CONTROL_REG_CONSTOBJECTOFFSET(uint32_t val)
 {
 	return ((val) << A4XX_HLSQ_DS_CONTROL_REG_CONSTOBJECTOFFSET__SHIFT) & A4XX_HLSQ_DS_CONTROL_REG_CONSTOBJECTOFFSET__MASK;
 }
+#define A4XX_HLSQ_DS_CONTROL_REG_SSBO_ENABLE			0x00008000
 #define A4XX_HLSQ_DS_CONTROL_REG_ENABLED			0x00010000
 #define A4XX_HLSQ_DS_CONTROL_REG_SHADEROBJOFFSET__MASK		0x00fe0000
 #define A4XX_HLSQ_DS_CONTROL_REG_SHADEROBJOFFSET__SHIFT		17
@@ -3652,12 +3652,13 @@ static inline uint32_t A4XX_HLSQ_GS_CONTROL_REG_CONSTLENGTH(uint32_t val)
 {
 	return ((val) << A4XX_HLSQ_GS_CONTROL_REG_CONSTLENGTH__SHIFT) & A4XX_HLSQ_GS_CONTROL_REG_CONSTLENGTH__MASK;
 }
-#define A4XX_HLSQ_GS_CONTROL_REG_CONSTOBJECTOFFSET__MASK	0x0000ff00
+#define A4XX_HLSQ_GS_CONTROL_REG_CONSTOBJECTOFFSET__MASK	0x00007f00
 #define A4XX_HLSQ_GS_CONTROL_REG_CONSTOBJECTOFFSET__SHIFT	8
 static inline uint32_t A4XX_HLSQ_GS_CONTROL_REG_CONSTOBJECTOFFSET(uint32_t val)
 {
 	return ((val) << A4XX_HLSQ_GS_CONTROL_REG_CONSTOBJECTOFFSET__SHIFT) & A4XX_HLSQ_GS_CONTROL_REG_CONSTOBJECTOFFSET__MASK;
 }
+#define A4XX_HLSQ_GS_CONTROL_REG_SSBO_ENABLE			0x00008000
 #define A4XX_HLSQ_GS_CONTROL_REG_ENABLED			0x00010000
 #define A4XX_HLSQ_GS_CONTROL_REG_SHADEROBJOFFSET__MASK		0x00fe0000
 #define A4XX_HLSQ_GS_CONTROL_REG_SHADEROBJOFFSET__SHIFT		17
@@ -3672,23 +3673,103 @@ static inline uint32_t A4XX_HLSQ_GS_CONTROL_REG_INSTRLENGTH(uint32_t val)
 	return ((val) << A4XX_HLSQ_GS_CONTROL_REG_INSTRLENGTH__SHIFT) & A4XX_HLSQ_GS_CONTROL_REG_INSTRLENGTH__MASK;
 }
 
-#define REG_A4XX_HLSQ_CS_CONTROL				0x000023ca
+#define REG_A4XX_HLSQ_CS_CONTROL_REG				0x000023ca
+#define A4XX_HLSQ_CS_CONTROL_REG_CONSTLENGTH__MASK		0x000000ff
+#define A4XX_HLSQ_CS_CONTROL_REG_CONSTLENGTH__SHIFT		0
+static inline uint32_t A4XX_HLSQ_CS_CONTROL_REG_CONSTLENGTH(uint32_t val)
+{
+	return ((val) << A4XX_HLSQ_CS_CONTROL_REG_CONSTLENGTH__SHIFT) & A4XX_HLSQ_CS_CONTROL_REG_CONSTLENGTH__MASK;
+}
+#define A4XX_HLSQ_CS_CONTROL_REG_CONSTOBJECTOFFSET__MASK	0x00007f00
+#define A4XX_HLSQ_CS_CONTROL_REG_CONSTOBJECTOFFSET__SHIFT	8
+static inline uint32_t A4XX_HLSQ_CS_CONTROL_REG_CONSTOBJECTOFFSET(uint32_t val)
+{
+	return ((val) << A4XX_HLSQ_CS_CONTROL_REG_CONSTOBJECTOFFSET__SHIFT) & A4XX_HLSQ_CS_CONTROL_REG_CONSTOBJECTOFFSET__MASK;
+}
+#define A4XX_HLSQ_CS_CONTROL_REG_SSBO_ENABLE			0x00008000
+#define A4XX_HLSQ_CS_CONTROL_REG_ENABLED			0x00010000
+#define A4XX_HLSQ_CS_CONTROL_REG_SHADEROBJOFFSET__MASK		0x00fe0000
+#define A4XX_HLSQ_CS_CONTROL_REG_SHADEROBJOFFSET__SHIFT		17
+static inline uint32_t A4XX_HLSQ_CS_CONTROL_REG_SHADEROBJOFFSET(uint32_t val)
+{
+	return ((val) << A4XX_HLSQ_CS_CONTROL_REG_SHADEROBJOFFSET__SHIFT) & A4XX_HLSQ_CS_CONTROL_REG_SHADEROBJOFFSET__MASK;
+}
+#define A4XX_HLSQ_CS_CONTROL_REG_INSTRLENGTH__MASK		0xff000000
+#define A4XX_HLSQ_CS_CONTROL_REG_INSTRLENGTH__SHIFT		24
+static inline uint32_t A4XX_HLSQ_CS_CONTROL_REG_INSTRLENGTH(uint32_t val)
+{
+	return ((val) << A4XX_HLSQ_CS_CONTROL_REG_INSTRLENGTH__SHIFT) & A4XX_HLSQ_CS_CONTROL_REG_INSTRLENGTH__MASK;
+}
 
 #define REG_A4XX_HLSQ_CL_NDRANGE_0				0x000023cd
+#define A4XX_HLSQ_CL_NDRANGE_0_KERNELDIM__MASK			0x00000003
+#define A4XX_HLSQ_CL_NDRANGE_0_KERNELDIM__SHIFT			0
+static inline uint32_t A4XX_HLSQ_CL_NDRANGE_0_KERNELDIM(uint32_t val)
+{
+	return ((val) << A4XX_HLSQ_CL_NDRANGE_0_KERNELDIM__SHIFT) & A4XX_HLSQ_CL_NDRANGE_0_KERNELDIM__MASK;
+}
+#define A4XX_HLSQ_CL_NDRANGE_0_LOCALSIZEX__MASK			0x00000ffc
+#define A4XX_HLSQ_CL_NDRANGE_0_LOCALSIZEX__SHIFT		2
+static inline uint32_t A4XX_HLSQ_CL_NDRANGE_0_LOCALSIZEX(uint32_t val)
+{
+	return ((val) << A4XX_HLSQ_CL_NDRANGE_0_LOCALSIZEX__SHIFT) & A4XX_HLSQ_CL_NDRANGE_0_LOCALSIZEX__MASK;
+}
+#define A4XX_HLSQ_CL_NDRANGE_0_LOCALSIZEY__MASK			0x003ff000
+#define A4XX_HLSQ_CL_NDRANGE_0_LOCALSIZEY__SHIFT		12
+static inline uint32_t A4XX_HLSQ_CL_NDRANGE_0_LOCALSIZEY(uint32_t val)
+{
+	return ((val) << A4XX_HLSQ_CL_NDRANGE_0_LOCALSIZEY__SHIFT) & A4XX_HLSQ_CL_NDRANGE_0_LOCALSIZEY__MASK;
+}
+#define A4XX_HLSQ_CL_NDRANGE_0_LOCALSIZEZ__MASK			0xffc00000
+#define A4XX_HLSQ_CL_NDRANGE_0_LOCALSIZEZ__SHIFT		22
+static inline uint32_t A4XX_HLSQ_CL_NDRANGE_0_LOCALSIZEZ(uint32_t val)
+{
+	return ((val) << A4XX_HLSQ_CL_NDRANGE_0_LOCALSIZEZ__SHIFT) & A4XX_HLSQ_CL_NDRANGE_0_LOCALSIZEZ__MASK;
+}
 
 #define REG_A4XX_HLSQ_CL_NDRANGE_1				0x000023ce
+#define A4XX_HLSQ_CL_NDRANGE_1_SIZE_X__MASK			0xffffffff
+#define A4XX_HLSQ_CL_NDRANGE_1_SIZE_X__SHIFT			0
+static inline uint32_t A4XX_HLSQ_CL_NDRANGE_1_SIZE_X(uint32_t val)
+{
+	return ((val) << A4XX_HLSQ_CL_NDRANGE_1_SIZE_X__SHIFT) & A4XX_HLSQ_CL_NDRANGE_1_SIZE_X__MASK;
+}
 
 #define REG_A4XX_HLSQ_CL_NDRANGE_2				0x000023cf
 
 #define REG_A4XX_HLSQ_CL_NDRANGE_3				0x000023d0
+#define A4XX_HLSQ_CL_NDRANGE_3_SIZE_Y__MASK			0xffffffff
+#define A4XX_HLSQ_CL_NDRANGE_3_SIZE_Y__SHIFT			0
+static inline uint32_t A4XX_HLSQ_CL_NDRANGE_3_SIZE_Y(uint32_t val)
+{
+	return ((val) << A4XX_HLSQ_CL_NDRANGE_3_SIZE_Y__SHIFT) & A4XX_HLSQ_CL_NDRANGE_3_SIZE_Y__MASK;
+}
 
 #define REG_A4XX_HLSQ_CL_NDRANGE_4				0x000023d1
 
 #define REG_A4XX_HLSQ_CL_NDRANGE_5				0x000023d2
+#define A4XX_HLSQ_CL_NDRANGE_5_SIZE_Z__MASK			0xffffffff
+#define A4XX_HLSQ_CL_NDRANGE_5_SIZE_Z__SHIFT			0
+static inline uint32_t A4XX_HLSQ_CL_NDRANGE_5_SIZE_Z(uint32_t val)
+{
+	return ((val) << A4XX_HLSQ_CL_NDRANGE_5_SIZE_Z__SHIFT) & A4XX_HLSQ_CL_NDRANGE_5_SIZE_Z__MASK;
+}
 
 #define REG_A4XX_HLSQ_CL_NDRANGE_6				0x000023d3
 
 #define REG_A4XX_HLSQ_CL_CONTROL_0				0x000023d4
+#define A4XX_HLSQ_CL_CONTROL_0_WGIDCONSTID__MASK		0x000000ff
+#define A4XX_HLSQ_CL_CONTROL_0_WGIDCONSTID__SHIFT		0
+static inline uint32_t A4XX_HLSQ_CL_CONTROL_0_WGIDCONSTID(uint32_t val)
+{
+	return ((val) << A4XX_HLSQ_CL_CONTROL_0_WGIDCONSTID__SHIFT) & A4XX_HLSQ_CL_CONTROL_0_WGIDCONSTID__MASK;
+}
+#define A4XX_HLSQ_CL_CONTROL_0_LOCALIDREGID__MASK		0xff000000
+#define A4XX_HLSQ_CL_CONTROL_0_LOCALIDREGID__SHIFT		24
+static inline uint32_t A4XX_HLSQ_CL_CONTROL_0_LOCALIDREGID(uint32_t val)
+{
+	return ((val) << A4XX_HLSQ_CL_CONTROL_0_LOCALIDREGID__SHIFT) & A4XX_HLSQ_CL_CONTROL_0_LOCALIDREGID__MASK;
+}
 
 #define REG_A4XX_HLSQ_CL_CONTROL_1				0x000023d5
 
@@ -4087,5 +4168,71 @@ static inline uint32_t A4XX_TEX_CONST_4_BASE(uint32_t val)
 
 #define REG_A4XX_TEX_CONST_7					0x00000007
 
+#define REG_A4XX_SSBO_0_0					0x00000000
+#define A4XX_SSBO_0_0_BASE__MASK				0xffffffe0
+#define A4XX_SSBO_0_0_BASE__SHIFT				5
+static inline uint32_t A4XX_SSBO_0_0_BASE(uint32_t val)
+{
+	return ((val >> 5) << A4XX_SSBO_0_0_BASE__SHIFT) & A4XX_SSBO_0_0_BASE__MASK;
+}
+
+#define REG_A4XX_SSBO_0_1					0x00000001
+#define A4XX_SSBO_0_1_PITCH__MASK				0x003fffff
+#define A4XX_SSBO_0_1_PITCH__SHIFT				0
+static inline uint32_t A4XX_SSBO_0_1_PITCH(uint32_t val)
+{
+	return ((val) << A4XX_SSBO_0_1_PITCH__SHIFT) & A4XX_SSBO_0_1_PITCH__MASK;
+}
+
+#define REG_A4XX_SSBO_0_2					0x00000002
+#define A4XX_SSBO_0_2_ARRAY_PITCH__MASK				0x03fff000
+#define A4XX_SSBO_0_2_ARRAY_PITCH__SHIFT			12
+static inline uint32_t A4XX_SSBO_0_2_ARRAY_PITCH(uint32_t val)
+{
+	return ((val >> 12) << A4XX_SSBO_0_2_ARRAY_PITCH__SHIFT) & A4XX_SSBO_0_2_ARRAY_PITCH__MASK;
+}
+
+#define REG_A4XX_SSBO_0_3					0x00000003
+#define A4XX_SSBO_0_3_CPP__MASK					0x0000003f
+#define A4XX_SSBO_0_3_CPP__SHIFT				0
+static inline uint32_t A4XX_SSBO_0_3_CPP(uint32_t val)
+{
+	return ((val) << A4XX_SSBO_0_3_CPP__SHIFT) & A4XX_SSBO_0_3_CPP__MASK;
+}
+
+#define REG_A4XX_SSBO_1_0					0x00000000
+#define A4XX_SSBO_1_0_CPP__MASK					0x0000001f
+#define A4XX_SSBO_1_0_CPP__SHIFT				0
+static inline uint32_t A4XX_SSBO_1_0_CPP(uint32_t val)
+{
+	return ((val) << A4XX_SSBO_1_0_CPP__SHIFT) & A4XX_SSBO_1_0_CPP__MASK;
+}
+#define A4XX_SSBO_1_0_FMT__MASK					0x0000ff00
+#define A4XX_SSBO_1_0_FMT__SHIFT				8
+static inline uint32_t A4XX_SSBO_1_0_FMT(enum a4xx_color_fmt val)
+{
+	return ((val) << A4XX_SSBO_1_0_FMT__SHIFT) & A4XX_SSBO_1_0_FMT__MASK;
+}
+#define A4XX_SSBO_1_0_WIDTH__MASK				0xffff0000
+#define A4XX_SSBO_1_0_WIDTH__SHIFT				16
+static inline uint32_t A4XX_SSBO_1_0_WIDTH(uint32_t val)
+{
+	return ((val) << A4XX_SSBO_1_0_WIDTH__SHIFT) & A4XX_SSBO_1_0_WIDTH__MASK;
+}
+
+#define REG_A4XX_SSBO_1_1					0x00000001
+#define A4XX_SSBO_1_1_HEIGHT__MASK				0x0000ffff
+#define A4XX_SSBO_1_1_HEIGHT__SHIFT				0
+static inline uint32_t A4XX_SSBO_1_1_HEIGHT(uint32_t val)
+{
+	return ((val) << A4XX_SSBO_1_1_HEIGHT__SHIFT) & A4XX_SSBO_1_1_HEIGHT__MASK;
+}
+#define A4XX_SSBO_1_1_DEPTH__MASK				0xffff0000
+#define A4XX_SSBO_1_1_DEPTH__SHIFT				16
+static inline uint32_t A4XX_SSBO_1_1_DEPTH(uint32_t val)
+{
+	return ((val) << A4XX_SSBO_1_1_DEPTH__SHIFT) & A4XX_SSBO_1_1_DEPTH__MASK;
+}
+
 
 #endif /* A4XX_XML */
diff --git a/drivers/gpu/drm/msm/adreno/a4xx_gpu.c b/drivers/gpu/drm/msm/adreno/a4xx_gpu.c
index 16d3d596638e..7c4e6dc1ed59 100644
--- a/drivers/gpu/drm/msm/adreno/a4xx_gpu.c
+++ b/drivers/gpu/drm/msm/adreno/a4xx_gpu.c
@@ -455,15 +455,19 @@ static const unsigned int a4xx_registers[] = {
 	~0 /* sentinel */
 };
 
-#ifdef CONFIG_DEBUG_FS
-static void a4xx_show(struct msm_gpu *gpu, struct seq_file *m)
+static struct msm_gpu_state *a4xx_gpu_state_get(struct msm_gpu *gpu)
 {
-	seq_printf(m, "status:   %08x\n",
-			gpu_read(gpu, REG_A4XX_RBBM_STATUS));
-	adreno_show(gpu, m);
+	struct msm_gpu_state *state = kzalloc(sizeof(*state), GFP_KERNEL);
+
+	if (!state)
+		return ERR_PTR(-ENOMEM);
+
+	adreno_gpu_state_get(gpu, state);
 
+	state->rbbm_status = gpu_read(gpu, REG_A4XX_RBBM_STATUS);
+
+	return state;
 }
-#endif
 
 /* Register offset defines for A4XX, in order of enum adreno_regs */
 static const unsigned int a4xx_register_offsets[REG_ADRENO_REGISTER_MAX] = {
@@ -538,9 +542,11 @@ static const struct adreno_gpu_funcs funcs = {
 		.active_ring = adreno_active_ring,
 		.irq = a4xx_irq,
 		.destroy = a4xx_destroy,
-#ifdef CONFIG_DEBUG_FS
-		.show = a4xx_show,
+#if defined(CONFIG_DEBUG_FS) || defined(CONFIG_DEV_COREDUMP)
+		.show = adreno_show,
 #endif
+		.gpu_state_get = a4xx_gpu_state_get,
+		.gpu_state_put = adreno_gpu_state_put,
 	},
 	.get_timestamp = a4xx_get_timestamp,
 };
diff --git a/drivers/gpu/drm/msm/adreno/a5xx.xml.h b/drivers/gpu/drm/msm/adreno/a5xx.xml.h
index e0e6711f4f78..182d37ff3794 100644
--- a/drivers/gpu/drm/msm/adreno/a5xx.xml.h
+++ b/drivers/gpu/drm/msm/adreno/a5xx.xml.h
@@ -8,17 +8,19 @@ http://github.com/freedreno/envytools/
 git clone https://github.com/freedreno/envytools.git
 
 The rules-ng-ng source files this header was generated from are:
-- /home/robclark/src/freedreno/envytools/rnndb/adreno.xml               (    431 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml  (   1572 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/a2xx.xml          (  37162 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_common.xml (  13324 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_pm4.xml    (  31866 bytes, from 2017-06-06 18:26:14)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/a3xx.xml          (  83840 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/a4xx.xml          ( 111898 bytes, from 2017-06-06 18:23:59)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/a5xx.xml          ( 139480 bytes, from 2017-06-16 12:44:39)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/ocmem.xml         (   1773 bytes, from 2017-05-17 13:21:27)
-
-Copyright (C) 2013-2017 by the following authors:
+- /home/robclark/src/envytools/rnndb/adreno.xml               (    501 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/freedreno_copyright.xml  (   1572 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/adreno/a2xx.xml          (  36805 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/adreno/adreno_common.xml (  13634 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/adreno/adreno_pm4.xml    (  42393 bytes, from 2018-08-06 18:45:45)
+- /home/robclark/src/envytools/rnndb/adreno/a3xx.xml          (  83840 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/adreno/a4xx.xml          ( 112086 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/adreno/a5xx.xml          ( 147240 bytes, from 2018-08-06 18:45:45)
+- /home/robclark/src/envytools/rnndb/adreno/a6xx.xml          ( 101627 bytes, from 2018-08-06 18:45:45)
+- /home/robclark/src/envytools/rnndb/adreno/a6xx_gmu.xml      (  10431 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/adreno/ocmem.xml         (   1773 bytes, from 2018-07-03 19:37:13)
+
+Copyright (C) 2013-2018 by the following authors:
 - Rob Clark <robdclark@gmail.com> (robclark)
 - Ilia Mirkin <imirkin@alum.mit.edu> (imirkin)
 
@@ -119,6 +121,11 @@ enum a5xx_vtx_fmt {
 	VFMT5_8_8_8_8_SNORM = 50,
 	VFMT5_8_8_8_8_UINT = 51,
 	VFMT5_8_8_8_8_SINT = 52,
+	VFMT5_10_10_10_2_UNORM = 54,
+	VFMT5_10_10_10_2_SNORM = 57,
+	VFMT5_10_10_10_2_UINT = 58,
+	VFMT5_10_10_10_2_SINT = 59,
+	VFMT5_11_11_10_FLOAT = 66,
 	VFMT5_16_16_UNORM = 67,
 	VFMT5_16_16_SNORM = 68,
 	VFMT5_16_16_FLOAT = 69,
@@ -204,14 +211,45 @@ enum a5xx_tex_fmt {
 	TFMT5_32_32_FLOAT = 103,
 	TFMT5_32_32_UINT = 104,
 	TFMT5_32_32_SINT = 105,
+	TFMT5_32_32_32_UINT = 114,
+	TFMT5_32_32_32_SINT = 115,
+	TFMT5_32_32_32_FLOAT = 116,
 	TFMT5_32_32_32_32_FLOAT = 130,
 	TFMT5_32_32_32_32_UINT = 131,
 	TFMT5_32_32_32_32_SINT = 132,
 	TFMT5_X8Z24_UNORM = 160,
+	TFMT5_ETC2_RG11_UNORM = 171,
+	TFMT5_ETC2_RG11_SNORM = 172,
+	TFMT5_ETC2_R11_UNORM = 173,
+	TFMT5_ETC2_R11_SNORM = 174,
+	TFMT5_ETC1 = 175,
+	TFMT5_ETC2_RGB8 = 176,
+	TFMT5_ETC2_RGBA8 = 177,
+	TFMT5_ETC2_RGB8A1 = 178,
+	TFMT5_DXT1 = 179,
+	TFMT5_DXT3 = 180,
+	TFMT5_DXT5 = 181,
 	TFMT5_RGTC1_UNORM = 183,
 	TFMT5_RGTC1_SNORM = 184,
 	TFMT5_RGTC2_UNORM = 187,
 	TFMT5_RGTC2_SNORM = 188,
+	TFMT5_BPTC_UFLOAT = 190,
+	TFMT5_BPTC_FLOAT = 191,
+	TFMT5_BPTC = 192,
+	TFMT5_ASTC_4x4 = 193,
+	TFMT5_ASTC_5x4 = 194,
+	TFMT5_ASTC_5x5 = 195,
+	TFMT5_ASTC_6x5 = 196,
+	TFMT5_ASTC_6x6 = 197,
+	TFMT5_ASTC_8x5 = 198,
+	TFMT5_ASTC_8x6 = 199,
+	TFMT5_ASTC_8x8 = 200,
+	TFMT5_ASTC_10x5 = 201,
+	TFMT5_ASTC_10x6 = 202,
+	TFMT5_ASTC_10x8 = 203,
+	TFMT5_ASTC_10x10 = 204,
+	TFMT5_ASTC_12x10 = 205,
+	TFMT5_ASTC_12x12 = 206,
 };
 
 enum a5xx_tex_fetchsize {
@@ -239,7 +277,7 @@ enum a5xx_blit_buf {
 	BLIT_MRT6 = 6,
 	BLIT_MRT7 = 7,
 	BLIT_ZS = 8,
-	BLIT_Z32 = 9,
+	BLIT_S = 9,
 };
 
 enum a5xx_cp_perfcounter_select {
@@ -899,6 +937,12 @@ enum a5xx_tex_type {
 
 #define REG_A5XX_CP_DRAW_STATE_DATA				0x0000080c
 
+#define REG_A5XX_CP_ME_NRT_ADDR_LO				0x0000080d
+
+#define REG_A5XX_CP_ME_NRT_ADDR_HI				0x0000080e
+
+#define REG_A5XX_CP_ME_NRT_DATA					0x00000810
+
 #define REG_A5XX_CP_CRASH_SCRIPT_BASE_LO			0x00000817
 
 #define REG_A5XX_CP_CRASH_SCRIPT_BASE_HI			0x00000818
@@ -2072,9 +2116,17 @@ static inline uint32_t A5XX_VSC_RESOLVE_CNTL_Y(uint32_t val)
 
 #define REG_A5XX_PC_MODE_CNTL					0x00000d02
 
-#define REG_A5XX_UNKNOWN_0D08					0x00000d08
+#define REG_A5XX_PC_INDEX_BUF_LO				0x00000d04
+
+#define REG_A5XX_PC_INDEX_BUF_HI				0x00000d05
+
+#define REG_A5XX_PC_START_INDEX					0x00000d06
 
-#define REG_A5XX_UNKNOWN_0D09					0x00000d09
+#define REG_A5XX_PC_MAX_INDEX					0x00000d07
+
+#define REG_A5XX_PC_TESSFACTOR_ADDR_LO				0x00000d08
+
+#define REG_A5XX_PC_TESSFACTOR_ADDR_HI				0x00000d09
 
 #define REG_A5XX_PC_PERFCTR_PC_SEL_0				0x00000d10
 
@@ -2327,6 +2379,14 @@ static inline uint32_t A5XX_VSC_RESOLVE_CNTL_Y(uint32_t val)
 
 #define REG_A5XX_VBIF_PERF_CNT_EN3				0x000030c3
 
+#define REG_A5XX_VBIF_PERF_CNT_CLR0				0x000030c8
+
+#define REG_A5XX_VBIF_PERF_CNT_CLR1				0x000030c9
+
+#define REG_A5XX_VBIF_PERF_CNT_CLR2				0x000030ca
+
+#define REG_A5XX_VBIF_PERF_CNT_CLR3				0x000030cb
+
 #define REG_A5XX_VBIF_PERF_CNT_SEL0				0x000030d0
 
 #define REG_A5XX_VBIF_PERF_CNT_SEL1				0x000030d1
@@ -2590,6 +2650,7 @@ static inline uint32_t A5XX_VSC_RESOLVE_CNTL_Y(uint32_t val)
 #define REG_A5XX_GPU_CS_AMP_CALIBRATION_CONTROL1		0x0000c557
 
 #define REG_A5XX_GRAS_CL_CNTL					0x0000e000
+#define A5XX_GRAS_CL_CNTL_ZERO_GB_SCALE_Z			0x00000040
 
 #define REG_A5XX_UNKNOWN_E001					0x0000e001
 
@@ -2700,7 +2761,7 @@ static inline uint32_t A5XX_GRAS_SU_POINT_SIZE(float val)
 	return ((((int32_t)(val * 16.0))) << A5XX_GRAS_SU_POINT_SIZE__SHIFT) & A5XX_GRAS_SU_POINT_SIZE__MASK;
 }
 
-#define REG_A5XX_UNKNOWN_E093					0x0000e093
+#define REG_A5XX_GRAS_SU_LAYERED				0x0000e093
 
 #define REG_A5XX_GRAS_SU_DEPTH_PLANE_CNTL			0x0000e094
 #define A5XX_GRAS_SU_DEPTH_PLANE_CNTL_FRAG_WRITES_Z		0x00000001
@@ -2936,7 +2997,9 @@ static inline uint32_t A5XX_RB_DEST_MSAA_CNTL_SAMPLES(enum a3xx_msaa_samples val
 #define A5XX_RB_RENDER_CONTROL0_WCOORD				0x00000200
 
 #define REG_A5XX_RB_RENDER_CONTROL1				0x0000e145
+#define A5XX_RB_RENDER_CONTROL1_SAMPLEMASK			0x00000001
 #define A5XX_RB_RENDER_CONTROL1_FACENESS			0x00000002
+#define A5XX_RB_RENDER_CONTROL1_SAMPLEID			0x00000004
 
 #define REG_A5XX_RB_FS_OUTPUT_CNTL				0x0000e146
 #define A5XX_RB_FS_OUTPUT_CNTL_MRT__MASK			0x0000000f
@@ -3002,6 +3065,13 @@ static inline uint32_t REG_A5XX_RB_MRT(uint32_t i0) { return 0x0000e150 + 0x7*i0
 static inline uint32_t REG_A5XX_RB_MRT_CONTROL(uint32_t i0) { return 0x0000e150 + 0x7*i0; }
 #define A5XX_RB_MRT_CONTROL_BLEND				0x00000001
 #define A5XX_RB_MRT_CONTROL_BLEND2				0x00000002
+#define A5XX_RB_MRT_CONTROL_ROP_ENABLE				0x00000004
+#define A5XX_RB_MRT_CONTROL_ROP_CODE__MASK			0x00000078
+#define A5XX_RB_MRT_CONTROL_ROP_CODE__SHIFT			3
+static inline uint32_t A5XX_RB_MRT_CONTROL_ROP_CODE(enum a3xx_rop_code val)
+{
+	return ((val) << A5XX_RB_MRT_CONTROL_ROP_CODE__SHIFT) & A5XX_RB_MRT_CONTROL_ROP_CODE__MASK;
+}
 #define A5XX_RB_MRT_CONTROL_COMPONENT_ENABLE__MASK		0x00000780
 #define A5XX_RB_MRT_CONTROL_COMPONENT_ENABLE__SHIFT		7
 static inline uint32_t A5XX_RB_MRT_CONTROL_COMPONENT_ENABLE(uint32_t val)
@@ -3060,6 +3130,12 @@ static inline uint32_t A5XX_RB_MRT_BUF_INFO_COLOR_TILE_MODE(enum a5xx_tile_mode
 {
 	return ((val) << A5XX_RB_MRT_BUF_INFO_COLOR_TILE_MODE__SHIFT) & A5XX_RB_MRT_BUF_INFO_COLOR_TILE_MODE__MASK;
 }
+#define A5XX_RB_MRT_BUF_INFO_DITHER_MODE__MASK			0x00001800
+#define A5XX_RB_MRT_BUF_INFO_DITHER_MODE__SHIFT			11
+static inline uint32_t A5XX_RB_MRT_BUF_INFO_DITHER_MODE(enum adreno_rb_dither_mode val)
+{
+	return ((val) << A5XX_RB_MRT_BUF_INFO_DITHER_MODE__SHIFT) & A5XX_RB_MRT_BUF_INFO_DITHER_MODE__MASK;
+}
 #define A5XX_RB_MRT_BUF_INFO_COLOR_SWAP__MASK			0x00006000
 #define A5XX_RB_MRT_BUF_INFO_COLOR_SWAP__SHIFT			13
 static inline uint32_t A5XX_RB_MRT_BUF_INFO_COLOR_SWAP(enum a3xx_color_swap val)
@@ -3223,6 +3299,7 @@ static inline uint32_t A5XX_RB_BLEND_CNTL_ENABLE_BLEND(uint32_t val)
 	return ((val) << A5XX_RB_BLEND_CNTL_ENABLE_BLEND__SHIFT) & A5XX_RB_BLEND_CNTL_ENABLE_BLEND__MASK;
 }
 #define A5XX_RB_BLEND_CNTL_INDEPENDENT_BLEND			0x00000100
+#define A5XX_RB_BLEND_CNTL_ALPHA_TO_COVERAGE			0x00000400
 #define A5XX_RB_BLEND_CNTL_SAMPLE_MASK__MASK			0xffff0000
 #define A5XX_RB_BLEND_CNTL_SAMPLE_MASK__SHIFT			16
 static inline uint32_t A5XX_RB_BLEND_CNTL_SAMPLE_MASK(uint32_t val)
@@ -3369,7 +3446,25 @@ static inline uint32_t A5XX_RB_STENCILREFMASK_STENCILWRITEMASK(uint32_t val)
 	return ((val) << A5XX_RB_STENCILREFMASK_STENCILWRITEMASK__SHIFT) & A5XX_RB_STENCILREFMASK_STENCILWRITEMASK__MASK;
 }
 
-#define REG_A5XX_UNKNOWN_E1C7					0x0000e1c7
+#define REG_A5XX_RB_STENCILREFMASK_BF				0x0000e1c7
+#define A5XX_RB_STENCILREFMASK_BF_STENCILREF__MASK		0x000000ff
+#define A5XX_RB_STENCILREFMASK_BF_STENCILREF__SHIFT		0
+static inline uint32_t A5XX_RB_STENCILREFMASK_BF_STENCILREF(uint32_t val)
+{
+	return ((val) << A5XX_RB_STENCILREFMASK_BF_STENCILREF__SHIFT) & A5XX_RB_STENCILREFMASK_BF_STENCILREF__MASK;
+}
+#define A5XX_RB_STENCILREFMASK_BF_STENCILMASK__MASK		0x0000ff00
+#define A5XX_RB_STENCILREFMASK_BF_STENCILMASK__SHIFT		8
+static inline uint32_t A5XX_RB_STENCILREFMASK_BF_STENCILMASK(uint32_t val)
+{
+	return ((val) << A5XX_RB_STENCILREFMASK_BF_STENCILMASK__SHIFT) & A5XX_RB_STENCILREFMASK_BF_STENCILMASK__MASK;
+}
+#define A5XX_RB_STENCILREFMASK_BF_STENCILWRITEMASK__MASK	0x00ff0000
+#define A5XX_RB_STENCILREFMASK_BF_STENCILWRITEMASK__SHIFT	16
+static inline uint32_t A5XX_RB_STENCILREFMASK_BF_STENCILWRITEMASK(uint32_t val)
+{
+	return ((val) << A5XX_RB_STENCILREFMASK_BF_STENCILWRITEMASK__SHIFT) & A5XX_RB_STENCILREFMASK_BF_STENCILWRITEMASK__MASK;
+}
 
 #define REG_A5XX_RB_WINDOW_OFFSET				0x0000e1d0
 #define A5XX_RB_WINDOW_OFFSET_WINDOW_OFFSET_DISABLE		0x80000000
@@ -3428,6 +3523,7 @@ static inline uint32_t A5XX_RB_RESOLVE_CNTL_2_Y(uint32_t val)
 }
 
 #define REG_A5XX_RB_RESOLVE_CNTL_3				0x0000e213
+#define A5XX_RB_RESOLVE_CNTL_3_TILED				0x00000001
 
 #define REG_A5XX_RB_BLIT_DST_LO					0x0000e214
 
@@ -3459,6 +3555,7 @@ static inline uint32_t A5XX_RB_BLIT_DST_ARRAY_PITCH(uint32_t val)
 
 #define REG_A5XX_RB_CLEAR_CNTL					0x0000e21c
 #define A5XX_RB_CLEAR_CNTL_FAST_CLEAR				0x00000002
+#define A5XX_RB_CLEAR_CNTL_MSAA_RESOLVE				0x00000004
 #define A5XX_RB_CLEAR_CNTL_MASK__MASK				0x000000f0
 #define A5XX_RB_CLEAR_CNTL_MASK__SHIFT				4
 static inline uint32_t A5XX_RB_CLEAR_CNTL_MASK(uint32_t val)
@@ -3627,22 +3724,69 @@ static inline uint32_t A5XX_PC_PRIMITIVE_CNTL_STRIDE_IN_VPC(uint32_t val)
 {
 	return ((val) << A5XX_PC_PRIMITIVE_CNTL_STRIDE_IN_VPC__SHIFT) & A5XX_PC_PRIMITIVE_CNTL_STRIDE_IN_VPC__MASK;
 }
+#define A5XX_PC_PRIMITIVE_CNTL_PRIMITIVE_RESTART		0x00000100
+#define A5XX_PC_PRIMITIVE_CNTL_COUNT_PRIMITIVES			0x00000200
 #define A5XX_PC_PRIMITIVE_CNTL_PROVOKING_VTX_LAST		0x00000400
 
 #define REG_A5XX_PC_PRIM_VTX_CNTL				0x0000e385
 #define A5XX_PC_PRIM_VTX_CNTL_PSIZE				0x00000800
 
 #define REG_A5XX_PC_RASTER_CNTL					0x0000e388
+#define A5XX_PC_RASTER_CNTL_POLYMODE_FRONT_PTYPE__MASK		0x00000007
+#define A5XX_PC_RASTER_CNTL_POLYMODE_FRONT_PTYPE__SHIFT		0
+static inline uint32_t A5XX_PC_RASTER_CNTL_POLYMODE_FRONT_PTYPE(enum adreno_pa_su_sc_draw val)
+{
+	return ((val) << A5XX_PC_RASTER_CNTL_POLYMODE_FRONT_PTYPE__SHIFT) & A5XX_PC_RASTER_CNTL_POLYMODE_FRONT_PTYPE__MASK;
+}
+#define A5XX_PC_RASTER_CNTL_POLYMODE_BACK_PTYPE__MASK		0x00000038
+#define A5XX_PC_RASTER_CNTL_POLYMODE_BACK_PTYPE__SHIFT		3
+static inline uint32_t A5XX_PC_RASTER_CNTL_POLYMODE_BACK_PTYPE(enum adreno_pa_su_sc_draw val)
+{
+	return ((val) << A5XX_PC_RASTER_CNTL_POLYMODE_BACK_PTYPE__SHIFT) & A5XX_PC_RASTER_CNTL_POLYMODE_BACK_PTYPE__MASK;
+}
+#define A5XX_PC_RASTER_CNTL_POLYMODE_ENABLE			0x00000040
 
 #define REG_A5XX_UNKNOWN_E389					0x0000e389
 
 #define REG_A5XX_PC_RESTART_INDEX				0x0000e38c
 
-#define REG_A5XX_UNKNOWN_E38D					0x0000e38d
+#define REG_A5XX_PC_GS_LAYERED					0x0000e38d
 
 #define REG_A5XX_PC_GS_PARAM					0x0000e38e
+#define A5XX_PC_GS_PARAM_MAX_VERTICES__MASK			0x000003ff
+#define A5XX_PC_GS_PARAM_MAX_VERTICES__SHIFT			0
+static inline uint32_t A5XX_PC_GS_PARAM_MAX_VERTICES(uint32_t val)
+{
+	return ((val) << A5XX_PC_GS_PARAM_MAX_VERTICES__SHIFT) & A5XX_PC_GS_PARAM_MAX_VERTICES__MASK;
+}
+#define A5XX_PC_GS_PARAM_INVOCATIONS__MASK			0x0000f800
+#define A5XX_PC_GS_PARAM_INVOCATIONS__SHIFT			11
+static inline uint32_t A5XX_PC_GS_PARAM_INVOCATIONS(uint32_t val)
+{
+	return ((val) << A5XX_PC_GS_PARAM_INVOCATIONS__SHIFT) & A5XX_PC_GS_PARAM_INVOCATIONS__MASK;
+}
+#define A5XX_PC_GS_PARAM_PRIMTYPE__MASK				0x01800000
+#define A5XX_PC_GS_PARAM_PRIMTYPE__SHIFT			23
+static inline uint32_t A5XX_PC_GS_PARAM_PRIMTYPE(enum adreno_pa_su_sc_draw val)
+{
+	return ((val) << A5XX_PC_GS_PARAM_PRIMTYPE__SHIFT) & A5XX_PC_GS_PARAM_PRIMTYPE__MASK;
+}
 
 #define REG_A5XX_PC_HS_PARAM					0x0000e38f
+#define A5XX_PC_HS_PARAM_VERTICES_OUT__MASK			0x0000003f
+#define A5XX_PC_HS_PARAM_VERTICES_OUT__SHIFT			0
+static inline uint32_t A5XX_PC_HS_PARAM_VERTICES_OUT(uint32_t val)
+{
+	return ((val) << A5XX_PC_HS_PARAM_VERTICES_OUT__SHIFT) & A5XX_PC_HS_PARAM_VERTICES_OUT__MASK;
+}
+#define A5XX_PC_HS_PARAM_SPACING__MASK				0x00600000
+#define A5XX_PC_HS_PARAM_SPACING__SHIFT				21
+static inline uint32_t A5XX_PC_HS_PARAM_SPACING(enum a4xx_tess_spacing val)
+{
+	return ((val) << A5XX_PC_HS_PARAM_SPACING__SHIFT) & A5XX_PC_HS_PARAM_SPACING__MASK;
+}
+#define A5XX_PC_HS_PARAM_CW					0x00800000
+#define A5XX_PC_HS_PARAM_CONNECTED				0x01000000
 
 #define REG_A5XX_PC_POWER_CNTL					0x0000e3b0
 
@@ -3667,10 +3811,40 @@ static inline uint32_t A5XX_VFD_CONTROL_1_REGID4INST(uint32_t val)
 {
 	return ((val) << A5XX_VFD_CONTROL_1_REGID4INST__SHIFT) & A5XX_VFD_CONTROL_1_REGID4INST__MASK;
 }
+#define A5XX_VFD_CONTROL_1_REGID4PRIMID__MASK			0x00ff0000
+#define A5XX_VFD_CONTROL_1_REGID4PRIMID__SHIFT			16
+static inline uint32_t A5XX_VFD_CONTROL_1_REGID4PRIMID(uint32_t val)
+{
+	return ((val) << A5XX_VFD_CONTROL_1_REGID4PRIMID__SHIFT) & A5XX_VFD_CONTROL_1_REGID4PRIMID__MASK;
+}
 
 #define REG_A5XX_VFD_CONTROL_2					0x0000e402
+#define A5XX_VFD_CONTROL_2_REGID_PATCHID__MASK			0x000000ff
+#define A5XX_VFD_CONTROL_2_REGID_PATCHID__SHIFT			0
+static inline uint32_t A5XX_VFD_CONTROL_2_REGID_PATCHID(uint32_t val)
+{
+	return ((val) << A5XX_VFD_CONTROL_2_REGID_PATCHID__SHIFT) & A5XX_VFD_CONTROL_2_REGID_PATCHID__MASK;
+}
 
 #define REG_A5XX_VFD_CONTROL_3					0x0000e403
+#define A5XX_VFD_CONTROL_3_REGID_PATCHID__MASK			0x0000ff00
+#define A5XX_VFD_CONTROL_3_REGID_PATCHID__SHIFT			8
+static inline uint32_t A5XX_VFD_CONTROL_3_REGID_PATCHID(uint32_t val)
+{
+	return ((val) << A5XX_VFD_CONTROL_3_REGID_PATCHID__SHIFT) & A5XX_VFD_CONTROL_3_REGID_PATCHID__MASK;
+}
+#define A5XX_VFD_CONTROL_3_REGID_TESSX__MASK			0x00ff0000
+#define A5XX_VFD_CONTROL_3_REGID_TESSX__SHIFT			16
+static inline uint32_t A5XX_VFD_CONTROL_3_REGID_TESSX(uint32_t val)
+{
+	return ((val) << A5XX_VFD_CONTROL_3_REGID_TESSX__SHIFT) & A5XX_VFD_CONTROL_3_REGID_TESSX__MASK;
+}
+#define A5XX_VFD_CONTROL_3_REGID_TESSY__MASK			0xff000000
+#define A5XX_VFD_CONTROL_3_REGID_TESSY__SHIFT			24
+static inline uint32_t A5XX_VFD_CONTROL_3_REGID_TESSY(uint32_t val)
+{
+	return ((val) << A5XX_VFD_CONTROL_3_REGID_TESSY__SHIFT) & A5XX_VFD_CONTROL_3_REGID_TESSY__MASK;
+}
 
 #define REG_A5XX_VFD_CONTROL_4					0x0000e404
 
@@ -3700,12 +3874,18 @@ static inline uint32_t A5XX_VFD_DECODE_INSTR_IDX(uint32_t val)
 	return ((val) << A5XX_VFD_DECODE_INSTR_IDX__SHIFT) & A5XX_VFD_DECODE_INSTR_IDX__MASK;
 }
 #define A5XX_VFD_DECODE_INSTR_INSTANCED				0x00020000
-#define A5XX_VFD_DECODE_INSTR_FORMAT__MASK			0x3ff00000
+#define A5XX_VFD_DECODE_INSTR_FORMAT__MASK			0x0ff00000
 #define A5XX_VFD_DECODE_INSTR_FORMAT__SHIFT			20
 static inline uint32_t A5XX_VFD_DECODE_INSTR_FORMAT(enum a5xx_vtx_fmt val)
 {
 	return ((val) << A5XX_VFD_DECODE_INSTR_FORMAT__SHIFT) & A5XX_VFD_DECODE_INSTR_FORMAT__MASK;
 }
+#define A5XX_VFD_DECODE_INSTR_SWAP__MASK			0x30000000
+#define A5XX_VFD_DECODE_INSTR_SWAP__SHIFT			28
+static inline uint32_t A5XX_VFD_DECODE_INSTR_SWAP(enum a3xx_color_swap val)
+{
+	return ((val) << A5XX_VFD_DECODE_INSTR_SWAP__SHIFT) & A5XX_VFD_DECODE_INSTR_SWAP__MASK;
+}
 #define A5XX_VFD_DECODE_INSTR_UNK30				0x40000000
 #define A5XX_VFD_DECODE_INSTR_FLOAT				0x80000000
 
@@ -3960,6 +4140,7 @@ static inline uint32_t A5XX_SP_FS_CTRL_REG0_BRANCHSTACK(uint32_t val)
 #define REG_A5XX_SP_BLEND_CNTL					0x0000e5c9
 #define A5XX_SP_BLEND_CNTL_ENABLED				0x00000001
 #define A5XX_SP_BLEND_CNTL_UNK8					0x00000100
+#define A5XX_SP_BLEND_CNTL_ALPHA_TO_COVERAGE			0x00000400
 
 #define REG_A5XX_SP_FS_OUTPUT_CNTL				0x0000e5ca
 #define A5XX_SP_FS_OUTPUT_CNTL_MRT__MASK			0x0000000f
@@ -4001,16 +4182,12 @@ static inline uint32_t A5XX_SP_FS_MRT_REG_COLOR_FORMAT(enum a5xx_color_fmt val)
 {
 	return ((val) << A5XX_SP_FS_MRT_REG_COLOR_FORMAT__SHIFT) & A5XX_SP_FS_MRT_REG_COLOR_FORMAT__MASK;
 }
+#define A5XX_SP_FS_MRT_REG_COLOR_SINT				0x00000100
+#define A5XX_SP_FS_MRT_REG_COLOR_UINT				0x00000200
 #define A5XX_SP_FS_MRT_REG_COLOR_SRGB				0x00000400
 
 #define REG_A5XX_UNKNOWN_E5DB					0x0000e5db
 
-#define REG_A5XX_UNKNOWN_E5F2					0x0000e5f2
-
-#define REG_A5XX_SP_CS_OBJ_START_LO				0x0000e5f3
-
-#define REG_A5XX_SP_CS_OBJ_START_HI				0x0000e5f4
-
 #define REG_A5XX_SP_CS_CTRL_REG0				0x0000e5f0
 #define A5XX_SP_CS_CTRL_REG0_THREADSIZE__MASK			0x00000008
 #define A5XX_SP_CS_CTRL_REG0_THREADSIZE__SHIFT			3
@@ -4039,7 +4216,39 @@ static inline uint32_t A5XX_SP_CS_CTRL_REG0_BRANCHSTACK(uint32_t val)
 	return ((val) << A5XX_SP_CS_CTRL_REG0_BRANCHSTACK__SHIFT) & A5XX_SP_CS_CTRL_REG0_BRANCHSTACK__MASK;
 }
 
-#define REG_A5XX_UNKNOWN_E600					0x0000e600
+#define REG_A5XX_UNKNOWN_E5F2					0x0000e5f2
+
+#define REG_A5XX_SP_CS_OBJ_START_LO				0x0000e5f3
+
+#define REG_A5XX_SP_CS_OBJ_START_HI				0x0000e5f4
+
+#define REG_A5XX_SP_HS_CTRL_REG0				0x0000e600
+#define A5XX_SP_HS_CTRL_REG0_THREADSIZE__MASK			0x00000008
+#define A5XX_SP_HS_CTRL_REG0_THREADSIZE__SHIFT			3
+static inline uint32_t A5XX_SP_HS_CTRL_REG0_THREADSIZE(enum a3xx_threadsize val)
+{
+	return ((val) << A5XX_SP_HS_CTRL_REG0_THREADSIZE__SHIFT) & A5XX_SP_HS_CTRL_REG0_THREADSIZE__MASK;
+}
+#define A5XX_SP_HS_CTRL_REG0_HALFREGFOOTPRINT__MASK		0x000003f0
+#define A5XX_SP_HS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT		4
+static inline uint32_t A5XX_SP_HS_CTRL_REG0_HALFREGFOOTPRINT(uint32_t val)
+{
+	return ((val) << A5XX_SP_HS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT) & A5XX_SP_HS_CTRL_REG0_HALFREGFOOTPRINT__MASK;
+}
+#define A5XX_SP_HS_CTRL_REG0_FULLREGFOOTPRINT__MASK		0x0000fc00
+#define A5XX_SP_HS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT		10
+static inline uint32_t A5XX_SP_HS_CTRL_REG0_FULLREGFOOTPRINT(uint32_t val)
+{
+	return ((val) << A5XX_SP_HS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT) & A5XX_SP_HS_CTRL_REG0_FULLREGFOOTPRINT__MASK;
+}
+#define A5XX_SP_HS_CTRL_REG0_VARYING				0x00010000
+#define A5XX_SP_HS_CTRL_REG0_PIXLODENABLE			0x00100000
+#define A5XX_SP_HS_CTRL_REG0_BRANCHSTACK__MASK			0xfe000000
+#define A5XX_SP_HS_CTRL_REG0_BRANCHSTACK__SHIFT			25
+static inline uint32_t A5XX_SP_HS_CTRL_REG0_BRANCHSTACK(uint32_t val)
+{
+	return ((val) << A5XX_SP_HS_CTRL_REG0_BRANCHSTACK__SHIFT) & A5XX_SP_HS_CTRL_REG0_BRANCHSTACK__MASK;
+}
 
 #define REG_A5XX_UNKNOWN_E602					0x0000e602
 
@@ -4047,13 +4256,67 @@ static inline uint32_t A5XX_SP_CS_CTRL_REG0_BRANCHSTACK(uint32_t val)
 
 #define REG_A5XX_SP_HS_OBJ_START_HI				0x0000e604
 
+#define REG_A5XX_SP_DS_CTRL_REG0				0x0000e610
+#define A5XX_SP_DS_CTRL_REG0_THREADSIZE__MASK			0x00000008
+#define A5XX_SP_DS_CTRL_REG0_THREADSIZE__SHIFT			3
+static inline uint32_t A5XX_SP_DS_CTRL_REG0_THREADSIZE(enum a3xx_threadsize val)
+{
+	return ((val) << A5XX_SP_DS_CTRL_REG0_THREADSIZE__SHIFT) & A5XX_SP_DS_CTRL_REG0_THREADSIZE__MASK;
+}
+#define A5XX_SP_DS_CTRL_REG0_HALFREGFOOTPRINT__MASK		0x000003f0
+#define A5XX_SP_DS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT		4
+static inline uint32_t A5XX_SP_DS_CTRL_REG0_HALFREGFOOTPRINT(uint32_t val)
+{
+	return ((val) << A5XX_SP_DS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT) & A5XX_SP_DS_CTRL_REG0_HALFREGFOOTPRINT__MASK;
+}
+#define A5XX_SP_DS_CTRL_REG0_FULLREGFOOTPRINT__MASK		0x0000fc00
+#define A5XX_SP_DS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT		10
+static inline uint32_t A5XX_SP_DS_CTRL_REG0_FULLREGFOOTPRINT(uint32_t val)
+{
+	return ((val) << A5XX_SP_DS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT) & A5XX_SP_DS_CTRL_REG0_FULLREGFOOTPRINT__MASK;
+}
+#define A5XX_SP_DS_CTRL_REG0_VARYING				0x00010000
+#define A5XX_SP_DS_CTRL_REG0_PIXLODENABLE			0x00100000
+#define A5XX_SP_DS_CTRL_REG0_BRANCHSTACK__MASK			0xfe000000
+#define A5XX_SP_DS_CTRL_REG0_BRANCHSTACK__SHIFT			25
+static inline uint32_t A5XX_SP_DS_CTRL_REG0_BRANCHSTACK(uint32_t val)
+{
+	return ((val) << A5XX_SP_DS_CTRL_REG0_BRANCHSTACK__SHIFT) & A5XX_SP_DS_CTRL_REG0_BRANCHSTACK__MASK;
+}
+
 #define REG_A5XX_UNKNOWN_E62B					0x0000e62b
 
 #define REG_A5XX_SP_DS_OBJ_START_LO				0x0000e62c
 
 #define REG_A5XX_SP_DS_OBJ_START_HI				0x0000e62d
 
-#define REG_A5XX_UNKNOWN_E640					0x0000e640
+#define REG_A5XX_SP_GS_CTRL_REG0				0x0000e640
+#define A5XX_SP_GS_CTRL_REG0_THREADSIZE__MASK			0x00000008
+#define A5XX_SP_GS_CTRL_REG0_THREADSIZE__SHIFT			3
+static inline uint32_t A5XX_SP_GS_CTRL_REG0_THREADSIZE(enum a3xx_threadsize val)
+{
+	return ((val) << A5XX_SP_GS_CTRL_REG0_THREADSIZE__SHIFT) & A5XX_SP_GS_CTRL_REG0_THREADSIZE__MASK;
+}
+#define A5XX_SP_GS_CTRL_REG0_HALFREGFOOTPRINT__MASK		0x000003f0
+#define A5XX_SP_GS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT		4
+static inline uint32_t A5XX_SP_GS_CTRL_REG0_HALFREGFOOTPRINT(uint32_t val)
+{
+	return ((val) << A5XX_SP_GS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT) & A5XX_SP_GS_CTRL_REG0_HALFREGFOOTPRINT__MASK;
+}
+#define A5XX_SP_GS_CTRL_REG0_FULLREGFOOTPRINT__MASK		0x0000fc00
+#define A5XX_SP_GS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT		10
+static inline uint32_t A5XX_SP_GS_CTRL_REG0_FULLREGFOOTPRINT(uint32_t val)
+{
+	return ((val) << A5XX_SP_GS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT) & A5XX_SP_GS_CTRL_REG0_FULLREGFOOTPRINT__MASK;
+}
+#define A5XX_SP_GS_CTRL_REG0_VARYING				0x00010000
+#define A5XX_SP_GS_CTRL_REG0_PIXLODENABLE			0x00100000
+#define A5XX_SP_GS_CTRL_REG0_BRANCHSTACK__MASK			0xfe000000
+#define A5XX_SP_GS_CTRL_REG0_BRANCHSTACK__SHIFT			25
+static inline uint32_t A5XX_SP_GS_CTRL_REG0_BRANCHSTACK(uint32_t val)
+{
+	return ((val) << A5XX_SP_GS_CTRL_REG0_BRANCHSTACK__SHIFT) & A5XX_SP_GS_CTRL_REG0_BRANCHSTACK__MASK;
+}
 
 #define REG_A5XX_UNKNOWN_E65B					0x0000e65b
 
@@ -4173,6 +4436,18 @@ static inline uint32_t A5XX_HLSQ_CONTROL_2_REG_FACEREGID(uint32_t val)
 {
 	return ((val) << A5XX_HLSQ_CONTROL_2_REG_FACEREGID__SHIFT) & A5XX_HLSQ_CONTROL_2_REG_FACEREGID__MASK;
 }
+#define A5XX_HLSQ_CONTROL_2_REG_SAMPLEID__MASK			0x0000ff00
+#define A5XX_HLSQ_CONTROL_2_REG_SAMPLEID__SHIFT			8
+static inline uint32_t A5XX_HLSQ_CONTROL_2_REG_SAMPLEID(uint32_t val)
+{
+	return ((val) << A5XX_HLSQ_CONTROL_2_REG_SAMPLEID__SHIFT) & A5XX_HLSQ_CONTROL_2_REG_SAMPLEID__MASK;
+}
+#define A5XX_HLSQ_CONTROL_2_REG_SAMPLEMASK__MASK		0x00ff0000
+#define A5XX_HLSQ_CONTROL_2_REG_SAMPLEMASK__SHIFT		16
+static inline uint32_t A5XX_HLSQ_CONTROL_2_REG_SAMPLEMASK(uint32_t val)
+{
+	return ((val) << A5XX_HLSQ_CONTROL_2_REG_SAMPLEMASK__SHIFT) & A5XX_HLSQ_CONTROL_2_REG_SAMPLEMASK__MASK;
+}
 
 #define REG_A5XX_HLSQ_CONTROL_3_REG				0x0000e787
 #define A5XX_HLSQ_CONTROL_3_REG_FRAGCOORDXYREGID__MASK		0x000000ff
@@ -4375,34 +4650,52 @@ static inline uint32_t A5XX_HLSQ_CS_NDRANGE_0_LOCALSIZEZ(uint32_t val)
 }
 
 #define REG_A5XX_HLSQ_CS_NDRANGE_1				0x0000e7b1
-#define A5XX_HLSQ_CS_NDRANGE_1_SIZE_X__MASK			0xffffffff
-#define A5XX_HLSQ_CS_NDRANGE_1_SIZE_X__SHIFT			0
-static inline uint32_t A5XX_HLSQ_CS_NDRANGE_1_SIZE_X(uint32_t val)
+#define A5XX_HLSQ_CS_NDRANGE_1_GLOBALSIZE_X__MASK		0xffffffff
+#define A5XX_HLSQ_CS_NDRANGE_1_GLOBALSIZE_X__SHIFT		0
+static inline uint32_t A5XX_HLSQ_CS_NDRANGE_1_GLOBALSIZE_X(uint32_t val)
 {
-	return ((val) << A5XX_HLSQ_CS_NDRANGE_1_SIZE_X__SHIFT) & A5XX_HLSQ_CS_NDRANGE_1_SIZE_X__MASK;
+	return ((val) << A5XX_HLSQ_CS_NDRANGE_1_GLOBALSIZE_X__SHIFT) & A5XX_HLSQ_CS_NDRANGE_1_GLOBALSIZE_X__MASK;
 }
 
 #define REG_A5XX_HLSQ_CS_NDRANGE_2				0x0000e7b2
+#define A5XX_HLSQ_CS_NDRANGE_2_GLOBALOFF_X__MASK		0xffffffff
+#define A5XX_HLSQ_CS_NDRANGE_2_GLOBALOFF_X__SHIFT		0
+static inline uint32_t A5XX_HLSQ_CS_NDRANGE_2_GLOBALOFF_X(uint32_t val)
+{
+	return ((val) << A5XX_HLSQ_CS_NDRANGE_2_GLOBALOFF_X__SHIFT) & A5XX_HLSQ_CS_NDRANGE_2_GLOBALOFF_X__MASK;
+}
 
 #define REG_A5XX_HLSQ_CS_NDRANGE_3				0x0000e7b3
-#define A5XX_HLSQ_CS_NDRANGE_3_SIZE_Y__MASK			0xffffffff
-#define A5XX_HLSQ_CS_NDRANGE_3_SIZE_Y__SHIFT			0
-static inline uint32_t A5XX_HLSQ_CS_NDRANGE_3_SIZE_Y(uint32_t val)
+#define A5XX_HLSQ_CS_NDRANGE_3_GLOBALSIZE_Y__MASK		0xffffffff
+#define A5XX_HLSQ_CS_NDRANGE_3_GLOBALSIZE_Y__SHIFT		0
+static inline uint32_t A5XX_HLSQ_CS_NDRANGE_3_GLOBALSIZE_Y(uint32_t val)
 {
-	return ((val) << A5XX_HLSQ_CS_NDRANGE_3_SIZE_Y__SHIFT) & A5XX_HLSQ_CS_NDRANGE_3_SIZE_Y__MASK;
+	return ((val) << A5XX_HLSQ_CS_NDRANGE_3_GLOBALSIZE_Y__SHIFT) & A5XX_HLSQ_CS_NDRANGE_3_GLOBALSIZE_Y__MASK;
 }
 
 #define REG_A5XX_HLSQ_CS_NDRANGE_4				0x0000e7b4
+#define A5XX_HLSQ_CS_NDRANGE_4_GLOBALOFF_Y__MASK		0xffffffff
+#define A5XX_HLSQ_CS_NDRANGE_4_GLOBALOFF_Y__SHIFT		0
+static inline uint32_t A5XX_HLSQ_CS_NDRANGE_4_GLOBALOFF_Y(uint32_t val)
+{
+	return ((val) << A5XX_HLSQ_CS_NDRANGE_4_GLOBALOFF_Y__SHIFT) & A5XX_HLSQ_CS_NDRANGE_4_GLOBALOFF_Y__MASK;
+}
 
 #define REG_A5XX_HLSQ_CS_NDRANGE_5				0x0000e7b5
-#define A5XX_HLSQ_CS_NDRANGE_5_SIZE_Z__MASK			0xffffffff
-#define A5XX_HLSQ_CS_NDRANGE_5_SIZE_Z__SHIFT			0
-static inline uint32_t A5XX_HLSQ_CS_NDRANGE_5_SIZE_Z(uint32_t val)
+#define A5XX_HLSQ_CS_NDRANGE_5_GLOBALSIZE_Z__MASK		0xffffffff
+#define A5XX_HLSQ_CS_NDRANGE_5_GLOBALSIZE_Z__SHIFT		0
+static inline uint32_t A5XX_HLSQ_CS_NDRANGE_5_GLOBALSIZE_Z(uint32_t val)
 {
-	return ((val) << A5XX_HLSQ_CS_NDRANGE_5_SIZE_Z__SHIFT) & A5XX_HLSQ_CS_NDRANGE_5_SIZE_Z__MASK;
+	return ((val) << A5XX_HLSQ_CS_NDRANGE_5_GLOBALSIZE_Z__SHIFT) & A5XX_HLSQ_CS_NDRANGE_5_GLOBALSIZE_Z__MASK;
 }
 
 #define REG_A5XX_HLSQ_CS_NDRANGE_6				0x0000e7b6
+#define A5XX_HLSQ_CS_NDRANGE_6_GLOBALOFF_Z__MASK		0xffffffff
+#define A5XX_HLSQ_CS_NDRANGE_6_GLOBALOFF_Z__SHIFT		0
+static inline uint32_t A5XX_HLSQ_CS_NDRANGE_6_GLOBALOFF_Z(uint32_t val)
+{
+	return ((val) << A5XX_HLSQ_CS_NDRANGE_6_GLOBALOFF_Z__SHIFT) & A5XX_HLSQ_CS_NDRANGE_6_GLOBALOFF_Z__MASK;
+}
 
 #define REG_A5XX_HLSQ_CS_CNTL_0					0x0000e7b7
 #define A5XX_HLSQ_CS_CNTL_0_WGIDCONSTID__MASK			0x000000ff
@@ -4468,6 +4761,8 @@ static inline uint32_t A5XX_HLSQ_CS_CNTL_0_LOCALIDREGID(uint32_t val)
 
 #define REG_A5XX_HLSQ_CS_INSTRLEN				0x0000e7dd
 
+#define REG_A5XX_RB_2D_BLIT_CNTL				0x00002100
+
 #define REG_A5XX_RB_2D_SRC_SOLID_DW0				0x00002101
 
 #define REG_A5XX_RB_2D_SRC_SOLID_DW1				0x00002102
@@ -4483,12 +4778,19 @@ static inline uint32_t A5XX_RB_2D_SRC_INFO_COLOR_FORMAT(enum a5xx_color_fmt val)
 {
 	return ((val) << A5XX_RB_2D_SRC_INFO_COLOR_FORMAT__SHIFT) & A5XX_RB_2D_SRC_INFO_COLOR_FORMAT__MASK;
 }
+#define A5XX_RB_2D_SRC_INFO_TILE_MODE__MASK			0x00000300
+#define A5XX_RB_2D_SRC_INFO_TILE_MODE__SHIFT			8
+static inline uint32_t A5XX_RB_2D_SRC_INFO_TILE_MODE(enum a5xx_tile_mode val)
+{
+	return ((val) << A5XX_RB_2D_SRC_INFO_TILE_MODE__SHIFT) & A5XX_RB_2D_SRC_INFO_TILE_MODE__MASK;
+}
 #define A5XX_RB_2D_SRC_INFO_COLOR_SWAP__MASK			0x00000c00
 #define A5XX_RB_2D_SRC_INFO_COLOR_SWAP__SHIFT			10
 static inline uint32_t A5XX_RB_2D_SRC_INFO_COLOR_SWAP(enum a3xx_color_swap val)
 {
 	return ((val) << A5XX_RB_2D_SRC_INFO_COLOR_SWAP__SHIFT) & A5XX_RB_2D_SRC_INFO_COLOR_SWAP__MASK;
 }
+#define A5XX_RB_2D_SRC_INFO_FLAGS				0x00001000
 
 #define REG_A5XX_RB_2D_SRC_LO					0x00002108
 
@@ -4515,12 +4817,19 @@ static inline uint32_t A5XX_RB_2D_DST_INFO_COLOR_FORMAT(enum a5xx_color_fmt val)
 {
 	return ((val) << A5XX_RB_2D_DST_INFO_COLOR_FORMAT__SHIFT) & A5XX_RB_2D_DST_INFO_COLOR_FORMAT__MASK;
 }
+#define A5XX_RB_2D_DST_INFO_TILE_MODE__MASK			0x00000300
+#define A5XX_RB_2D_DST_INFO_TILE_MODE__SHIFT			8
+static inline uint32_t A5XX_RB_2D_DST_INFO_TILE_MODE(enum a5xx_tile_mode val)
+{
+	return ((val) << A5XX_RB_2D_DST_INFO_TILE_MODE__SHIFT) & A5XX_RB_2D_DST_INFO_TILE_MODE__MASK;
+}
 #define A5XX_RB_2D_DST_INFO_COLOR_SWAP__MASK			0x00000c00
 #define A5XX_RB_2D_DST_INFO_COLOR_SWAP__SHIFT			10
 static inline uint32_t A5XX_RB_2D_DST_INFO_COLOR_SWAP(enum a3xx_color_swap val)
 {
 	return ((val) << A5XX_RB_2D_DST_INFO_COLOR_SWAP__SHIFT) & A5XX_RB_2D_DST_INFO_COLOR_SWAP__MASK;
 }
+#define A5XX_RB_2D_DST_INFO_FLAGS				0x00001000
 
 #define REG_A5XX_RB_2D_DST_LO					0x00002111
 
@@ -4548,6 +4857,8 @@ static inline uint32_t A5XX_RB_2D_DST_SIZE_ARRAY_PITCH(uint32_t val)
 
 #define REG_A5XX_RB_2D_DST_FLAGS_HI				0x00002144
 
+#define REG_A5XX_GRAS_2D_BLIT_CNTL				0x00002180
+
 #define REG_A5XX_GRAS_2D_SRC_INFO				0x00002181
 #define A5XX_GRAS_2D_SRC_INFO_COLOR_FORMAT__MASK		0x000000ff
 #define A5XX_GRAS_2D_SRC_INFO_COLOR_FORMAT__SHIFT		0
@@ -4555,12 +4866,19 @@ static inline uint32_t A5XX_GRAS_2D_SRC_INFO_COLOR_FORMAT(enum a5xx_color_fmt va
 {
 	return ((val) << A5XX_GRAS_2D_SRC_INFO_COLOR_FORMAT__SHIFT) & A5XX_GRAS_2D_SRC_INFO_COLOR_FORMAT__MASK;
 }
+#define A5XX_GRAS_2D_SRC_INFO_TILE_MODE__MASK			0x00000300
+#define A5XX_GRAS_2D_SRC_INFO_TILE_MODE__SHIFT			8
+static inline uint32_t A5XX_GRAS_2D_SRC_INFO_TILE_MODE(enum a5xx_tile_mode val)
+{
+	return ((val) << A5XX_GRAS_2D_SRC_INFO_TILE_MODE__SHIFT) & A5XX_GRAS_2D_SRC_INFO_TILE_MODE__MASK;
+}
 #define A5XX_GRAS_2D_SRC_INFO_COLOR_SWAP__MASK			0x00000c00
 #define A5XX_GRAS_2D_SRC_INFO_COLOR_SWAP__SHIFT			10
 static inline uint32_t A5XX_GRAS_2D_SRC_INFO_COLOR_SWAP(enum a3xx_color_swap val)
 {
 	return ((val) << A5XX_GRAS_2D_SRC_INFO_COLOR_SWAP__SHIFT) & A5XX_GRAS_2D_SRC_INFO_COLOR_SWAP__MASK;
 }
+#define A5XX_GRAS_2D_SRC_INFO_FLAGS				0x00001000
 
 #define REG_A5XX_GRAS_2D_DST_INFO				0x00002182
 #define A5XX_GRAS_2D_DST_INFO_COLOR_FORMAT__MASK		0x000000ff
@@ -4569,12 +4887,19 @@ static inline uint32_t A5XX_GRAS_2D_DST_INFO_COLOR_FORMAT(enum a5xx_color_fmt va
 {
 	return ((val) << A5XX_GRAS_2D_DST_INFO_COLOR_FORMAT__SHIFT) & A5XX_GRAS_2D_DST_INFO_COLOR_FORMAT__MASK;
 }
+#define A5XX_GRAS_2D_DST_INFO_TILE_MODE__MASK			0x00000300
+#define A5XX_GRAS_2D_DST_INFO_TILE_MODE__SHIFT			8
+static inline uint32_t A5XX_GRAS_2D_DST_INFO_TILE_MODE(enum a5xx_tile_mode val)
+{
+	return ((val) << A5XX_GRAS_2D_DST_INFO_TILE_MODE__SHIFT) & A5XX_GRAS_2D_DST_INFO_TILE_MODE__MASK;
+}
 #define A5XX_GRAS_2D_DST_INFO_COLOR_SWAP__MASK			0x00000c00
 #define A5XX_GRAS_2D_DST_INFO_COLOR_SWAP__SHIFT			10
 static inline uint32_t A5XX_GRAS_2D_DST_INFO_COLOR_SWAP(enum a3xx_color_swap val)
 {
 	return ((val) << A5XX_GRAS_2D_DST_INFO_COLOR_SWAP__SHIFT) & A5XX_GRAS_2D_DST_INFO_COLOR_SWAP__MASK;
 }
+#define A5XX_GRAS_2D_DST_INFO_FLAGS				0x00001000
 
 #define REG_A5XX_UNKNOWN_2100					0x00002100
 
@@ -4698,6 +5023,12 @@ static inline uint32_t A5XX_TEX_CONST_0_MIPLVLS(uint32_t val)
 {
 	return ((val) << A5XX_TEX_CONST_0_MIPLVLS__SHIFT) & A5XX_TEX_CONST_0_MIPLVLS__MASK;
 }
+#define A5XX_TEX_CONST_0_SAMPLES__MASK				0x00300000
+#define A5XX_TEX_CONST_0_SAMPLES__SHIFT				20
+static inline uint32_t A5XX_TEX_CONST_0_SAMPLES(enum a3xx_msaa_samples val)
+{
+	return ((val) << A5XX_TEX_CONST_0_SAMPLES__SHIFT) & A5XX_TEX_CONST_0_SAMPLES__MASK;
+}
 #define A5XX_TEX_CONST_0_FMT__MASK				0x3fc00000
 #define A5XX_TEX_CONST_0_FMT__SHIFT				22
 static inline uint32_t A5XX_TEX_CONST_0_FMT(enum a5xx_tex_fmt val)
@@ -4788,5 +5119,81 @@ static inline uint32_t A5XX_TEX_CONST_5_DEPTH(uint32_t val)
 
 #define REG_A5XX_TEX_CONST_11					0x0000000b
 
+#define REG_A5XX_SSBO_0_0					0x00000000
+#define A5XX_SSBO_0_0_BASE_LO__MASK				0xffffffe0
+#define A5XX_SSBO_0_0_BASE_LO__SHIFT				5
+static inline uint32_t A5XX_SSBO_0_0_BASE_LO(uint32_t val)
+{
+	return ((val >> 5) << A5XX_SSBO_0_0_BASE_LO__SHIFT) & A5XX_SSBO_0_0_BASE_LO__MASK;
+}
+
+#define REG_A5XX_SSBO_0_1					0x00000001
+#define A5XX_SSBO_0_1_PITCH__MASK				0x003fffff
+#define A5XX_SSBO_0_1_PITCH__SHIFT				0
+static inline uint32_t A5XX_SSBO_0_1_PITCH(uint32_t val)
+{
+	return ((val) << A5XX_SSBO_0_1_PITCH__SHIFT) & A5XX_SSBO_0_1_PITCH__MASK;
+}
+
+#define REG_A5XX_SSBO_0_2					0x00000002
+#define A5XX_SSBO_0_2_ARRAY_PITCH__MASK				0x03fff000
+#define A5XX_SSBO_0_2_ARRAY_PITCH__SHIFT			12
+static inline uint32_t A5XX_SSBO_0_2_ARRAY_PITCH(uint32_t val)
+{
+	return ((val >> 12) << A5XX_SSBO_0_2_ARRAY_PITCH__SHIFT) & A5XX_SSBO_0_2_ARRAY_PITCH__MASK;
+}
+
+#define REG_A5XX_SSBO_0_3					0x00000003
+#define A5XX_SSBO_0_3_CPP__MASK					0x0000003f
+#define A5XX_SSBO_0_3_CPP__SHIFT				0
+static inline uint32_t A5XX_SSBO_0_3_CPP(uint32_t val)
+{
+	return ((val) << A5XX_SSBO_0_3_CPP__SHIFT) & A5XX_SSBO_0_3_CPP__MASK;
+}
+
+#define REG_A5XX_SSBO_1_0					0x00000000
+#define A5XX_SSBO_1_0_FMT__MASK					0x0000ff00
+#define A5XX_SSBO_1_0_FMT__SHIFT				8
+static inline uint32_t A5XX_SSBO_1_0_FMT(enum a5xx_tex_fmt val)
+{
+	return ((val) << A5XX_SSBO_1_0_FMT__SHIFT) & A5XX_SSBO_1_0_FMT__MASK;
+}
+#define A5XX_SSBO_1_0_WIDTH__MASK				0xffff0000
+#define A5XX_SSBO_1_0_WIDTH__SHIFT				16
+static inline uint32_t A5XX_SSBO_1_0_WIDTH(uint32_t val)
+{
+	return ((val) << A5XX_SSBO_1_0_WIDTH__SHIFT) & A5XX_SSBO_1_0_WIDTH__MASK;
+}
+
+#define REG_A5XX_SSBO_1_1					0x00000001
+#define A5XX_SSBO_1_1_HEIGHT__MASK				0x0000ffff
+#define A5XX_SSBO_1_1_HEIGHT__SHIFT				0
+static inline uint32_t A5XX_SSBO_1_1_HEIGHT(uint32_t val)
+{
+	return ((val) << A5XX_SSBO_1_1_HEIGHT__SHIFT) & A5XX_SSBO_1_1_HEIGHT__MASK;
+}
+#define A5XX_SSBO_1_1_DEPTH__MASK				0xffff0000
+#define A5XX_SSBO_1_1_DEPTH__SHIFT				16
+static inline uint32_t A5XX_SSBO_1_1_DEPTH(uint32_t val)
+{
+	return ((val) << A5XX_SSBO_1_1_DEPTH__SHIFT) & A5XX_SSBO_1_1_DEPTH__MASK;
+}
+
+#define REG_A5XX_SSBO_2_0					0x00000000
+#define A5XX_SSBO_2_0_BASE_LO__MASK				0xffffffff
+#define A5XX_SSBO_2_0_BASE_LO__SHIFT				0
+static inline uint32_t A5XX_SSBO_2_0_BASE_LO(uint32_t val)
+{
+	return ((val) << A5XX_SSBO_2_0_BASE_LO__SHIFT) & A5XX_SSBO_2_0_BASE_LO__MASK;
+}
+
+#define REG_A5XX_SSBO_2_1					0x00000001
+#define A5XX_SSBO_2_1_BASE_HI__MASK				0xffffffff
+#define A5XX_SSBO_2_1_BASE_HI__SHIFT				0
+static inline uint32_t A5XX_SSBO_2_1_BASE_HI(uint32_t val)
+{
+	return ((val) << A5XX_SSBO_2_1_BASE_HI__SHIFT) & A5XX_SSBO_2_1_BASE_HI__MASK;
+}
+
 
 #endif /* A5XX_XML */
diff --git a/drivers/gpu/drm/msm/adreno/a5xx_gpu.c b/drivers/gpu/drm/msm/adreno/a5xx_gpu.c
index d39400e5bc42..ab1d9308c311 100644
--- a/drivers/gpu/drm/msm/adreno/a5xx_gpu.c
+++ b/drivers/gpu/drm/msm/adreno/a5xx_gpu.c
@@ -11,6 +11,7 @@
  *
  */
 
+#include <linux/kernel.h>
 #include <linux/types.h>
 #include <linux/cpumask.h>
 #include <linux/qcom_scm.h>
@@ -19,6 +20,8 @@
 #include <linux/soc/qcom/mdt_loader.h>
 #include <linux/pm_opp.h>
 #include <linux/nvmem-consumer.h>
+#include <linux/iopoll.h>
+#include <linux/slab.h>
 #include "msm_gem.h"
 #include "msm_mmu.h"
 #include "a5xx_gpu.h"
@@ -91,12 +94,13 @@ static int zap_shader_load_mdt(struct msm_gpu *gpu, const char *fwname)
 		ret = qcom_mdt_load(dev, fw, fwname, GPU_PAS_ID,
 				mem_region, mem_phys, mem_size, NULL);
 	} else {
-		char newname[strlen("qcom/") + strlen(fwname) + 1];
+		char *newname;
 
-		sprintf(newname, "qcom/%s", fwname);
+		newname = kasprintf(GFP_KERNEL, "qcom/%s", fwname);
 
 		ret = qcom_mdt_load(dev, fw, newname, GPU_PAS_ID,
 				mem_region, mem_phys, mem_size, NULL);
+		kfree(newname);
 	}
 	if (ret)
 		goto out;
@@ -1123,8 +1127,9 @@ static const u32 a5xx_registers[] = {
 	0xE800, 0xE806, 0xE810, 0xE89A, 0xE8A0, 0xE8A4, 0xE8AA, 0xE8EB,
 	0xE900, 0xE905, 0xEB80, 0xEB8F, 0xEBB0, 0xEBB0, 0xEC00, 0xEC05,
 	0xEC08, 0xECE9, 0xECF0, 0xECF0, 0xEA80, 0xEA80, 0xEA82, 0xEAA3,
-	0xEAA5, 0xEAC2, 0xA800, 0xA8FF, 0xAC60, 0xAC60, 0xB000, 0xB97F,
-	0xB9A0, 0xB9BF, ~0
+	0xEAA5, 0xEAC2, 0xA800, 0xA800, 0xA820, 0xA828, 0xA840, 0xA87D,
+	0XA880, 0xA88D, 0xA890, 0xA8A3, 0xA8D0, 0xA8D8, 0xA8E0, 0xA8F5,
+	0xAC60, 0xAC60, ~0,
 };
 
 static void a5xx_dump(struct msm_gpu *gpu)
@@ -1195,19 +1200,231 @@ static int a5xx_get_timestamp(struct msm_gpu *gpu, uint64_t *value)
 	return 0;
 }
 
-#ifdef CONFIG_DEBUG_FS
-static void a5xx_show(struct msm_gpu *gpu, struct seq_file *m)
+struct a5xx_crashdumper {
+	void *ptr;
+	struct drm_gem_object *bo;
+	u64 iova;
+};
+
+struct a5xx_gpu_state {
+	struct msm_gpu_state base;
+	u32 *hlsqregs;
+};
+
+#define gpu_poll_timeout(gpu, addr, val, cond, interval, timeout) \
+	readl_poll_timeout((gpu)->mmio + ((addr) << 2), val, cond, \
+		interval, timeout)
+
+static int a5xx_crashdumper_init(struct msm_gpu *gpu,
+		struct a5xx_crashdumper *dumper)
 {
-	seq_printf(m, "status:   %08x\n",
-			gpu_read(gpu, REG_A5XX_RBBM_STATUS));
+	dumper->ptr = msm_gem_kernel_new_locked(gpu->dev,
+		SZ_1M, MSM_BO_UNCACHED, gpu->aspace,
+		&dumper->bo, &dumper->iova);
 
-	/*
-	 * Temporarily disable hardware clock gating before going into
-	 * adreno_show to avoid issues while reading the registers
-	 */
+	if (IS_ERR(dumper->ptr))
+		return PTR_ERR(dumper->ptr);
+
+	return 0;
+}
+
+static void a5xx_crashdumper_free(struct msm_gpu *gpu,
+		struct a5xx_crashdumper *dumper)
+{
+	msm_gem_put_iova(dumper->bo, gpu->aspace);
+	msm_gem_put_vaddr(dumper->bo);
+
+	drm_gem_object_unreference(dumper->bo);
+}
+
+static int a5xx_crashdumper_run(struct msm_gpu *gpu,
+		struct a5xx_crashdumper *dumper)
+{
+	u32 val;
+
+	if (IS_ERR_OR_NULL(dumper->ptr))
+		return -EINVAL;
+
+	gpu_write64(gpu, REG_A5XX_CP_CRASH_SCRIPT_BASE_LO,
+		REG_A5XX_CP_CRASH_SCRIPT_BASE_HI, dumper->iova);
+
+	gpu_write(gpu, REG_A5XX_CP_CRASH_DUMP_CNTL, 1);
+
+	return gpu_poll_timeout(gpu, REG_A5XX_CP_CRASH_DUMP_CNTL, val,
+		val & 0x04, 100, 10000);
+}
+
+/*
+ * These are a list of the registers that need to be read through the HLSQ
+ * aperture through the crashdumper.  These are not nominally accessible from
+ * the CPU on a secure platform.
+ */
+static const struct {
+	u32 type;
+	u32 regoffset;
+	u32 count;
+} a5xx_hlsq_aperture_regs[] = {
+	{ 0x35, 0xe00, 0x32 },   /* HSLQ non-context */
+	{ 0x31, 0x2080, 0x1 },   /* HLSQ 2D context 0 */
+	{ 0x33, 0x2480, 0x1 },   /* HLSQ 2D context 1 */
+	{ 0x32, 0xe780, 0x62 },  /* HLSQ 3D context 0 */
+	{ 0x34, 0xef80, 0x62 },  /* HLSQ 3D context 1 */
+	{ 0x3f, 0x0ec0, 0x40 },  /* SP non-context */
+	{ 0x3d, 0x2040, 0x1 },   /* SP 2D context 0 */
+	{ 0x3b, 0x2440, 0x1 },   /* SP 2D context 1 */
+	{ 0x3e, 0xe580, 0x170 }, /* SP 3D context 0 */
+	{ 0x3c, 0xed80, 0x170 }, /* SP 3D context 1 */
+	{ 0x3a, 0x0f00, 0x1c },  /* TP non-context */
+	{ 0x38, 0x2000, 0xa },   /* TP 2D context 0 */
+	{ 0x36, 0x2400, 0xa },   /* TP 2D context 1 */
+	{ 0x39, 0xe700, 0x80 },  /* TP 3D context 0 */
+	{ 0x37, 0xef00, 0x80 },  /* TP 3D context 1 */
+};
+
+static void a5xx_gpu_state_get_hlsq_regs(struct msm_gpu *gpu,
+		struct a5xx_gpu_state *a5xx_state)
+{
+	struct a5xx_crashdumper dumper = { 0 };
+	u32 offset, count = 0;
+	u64 *ptr;
+	int i;
+
+	if (a5xx_crashdumper_init(gpu, &dumper))
+		return;
+
+	/* The script will be written at offset 0 */
+	ptr = dumper.ptr;
+
+	/* Start writing the data at offset 256k */
+	offset = dumper.iova + (256 * SZ_1K);
+
+	/* Count how many additional registers to get from the HLSQ aperture */
+	for (i = 0; i < ARRAY_SIZE(a5xx_hlsq_aperture_regs); i++)
+		count += a5xx_hlsq_aperture_regs[i].count;
+
+	a5xx_state->hlsqregs = kcalloc(count, sizeof(u32), GFP_KERNEL);
+	if (!a5xx_state->hlsqregs)
+		return;
+
+	/* Build the crashdump script */
+	for (i = 0; i < ARRAY_SIZE(a5xx_hlsq_aperture_regs); i++) {
+		u32 type = a5xx_hlsq_aperture_regs[i].type;
+		u32 c = a5xx_hlsq_aperture_regs[i].count;
+
+		/* Write the register to select the desired bank */
+		*ptr++ = ((u64) type << 8);
+		*ptr++ = (((u64) REG_A5XX_HLSQ_DBG_READ_SEL) << 44) |
+			(1 << 21) | 1;
+
+		*ptr++ = offset;
+		*ptr++ = (((u64) REG_A5XX_HLSQ_DBG_AHB_READ_APERTURE) << 44)
+			| c;
+
+		offset += c * sizeof(u32);
+	}
+
+	/* Write two zeros to close off the script */
+	*ptr++ = 0;
+	*ptr++ = 0;
+
+	if (a5xx_crashdumper_run(gpu, &dumper)) {
+		kfree(a5xx_state->hlsqregs);
+		a5xx_crashdumper_free(gpu, &dumper);
+		return;
+	}
+
+	/* Copy the data from the crashdumper to the state */
+	memcpy(a5xx_state->hlsqregs, dumper.ptr + (256 * SZ_1K),
+		count * sizeof(u32));
+
+	a5xx_crashdumper_free(gpu, &dumper);
+}
+
+static struct msm_gpu_state *a5xx_gpu_state_get(struct msm_gpu *gpu)
+{
+	struct a5xx_gpu_state *a5xx_state = kzalloc(sizeof(*a5xx_state),
+			GFP_KERNEL);
+
+	if (!a5xx_state)
+		return ERR_PTR(-ENOMEM);
+
+	/* Temporarily disable hardware clock gating before reading the hw */
 	a5xx_set_hwcg(gpu, false);
-	adreno_show(gpu, m);
+
+	/* First get the generic state from the adreno core */
+	adreno_gpu_state_get(gpu, &(a5xx_state->base));
+
+	a5xx_state->base.rbbm_status = gpu_read(gpu, REG_A5XX_RBBM_STATUS);
+
+	/* Get the HLSQ regs with the help of the crashdumper */
+	a5xx_gpu_state_get_hlsq_regs(gpu, a5xx_state);
+
 	a5xx_set_hwcg(gpu, true);
+
+	return &a5xx_state->base;
+}
+
+static void a5xx_gpu_state_destroy(struct kref *kref)
+{
+	struct msm_gpu_state *state = container_of(kref,
+		struct msm_gpu_state, ref);
+	struct a5xx_gpu_state *a5xx_state = container_of(state,
+		struct a5xx_gpu_state, base);
+
+	kfree(a5xx_state->hlsqregs);
+
+	adreno_gpu_state_destroy(state);
+	kfree(a5xx_state);
+}
+
+int a5xx_gpu_state_put(struct msm_gpu_state *state)
+{
+	if (IS_ERR_OR_NULL(state))
+		return 1;
+
+	return kref_put(&state->ref, a5xx_gpu_state_destroy);
+}
+
+
+#if defined(CONFIG_DEBUG_FS) || defined(CONFIG_DEV_COREDUMP)
+void a5xx_show(struct msm_gpu *gpu, struct msm_gpu_state *state,
+		struct drm_printer *p)
+{
+	int i, j;
+	u32 pos = 0;
+	struct a5xx_gpu_state *a5xx_state = container_of(state,
+		struct a5xx_gpu_state, base);
+
+	if (IS_ERR_OR_NULL(state))
+		return;
+
+	adreno_show(gpu, state, p);
+
+	/* Dump the additional a5xx HLSQ registers */
+	if (!a5xx_state->hlsqregs)
+		return;
+
+	drm_printf(p, "registers-hlsq:\n");
+
+	for (i = 0; i < ARRAY_SIZE(a5xx_hlsq_aperture_regs); i++) {
+		u32 o = a5xx_hlsq_aperture_regs[i].regoffset;
+		u32 c = a5xx_hlsq_aperture_regs[i].count;
+
+		for (j = 0; j < c; j++, pos++, o++) {
+			/*
+			 * To keep the crashdump simple we pull the entire range
+			 * for each register type but not all of the registers
+			 * in the range are valid. Fortunately invalid registers
+			 * stick out like a sore thumb with a value of
+			 * 0xdeadbeef
+			 */
+			if (a5xx_state->hlsqregs[pos] == 0xdeadbeef)
+				continue;
+
+			drm_printf(p, "  - { offset: 0x%04x, value: 0x%08x }\n",
+				o << 2, a5xx_state->hlsqregs[pos]);
+		}
+	}
 }
 #endif
 
@@ -1239,11 +1456,15 @@ static const struct adreno_gpu_funcs funcs = {
 		.active_ring = a5xx_active_ring,
 		.irq = a5xx_irq,
 		.destroy = a5xx_destroy,
-#ifdef CONFIG_DEBUG_FS
+#if defined(CONFIG_DEBUG_FS) || defined(CONFIG_DEV_COREDUMP)
 		.show = a5xx_show,
+#endif
+#if defined(CONFIG_DEBUG_FS)
 		.debugfs_init = a5xx_debugfs_init,
 #endif
 		.gpu_busy = a5xx_gpu_busy,
+		.gpu_state_get = a5xx_gpu_state_get,
+		.gpu_state_put = a5xx_gpu_state_put,
 	},
 	.get_timestamp = a5xx_get_timestamp,
 };
diff --git a/drivers/gpu/drm/msm/adreno/a6xx.xml.h b/drivers/gpu/drm/msm/adreno/a6xx.xml.h
new file mode 100644
index 000000000000..87eab51f7000
--- /dev/null
+++ b/drivers/gpu/drm/msm/adreno/a6xx.xml.h
@@ -0,0 +1,4562 @@
+#ifndef A6XX_XML
+#define A6XX_XML
+
+/* Autogenerated file, DO NOT EDIT manually!
+
+This file was generated by the rules-ng-ng headergen tool in this git repository:
+http://github.com/freedreno/envytools/
+git clone https://github.com/freedreno/envytools.git
+
+The rules-ng-ng source files this header was generated from are:
+- /home/robclark/src/envytools/rnndb/adreno.xml               (    501 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/freedreno_copyright.xml  (   1572 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/adreno/a2xx.xml          (  36805 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/adreno/adreno_common.xml (  13634 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/adreno/adreno_pm4.xml    (  42393 bytes, from 2018-08-06 18:45:45)
+- /home/robclark/src/envytools/rnndb/adreno/a3xx.xml          (  83840 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/adreno/a4xx.xml          ( 112086 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/adreno/a5xx.xml          ( 147240 bytes, from 2018-08-06 18:45:45)
+- /home/robclark/src/envytools/rnndb/adreno/a6xx.xml          ( 101627 bytes, from 2018-08-06 18:45:45)
+- /home/robclark/src/envytools/rnndb/adreno/a6xx_gmu.xml      (  10431 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/adreno/ocmem.xml         (   1773 bytes, from 2018-07-03 19:37:13)
+
+Copyright (C) 2013-2018 by the following authors:
+- Rob Clark <robdclark@gmail.com> (robclark)
+- Ilia Mirkin <imirkin@alum.mit.edu> (imirkin)
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice (including the
+next paragraph) shall be included in all copies or substantial
+portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/
+
+
+enum a6xx_color_fmt {
+	RB6_A8_UNORM = 2,
+	RB6_R8_UNORM = 3,
+	RB6_R8_SNORM = 4,
+	RB6_R8_UINT = 5,
+	RB6_R8_SINT = 6,
+	RB6_R4G4B4A4_UNORM = 8,
+	RB6_R5G5B5A1_UNORM = 10,
+	RB6_R5G6B5_UNORM = 14,
+	RB6_R8G8_UNORM = 15,
+	RB6_R8G8_SNORM = 16,
+	RB6_R8G8_UINT = 17,
+	RB6_R8G8_SINT = 18,
+	RB6_R16_UNORM = 21,
+	RB6_R16_SNORM = 22,
+	RB6_R16_FLOAT = 23,
+	RB6_R16_UINT = 24,
+	RB6_R16_SINT = 25,
+	RB6_R8G8B8A8_UNORM = 48,
+	RB6_R8G8B8_UNORM = 49,
+	RB6_R8G8B8A8_SNORM = 50,
+	RB6_R8G8B8A8_UINT = 51,
+	RB6_R8G8B8A8_SINT = 52,
+	RB6_R10G10B10A2_UNORM = 55,
+	RB6_R10G10B10A2_UINT = 58,
+	RB6_R11G11B10_FLOAT = 66,
+	RB6_R16G16_UNORM = 67,
+	RB6_R16G16_SNORM = 68,
+	RB6_R16G16_FLOAT = 69,
+	RB6_R16G16_UINT = 70,
+	RB6_R16G16_SINT = 71,
+	RB6_R32_FLOAT = 74,
+	RB6_R32_UINT = 75,
+	RB6_R32_SINT = 76,
+	RB6_R16G16B16A16_UNORM = 96,
+	RB6_R16G16B16A16_SNORM = 97,
+	RB6_R16G16B16A16_FLOAT = 98,
+	RB6_R16G16B16A16_UINT = 99,
+	RB6_R16G16B16A16_SINT = 100,
+	RB6_R32G32_FLOAT = 103,
+	RB6_R32G32_UINT = 104,
+	RB6_R32G32_SINT = 105,
+	RB6_R32G32B32A32_FLOAT = 130,
+	RB6_R32G32B32A32_UINT = 131,
+	RB6_R32G32B32A32_SINT = 132,
+	RB6_X8Z24_UNORM = 160,
+};
+
+enum a6xx_tile_mode {
+	TILE6_LINEAR = 0,
+	TILE6_2 = 2,
+	TILE6_3 = 3,
+};
+
+enum a6xx_vtx_fmt {
+	VFMT6_8_UNORM = 3,
+	VFMT6_8_SNORM = 4,
+	VFMT6_8_UINT = 5,
+	VFMT6_8_SINT = 6,
+	VFMT6_8_8_UNORM = 15,
+	VFMT6_8_8_SNORM = 16,
+	VFMT6_8_8_UINT = 17,
+	VFMT6_8_8_SINT = 18,
+	VFMT6_16_UNORM = 21,
+	VFMT6_16_SNORM = 22,
+	VFMT6_16_FLOAT = 23,
+	VFMT6_16_UINT = 24,
+	VFMT6_16_SINT = 25,
+	VFMT6_8_8_8_UNORM = 33,
+	VFMT6_8_8_8_SNORM = 34,
+	VFMT6_8_8_8_UINT = 35,
+	VFMT6_8_8_8_SINT = 36,
+	VFMT6_8_8_8_8_UNORM = 48,
+	VFMT6_8_8_8_8_SNORM = 50,
+	VFMT6_8_8_8_8_UINT = 51,
+	VFMT6_8_8_8_8_SINT = 52,
+	VFMT6_10_10_10_2_UNORM = 54,
+	VFMT6_10_10_10_2_SNORM = 57,
+	VFMT6_10_10_10_2_UINT = 58,
+	VFMT6_10_10_10_2_SINT = 59,
+	VFMT6_11_11_10_FLOAT = 66,
+	VFMT6_16_16_UNORM = 67,
+	VFMT6_16_16_SNORM = 68,
+	VFMT6_16_16_FLOAT = 69,
+	VFMT6_16_16_UINT = 70,
+	VFMT6_16_16_SINT = 71,
+	VFMT6_32_UNORM = 72,
+	VFMT6_32_SNORM = 73,
+	VFMT6_32_FLOAT = 74,
+	VFMT6_32_UINT = 75,
+	VFMT6_32_SINT = 76,
+	VFMT6_32_FIXED = 77,
+	VFMT6_16_16_16_UNORM = 88,
+	VFMT6_16_16_16_SNORM = 89,
+	VFMT6_16_16_16_FLOAT = 90,
+	VFMT6_16_16_16_UINT = 91,
+	VFMT6_16_16_16_SINT = 92,
+	VFMT6_16_16_16_16_UNORM = 96,
+	VFMT6_16_16_16_16_SNORM = 97,
+	VFMT6_16_16_16_16_FLOAT = 98,
+	VFMT6_16_16_16_16_UINT = 99,
+	VFMT6_16_16_16_16_SINT = 100,
+	VFMT6_32_32_UNORM = 101,
+	VFMT6_32_32_SNORM = 102,
+	VFMT6_32_32_FLOAT = 103,
+	VFMT6_32_32_UINT = 104,
+	VFMT6_32_32_SINT = 105,
+	VFMT6_32_32_FIXED = 106,
+	VFMT6_32_32_32_UNORM = 112,
+	VFMT6_32_32_32_SNORM = 113,
+	VFMT6_32_32_32_UINT = 114,
+	VFMT6_32_32_32_SINT = 115,
+	VFMT6_32_32_32_FLOAT = 116,
+	VFMT6_32_32_32_FIXED = 117,
+	VFMT6_32_32_32_32_UNORM = 128,
+	VFMT6_32_32_32_32_SNORM = 129,
+	VFMT6_32_32_32_32_FLOAT = 130,
+	VFMT6_32_32_32_32_UINT = 131,
+	VFMT6_32_32_32_32_SINT = 132,
+	VFMT6_32_32_32_32_FIXED = 133,
+};
+
+enum a6xx_tex_fmt {
+	TFMT6_A8_UNORM = 2,
+	TFMT6_8_UNORM = 3,
+	TFMT6_8_SNORM = 4,
+	TFMT6_8_UINT = 5,
+	TFMT6_8_SINT = 6,
+	TFMT6_4_4_4_4_UNORM = 8,
+	TFMT6_5_5_5_1_UNORM = 10,
+	TFMT6_5_6_5_UNORM = 14,
+	TFMT6_8_8_UNORM = 15,
+	TFMT6_8_8_SNORM = 16,
+	TFMT6_8_8_UINT = 17,
+	TFMT6_8_8_SINT = 18,
+	TFMT6_L8_A8_UNORM = 19,
+	TFMT6_16_UNORM = 21,
+	TFMT6_16_SNORM = 22,
+	TFMT6_16_FLOAT = 23,
+	TFMT6_16_UINT = 24,
+	TFMT6_16_SINT = 25,
+	TFMT6_8_8_8_8_UNORM = 48,
+	TFMT6_8_8_8_UNORM = 49,
+	TFMT6_8_8_8_8_SNORM = 50,
+	TFMT6_8_8_8_8_UINT = 51,
+	TFMT6_8_8_8_8_SINT = 52,
+	TFMT6_9_9_9_E5_FLOAT = 53,
+	TFMT6_10_10_10_2_UNORM = 54,
+	TFMT6_10_10_10_2_UINT = 58,
+	TFMT6_11_11_10_FLOAT = 66,
+	TFMT6_16_16_UNORM = 67,
+	TFMT6_16_16_SNORM = 68,
+	TFMT6_16_16_FLOAT = 69,
+	TFMT6_16_16_UINT = 70,
+	TFMT6_16_16_SINT = 71,
+	TFMT6_32_FLOAT = 74,
+	TFMT6_32_UINT = 75,
+	TFMT6_32_SINT = 76,
+	TFMT6_16_16_16_16_UNORM = 96,
+	TFMT6_16_16_16_16_SNORM = 97,
+	TFMT6_16_16_16_16_FLOAT = 98,
+	TFMT6_16_16_16_16_UINT = 99,
+	TFMT6_16_16_16_16_SINT = 100,
+	TFMT6_32_32_FLOAT = 103,
+	TFMT6_32_32_UINT = 104,
+	TFMT6_32_32_SINT = 105,
+	TFMT6_32_32_32_UINT = 114,
+	TFMT6_32_32_32_SINT = 115,
+	TFMT6_32_32_32_FLOAT = 116,
+	TFMT6_32_32_32_32_FLOAT = 130,
+	TFMT6_32_32_32_32_UINT = 131,
+	TFMT6_32_32_32_32_SINT = 132,
+	TFMT6_X8Z24_UNORM = 160,
+	TFMT6_ETC2_RG11_UNORM = 171,
+	TFMT6_ETC2_RG11_SNORM = 172,
+	TFMT6_ETC2_R11_UNORM = 173,
+	TFMT6_ETC2_R11_SNORM = 174,
+	TFMT6_ETC1 = 175,
+	TFMT6_ETC2_RGB8 = 176,
+	TFMT6_ETC2_RGBA8 = 177,
+	TFMT6_ETC2_RGB8A1 = 178,
+	TFMT6_DXT1 = 179,
+	TFMT6_DXT3 = 180,
+	TFMT6_DXT5 = 181,
+	TFMT6_RGTC1_UNORM = 183,
+	TFMT6_RGTC1_SNORM = 184,
+	TFMT6_RGTC2_UNORM = 187,
+	TFMT6_RGTC2_SNORM = 188,
+	TFMT6_BPTC_UFLOAT = 190,
+	TFMT6_BPTC_FLOAT = 191,
+	TFMT6_BPTC = 192,
+	TFMT6_ASTC_4x4 = 193,
+	TFMT6_ASTC_5x4 = 194,
+	TFMT6_ASTC_5x5 = 195,
+	TFMT6_ASTC_6x5 = 196,
+	TFMT6_ASTC_6x6 = 197,
+	TFMT6_ASTC_8x5 = 198,
+	TFMT6_ASTC_8x6 = 199,
+	TFMT6_ASTC_8x8 = 200,
+	TFMT6_ASTC_10x5 = 201,
+	TFMT6_ASTC_10x6 = 202,
+	TFMT6_ASTC_10x8 = 203,
+	TFMT6_ASTC_10x10 = 204,
+	TFMT6_ASTC_12x10 = 205,
+	TFMT6_ASTC_12x12 = 206,
+};
+
+enum a6xx_tex_fetchsize {
+	TFETCH6_1_BYTE = 0,
+	TFETCH6_2_BYTE = 1,
+	TFETCH6_4_BYTE = 2,
+	TFETCH6_8_BYTE = 3,
+	TFETCH6_16_BYTE = 4,
+};
+
+enum a6xx_depth_format {
+	DEPTH6_NONE = 0,
+	DEPTH6_16 = 1,
+	DEPTH6_24_8 = 2,
+	DEPTH6_32 = 4,
+};
+
+enum a6xx_cp_perfcounter_select {
+	PERF_CP_ALWAYS_COUNT = 0,
+};
+
+enum a6xx_tex_filter {
+	A6XX_TEX_NEAREST = 0,
+	A6XX_TEX_LINEAR = 1,
+	A6XX_TEX_ANISO = 2,
+};
+
+enum a6xx_tex_clamp {
+	A6XX_TEX_REPEAT = 0,
+	A6XX_TEX_CLAMP_TO_EDGE = 1,
+	A6XX_TEX_MIRROR_REPEAT = 2,
+	A6XX_TEX_CLAMP_TO_BORDER = 3,
+	A6XX_TEX_MIRROR_CLAMP = 4,
+};
+
+enum a6xx_tex_aniso {
+	A6XX_TEX_ANISO_1 = 0,
+	A6XX_TEX_ANISO_2 = 1,
+	A6XX_TEX_ANISO_4 = 2,
+	A6XX_TEX_ANISO_8 = 3,
+	A6XX_TEX_ANISO_16 = 4,
+};
+
+enum a6xx_tex_swiz {
+	A6XX_TEX_X = 0,
+	A6XX_TEX_Y = 1,
+	A6XX_TEX_Z = 2,
+	A6XX_TEX_W = 3,
+	A6XX_TEX_ZERO = 4,
+	A6XX_TEX_ONE = 5,
+};
+
+enum a6xx_tex_type {
+	A6XX_TEX_1D = 0,
+	A6XX_TEX_2D = 1,
+	A6XX_TEX_CUBE = 2,
+	A6XX_TEX_3D = 3,
+};
+
+#define A6XX_RBBM_INT_0_MASK_RBBM_GPU_IDLE			0x00000001
+#define A6XX_RBBM_INT_0_MASK_CP_AHB_ERROR			0x00000002
+#define A6XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNCFIFO_OVERFLOW	0x00000040
+#define A6XX_RBBM_INT_0_MASK_RBBM_GPC_ERROR			0x00000080
+#define A6XX_RBBM_INT_0_MASK_CP_SW				0x00000100
+#define A6XX_RBBM_INT_0_MASK_CP_HW_ERROR			0x00000200
+#define A6XX_RBBM_INT_0_MASK_CP_CCU_FLUSH_DEPTH_TS		0x00000400
+#define A6XX_RBBM_INT_0_MASK_CP_CCU_FLUSH_COLOR_TS		0x00000800
+#define A6XX_RBBM_INT_0_MASK_CP_CCU_RESOLVE_TS			0x00001000
+#define A6XX_RBBM_INT_0_MASK_CP_IB2				0x00002000
+#define A6XX_RBBM_INT_0_MASK_CP_IB1				0x00004000
+#define A6XX_RBBM_INT_0_MASK_CP_RB				0x00008000
+#define A6XX_RBBM_INT_0_MASK_CP_RB_DONE_TS			0x00020000
+#define A6XX_RBBM_INT_0_MASK_CP_WT_DONE_TS			0x00040000
+#define A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS			0x00100000
+#define A6XX_RBBM_INT_0_MASK_RBBM_ATB_BUS_OVERFLOW		0x00400000
+#define A6XX_RBBM_INT_0_MASK_RBBM_HANG_DETECT			0x00800000
+#define A6XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS			0x01000000
+#define A6XX_RBBM_INT_0_MASK_UCHE_TRAP_INTR			0x02000000
+#define A6XX_RBBM_INT_0_MASK_DEBBUS_INTR_0			0x04000000
+#define A6XX_RBBM_INT_0_MASK_DEBBUS_INTR_1			0x08000000
+#define A6XX_RBBM_INT_0_MASK_ISDB_CPU_IRQ			0x40000000
+#define A6XX_RBBM_INT_0_MASK_ISDB_UNDER_DEBUG			0x80000000
+#define A6XX_CP_INT_CP_OPCODE_ERROR				0x00000001
+#define A6XX_CP_INT_CP_UCODE_ERROR				0x00000002
+#define A6XX_CP_INT_CP_HW_FAULT_ERROR				0x00000004
+#define A6XX_CP_INT_CP_REGISTER_PROTECTION_ERROR		0x00000010
+#define A6XX_CP_INT_CP_AHB_ERROR				0x00000020
+#define A6XX_CP_INT_CP_VSD_PARITY_ERROR				0x00000040
+#define A6XX_CP_INT_CP_ILLEGAL_INSTR_ERROR			0x00000080
+#define REG_A6XX_CP_RB_BASE					0x00000800
+
+#define REG_A6XX_CP_RB_BASE_HI					0x00000801
+
+#define REG_A6XX_CP_RB_CNTL					0x00000802
+
+#define REG_A6XX_CP_RB_RPTR_ADDR_LO				0x00000804
+
+#define REG_A6XX_CP_RB_RPTR_ADDR_HI				0x00000805
+
+#define REG_A6XX_CP_RB_RPTR					0x00000806
+
+#define REG_A6XX_CP_RB_WPTR					0x00000807
+
+#define REG_A6XX_CP_SQE_CNTL					0x00000808
+
+#define REG_A6XX_CP_HW_FAULT					0x00000821
+
+#define REG_A6XX_CP_INTERRUPT_STATUS				0x00000823
+
+#define REG_A6XX_CP_PROTECT_STATUS				0x00000824
+
+#define REG_A6XX_CP_SQE_INSTR_BASE_LO				0x00000830
+
+#define REG_A6XX_CP_SQE_INSTR_BASE_HI				0x00000831
+
+#define REG_A6XX_CP_MISC_CNTL					0x00000840
+
+#define REG_A6XX_CP_ROQ_THRESHOLDS_1				0x000008c1
+
+#define REG_A6XX_CP_ROQ_THRESHOLDS_2				0x000008c2
+
+#define REG_A6XX_CP_MEM_POOL_SIZE				0x000008c3
+
+#define REG_A6XX_CP_CHICKEN_DBG					0x00000841
+
+#define REG_A6XX_CP_ADDR_MODE_CNTL				0x00000842
+
+#define REG_A6XX_CP_DBG_ECO_CNTL				0x00000843
+
+#define REG_A6XX_CP_PROTECT_CNTL				0x0000084f
+
+static inline uint32_t REG_A6XX_CP_SCRATCH(uint32_t i0) { return 0x00000883 + 0x1*i0; }
+
+static inline uint32_t REG_A6XX_CP_SCRATCH_REG(uint32_t i0) { return 0x00000883 + 0x1*i0; }
+
+static inline uint32_t REG_A6XX_CP_PROTECT(uint32_t i0) { return 0x00000850 + 0x1*i0; }
+
+static inline uint32_t REG_A6XX_CP_PROTECT_REG(uint32_t i0) { return 0x00000850 + 0x1*i0; }
+#define A6XX_CP_PROTECT_REG_BASE_ADDR__MASK			0x0003ffff
+#define A6XX_CP_PROTECT_REG_BASE_ADDR__SHIFT			0
+static inline uint32_t A6XX_CP_PROTECT_REG_BASE_ADDR(uint32_t val)
+{
+	return ((val) << A6XX_CP_PROTECT_REG_BASE_ADDR__SHIFT) & A6XX_CP_PROTECT_REG_BASE_ADDR__MASK;
+}
+#define A6XX_CP_PROTECT_REG_MASK_LEN__MASK			0x7ffc0000
+#define A6XX_CP_PROTECT_REG_MASK_LEN__SHIFT			18
+static inline uint32_t A6XX_CP_PROTECT_REG_MASK_LEN(uint32_t val)
+{
+	return ((val) << A6XX_CP_PROTECT_REG_MASK_LEN__SHIFT) & A6XX_CP_PROTECT_REG_MASK_LEN__MASK;
+}
+#define A6XX_CP_PROTECT_REG_READ				0x80000000
+
+#define REG_A6XX_CP_CONTEXT_SWITCH_CNTL				0x000008a0
+
+#define REG_A6XX_CP_CONTEXT_SWITCH_SMMU_INFO_LO			0x000008a1
+
+#define REG_A6XX_CP_CONTEXT_SWITCH_SMMU_INFO_HI			0x000008a2
+
+#define REG_A6XX_CP_CONTEXT_SWITCH_PRIV_NON_SECURE_RESTORE_ADDR_LO	0x000008a3
+
+#define REG_A6XX_CP_CONTEXT_SWITCH_PRIV_NON_SECURE_RESTORE_ADDR_HI	0x000008a4
+
+#define REG_A6XX_CP_CONTEXT_SWITCH_PRIV_SECURE_RESTORE_ADDR_LO	0x000008a5
+
+#define REG_A6XX_CP_CONTEXT_SWITCH_PRIV_SECURE_RESTORE_ADDR_HI	0x000008a6
+
+#define REG_A6XX_CP_CONTEXT_SWITCH_NON_PRIV_RESTORE_ADDR_LO	0x000008a7
+
+#define REG_A6XX_CP_CONTEXT_SWITCH_NON_PRIV_RESTORE_ADDR_HI	0x000008a8
+
+#define REG_A6XX_CP_PERFCTR_CP_SEL_0				0x000008d0
+
+#define REG_A6XX_CP_PERFCTR_CP_SEL_1				0x000008d1
+
+#define REG_A6XX_CP_PERFCTR_CP_SEL_2				0x000008d2
+
+#define REG_A6XX_CP_PERFCTR_CP_SEL_3				0x000008d3
+
+#define REG_A6XX_CP_PERFCTR_CP_SEL_4				0x000008d4
+
+#define REG_A6XX_CP_PERFCTR_CP_SEL_5				0x000008d5
+
+#define REG_A6XX_CP_PERFCTR_CP_SEL_6				0x000008d6
+
+#define REG_A6XX_CP_PERFCTR_CP_SEL_7				0x000008d7
+
+#define REG_A6XX_CP_PERFCTR_CP_SEL_8				0x000008d8
+
+#define REG_A6XX_CP_PERFCTR_CP_SEL_9				0x000008d9
+
+#define REG_A6XX_CP_PERFCTR_CP_SEL_10				0x000008da
+
+#define REG_A6XX_CP_PERFCTR_CP_SEL_11				0x000008db
+
+#define REG_A6XX_CP_PERFCTR_CP_SEL_12				0x000008dc
+
+#define REG_A6XX_CP_PERFCTR_CP_SEL_13				0x000008dd
+
+#define REG_A6XX_CP_CRASH_SCRIPT_BASE_LO			0x00000900
+
+#define REG_A6XX_CP_CRASH_SCRIPT_BASE_HI			0x00000901
+
+#define REG_A6XX_CP_CRASH_DUMP_CNTL				0x00000902
+
+#define REG_A6XX_CP_CRASH_DUMP_STATUS				0x00000903
+
+#define REG_A6XX_CP_SQE_STAT_ADDR				0x00000908
+
+#define REG_A6XX_CP_SQE_STAT_DATA				0x00000909
+
+#define REG_A6XX_CP_DRAW_STATE_ADDR				0x0000090a
+
+#define REG_A6XX_CP_DRAW_STATE_DATA				0x0000090b
+
+#define REG_A6XX_CP_ROQ_DBG_ADDR				0x0000090c
+
+#define REG_A6XX_CP_ROQ_DBG_DATA				0x0000090d
+
+#define REG_A6XX_CP_MEM_POOL_DBG_ADDR				0x0000090e
+
+#define REG_A6XX_CP_MEM_POOL_DBG_DATA				0x0000090f
+
+#define REG_A6XX_CP_SQE_UCODE_DBG_ADDR				0x00000910
+
+#define REG_A6XX_CP_SQE_UCODE_DBG_DATA				0x00000911
+
+#define REG_A6XX_CP_IB1_BASE					0x00000928
+
+#define REG_A6XX_CP_IB1_BASE_HI					0x00000929
+
+#define REG_A6XX_CP_IB1_REM_SIZE				0x0000092a
+
+#define REG_A6XX_CP_IB2_BASE					0x0000092b
+
+#define REG_A6XX_CP_IB2_BASE_HI					0x0000092c
+
+#define REG_A6XX_CP_IB2_REM_SIZE				0x0000092d
+
+#define REG_A6XX_CP_ALWAYS_ON_COUNTER_LO			0x00000980
+
+#define REG_A6XX_CP_ALWAYS_ON_COUNTER_HI			0x00000981
+
+#define REG_A6XX_CP_AHB_CNTL					0x0000098d
+
+#define REG_A6XX_CP_APERTURE_CNTL_HOST				0x00000a00
+
+#define REG_A6XX_CP_APERTURE_CNTL_CD				0x00000a03
+
+#define REG_A6XX_VSC_ADDR_MODE_CNTL				0x00000c01
+
+#define REG_A6XX_RBBM_INT_0_STATUS				0x00000201
+
+#define REG_A6XX_RBBM_STATUS					0x00000210
+#define A6XX_RBBM_STATUS_GPU_BUSY_IGN_AHB			0x00800000
+#define A6XX_RBBM_STATUS_GPU_BUSY_IGN_AHB_CP			0x00400000
+#define A6XX_RBBM_STATUS_HLSQ_BUSY				0x00200000
+#define A6XX_RBBM_STATUS_VSC_BUSY				0x00100000
+#define A6XX_RBBM_STATUS_TPL1_BUSY				0x00080000
+#define A6XX_RBBM_STATUS_SP_BUSY				0x00040000
+#define A6XX_RBBM_STATUS_UCHE_BUSY				0x00020000
+#define A6XX_RBBM_STATUS_VPC_BUSY				0x00010000
+#define A6XX_RBBM_STATUS_VFD_BUSY				0x00008000
+#define A6XX_RBBM_STATUS_TESS_BUSY				0x00004000
+#define A6XX_RBBM_STATUS_PC_VSD_BUSY				0x00002000
+#define A6XX_RBBM_STATUS_PC_DCALL_BUSY				0x00001000
+#define A6XX_RBBM_STATUS_COM_DCOM_BUSY				0x00000800
+#define A6XX_RBBM_STATUS_LRZ_BUSY				0x00000400
+#define A6XX_RBBM_STATUS_A2D_BUSY				0x00000200
+#define A6XX_RBBM_STATUS_CCU_BUSY				0x00000100
+#define A6XX_RBBM_STATUS_RB_BUSY				0x00000080
+#define A6XX_RBBM_STATUS_RAS_BUSY				0x00000040
+#define A6XX_RBBM_STATUS_TSE_BUSY				0x00000020
+#define A6XX_RBBM_STATUS_VBIF_BUSY				0x00000010
+#define A6XX_RBBM_STATUS_GFX_DBGC_BUSY				0x00000008
+#define A6XX_RBBM_STATUS_CP_BUSY				0x00000004
+#define A6XX_RBBM_STATUS_CP_AHB_BUSY_CP_MASTER			0x00000002
+#define A6XX_RBBM_STATUS_CP_AHB_BUSY_CX_MASTER			0x00000001
+
+#define REG_A6XX_RBBM_STATUS3					0x00000213
+
+#define REG_A6XX_RBBM_VBIF_GX_RESET_STATUS			0x00000215
+
+#define REG_A6XX_RBBM_PERFCTR_CP_0_LO				0x00000400
+
+#define REG_A6XX_RBBM_PERFCTR_CP_0_HI				0x00000401
+
+#define REG_A6XX_RBBM_PERFCTR_CP_1_LO				0x00000402
+
+#define REG_A6XX_RBBM_PERFCTR_CP_1_HI				0x00000403
+
+#define REG_A6XX_RBBM_PERFCTR_CP_2_LO				0x00000404
+
+#define REG_A6XX_RBBM_PERFCTR_CP_2_HI				0x00000405
+
+#define REG_A6XX_RBBM_PERFCTR_CP_3_LO				0x00000406
+
+#define REG_A6XX_RBBM_PERFCTR_CP_3_HI				0x00000407
+
+#define REG_A6XX_RBBM_PERFCTR_CP_4_LO				0x00000408
+
+#define REG_A6XX_RBBM_PERFCTR_CP_4_HI				0x00000409
+
+#define REG_A6XX_RBBM_PERFCTR_CP_5_LO				0x0000040a
+
+#define REG_A6XX_RBBM_PERFCTR_CP_5_HI				0x0000040b
+
+#define REG_A6XX_RBBM_PERFCTR_CP_6_LO				0x0000040c
+
+#define REG_A6XX_RBBM_PERFCTR_CP_6_HI				0x0000040d
+
+#define REG_A6XX_RBBM_PERFCTR_CP_7_LO				0x0000040e
+
+#define REG_A6XX_RBBM_PERFCTR_CP_7_HI				0x0000040f
+
+#define REG_A6XX_RBBM_PERFCTR_CP_8_LO				0x00000410
+
+#define REG_A6XX_RBBM_PERFCTR_CP_8_HI				0x00000411
+
+#define REG_A6XX_RBBM_PERFCTR_CP_9_LO				0x00000412
+
+#define REG_A6XX_RBBM_PERFCTR_CP_9_HI				0x00000413
+
+#define REG_A6XX_RBBM_PERFCTR_CP_10_LO				0x00000414
+
+#define REG_A6XX_RBBM_PERFCTR_CP_10_HI				0x00000415
+
+#define REG_A6XX_RBBM_PERFCTR_CP_11_LO				0x00000416
+
+#define REG_A6XX_RBBM_PERFCTR_CP_11_HI				0x00000417
+
+#define REG_A6XX_RBBM_PERFCTR_CP_12_LO				0x00000418
+
+#define REG_A6XX_RBBM_PERFCTR_CP_12_HI				0x00000419
+
+#define REG_A6XX_RBBM_PERFCTR_CP_13_LO				0x0000041a
+
+#define REG_A6XX_RBBM_PERFCTR_CP_13_HI				0x0000041b
+
+#define REG_A6XX_RBBM_PERFCTR_RBBM_0_LO				0x0000041c
+
+#define REG_A6XX_RBBM_PERFCTR_RBBM_0_HI				0x0000041d
+
+#define REG_A6XX_RBBM_PERFCTR_RBBM_1_LO				0x0000041e
+
+#define REG_A6XX_RBBM_PERFCTR_RBBM_1_HI				0x0000041f
+
+#define REG_A6XX_RBBM_PERFCTR_RBBM_2_LO				0x00000420
+
+#define REG_A6XX_RBBM_PERFCTR_RBBM_2_HI				0x00000421
+
+#define REG_A6XX_RBBM_PERFCTR_RBBM_3_LO				0x00000422
+
+#define REG_A6XX_RBBM_PERFCTR_RBBM_3_HI				0x00000423
+
+#define REG_A6XX_RBBM_PERFCTR_PC_0_LO				0x00000424
+
+#define REG_A6XX_RBBM_PERFCTR_PC_0_HI				0x00000425
+
+#define REG_A6XX_RBBM_PERFCTR_PC_1_LO				0x00000426
+
+#define REG_A6XX_RBBM_PERFCTR_PC_1_HI				0x00000427
+
+#define REG_A6XX_RBBM_PERFCTR_PC_2_LO				0x00000428
+
+#define REG_A6XX_RBBM_PERFCTR_PC_2_HI				0x00000429
+
+#define REG_A6XX_RBBM_PERFCTR_PC_3_LO				0x0000042a
+
+#define REG_A6XX_RBBM_PERFCTR_PC_3_HI				0x0000042b
+
+#define REG_A6XX_RBBM_PERFCTR_PC_4_LO				0x0000042c
+
+#define REG_A6XX_RBBM_PERFCTR_PC_4_HI				0x0000042d
+
+#define REG_A6XX_RBBM_PERFCTR_PC_5_LO				0x0000042e
+
+#define REG_A6XX_RBBM_PERFCTR_PC_5_HI				0x0000042f
+
+#define REG_A6XX_RBBM_PERFCTR_PC_6_LO				0x00000430
+
+#define REG_A6XX_RBBM_PERFCTR_PC_6_HI				0x00000431
+
+#define REG_A6XX_RBBM_PERFCTR_PC_7_LO				0x00000432
+
+#define REG_A6XX_RBBM_PERFCTR_PC_7_HI				0x00000433
+
+#define REG_A6XX_RBBM_PERFCTR_VFD_0_LO				0x00000434
+
+#define REG_A6XX_RBBM_PERFCTR_VFD_0_HI				0x00000435
+
+#define REG_A6XX_RBBM_PERFCTR_VFD_1_LO				0x00000436
+
+#define REG_A6XX_RBBM_PERFCTR_VFD_1_HI				0x00000437
+
+#define REG_A6XX_RBBM_PERFCTR_VFD_2_LO				0x00000438
+
+#define REG_A6XX_RBBM_PERFCTR_VFD_2_HI				0x00000439
+
+#define REG_A6XX_RBBM_PERFCTR_VFD_3_LO				0x0000043a
+
+#define REG_A6XX_RBBM_PERFCTR_VFD_3_HI				0x0000043b
+
+#define REG_A6XX_RBBM_PERFCTR_VFD_4_LO				0x0000043c
+
+#define REG_A6XX_RBBM_PERFCTR_VFD_4_HI				0x0000043d
+
+#define REG_A6XX_RBBM_PERFCTR_VFD_5_LO				0x0000043e
+
+#define REG_A6XX_RBBM_PERFCTR_VFD_5_HI				0x0000043f
+
+#define REG_A6XX_RBBM_PERFCTR_VFD_6_LO				0x00000440
+
+#define REG_A6XX_RBBM_PERFCTR_VFD_6_HI				0x00000441
+
+#define REG_A6XX_RBBM_PERFCTR_VFD_7_LO				0x00000442
+
+#define REG_A6XX_RBBM_PERFCTR_VFD_7_HI				0x00000443
+
+#define REG_A6XX_RBBM_PERFCTR_HLSQ_0_LO				0x00000444
+
+#define REG_A6XX_RBBM_PERFCTR_HLSQ_0_HI				0x00000445
+
+#define REG_A6XX_RBBM_PERFCTR_HLSQ_1_LO				0x00000446
+
+#define REG_A6XX_RBBM_PERFCTR_HLSQ_1_HI				0x00000447
+
+#define REG_A6XX_RBBM_PERFCTR_HLSQ_2_LO				0x00000448
+
+#define REG_A6XX_RBBM_PERFCTR_HLSQ_2_HI				0x00000449
+
+#define REG_A6XX_RBBM_PERFCTR_HLSQ_3_LO				0x0000044a
+
+#define REG_A6XX_RBBM_PERFCTR_HLSQ_3_HI				0x0000044b
+
+#define REG_A6XX_RBBM_PERFCTR_HLSQ_4_LO				0x0000044c
+
+#define REG_A6XX_RBBM_PERFCTR_HLSQ_4_HI				0x0000044d
+
+#define REG_A6XX_RBBM_PERFCTR_HLSQ_5_LO				0x0000044e
+
+#define REG_A6XX_RBBM_PERFCTR_HLSQ_5_HI				0x0000044f
+
+#define REG_A6XX_RBBM_PERFCTR_VPC_0_LO				0x00000450
+
+#define REG_A6XX_RBBM_PERFCTR_VPC_0_HI				0x00000451
+
+#define REG_A6XX_RBBM_PERFCTR_VPC_1_LO				0x00000452
+
+#define REG_A6XX_RBBM_PERFCTR_VPC_1_HI				0x00000453
+
+#define REG_A6XX_RBBM_PERFCTR_VPC_2_LO				0x00000454
+
+#define REG_A6XX_RBBM_PERFCTR_VPC_2_HI				0x00000455
+
+#define REG_A6XX_RBBM_PERFCTR_VPC_3_LO				0x00000456
+
+#define REG_A6XX_RBBM_PERFCTR_VPC_3_HI				0x00000457
+
+#define REG_A6XX_RBBM_PERFCTR_VPC_4_LO				0x00000458
+
+#define REG_A6XX_RBBM_PERFCTR_VPC_4_HI				0x00000459
+
+#define REG_A6XX_RBBM_PERFCTR_VPC_5_LO				0x0000045a
+
+#define REG_A6XX_RBBM_PERFCTR_VPC_5_HI				0x0000045b
+
+#define REG_A6XX_RBBM_PERFCTR_CCU_0_LO				0x0000045c
+
+#define REG_A6XX_RBBM_PERFCTR_CCU_0_HI				0x0000045d
+
+#define REG_A6XX_RBBM_PERFCTR_CCU_1_LO				0x0000045e
+
+#define REG_A6XX_RBBM_PERFCTR_CCU_1_HI				0x0000045f
+
+#define REG_A6XX_RBBM_PERFCTR_CCU_2_LO				0x00000460
+
+#define REG_A6XX_RBBM_PERFCTR_CCU_2_HI				0x00000461
+
+#define REG_A6XX_RBBM_PERFCTR_CCU_3_LO				0x00000462
+
+#define REG_A6XX_RBBM_PERFCTR_CCU_3_HI				0x00000463
+
+#define REG_A6XX_RBBM_PERFCTR_CCU_4_LO				0x00000464
+
+#define REG_A6XX_RBBM_PERFCTR_CCU_4_HI				0x00000465
+
+#define REG_A6XX_RBBM_PERFCTR_TSE_0_LO				0x00000466
+
+#define REG_A6XX_RBBM_PERFCTR_TSE_0_HI				0x00000467
+
+#define REG_A6XX_RBBM_PERFCTR_TSE_1_LO				0x00000468
+
+#define REG_A6XX_RBBM_PERFCTR_TSE_1_HI				0x00000469
+
+#define REG_A6XX_RBBM_PERFCTR_TSE_2_LO				0x0000046a
+
+#define REG_A6XX_RBBM_PERFCTR_CCU_4_HI				0x00000465
+
+#define REG_A6XX_RBBM_PERFCTR_TSE_0_LO				0x00000466
+
+#define REG_A6XX_RBBM_PERFCTR_TSE_0_HI				0x00000467
+
+#define REG_A6XX_RBBM_PERFCTR_TSE_1_LO				0x00000468
+
+#define REG_A6XX_RBBM_PERFCTR_TSE_1_HI				0x00000469
+
+#define REG_A6XX_RBBM_PERFCTR_TSE_2_LO				0x0000046a
+
+#define REG_A6XX_RBBM_PERFCTR_TSE_2_HI				0x0000046b
+
+#define REG_A6XX_RBBM_PERFCTR_TSE_3_LO				0x0000046c
+
+#define REG_A6XX_RBBM_PERFCTR_TSE_3_HI				0x0000046d
+
+#define REG_A6XX_RBBM_PERFCTR_RAS_0_LO				0x0000046e
+
+#define REG_A6XX_RBBM_PERFCTR_RAS_0_HI				0x0000046f
+
+#define REG_A6XX_RBBM_PERFCTR_RAS_1_LO				0x00000470
+
+#define REG_A6XX_RBBM_PERFCTR_RAS_1_HI				0x00000471
+
+#define REG_A6XX_RBBM_PERFCTR_RAS_2_LO				0x00000472
+
+#define REG_A6XX_RBBM_PERFCTR_RAS_2_HI				0x00000473
+
+#define REG_A6XX_RBBM_PERFCTR_RAS_3_LO				0x00000474
+
+#define REG_A6XX_RBBM_PERFCTR_RAS_3_HI				0x00000475
+
+#define REG_A6XX_RBBM_PERFCTR_UCHE_0_LO				0x00000476
+
+#define REG_A6XX_RBBM_PERFCTR_UCHE_0_HI				0x00000477
+
+#define REG_A6XX_RBBM_PERFCTR_UCHE_1_LO				0x00000478
+
+#define REG_A6XX_RBBM_PERFCTR_UCHE_1_HI				0x00000479
+
+#define REG_A6XX_RBBM_PERFCTR_UCHE_2_LO				0x0000047a
+
+#define REG_A6XX_RBBM_PERFCTR_UCHE_2_HI				0x0000047b
+
+#define REG_A6XX_RBBM_PERFCTR_UCHE_3_LO				0x0000047c
+
+#define REG_A6XX_RBBM_PERFCTR_UCHE_3_HI				0x0000047d
+
+#define REG_A6XX_RBBM_PERFCTR_UCHE_4_LO				0x0000047e
+
+#define REG_A6XX_RBBM_PERFCTR_UCHE_4_HI				0x0000047f
+
+#define REG_A6XX_RBBM_PERFCTR_UCHE_5_LO				0x00000480
+
+#define REG_A6XX_RBBM_PERFCTR_UCHE_5_HI				0x00000481
+
+#define REG_A6XX_RBBM_PERFCTR_UCHE_6_LO				0x00000482
+
+#define REG_A6XX_RBBM_PERFCTR_UCHE_6_HI				0x00000483
+
+#define REG_A6XX_RBBM_PERFCTR_UCHE_7_LO				0x00000484
+
+#define REG_A6XX_RBBM_PERFCTR_UCHE_7_HI				0x00000485
+
+#define REG_A6XX_RBBM_PERFCTR_UCHE_8_LO				0x00000486
+
+#define REG_A6XX_RBBM_PERFCTR_UCHE_8_HI				0x00000487
+
+#define REG_A6XX_RBBM_PERFCTR_UCHE_9_LO				0x00000488
+
+#define REG_A6XX_RBBM_PERFCTR_UCHE_9_HI				0x00000489
+
+#define REG_A6XX_RBBM_PERFCTR_UCHE_10_LO			0x0000048a
+
+#define REG_A6XX_RBBM_PERFCTR_UCHE_10_HI			0x0000048b
+
+#define REG_A6XX_RBBM_PERFCTR_UCHE_11_LO			0x0000048c
+
+#define REG_A6XX_RBBM_PERFCTR_UCHE_11_HI			0x0000048d
+
+#define REG_A6XX_RBBM_PERFCTR_TP_0_LO				0x0000048e
+
+#define REG_A6XX_RBBM_PERFCTR_TP_0_HI				0x0000048f
+
+#define REG_A6XX_RBBM_PERFCTR_TP_1_LO				0x00000490
+
+#define REG_A6XX_RBBM_PERFCTR_TP_1_HI				0x00000491
+
+#define REG_A6XX_RBBM_PERFCTR_TP_2_LO				0x00000492
+
+#define REG_A6XX_RBBM_PERFCTR_TP_2_HI				0x00000493
+
+#define REG_A6XX_RBBM_PERFCTR_TP_3_LO				0x00000494
+
+#define REG_A6XX_RBBM_PERFCTR_TP_3_HI				0x00000495
+
+#define REG_A6XX_RBBM_PERFCTR_TP_4_LO				0x00000496
+
+#define REG_A6XX_RBBM_PERFCTR_TP_4_HI				0x00000497
+
+#define REG_A6XX_RBBM_PERFCTR_TP_5_LO				0x00000498
+
+#define REG_A6XX_RBBM_PERFCTR_TP_5_HI				0x00000499
+
+#define REG_A6XX_RBBM_PERFCTR_TP_6_LO				0x0000049a
+
+#define REG_A6XX_RBBM_PERFCTR_TP_6_HI				0x0000049b
+
+#define REG_A6XX_RBBM_PERFCTR_TP_7_LO				0x0000049c
+
+#define REG_A6XX_RBBM_PERFCTR_TP_7_HI				0x0000049d
+
+#define REG_A6XX_RBBM_PERFCTR_TP_8_LO				0x0000049e
+
+#define REG_A6XX_RBBM_PERFCTR_TP_8_HI				0x0000049f
+
+#define REG_A6XX_RBBM_PERFCTR_TP_9_LO				0x000004a0
+
+#define REG_A6XX_RBBM_PERFCTR_TP_9_HI				0x000004a1
+
+#define REG_A6XX_RBBM_PERFCTR_TP_10_LO				0x000004a2
+
+#define REG_A6XX_RBBM_PERFCTR_TP_10_HI				0x000004a3
+
+#define REG_A6XX_RBBM_PERFCTR_TP_11_LO				0x000004a4
+
+#define REG_A6XX_RBBM_PERFCTR_TP_11_HI				0x000004a5
+
+#define REG_A6XX_RBBM_PERFCTR_SP_0_LO				0x000004a6
+
+#define REG_A6XX_RBBM_PERFCTR_SP_0_HI				0x000004a7
+
+#define REG_A6XX_RBBM_PERFCTR_SP_1_LO				0x000004a8
+
+#define REG_A6XX_RBBM_PERFCTR_SP_1_HI				0x000004a9
+
+#define REG_A6XX_RBBM_PERFCTR_SP_2_LO				0x000004aa
+
+#define REG_A6XX_RBBM_PERFCTR_SP_2_HI				0x000004ab
+
+#define REG_A6XX_RBBM_PERFCTR_SP_3_LO				0x000004ac
+
+#define REG_A6XX_RBBM_PERFCTR_SP_3_HI				0x000004ad
+
+#define REG_A6XX_RBBM_PERFCTR_SP_4_LO				0x000004ae
+
+#define REG_A6XX_RBBM_PERFCTR_SP_4_HI				0x000004af
+
+#define REG_A6XX_RBBM_PERFCTR_SP_5_LO				0x000004b0
+
+#define REG_A6XX_RBBM_PERFCTR_SP_5_HI				0x000004b1
+
+#define REG_A6XX_RBBM_PERFCTR_SP_6_LO				0x000004b2
+
+#define REG_A6XX_RBBM_PERFCTR_SP_6_HI				0x000004b3
+
+#define REG_A6XX_RBBM_PERFCTR_SP_7_LO				0x000004b4
+
+#define REG_A6XX_RBBM_PERFCTR_SP_7_HI				0x000004b5
+
+#define REG_A6XX_RBBM_PERFCTR_SP_8_LO				0x000004b6
+
+#define REG_A6XX_RBBM_PERFCTR_SP_8_HI				0x000004b7
+
+#define REG_A6XX_RBBM_PERFCTR_SP_9_LO				0x000004b8
+
+#define REG_A6XX_RBBM_PERFCTR_SP_9_HI				0x000004b9
+
+#define REG_A6XX_RBBM_PERFCTR_SP_10_LO				0x000004ba
+
+#define REG_A6XX_RBBM_PERFCTR_SP_10_HI				0x000004bb
+
+#define REG_A6XX_RBBM_PERFCTR_SP_11_LO				0x000004bc
+
+#define REG_A6XX_RBBM_PERFCTR_SP_11_HI				0x000004bd
+
+#define REG_A6XX_RBBM_PERFCTR_SP_12_LO				0x000004be
+
+#define REG_A6XX_RBBM_PERFCTR_SP_12_HI				0x000004bf
+
+#define REG_A6XX_RBBM_PERFCTR_SP_13_LO				0x000004c0
+
+#define REG_A6XX_RBBM_PERFCTR_SP_13_HI				0x000004c1
+
+#define REG_A6XX_RBBM_PERFCTR_SP_14_LO				0x000004c2
+
+#define REG_A6XX_RBBM_PERFCTR_SP_14_HI				0x000004c3
+
+#define REG_A6XX_RBBM_PERFCTR_SP_15_LO				0x000004c4
+
+#define REG_A6XX_RBBM_PERFCTR_SP_15_HI				0x000004c5
+
+#define REG_A6XX_RBBM_PERFCTR_SP_16_LO				0x000004c6
+
+#define REG_A6XX_RBBM_PERFCTR_SP_16_HI				0x000004c7
+
+#define REG_A6XX_RBBM_PERFCTR_SP_17_LO				0x000004c8
+
+#define REG_A6XX_RBBM_PERFCTR_SP_17_HI				0x000004c9
+
+#define REG_A6XX_RBBM_PERFCTR_SP_18_LO				0x000004ca
+
+#define REG_A6XX_RBBM_PERFCTR_SP_18_HI				0x000004cb
+
+#define REG_A6XX_RBBM_PERFCTR_SP_19_LO				0x000004cc
+
+#define REG_A6XX_RBBM_PERFCTR_SP_19_HI				0x000004cd
+
+#define REG_A6XX_RBBM_PERFCTR_SP_20_LO				0x000004ce
+
+#define REG_A6XX_RBBM_PERFCTR_SP_20_HI				0x000004cf
+
+#define REG_A6XX_RBBM_PERFCTR_SP_21_LO				0x000004d0
+
+#define REG_A6XX_RBBM_PERFCTR_SP_21_HI				0x000004d1
+
+#define REG_A6XX_RBBM_PERFCTR_SP_22_LO				0x000004d2
+
+#define REG_A6XX_RBBM_PERFCTR_SP_22_HI				0x000004d3
+
+#define REG_A6XX_RBBM_PERFCTR_SP_23_LO				0x000004d4
+
+#define REG_A6XX_RBBM_PERFCTR_SP_23_HI				0x000004d5
+
+#define REG_A6XX_RBBM_PERFCTR_RB_0_LO				0x000004d6
+
+#define REG_A6XX_RBBM_PERFCTR_RB_0_HI				0x000004d7
+
+#define REG_A6XX_RBBM_PERFCTR_RB_1_LO				0x000004d8
+
+#define REG_A6XX_RBBM_PERFCTR_RB_1_HI				0x000004d9
+
+#define REG_A6XX_RBBM_PERFCTR_RB_2_LO				0x000004da
+
+#define REG_A6XX_RBBM_PERFCTR_RB_2_HI				0x000004db
+
+#define REG_A6XX_RBBM_PERFCTR_RB_3_LO				0x000004dc
+
+#define REG_A6XX_RBBM_PERFCTR_RB_3_HI				0x000004dd
+
+#define REG_A6XX_RBBM_PERFCTR_RB_4_LO				0x000004de
+
+#define REG_A6XX_RBBM_PERFCTR_RB_4_HI				0x000004df
+
+#define REG_A6XX_RBBM_PERFCTR_RB_5_LO				0x000004e0
+
+#define REG_A6XX_RBBM_PERFCTR_RB_5_HI				0x000004e1
+
+#define REG_A6XX_RBBM_PERFCTR_RB_6_LO				0x000004e2
+
+#define REG_A6XX_RBBM_PERFCTR_RB_6_HI				0x000004e3
+
+#define REG_A6XX_RBBM_PERFCTR_RB_7_LO				0x000004e4
+
+#define REG_A6XX_RBBM_PERFCTR_RB_7_HI				0x000004e5
+
+#define REG_A6XX_RBBM_PERFCTR_VSC_0_LO				0x000004e6
+
+#define REG_A6XX_RBBM_PERFCTR_VSC_0_HI				0x000004e7
+
+#define REG_A6XX_RBBM_PERFCTR_VSC_1_LO				0x000004e8
+
+#define REG_A6XX_RBBM_PERFCTR_VSC_1_HI				0x000004e9
+
+#define REG_A6XX_RBBM_PERFCTR_LRZ_0_LO				0x000004ea
+
+#define REG_A6XX_RBBM_PERFCTR_LRZ_0_HI				0x000004eb
+
+#define REG_A6XX_RBBM_PERFCTR_LRZ_1_LO				0x000004ec
+
+#define REG_A6XX_RBBM_PERFCTR_LRZ_1_HI				0x000004ed
+
+#define REG_A6XX_RBBM_PERFCTR_LRZ_2_LO				0x000004ee
+
+#define REG_A6XX_RBBM_PERFCTR_LRZ_2_HI				0x000004ef
+
+#define REG_A6XX_RBBM_PERFCTR_LRZ_3_LO				0x000004f0
+
+#define REG_A6XX_RBBM_PERFCTR_LRZ_3_HI				0x000004f1
+
+#define REG_A6XX_RBBM_PERFCTR_CMP_0_LO				0x000004f2
+
+#define REG_A6XX_RBBM_PERFCTR_CMP_0_HI				0x000004f3
+
+#define REG_A6XX_RBBM_PERFCTR_CMP_1_LO				0x000004f4
+
+#define REG_A6XX_RBBM_PERFCTR_CMP_1_HI				0x000004f5
+
+#define REG_A6XX_RBBM_PERFCTR_CMP_2_LO				0x000004f6
+
+#define REG_A6XX_RBBM_PERFCTR_CMP_2_HI				0x000004f7
+
+#define REG_A6XX_RBBM_PERFCTR_CMP_3_LO				0x000004f8
+
+#define REG_A6XX_RBBM_PERFCTR_CMP_3_HI				0x000004f9
+
+#define REG_A6XX_RBBM_PERFCTR_CNTL				0x00000500
+
+#define REG_A6XX_RBBM_PERFCTR_LOAD_CMD0				0x00000501
+
+#define REG_A6XX_RBBM_PERFCTR_LOAD_CMD1				0x00000502
+
+#define REG_A6XX_RBBM_PERFCTR_LOAD_CMD2				0x00000503
+
+#define REG_A6XX_RBBM_PERFCTR_LOAD_CMD3				0x00000504
+
+#define REG_A6XX_RBBM_PERFCTR_LOAD_VALUE_LO			0x00000505
+
+#define REG_A6XX_RBBM_PERFCTR_LOAD_VALUE_HI			0x00000506
+
+#define REG_A6XX_RBBM_PERFCTR_RBBM_SEL_0			0x00000507
+
+#define REG_A6XX_RBBM_PERFCTR_RBBM_SEL_1			0x00000508
+
+#define REG_A6XX_RBBM_PERFCTR_RBBM_SEL_2			0x00000509
+
+#define REG_A6XX_RBBM_PERFCTR_RBBM_SEL_3			0x0000050a
+
+#define REG_A6XX_RBBM_PERFCTR_GPU_BUSY_MASKED			0x0000050b
+
+#define REG_A6XX_RBBM_ISDB_CNT					0x00000533
+
+#define REG_A6XX_RBBM_SECVID_TRUST_CNTL				0x0000f400
+
+#define REG_A6XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO		0x0000f800
+
+#define REG_A6XX_RBBM_SECVID_TSB_TRUSTED_BASE_HI		0x0000f801
+
+#define REG_A6XX_RBBM_SECVID_TSB_TRUSTED_SIZE			0x0000f802
+
+#define REG_A6XX_RBBM_SECVID_TSB_CNTL				0x0000f803
+
+#define REG_A6XX_RBBM_SECVID_TSB_ADDR_MODE_CNTL			0x0000f810
+
+#define REG_A6XX_RBBM_VBIF_CLIENT_QOS_CNTL			0x00000010
+
+#define REG_A6XX_RBBM_INTERFACE_HANG_INT_CNTL			0x0000001f
+
+#define REG_A6XX_RBBM_INT_CLEAR_CMD				0x00000037
+
+#define REG_A6XX_RBBM_INT_0_MASK				0x00000038
+
+#define REG_A6XX_RBBM_SP_HYST_CNT				0x00000042
+
+#define REG_A6XX_RBBM_SW_RESET_CMD				0x00000043
+
+#define REG_A6XX_RBBM_RAC_THRESHOLD_CNT				0x00000044
+
+#define REG_A6XX_RBBM_BLOCK_SW_RESET_CMD			0x00000045
+
+#define REG_A6XX_RBBM_BLOCK_SW_RESET_CMD2			0x00000046
+
+#define REG_A6XX_RBBM_CLOCK_CNTL				0x000000ae
+
+#define REG_A6XX_RBBM_CLOCK_CNTL_SP0				0x000000b0
+
+#define REG_A6XX_RBBM_CLOCK_CNTL_SP1				0x000000b1
+
+#define REG_A6XX_RBBM_CLOCK_CNTL_SP2				0x000000b2
+
+#define REG_A6XX_RBBM_CLOCK_CNTL_SP3				0x000000b3
+
+#define REG_A6XX_RBBM_CLOCK_CNTL2_SP0				0x000000b4
+
+#define REG_A6XX_RBBM_CLOCK_CNTL2_SP1				0x000000b5
+
+#define REG_A6XX_RBBM_CLOCK_CNTL2_SP2				0x000000b6
+
+#define REG_A6XX_RBBM_CLOCK_CNTL2_SP3				0x000000b7
+
+#define REG_A6XX_RBBM_CLOCK_DELAY_SP0				0x000000b8
+
+#define REG_A6XX_RBBM_CLOCK_DELAY_SP1				0x000000b9
+
+#define REG_A6XX_RBBM_CLOCK_DELAY_SP2				0x000000ba
+
+#define REG_A6XX_RBBM_CLOCK_DELAY_SP3				0x000000bb
+
+#define REG_A6XX_RBBM_CLOCK_HYST_SP0				0x000000bc
+
+#define REG_A6XX_RBBM_CLOCK_HYST_SP1				0x000000bd
+
+#define REG_A6XX_RBBM_CLOCK_HYST_SP2				0x000000be
+
+#define REG_A6XX_RBBM_CLOCK_HYST_SP3				0x000000bf
+
+#define REG_A6XX_RBBM_CLOCK_CNTL_TP0				0x000000c0
+
+#define REG_A6XX_RBBM_CLOCK_CNTL_TP1				0x000000c1
+
+#define REG_A6XX_RBBM_CLOCK_CNTL_TP2				0x000000c2
+
+#define REG_A6XX_RBBM_CLOCK_CNTL_TP3				0x000000c3
+
+#define REG_A6XX_RBBM_CLOCK_CNTL2_TP0				0x000000c4
+
+#define REG_A6XX_RBBM_CLOCK_CNTL2_TP1				0x000000c5
+
+#define REG_A6XX_RBBM_CLOCK_CNTL2_TP2				0x000000c6
+
+#define REG_A6XX_RBBM_CLOCK_CNTL2_TP3				0x000000c7
+
+#define REG_A6XX_RBBM_CLOCK_CNTL3_TP0				0x000000c8
+
+#define REG_A6XX_RBBM_CLOCK_CNTL3_TP1				0x000000c9
+
+#define REG_A6XX_RBBM_CLOCK_CNTL3_TP2				0x000000ca
+
+#define REG_A6XX_RBBM_CLOCK_CNTL3_TP3				0x000000cb
+
+#define REG_A6XX_RBBM_CLOCK_CNTL4_TP0				0x000000cc
+
+#define REG_A6XX_RBBM_CLOCK_CNTL4_TP1				0x000000cd
+
+#define REG_A6XX_RBBM_CLOCK_CNTL4_TP2				0x000000ce
+
+#define REG_A6XX_RBBM_CLOCK_CNTL4_TP3				0x000000cf
+
+#define REG_A6XX_RBBM_CLOCK_DELAY_TP0				0x000000d0
+
+#define REG_A6XX_RBBM_CLOCK_DELAY_TP1				0x000000d1
+
+#define REG_A6XX_RBBM_CLOCK_DELAY_TP2				0x000000d2
+
+#define REG_A6XX_RBBM_CLOCK_DELAY_TP3				0x000000d3
+
+#define REG_A6XX_RBBM_CLOCK_DELAY2_TP0				0x000000d4
+
+#define REG_A6XX_RBBM_CLOCK_DELAY2_TP1				0x000000d5
+
+#define REG_A6XX_RBBM_CLOCK_DELAY2_TP2				0x000000d6
+
+#define REG_A6XX_RBBM_CLOCK_DELAY2_TP3				0x000000d7
+
+#define REG_A6XX_RBBM_CLOCK_DELAY3_TP0				0x000000d8
+
+#define REG_A6XX_RBBM_CLOCK_DELAY3_TP1				0x000000d9
+
+#define REG_A6XX_RBBM_CLOCK_DELAY3_TP2				0x000000da
+
+#define REG_A6XX_RBBM_CLOCK_DELAY3_TP3				0x000000db
+
+#define REG_A6XX_RBBM_CLOCK_DELAY4_TP0				0x000000dc
+
+#define REG_A6XX_RBBM_CLOCK_DELAY4_TP1				0x000000dd
+
+#define REG_A6XX_RBBM_CLOCK_DELAY4_TP2				0x000000de
+
+#define REG_A6XX_RBBM_CLOCK_DELAY4_TP3				0x000000df
+
+#define REG_A6XX_RBBM_CLOCK_HYST_TP0				0x000000e0
+
+#define REG_A6XX_RBBM_CLOCK_HYST_TP1				0x000000e1
+
+#define REG_A6XX_RBBM_CLOCK_HYST_TP2				0x000000e2
+
+#define REG_A6XX_RBBM_CLOCK_HYST_TP3				0x000000e3
+
+#define REG_A6XX_RBBM_CLOCK_HYST2_TP0				0x000000e4
+
+#define REG_A6XX_RBBM_CLOCK_HYST2_TP1				0x000000e5
+
+#define REG_A6XX_RBBM_CLOCK_HYST2_TP2				0x000000e6
+
+#define REG_A6XX_RBBM_CLOCK_HYST2_TP3				0x000000e7
+
+#define REG_A6XX_RBBM_CLOCK_HYST3_TP0				0x000000e8
+
+#define REG_A6XX_RBBM_CLOCK_HYST3_TP1				0x000000e9
+
+#define REG_A6XX_RBBM_CLOCK_HYST3_TP2				0x000000ea
+
+#define REG_A6XX_RBBM_CLOCK_HYST3_TP3				0x000000eb
+
+#define REG_A6XX_RBBM_CLOCK_HYST4_TP0				0x000000ec
+
+#define REG_A6XX_RBBM_CLOCK_HYST4_TP1				0x000000ed
+
+#define REG_A6XX_RBBM_CLOCK_HYST4_TP2				0x000000ee
+
+#define REG_A6XX_RBBM_CLOCK_HYST4_TP3				0x000000ef
+
+#define REG_A6XX_RBBM_CLOCK_CNTL_RB0				0x000000f0
+
+#define REG_A6XX_RBBM_CLOCK_CNTL_RB1				0x000000f1
+
+#define REG_A6XX_RBBM_CLOCK_CNTL_RB2				0x000000f2
+
+#define REG_A6XX_RBBM_CLOCK_CNTL_RB3				0x000000f3
+
+#define REG_A6XX_RBBM_CLOCK_CNTL2_RB0				0x000000f4
+
+#define REG_A6XX_RBBM_CLOCK_CNTL2_RB1				0x000000f5
+
+#define REG_A6XX_RBBM_CLOCK_CNTL2_RB2				0x000000f6
+
+#define REG_A6XX_RBBM_CLOCK_CNTL2_RB3				0x000000f7
+
+#define REG_A6XX_RBBM_CLOCK_CNTL_CCU0				0x000000f8
+
+#define REG_A6XX_RBBM_CLOCK_CNTL_CCU1				0x000000f9
+
+#define REG_A6XX_RBBM_CLOCK_CNTL_CCU2				0x000000fa
+
+#define REG_A6XX_RBBM_CLOCK_CNTL_CCU3				0x000000fb
+
+#define REG_A6XX_RBBM_CLOCK_HYST_RB_CCU0			0x00000100
+
+#define REG_A6XX_RBBM_CLOCK_HYST_RB_CCU1			0x00000101
+
+#define REG_A6XX_RBBM_CLOCK_HYST_RB_CCU2			0x00000102
+
+#define REG_A6XX_RBBM_CLOCK_HYST_RB_CCU3			0x00000103
+
+#define REG_A6XX_RBBM_CLOCK_CNTL_RAC				0x00000104
+
+#define REG_A6XX_RBBM_CLOCK_CNTL2_RAC				0x00000105
+
+#define REG_A6XX_RBBM_CLOCK_DELAY_RAC				0x00000106
+
+#define REG_A6XX_RBBM_CLOCK_HYST_RAC				0x00000107
+
+#define REG_A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM			0x00000108
+
+#define REG_A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM			0x00000109
+
+#define REG_A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM			0x0000010a
+
+#define REG_A6XX_RBBM_CLOCK_CNTL_UCHE				0x0000010b
+
+#define REG_A6XX_RBBM_CLOCK_CNTL2_UCHE				0x0000010c
+
+#define REG_A6XX_RBBM_CLOCK_CNTL3_UCHE				0x0000010d
+
+#define REG_A6XX_RBBM_CLOCK_CNTL4_UCHE				0x0000010e
+
+#define REG_A6XX_RBBM_CLOCK_DELAY_UCHE				0x0000010f
+
+#define REG_A6XX_RBBM_CLOCK_HYST_UCHE				0x00000110
+
+#define REG_A6XX_RBBM_CLOCK_MODE_VFD				0x00000111
+
+#define REG_A6XX_RBBM_CLOCK_DELAY_VFD				0x00000112
+
+#define REG_A6XX_RBBM_CLOCK_HYST_VFD				0x00000113
+
+#define REG_A6XX_RBBM_CLOCK_MODE_GPC				0x00000114
+
+#define REG_A6XX_RBBM_CLOCK_DELAY_GPC				0x00000115
+
+#define REG_A6XX_RBBM_CLOCK_HYST_GPC				0x00000116
+
+#define REG_A6XX_RBBM_CLOCK_DELAY_HLSQ_2			0x00000117
+
+#define REG_A6XX_RBBM_CLOCK_CNTL_GMU_GX				0x00000118
+
+#define REG_A6XX_RBBM_CLOCK_DELAY_GMU_GX			0x00000119
+
+#define REG_A6XX_RBBM_CLOCK_HYST_GMU_GX				0x0000011a
+
+#define REG_A6XX_RBBM_CLOCK_MODE_HLSQ				0x0000011b
+
+#define REG_A6XX_RBBM_CLOCK_DELAY_HLSQ				0x0000011c
+
+#define REG_A6XX_DBGC_CFG_DBGBUS_SEL_A				0x00000600
+
+#define REG_A6XX_DBGC_CFG_DBGBUS_SEL_B				0x00000601
+
+#define REG_A6XX_DBGC_CFG_DBGBUS_SEL_C				0x00000602
+
+#define REG_A6XX_DBGC_CFG_DBGBUS_SEL_D				0x00000603
+#define A6XX_DBGC_CFG_DBGBUS_SEL_D_PING_INDEX__MASK		0x000000ff
+#define A6XX_DBGC_CFG_DBGBUS_SEL_D_PING_INDEX__SHIFT		0
+static inline uint32_t A6XX_DBGC_CFG_DBGBUS_SEL_D_PING_INDEX(uint32_t val)
+{
+	return ((val) << A6XX_DBGC_CFG_DBGBUS_SEL_D_PING_INDEX__SHIFT) & A6XX_DBGC_CFG_DBGBUS_SEL_D_PING_INDEX__MASK;
+}
+#define A6XX_DBGC_CFG_DBGBUS_SEL_D_PING_BLK_SEL__MASK		0x0000ff00
+#define A6XX_DBGC_CFG_DBGBUS_SEL_D_PING_BLK_SEL__SHIFT		8
+static inline uint32_t A6XX_DBGC_CFG_DBGBUS_SEL_D_PING_BLK_SEL(uint32_t val)
+{
+	return ((val) << A6XX_DBGC_CFG_DBGBUS_SEL_D_PING_BLK_SEL__SHIFT) & A6XX_DBGC_CFG_DBGBUS_SEL_D_PING_BLK_SEL__MASK;
+}
+
+#define REG_A6XX_DBGC_CFG_DBGBUS_CNTLT				0x00000604
+#define A6XX_DBGC_CFG_DBGBUS_CNTLT_TRACEEN__MASK		0x0000003f
+#define A6XX_DBGC_CFG_DBGBUS_CNTLT_TRACEEN__SHIFT		0
+static inline uint32_t A6XX_DBGC_CFG_DBGBUS_CNTLT_TRACEEN(uint32_t val)
+{
+	return ((val) << A6XX_DBGC_CFG_DBGBUS_CNTLT_TRACEEN__SHIFT) & A6XX_DBGC_CFG_DBGBUS_CNTLT_TRACEEN__MASK;
+}
+#define A6XX_DBGC_CFG_DBGBUS_CNTLT_GRANU__MASK			0x00007000
+#define A6XX_DBGC_CFG_DBGBUS_CNTLT_GRANU__SHIFT			12
+static inline uint32_t A6XX_DBGC_CFG_DBGBUS_CNTLT_GRANU(uint32_t val)
+{
+	return ((val) << A6XX_DBGC_CFG_DBGBUS_CNTLT_GRANU__SHIFT) & A6XX_DBGC_CFG_DBGBUS_CNTLT_GRANU__MASK;
+}
+#define A6XX_DBGC_CFG_DBGBUS_CNTLT_SEGT__MASK			0xf0000000
+#define A6XX_DBGC_CFG_DBGBUS_CNTLT_SEGT__SHIFT			28
+static inline uint32_t A6XX_DBGC_CFG_DBGBUS_CNTLT_SEGT(uint32_t val)
+{
+	return ((val) << A6XX_DBGC_CFG_DBGBUS_CNTLT_SEGT__SHIFT) & A6XX_DBGC_CFG_DBGBUS_CNTLT_SEGT__MASK;
+}
+
+#define REG_A6XX_DBGC_CFG_DBGBUS_CNTLM				0x00000605
+#define A6XX_DBGC_CFG_DBGBUS_CNTLM_ENABLE__MASK			0x0f000000
+#define A6XX_DBGC_CFG_DBGBUS_CNTLM_ENABLE__SHIFT		24
+static inline uint32_t A6XX_DBGC_CFG_DBGBUS_CNTLM_ENABLE(uint32_t val)
+{
+	return ((val) << A6XX_DBGC_CFG_DBGBUS_CNTLM_ENABLE__SHIFT) & A6XX_DBGC_CFG_DBGBUS_CNTLM_ENABLE__MASK;
+}
+
+#define REG_A6XX_DBGC_CFG_DBGBUS_IVTL_0				0x00000608
+
+#define REG_A6XX_DBGC_CFG_DBGBUS_IVTL_1				0x00000609
+
+#define REG_A6XX_DBGC_CFG_DBGBUS_IVTL_2				0x0000060a
+
+#define REG_A6XX_DBGC_CFG_DBGBUS_IVTL_3				0x0000060b
+
+#define REG_A6XX_DBGC_CFG_DBGBUS_MASKL_0			0x0000060c
+
+#define REG_A6XX_DBGC_CFG_DBGBUS_MASKL_1			0x0000060d
+
+#define REG_A6XX_DBGC_CFG_DBGBUS_MASKL_2			0x0000060e
+
+#define REG_A6XX_DBGC_CFG_DBGBUS_MASKL_3			0x0000060f
+
+#define REG_A6XX_DBGC_CFG_DBGBUS_BYTEL_0			0x00000610
+#define A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL0__MASK		0x0000000f
+#define A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL0__SHIFT		0
+static inline uint32_t A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL0(uint32_t val)
+{
+	return ((val) << A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL0__SHIFT) & A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL0__MASK;
+}
+#define A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL1__MASK		0x000000f0
+#define A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL1__SHIFT		4
+static inline uint32_t A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL1(uint32_t val)
+{
+	return ((val) << A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL1__SHIFT) & A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL1__MASK;
+}
+#define A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL2__MASK		0x00000f00
+#define A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL2__SHIFT		8
+static inline uint32_t A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL2(uint32_t val)
+{
+	return ((val) << A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL2__SHIFT) & A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL2__MASK;
+}
+#define A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL3__MASK		0x0000f000
+#define A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL3__SHIFT		12
+static inline uint32_t A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL3(uint32_t val)
+{
+	return ((val) << A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL3__SHIFT) & A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL3__MASK;
+}
+#define A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL4__MASK		0x000f0000
+#define A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL4__SHIFT		16
+static inline uint32_t A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL4(uint32_t val)
+{
+	return ((val) << A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL4__SHIFT) & A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL4__MASK;
+}
+#define A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL5__MASK		0x00f00000
+#define A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL5__SHIFT		20
+static inline uint32_t A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL5(uint32_t val)
+{
+	return ((val) << A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL5__SHIFT) & A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL5__MASK;
+}
+#define A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL6__MASK		0x0f000000
+#define A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL6__SHIFT		24
+static inline uint32_t A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL6(uint32_t val)
+{
+	return ((val) << A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL6__SHIFT) & A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL6__MASK;
+}
+#define A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL7__MASK		0xf0000000
+#define A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL7__SHIFT		28
+static inline uint32_t A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL7(uint32_t val)
+{
+	return ((val) << A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL7__SHIFT) & A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL7__MASK;
+}
+
+#define REG_A6XX_DBGC_CFG_DBGBUS_BYTEL_1			0x00000611
+#define A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL8__MASK		0x0000000f
+#define A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL8__SHIFT		0
+static inline uint32_t A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL8(uint32_t val)
+{
+	return ((val) << A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL8__SHIFT) & A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL8__MASK;
+}
+#define A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL9__MASK		0x000000f0
+#define A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL9__SHIFT		4
+static inline uint32_t A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL9(uint32_t val)
+{
+	return ((val) << A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL9__SHIFT) & A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL9__MASK;
+}
+#define A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL10__MASK		0x00000f00
+#define A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL10__SHIFT		8
+static inline uint32_t A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL10(uint32_t val)
+{
+	return ((val) << A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL10__SHIFT) & A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL10__MASK;
+}
+#define A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL11__MASK		0x0000f000
+#define A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL11__SHIFT		12
+static inline uint32_t A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL11(uint32_t val)
+{
+	return ((val) << A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL11__SHIFT) & A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL11__MASK;
+}
+#define A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL12__MASK		0x000f0000
+#define A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL12__SHIFT		16
+static inline uint32_t A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL12(uint32_t val)
+{
+	return ((val) << A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL12__SHIFT) & A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL12__MASK;
+}
+#define A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL13__MASK		0x00f00000
+#define A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL13__SHIFT		20
+static inline uint32_t A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL13(uint32_t val)
+{
+	return ((val) << A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL13__SHIFT) & A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL13__MASK;
+}
+#define A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL14__MASK		0x0f000000
+#define A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL14__SHIFT		24
+static inline uint32_t A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL14(uint32_t val)
+{
+	return ((val) << A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL14__SHIFT) & A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL14__MASK;
+}
+#define A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL15__MASK		0xf0000000
+#define A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL15__SHIFT		28
+static inline uint32_t A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL15(uint32_t val)
+{
+	return ((val) << A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL15__SHIFT) & A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL15__MASK;
+}
+
+#define REG_A6XX_DBGC_CFG_DBGBUS_TRACE_BUF1			0x0000062f
+
+#define REG_A6XX_DBGC_CFG_DBGBUS_TRACE_BUF2			0x00000630
+
+#define REG_A6XX_VSC_PERFCTR_VSC_SEL_0				0x00000cd8
+
+#define REG_A6XX_VSC_PERFCTR_VSC_SEL_1				0x00000cd9
+
+#define REG_A6XX_GRAS_ADDR_MODE_CNTL				0x00008601
+
+#define REG_A6XX_GRAS_PERFCTR_TSE_SEL_0				0x00008610
+
+#define REG_A6XX_GRAS_PERFCTR_TSE_SEL_1				0x00008611
+
+#define REG_A6XX_GRAS_PERFCTR_TSE_SEL_2				0x00008612
+
+#define REG_A6XX_GRAS_PERFCTR_TSE_SEL_3				0x00008613
+
+#define REG_A6XX_GRAS_PERFCTR_RAS_SEL_0				0x00008614
+
+#define REG_A6XX_GRAS_PERFCTR_RAS_SEL_1				0x00008615
+
+#define REG_A6XX_GRAS_PERFCTR_RAS_SEL_2				0x00008616
+
+#define REG_A6XX_GRAS_PERFCTR_RAS_SEL_3				0x00008617
+
+#define REG_A6XX_GRAS_PERFCTR_LRZ_SEL_0				0x00008618
+
+#define REG_A6XX_GRAS_PERFCTR_LRZ_SEL_1				0x00008619
+
+#define REG_A6XX_GRAS_PERFCTR_LRZ_SEL_2				0x0000861a
+
+#define REG_A6XX_GRAS_PERFCTR_LRZ_SEL_3				0x0000861b
+
+#define REG_A6XX_RB_ADDR_MODE_CNTL				0x00008e05
+
+#define REG_A6XX_RB_NC_MODE_CNTL				0x00008e08
+
+#define REG_A6XX_RB_PERFCTR_RB_SEL_0				0x00008e10
+
+#define REG_A6XX_RB_PERFCTR_RB_SEL_1				0x00008e11
+
+#define REG_A6XX_RB_PERFCTR_RB_SEL_2				0x00008e12
+
+#define REG_A6XX_RB_PERFCTR_RB_SEL_3				0x00008e13
+
+#define REG_A6XX_RB_PERFCTR_RB_SEL_4				0x00008e14
+
+#define REG_A6XX_RB_PERFCTR_RB_SEL_5				0x00008e15
+
+#define REG_A6XX_RB_PERFCTR_RB_SEL_6				0x00008e16
+
+#define REG_A6XX_RB_PERFCTR_RB_SEL_7				0x00008e17
+
+#define REG_A6XX_RB_PERFCTR_CCU_SEL_0				0x00008e18
+
+#define REG_A6XX_RB_PERFCTR_CCU_SEL_1				0x00008e19
+
+#define REG_A6XX_RB_PERFCTR_CCU_SEL_2				0x00008e1a
+
+#define REG_A6XX_RB_PERFCTR_CCU_SEL_3				0x00008e1b
+
+#define REG_A6XX_RB_PERFCTR_CCU_SEL_4				0x00008e1c
+
+#define REG_A6XX_RB_PERFCTR_CMP_SEL_0				0x00008e2c
+
+#define REG_A6XX_RB_PERFCTR_CMP_SEL_1				0x00008e2d
+
+#define REG_A6XX_RB_PERFCTR_CMP_SEL_2				0x00008e2e
+
+#define REG_A6XX_RB_PERFCTR_CMP_SEL_3				0x00008e2f
+
+#define REG_A6XX_RB_RB_SUB_BLOCK_SEL_CNTL_CD			0x00008e3d
+
+#define REG_A6XX_RB_CONTEXT_SWITCH_GMEM_SAVE_RESTORE		0x00008e50
+
+#define REG_A6XX_PC_DBG_ECO_CNTL				0x00009e00
+
+#define REG_A6XX_PC_ADDR_MODE_CNTL				0x00009e01
+
+#define REG_A6XX_PC_PERFCTR_PC_SEL_0				0x00009e34
+
+#define REG_A6XX_PC_PERFCTR_PC_SEL_1				0x00009e35
+
+#define REG_A6XX_PC_PERFCTR_PC_SEL_2				0x00009e36
+
+#define REG_A6XX_PC_PERFCTR_PC_SEL_3				0x00009e37
+
+#define REG_A6XX_PC_PERFCTR_PC_SEL_4				0x00009e38
+
+#define REG_A6XX_PC_PERFCTR_PC_SEL_5				0x00009e39
+
+#define REG_A6XX_PC_PERFCTR_PC_SEL_6				0x00009e3a
+
+#define REG_A6XX_PC_PERFCTR_PC_SEL_7				0x00009e3b
+
+#define REG_A6XX_HLSQ_ADDR_MODE_CNTL				0x0000be05
+
+#define REG_A6XX_HLSQ_PERFCTR_HLSQ_SEL_0			0x0000be10
+
+#define REG_A6XX_HLSQ_PERFCTR_HLSQ_SEL_1			0x0000be11
+
+#define REG_A6XX_HLSQ_PERFCTR_HLSQ_SEL_2			0x0000be12
+
+#define REG_A6XX_HLSQ_PERFCTR_HLSQ_SEL_3			0x0000be13
+
+#define REG_A6XX_HLSQ_PERFCTR_HLSQ_SEL_4			0x0000be14
+
+#define REG_A6XX_HLSQ_PERFCTR_HLSQ_SEL_5			0x0000be15
+
+#define REG_A6XX_HLSQ_DBG_AHB_READ_APERTURE			0x0000c800
+
+#define REG_A6XX_HLSQ_DBG_READ_SEL				0x0000d000
+
+#define REG_A6XX_VFD_ADDR_MODE_CNTL				0x0000a601
+
+#define REG_A6XX_VFD_PERFCTR_VFD_SEL_0				0x0000a610
+
+#define REG_A6XX_VFD_PERFCTR_VFD_SEL_1				0x0000a611
+
+#define REG_A6XX_VFD_PERFCTR_VFD_SEL_2				0x0000a612
+
+#define REG_A6XX_VFD_PERFCTR_VFD_SEL_3				0x0000a613
+
+#define REG_A6XX_VFD_PERFCTR_VFD_SEL_4				0x0000a614
+
+#define REG_A6XX_VFD_PERFCTR_VFD_SEL_5				0x0000a615
+
+#define REG_A6XX_VFD_PERFCTR_VFD_SEL_6				0x0000a616
+
+#define REG_A6XX_VFD_PERFCTR_VFD_SEL_7				0x0000a617
+
+#define REG_A6XX_VPC_ADDR_MODE_CNTL				0x00009601
+
+#define REG_A6XX_VPC_PERFCTR_VPC_SEL_0				0x00009604
+
+#define REG_A6XX_VPC_PERFCTR_VPC_SEL_1				0x00009605
+
+#define REG_A6XX_VPC_PERFCTR_VPC_SEL_2				0x00009606
+
+#define REG_A6XX_VPC_PERFCTR_VPC_SEL_3				0x00009607
+
+#define REG_A6XX_VPC_PERFCTR_VPC_SEL_4				0x00009608
+
+#define REG_A6XX_VPC_PERFCTR_VPC_SEL_5				0x00009609
+
+#define REG_A6XX_UCHE_ADDR_MODE_CNTL				0x00000e00
+
+#define REG_A6XX_UCHE_MODE_CNTL					0x00000e01
+
+#define REG_A6XX_UCHE_WRITE_RANGE_MAX_LO			0x00000e05
+
+#define REG_A6XX_UCHE_WRITE_RANGE_MAX_HI			0x00000e06
+
+#define REG_A6XX_UCHE_WRITE_THRU_BASE_LO			0x00000e07
+
+#define REG_A6XX_UCHE_WRITE_THRU_BASE_HI			0x00000e08
+
+#define REG_A6XX_UCHE_TRAP_BASE_LO				0x00000e09
+
+#define REG_A6XX_UCHE_TRAP_BASE_HI				0x00000e0a
+
+#define REG_A6XX_UCHE_GMEM_RANGE_MIN_LO				0x00000e0b
+
+#define REG_A6XX_UCHE_GMEM_RANGE_MIN_HI				0x00000e0c
+
+#define REG_A6XX_UCHE_GMEM_RANGE_MAX_LO				0x00000e0d
+
+#define REG_A6XX_UCHE_GMEM_RANGE_MAX_HI				0x00000e0e
+
+#define REG_A6XX_UCHE_CACHE_WAYS				0x00000e17
+
+#define REG_A6XX_UCHE_FILTER_CNTL				0x00000e18
+
+#define REG_A6XX_UCHE_CLIENT_PF					0x00000e19
+#define A6XX_UCHE_CLIENT_PF_PERFSEL__MASK			0x000000ff
+#define A6XX_UCHE_CLIENT_PF_PERFSEL__SHIFT			0
+static inline uint32_t A6XX_UCHE_CLIENT_PF_PERFSEL(uint32_t val)
+{
+	return ((val) << A6XX_UCHE_CLIENT_PF_PERFSEL__SHIFT) & A6XX_UCHE_CLIENT_PF_PERFSEL__MASK;
+}
+
+#define REG_A6XX_UCHE_PERFCTR_UCHE_SEL_0			0x00000e1c
+
+#define REG_A6XX_UCHE_PERFCTR_UCHE_SEL_1			0x00000e1d
+
+#define REG_A6XX_UCHE_PERFCTR_UCHE_SEL_2			0x00000e1e
+
+#define REG_A6XX_UCHE_PERFCTR_UCHE_SEL_3			0x00000e1f
+
+#define REG_A6XX_UCHE_PERFCTR_UCHE_SEL_4			0x00000e20
+
+#define REG_A6XX_UCHE_PERFCTR_UCHE_SEL_5			0x00000e21
+
+#define REG_A6XX_UCHE_PERFCTR_UCHE_SEL_6			0x00000e22
+
+#define REG_A6XX_UCHE_PERFCTR_UCHE_SEL_7			0x00000e23
+
+#define REG_A6XX_UCHE_PERFCTR_UCHE_SEL_8			0x00000e24
+
+#define REG_A6XX_UCHE_PERFCTR_UCHE_SEL_9			0x00000e25
+
+#define REG_A6XX_UCHE_PERFCTR_UCHE_SEL_10			0x00000e26
+
+#define REG_A6XX_UCHE_PERFCTR_UCHE_SEL_11			0x00000e27
+
+#define REG_A6XX_SP_ADDR_MODE_CNTL				0x0000ae01
+
+#define REG_A6XX_SP_NC_MODE_CNTL				0x0000ae02
+
+#define REG_A6XX_SP_PERFCTR_SP_SEL_0				0x0000ae10
+
+#define REG_A6XX_SP_PERFCTR_SP_SEL_1				0x0000ae11
+
+#define REG_A6XX_SP_PERFCTR_SP_SEL_2				0x0000ae12
+
+#define REG_A6XX_SP_PERFCTR_SP_SEL_3				0x0000ae13
+
+#define REG_A6XX_SP_PERFCTR_SP_SEL_4				0x0000ae14
+
+#define REG_A6XX_SP_PERFCTR_SP_SEL_5				0x0000ae15
+
+#define REG_A6XX_SP_PERFCTR_SP_SEL_6				0x0000ae16
+
+#define REG_A6XX_SP_PERFCTR_SP_SEL_7				0x0000ae17
+
+#define REG_A6XX_SP_PERFCTR_SP_SEL_8				0x0000ae18
+
+#define REG_A6XX_SP_PERFCTR_SP_SEL_9				0x0000ae19
+
+#define REG_A6XX_SP_PERFCTR_SP_SEL_10				0x0000ae1a
+
+#define REG_A6XX_SP_PERFCTR_SP_SEL_11				0x0000ae1b
+
+#define REG_A6XX_SP_PERFCTR_SP_SEL_12				0x0000ae1c
+
+#define REG_A6XX_SP_PERFCTR_SP_SEL_13				0x0000ae1d
+
+#define REG_A6XX_SP_PERFCTR_SP_SEL_14				0x0000ae1e
+
+#define REG_A6XX_SP_PERFCTR_SP_SEL_15				0x0000ae1f
+
+#define REG_A6XX_SP_PERFCTR_SP_SEL_16				0x0000ae20
+
+#define REG_A6XX_SP_PERFCTR_SP_SEL_17				0x0000ae21
+
+#define REG_A6XX_SP_PERFCTR_SP_SEL_18				0x0000ae22
+
+#define REG_A6XX_SP_PERFCTR_SP_SEL_19				0x0000ae23
+
+#define REG_A6XX_SP_PERFCTR_SP_SEL_20				0x0000ae24
+
+#define REG_A6XX_SP_PERFCTR_SP_SEL_21				0x0000ae25
+
+#define REG_A6XX_SP_PERFCTR_SP_SEL_22				0x0000ae26
+
+#define REG_A6XX_SP_PERFCTR_SP_SEL_23				0x0000ae27
+
+#define REG_A6XX_TPL1_ADDR_MODE_CNTL				0x0000b601
+
+#define REG_A6XX_TPL1_NC_MODE_CNTL				0x0000b604
+
+#define REG_A6XX_TPL1_PERFCTR_TP_SEL_0				0x0000b610
+
+#define REG_A6XX_TPL1_PERFCTR_TP_SEL_1				0x0000b611
+
+#define REG_A6XX_TPL1_PERFCTR_TP_SEL_2				0x0000b612
+
+#define REG_A6XX_TPL1_PERFCTR_TP_SEL_3				0x0000b613
+
+#define REG_A6XX_TPL1_PERFCTR_TP_SEL_4				0x0000b614
+
+#define REG_A6XX_TPL1_PERFCTR_TP_SEL_5				0x0000b615
+
+#define REG_A6XX_TPL1_PERFCTR_TP_SEL_6				0x0000b616
+
+#define REG_A6XX_TPL1_PERFCTR_TP_SEL_7				0x0000b617
+
+#define REG_A6XX_TPL1_PERFCTR_TP_SEL_8				0x0000b618
+
+#define REG_A6XX_TPL1_PERFCTR_TP_SEL_9				0x0000b619
+
+#define REG_A6XX_TPL1_PERFCTR_TP_SEL_10				0x0000b61a
+
+#define REG_A6XX_TPL1_PERFCTR_TP_SEL_11				0x0000b61b
+
+#define REG_A6XX_VBIF_VERSION					0x00003000
+
+#define REG_A6XX_VBIF_GATE_OFF_WRREQ_EN				0x0000302a
+
+#define REG_A6XX_VBIF_XIN_HALT_CTRL0				0x00003080
+
+#define REG_A6XX_VBIF_XIN_HALT_CTRL1				0x00003081
+
+#define REG_A6XX_VBIF_PERF_CNT_SEL0				0x000030d0
+
+#define REG_A6XX_VBIF_PERF_CNT_SEL1				0x000030d1
+
+#define REG_A6XX_VBIF_PERF_CNT_SEL2				0x000030d2
+
+#define REG_A6XX_VBIF_PERF_CNT_SEL3				0x000030d3
+
+#define REG_A6XX_VBIF_PERF_CNT_LOW0				0x000030d8
+
+#define REG_A6XX_VBIF_PERF_CNT_LOW1				0x000030d9
+
+#define REG_A6XX_VBIF_PERF_CNT_LOW2				0x000030da
+
+#define REG_A6XX_VBIF_PERF_CNT_LOW3				0x000030db
+
+#define REG_A6XX_VBIF_PERF_CNT_HIGH0				0x000030e0
+
+#define REG_A6XX_VBIF_PERF_CNT_HIGH1				0x000030e1
+
+#define REG_A6XX_VBIF_PERF_CNT_HIGH2				0x000030e2
+
+#define REG_A6XX_VBIF_PERF_CNT_HIGH3				0x000030e3
+
+#define REG_A6XX_VBIF_PERF_PWR_CNT_EN0				0x00003100
+
+#define REG_A6XX_VBIF_PERF_PWR_CNT_EN1				0x00003101
+
+#define REG_A6XX_VBIF_PERF_PWR_CNT_EN2				0x00003102
+
+#define REG_A6XX_VBIF_PERF_PWR_CNT_LOW0				0x00003110
+
+#define REG_A6XX_VBIF_PERF_PWR_CNT_LOW1				0x00003111
+
+#define REG_A6XX_VBIF_PERF_PWR_CNT_LOW2				0x00003112
+
+#define REG_A6XX_VBIF_PERF_PWR_CNT_HIGH0			0x00003118
+
+#define REG_A6XX_VBIF_PERF_PWR_CNT_HIGH1			0x00003119
+
+#define REG_A6XX_VBIF_PERF_PWR_CNT_HIGH2			0x0000311a
+
+#define REG_A6XX_CX_DBGC_CFG_DBGBUS_SEL_A			0x00018400
+
+#define REG_A6XX_CX_DBGC_CFG_DBGBUS_SEL_B			0x00018401
+
+#define REG_A6XX_CX_DBGC_CFG_DBGBUS_SEL_C			0x00018402
+
+#define REG_A6XX_CX_DBGC_CFG_DBGBUS_SEL_D			0x00018403
+#define A6XX_CX_DBGC_CFG_DBGBUS_SEL_D_PING_INDEX__MASK		0x000000ff
+#define A6XX_CX_DBGC_CFG_DBGBUS_SEL_D_PING_INDEX__SHIFT		0
+static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_SEL_D_PING_INDEX(uint32_t val)
+{
+	return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_SEL_D_PING_INDEX__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_SEL_D_PING_INDEX__MASK;
+}
+#define A6XX_CX_DBGC_CFG_DBGBUS_SEL_D_PING_BLK_SEL__MASK	0x0000ff00
+#define A6XX_CX_DBGC_CFG_DBGBUS_SEL_D_PING_BLK_SEL__SHIFT	8
+static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_SEL_D_PING_BLK_SEL(uint32_t val)
+{
+	return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_SEL_D_PING_BLK_SEL__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_SEL_D_PING_BLK_SEL__MASK;
+}
+
+#define REG_A6XX_CX_DBGC_CFG_DBGBUS_CNTLT			0x00018404
+#define A6XX_CX_DBGC_CFG_DBGBUS_CNTLT_TRACEEN__MASK		0x0000003f
+#define A6XX_CX_DBGC_CFG_DBGBUS_CNTLT_TRACEEN__SHIFT		0
+static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_CNTLT_TRACEEN(uint32_t val)
+{
+	return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_CNTLT_TRACEEN__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_CNTLT_TRACEEN__MASK;
+}
+#define A6XX_CX_DBGC_CFG_DBGBUS_CNTLT_GRANU__MASK		0x00007000
+#define A6XX_CX_DBGC_CFG_DBGBUS_CNTLT_GRANU__SHIFT		12
+static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_CNTLT_GRANU(uint32_t val)
+{
+	return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_CNTLT_GRANU__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_CNTLT_GRANU__MASK;
+}
+#define A6XX_CX_DBGC_CFG_DBGBUS_CNTLT_SEGT__MASK		0xf0000000
+#define A6XX_CX_DBGC_CFG_DBGBUS_CNTLT_SEGT__SHIFT		28
+static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_CNTLT_SEGT(uint32_t val)
+{
+	return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_CNTLT_SEGT__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_CNTLT_SEGT__MASK;
+}
+
+#define REG_A6XX_CX_DBGC_CFG_DBGBUS_CNTLM			0x00018405
+#define A6XX_CX_DBGC_CFG_DBGBUS_CNTLM_ENABLE__MASK		0x0f000000
+#define A6XX_CX_DBGC_CFG_DBGBUS_CNTLM_ENABLE__SHIFT		24
+static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_CNTLM_ENABLE(uint32_t val)
+{
+	return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_CNTLM_ENABLE__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_CNTLM_ENABLE__MASK;
+}
+
+#define REG_A6XX_CX_DBGC_CFG_DBGBUS_IVTL_0			0x00018408
+
+#define REG_A6XX_CX_DBGC_CFG_DBGBUS_IVTL_1			0x00018409
+
+#define REG_A6XX_CX_DBGC_CFG_DBGBUS_IVTL_2			0x0001840a
+
+#define REG_A6XX_CX_DBGC_CFG_DBGBUS_IVTL_3			0x0001840b
+
+#define REG_A6XX_CX_DBGC_CFG_DBGBUS_MASKL_0			0x0001840c
+
+#define REG_A6XX_CX_DBGC_CFG_DBGBUS_MASKL_1			0x0001840d
+
+#define REG_A6XX_CX_DBGC_CFG_DBGBUS_MASKL_2			0x0001840e
+
+#define REG_A6XX_CX_DBGC_CFG_DBGBUS_MASKL_3			0x0001840f
+
+#define REG_A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0			0x00018410
+#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL0__MASK		0x0000000f
+#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL0__SHIFT		0
+static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL0(uint32_t val)
+{
+	return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL0__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL0__MASK;
+}
+#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL1__MASK		0x000000f0
+#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL1__SHIFT		4
+static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL1(uint32_t val)
+{
+	return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL1__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL1__MASK;
+}
+#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL2__MASK		0x00000f00
+#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL2__SHIFT		8
+static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL2(uint32_t val)
+{
+	return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL2__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL2__MASK;
+}
+#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL3__MASK		0x0000f000
+#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL3__SHIFT		12
+static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL3(uint32_t val)
+{
+	return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL3__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL3__MASK;
+}
+#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL4__MASK		0x000f0000
+#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL4__SHIFT		16
+static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL4(uint32_t val)
+{
+	return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL4__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL4__MASK;
+}
+#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL5__MASK		0x00f00000
+#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL5__SHIFT		20
+static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL5(uint32_t val)
+{
+	return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL5__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL5__MASK;
+}
+#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL6__MASK		0x0f000000
+#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL6__SHIFT		24
+static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL6(uint32_t val)
+{
+	return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL6__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL6__MASK;
+}
+#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL7__MASK		0xf0000000
+#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL7__SHIFT		28
+static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL7(uint32_t val)
+{
+	return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL7__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL7__MASK;
+}
+
+#define REG_A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1			0x00018411
+#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL8__MASK		0x0000000f
+#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL8__SHIFT		0
+static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL8(uint32_t val)
+{
+	return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL8__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL8__MASK;
+}
+#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL9__MASK		0x000000f0
+#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL9__SHIFT		4
+static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL9(uint32_t val)
+{
+	return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL9__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL9__MASK;
+}
+#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL10__MASK		0x00000f00
+#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL10__SHIFT		8
+static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL10(uint32_t val)
+{
+	return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL10__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL10__MASK;
+}
+#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL11__MASK		0x0000f000
+#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL11__SHIFT		12
+static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL11(uint32_t val)
+{
+	return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL11__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL11__MASK;
+}
+#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL12__MASK		0x000f0000
+#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL12__SHIFT		16
+static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL12(uint32_t val)
+{
+	return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL12__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL12__MASK;
+}
+#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL13__MASK		0x00f00000
+#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL13__SHIFT		20
+static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL13(uint32_t val)
+{
+	return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL13__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL13__MASK;
+}
+#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL14__MASK		0x0f000000
+#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL14__SHIFT		24
+static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL14(uint32_t val)
+{
+	return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL14__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL14__MASK;
+}
+#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL15__MASK		0xf0000000
+#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL15__SHIFT		28
+static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL15(uint32_t val)
+{
+	return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL15__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL15__MASK;
+}
+
+#define REG_A6XX_CX_DBGC_CFG_DBGBUS_TRACE_BUF1			0x0001842f
+
+#define REG_A6XX_CX_DBGC_CFG_DBGBUS_TRACE_BUF2			0x00018430
+
+#define REG_A6XX_PDC_GPU_ENABLE_PDC				0x00021140
+
+#define REG_A6XX_PDC_GPU_SEQ_START_ADDR				0x00021148
+
+#define REG_A6XX_PDC_GPU_TCS0_CONTROL				0x00021540
+
+#define REG_A6XX_PDC_GPU_TCS0_CMD_ENABLE_BANK			0x00021541
+
+#define REG_A6XX_PDC_GPU_TCS0_CMD_WAIT_FOR_CMPL_BANK		0x00021542
+
+#define REG_A6XX_PDC_GPU_TCS0_CMD0_MSGID			0x00021543
+
+#define REG_A6XX_PDC_GPU_TCS0_CMD0_ADDR				0x00021544
+
+#define REG_A6XX_PDC_GPU_TCS0_CMD0_DATA				0x00021545
+
+#define REG_A6XX_PDC_GPU_TCS1_CONTROL				0x00021572
+
+#define REG_A6XX_PDC_GPU_TCS1_CMD_ENABLE_BANK			0x00021573
+
+#define REG_A6XX_PDC_GPU_TCS1_CMD_WAIT_FOR_CMPL_BANK		0x00021574
+
+#define REG_A6XX_PDC_GPU_TCS1_CMD0_MSGID			0x00021575
+
+#define REG_A6XX_PDC_GPU_TCS1_CMD0_ADDR				0x00021576
+
+#define REG_A6XX_PDC_GPU_TCS1_CMD0_DATA				0x00021577
+
+#define REG_A6XX_PDC_GPU_TCS2_CONTROL				0x000215a4
+
+#define REG_A6XX_PDC_GPU_TCS2_CMD_ENABLE_BANK			0x000215a5
+
+#define REG_A6XX_PDC_GPU_TCS2_CMD_WAIT_FOR_CMPL_BANK		0x000215a6
+
+#define REG_A6XX_PDC_GPU_TCS2_CMD0_MSGID			0x000215a7
+
+#define REG_A6XX_PDC_GPU_TCS2_CMD0_ADDR				0x000215a8
+
+#define REG_A6XX_PDC_GPU_TCS2_CMD0_DATA				0x000215a9
+
+#define REG_A6XX_PDC_GPU_TCS3_CONTROL				0x000215d6
+
+#define REG_A6XX_PDC_GPU_TCS3_CMD_ENABLE_BANK			0x000215d7
+
+#define REG_A6XX_PDC_GPU_TCS3_CMD_WAIT_FOR_CMPL_BANK		0x000215d8
+
+#define REG_A6XX_PDC_GPU_TCS3_CMD0_MSGID			0x000215d9
+
+#define REG_A6XX_PDC_GPU_TCS3_CMD0_ADDR				0x000215da
+
+#define REG_A6XX_PDC_GPU_TCS3_CMD0_DATA				0x000215db
+
+#define REG_A6XX_PDC_GPU_SEQ_MEM_0				0x000a0000
+
+#define REG_A6XX_X1_WINDOW_OFFSET				0x000088d4
+#define A6XX_X1_WINDOW_OFFSET_WINDOW_OFFSET_DISABLE		0x80000000
+#define A6XX_X1_WINDOW_OFFSET_X__MASK				0x00007fff
+#define A6XX_X1_WINDOW_OFFSET_X__SHIFT				0
+static inline uint32_t A6XX_X1_WINDOW_OFFSET_X(uint32_t val)
+{
+	return ((val) << A6XX_X1_WINDOW_OFFSET_X__SHIFT) & A6XX_X1_WINDOW_OFFSET_X__MASK;
+}
+#define A6XX_X1_WINDOW_OFFSET_Y__MASK				0x7fff0000
+#define A6XX_X1_WINDOW_OFFSET_Y__SHIFT				16
+static inline uint32_t A6XX_X1_WINDOW_OFFSET_Y(uint32_t val)
+{
+	return ((val) << A6XX_X1_WINDOW_OFFSET_Y__SHIFT) & A6XX_X1_WINDOW_OFFSET_Y__MASK;
+}
+
+#define REG_A6XX_X2_WINDOW_OFFSET				0x0000b4d1
+#define A6XX_X2_WINDOW_OFFSET_WINDOW_OFFSET_DISABLE		0x80000000
+#define A6XX_X2_WINDOW_OFFSET_X__MASK				0x00007fff
+#define A6XX_X2_WINDOW_OFFSET_X__SHIFT				0
+static inline uint32_t A6XX_X2_WINDOW_OFFSET_X(uint32_t val)
+{
+	return ((val) << A6XX_X2_WINDOW_OFFSET_X__SHIFT) & A6XX_X2_WINDOW_OFFSET_X__MASK;
+}
+#define A6XX_X2_WINDOW_OFFSET_Y__MASK				0x7fff0000
+#define A6XX_X2_WINDOW_OFFSET_Y__SHIFT				16
+static inline uint32_t A6XX_X2_WINDOW_OFFSET_Y(uint32_t val)
+{
+	return ((val) << A6XX_X2_WINDOW_OFFSET_Y__SHIFT) & A6XX_X2_WINDOW_OFFSET_Y__MASK;
+}
+
+#define REG_A6XX_X3_WINDOW_OFFSET				0x0000b307
+#define A6XX_X3_WINDOW_OFFSET_WINDOW_OFFSET_DISABLE		0x80000000
+#define A6XX_X3_WINDOW_OFFSET_X__MASK				0x00007fff
+#define A6XX_X3_WINDOW_OFFSET_X__SHIFT				0
+static inline uint32_t A6XX_X3_WINDOW_OFFSET_X(uint32_t val)
+{
+	return ((val) << A6XX_X3_WINDOW_OFFSET_X__SHIFT) & A6XX_X3_WINDOW_OFFSET_X__MASK;
+}
+#define A6XX_X3_WINDOW_OFFSET_Y__MASK				0x7fff0000
+#define A6XX_X3_WINDOW_OFFSET_Y__SHIFT				16
+static inline uint32_t A6XX_X3_WINDOW_OFFSET_Y(uint32_t val)
+{
+	return ((val) << A6XX_X3_WINDOW_OFFSET_Y__SHIFT) & A6XX_X3_WINDOW_OFFSET_Y__MASK;
+}
+
+#define REG_A6XX_X1_BIN_SIZE					0x000080a1
+#define A6XX_X1_BIN_SIZE_WIDTH__MASK				0x000000ff
+#define A6XX_X1_BIN_SIZE_WIDTH__SHIFT				0
+static inline uint32_t A6XX_X1_BIN_SIZE_WIDTH(uint32_t val)
+{
+	return ((val >> 5) << A6XX_X1_BIN_SIZE_WIDTH__SHIFT) & A6XX_X1_BIN_SIZE_WIDTH__MASK;
+}
+#define A6XX_X1_BIN_SIZE_HEIGHT__MASK				0x0001ff00
+#define A6XX_X1_BIN_SIZE_HEIGHT__SHIFT				8
+static inline uint32_t A6XX_X1_BIN_SIZE_HEIGHT(uint32_t val)
+{
+	return ((val >> 4) << A6XX_X1_BIN_SIZE_HEIGHT__SHIFT) & A6XX_X1_BIN_SIZE_HEIGHT__MASK;
+}
+
+#define REG_A6XX_X2_BIN_SIZE					0x00008800
+#define A6XX_X2_BIN_SIZE_WIDTH__MASK				0x000000ff
+#define A6XX_X2_BIN_SIZE_WIDTH__SHIFT				0
+static inline uint32_t A6XX_X2_BIN_SIZE_WIDTH(uint32_t val)
+{
+	return ((val >> 5) << A6XX_X2_BIN_SIZE_WIDTH__SHIFT) & A6XX_X2_BIN_SIZE_WIDTH__MASK;
+}
+#define A6XX_X2_BIN_SIZE_HEIGHT__MASK				0x0001ff00
+#define A6XX_X2_BIN_SIZE_HEIGHT__SHIFT				8
+static inline uint32_t A6XX_X2_BIN_SIZE_HEIGHT(uint32_t val)
+{
+	return ((val >> 4) << A6XX_X2_BIN_SIZE_HEIGHT__SHIFT) & A6XX_X2_BIN_SIZE_HEIGHT__MASK;
+}
+
+#define REG_A6XX_X3_BIN_SIZE					0x000088d3
+#define A6XX_X3_BIN_SIZE_WIDTH__MASK				0x000000ff
+#define A6XX_X3_BIN_SIZE_WIDTH__SHIFT				0
+static inline uint32_t A6XX_X3_BIN_SIZE_WIDTH(uint32_t val)
+{
+	return ((val >> 5) << A6XX_X3_BIN_SIZE_WIDTH__SHIFT) & A6XX_X3_BIN_SIZE_WIDTH__MASK;
+}
+#define A6XX_X3_BIN_SIZE_HEIGHT__MASK				0x0001ff00
+#define A6XX_X3_BIN_SIZE_HEIGHT__SHIFT				8
+static inline uint32_t A6XX_X3_BIN_SIZE_HEIGHT(uint32_t val)
+{
+	return ((val >> 4) << A6XX_X3_BIN_SIZE_HEIGHT__SHIFT) & A6XX_X3_BIN_SIZE_HEIGHT__MASK;
+}
+
+#define REG_A6XX_VSC_BIN_SIZE					0x00000c02
+#define A6XX_VSC_BIN_SIZE_WIDTH__MASK				0x000000ff
+#define A6XX_VSC_BIN_SIZE_WIDTH__SHIFT				0
+static inline uint32_t A6XX_VSC_BIN_SIZE_WIDTH(uint32_t val)
+{
+	return ((val >> 5) << A6XX_VSC_BIN_SIZE_WIDTH__SHIFT) & A6XX_VSC_BIN_SIZE_WIDTH__MASK;
+}
+#define A6XX_VSC_BIN_SIZE_HEIGHT__MASK				0x0001ff00
+#define A6XX_VSC_BIN_SIZE_HEIGHT__SHIFT				8
+static inline uint32_t A6XX_VSC_BIN_SIZE_HEIGHT(uint32_t val)
+{
+	return ((val >> 4) << A6XX_VSC_BIN_SIZE_HEIGHT__SHIFT) & A6XX_VSC_BIN_SIZE_HEIGHT__MASK;
+}
+
+#define REG_A6XX_VSC_SIZE_ADDRESS_LO				0x00000c03
+
+#define REG_A6XX_VSC_SIZE_ADDRESS_HI				0x00000c04
+
+#define REG_A6XX_VSC_BIN_COUNT					0x00000c06
+#define A6XX_VSC_BIN_COUNT_NX__MASK				0x000007fe
+#define A6XX_VSC_BIN_COUNT_NX__SHIFT				1
+static inline uint32_t A6XX_VSC_BIN_COUNT_NX(uint32_t val)
+{
+	return ((val) << A6XX_VSC_BIN_COUNT_NX__SHIFT) & A6XX_VSC_BIN_COUNT_NX__MASK;
+}
+#define A6XX_VSC_BIN_COUNT_NY__MASK				0x001ff800
+#define A6XX_VSC_BIN_COUNT_NY__SHIFT				11
+static inline uint32_t A6XX_VSC_BIN_COUNT_NY(uint32_t val)
+{
+	return ((val) << A6XX_VSC_BIN_COUNT_NY__SHIFT) & A6XX_VSC_BIN_COUNT_NY__MASK;
+}
+
+static inline uint32_t REG_A6XX_VSC_PIPE_CONFIG(uint32_t i0) { return 0x00000c10 + 0x1*i0; }
+
+static inline uint32_t REG_A6XX_VSC_PIPE_CONFIG_REG(uint32_t i0) { return 0x00000c10 + 0x1*i0; }
+#define A6XX_VSC_PIPE_CONFIG_REG_X__MASK			0x000003ff
+#define A6XX_VSC_PIPE_CONFIG_REG_X__SHIFT			0
+static inline uint32_t A6XX_VSC_PIPE_CONFIG_REG_X(uint32_t val)
+{
+	return ((val) << A6XX_VSC_PIPE_CONFIG_REG_X__SHIFT) & A6XX_VSC_PIPE_CONFIG_REG_X__MASK;
+}
+#define A6XX_VSC_PIPE_CONFIG_REG_Y__MASK			0x000ffc00
+#define A6XX_VSC_PIPE_CONFIG_REG_Y__SHIFT			10
+static inline uint32_t A6XX_VSC_PIPE_CONFIG_REG_Y(uint32_t val)
+{
+	return ((val) << A6XX_VSC_PIPE_CONFIG_REG_Y__SHIFT) & A6XX_VSC_PIPE_CONFIG_REG_Y__MASK;
+}
+#define A6XX_VSC_PIPE_CONFIG_REG_W__MASK			0x03f00000
+#define A6XX_VSC_PIPE_CONFIG_REG_W__SHIFT			20
+static inline uint32_t A6XX_VSC_PIPE_CONFIG_REG_W(uint32_t val)
+{
+	return ((val) << A6XX_VSC_PIPE_CONFIG_REG_W__SHIFT) & A6XX_VSC_PIPE_CONFIG_REG_W__MASK;
+}
+#define A6XX_VSC_PIPE_CONFIG_REG_H__MASK			0xfc000000
+#define A6XX_VSC_PIPE_CONFIG_REG_H__SHIFT			26
+static inline uint32_t A6XX_VSC_PIPE_CONFIG_REG_H(uint32_t val)
+{
+	return ((val) << A6XX_VSC_PIPE_CONFIG_REG_H__SHIFT) & A6XX_VSC_PIPE_CONFIG_REG_H__MASK;
+}
+
+#define REG_A6XX_VSC_XXX_ADDRESS_LO				0x00000c30
+
+#define REG_A6XX_VSC_XXX_ADDRESS_HI				0x00000c31
+
+#define REG_A6XX_VSC_XXX_PITCH					0x00000c32
+
+#define REG_A6XX_VSC_PIPE_DATA_ADDRESS_LO			0x00000c34
+
+#define REG_A6XX_VSC_PIPE_DATA_ADDRESS_HI			0x00000c35
+
+#define REG_A6XX_VSC_PIPE_DATA_PITCH				0x00000c36
+
+static inline uint32_t REG_A6XX_VSC_SIZE(uint32_t i0) { return 0x00000c78 + 0x1*i0; }
+
+static inline uint32_t REG_A6XX_VSC_SIZE_REG(uint32_t i0) { return 0x00000c78 + 0x1*i0; }
+
+#define REG_A6XX_UCHE_UNKNOWN_0E12				0x00000e12
+
+#define REG_A6XX_GRAS_UNKNOWN_8001				0x00008001
+
+#define REG_A6XX_GRAS_UNKNOWN_8004				0x00008004
+
+#define REG_A6XX_GRAS_CNTL					0x00008005
+#define A6XX_GRAS_CNTL_VARYING					0x00000001
+#define A6XX_GRAS_CNTL_XCOORD					0x00000040
+#define A6XX_GRAS_CNTL_YCOORD					0x00000080
+#define A6XX_GRAS_CNTL_ZCOORD					0x00000100
+#define A6XX_GRAS_CNTL_WCOORD					0x00000200
+
+#define REG_A6XX_GRAS_CL_GUARDBAND_CLIP_ADJ			0x00008006
+#define A6XX_GRAS_CL_GUARDBAND_CLIP_ADJ_HORZ__MASK		0x000003ff
+#define A6XX_GRAS_CL_GUARDBAND_CLIP_ADJ_HORZ__SHIFT		0
+static inline uint32_t A6XX_GRAS_CL_GUARDBAND_CLIP_ADJ_HORZ(uint32_t val)
+{
+	return ((val) << A6XX_GRAS_CL_GUARDBAND_CLIP_ADJ_HORZ__SHIFT) & A6XX_GRAS_CL_GUARDBAND_CLIP_ADJ_HORZ__MASK;
+}
+#define A6XX_GRAS_CL_GUARDBAND_CLIP_ADJ_VERT__MASK		0x000ffc00
+#define A6XX_GRAS_CL_GUARDBAND_CLIP_ADJ_VERT__SHIFT		10
+static inline uint32_t A6XX_GRAS_CL_GUARDBAND_CLIP_ADJ_VERT(uint32_t val)
+{
+	return ((val) << A6XX_GRAS_CL_GUARDBAND_CLIP_ADJ_VERT__SHIFT) & A6XX_GRAS_CL_GUARDBAND_CLIP_ADJ_VERT__MASK;
+}
+
+#define REG_A6XX_GRAS_CL_VPORT_XOFFSET_0			0x00008010
+#define A6XX_GRAS_CL_VPORT_XOFFSET_0__MASK			0xffffffff
+#define A6XX_GRAS_CL_VPORT_XOFFSET_0__SHIFT			0
+static inline uint32_t A6XX_GRAS_CL_VPORT_XOFFSET_0(float val)
+{
+	return ((fui(val)) << A6XX_GRAS_CL_VPORT_XOFFSET_0__SHIFT) & A6XX_GRAS_CL_VPORT_XOFFSET_0__MASK;
+}
+
+#define REG_A6XX_GRAS_CL_VPORT_XSCALE_0				0x00008011
+#define A6XX_GRAS_CL_VPORT_XSCALE_0__MASK			0xffffffff
+#define A6XX_GRAS_CL_VPORT_XSCALE_0__SHIFT			0
+static inline uint32_t A6XX_GRAS_CL_VPORT_XSCALE_0(float val)
+{
+	return ((fui(val)) << A6XX_GRAS_CL_VPORT_XSCALE_0__SHIFT) & A6XX_GRAS_CL_VPORT_XSCALE_0__MASK;
+}
+
+#define REG_A6XX_GRAS_CL_VPORT_YOFFSET_0			0x00008012
+#define A6XX_GRAS_CL_VPORT_YOFFSET_0__MASK			0xffffffff
+#define A6XX_GRAS_CL_VPORT_YOFFSET_0__SHIFT			0
+static inline uint32_t A6XX_GRAS_CL_VPORT_YOFFSET_0(float val)
+{
+	return ((fui(val)) << A6XX_GRAS_CL_VPORT_YOFFSET_0__SHIFT) & A6XX_GRAS_CL_VPORT_YOFFSET_0__MASK;
+}
+
+#define REG_A6XX_GRAS_CL_VPORT_YSCALE_0				0x00008013
+#define A6XX_GRAS_CL_VPORT_YSCALE_0__MASK			0xffffffff
+#define A6XX_GRAS_CL_VPORT_YSCALE_0__SHIFT			0
+static inline uint32_t A6XX_GRAS_CL_VPORT_YSCALE_0(float val)
+{
+	return ((fui(val)) << A6XX_GRAS_CL_VPORT_YSCALE_0__SHIFT) & A6XX_GRAS_CL_VPORT_YSCALE_0__MASK;
+}
+
+#define REG_A6XX_GRAS_CL_VPORT_ZOFFSET_0			0x00008014
+#define A6XX_GRAS_CL_VPORT_ZOFFSET_0__MASK			0xffffffff
+#define A6XX_GRAS_CL_VPORT_ZOFFSET_0__SHIFT			0
+static inline uint32_t A6XX_GRAS_CL_VPORT_ZOFFSET_0(float val)
+{
+	return ((fui(val)) << A6XX_GRAS_CL_VPORT_ZOFFSET_0__SHIFT) & A6XX_GRAS_CL_VPORT_ZOFFSET_0__MASK;
+}
+
+#define REG_A6XX_GRAS_CL_VPORT_ZSCALE_0				0x00008015
+#define A6XX_GRAS_CL_VPORT_ZSCALE_0__MASK			0xffffffff
+#define A6XX_GRAS_CL_VPORT_ZSCALE_0__SHIFT			0
+static inline uint32_t A6XX_GRAS_CL_VPORT_ZSCALE_0(float val)
+{
+	return ((fui(val)) << A6XX_GRAS_CL_VPORT_ZSCALE_0__SHIFT) & A6XX_GRAS_CL_VPORT_ZSCALE_0__MASK;
+}
+
+#define REG_A6XX_GRAS_SU_CNTL					0x00008090
+#define A6XX_GRAS_SU_CNTL_CULL_FRONT				0x00000001
+#define A6XX_GRAS_SU_CNTL_CULL_BACK				0x00000002
+#define A6XX_GRAS_SU_CNTL_FRONT_CW				0x00000004
+#define A6XX_GRAS_SU_CNTL_LINEHALFWIDTH__MASK			0x000007f8
+#define A6XX_GRAS_SU_CNTL_LINEHALFWIDTH__SHIFT			3
+static inline uint32_t A6XX_GRAS_SU_CNTL_LINEHALFWIDTH(float val)
+{
+	return ((((int32_t)(val * 4.0))) << A6XX_GRAS_SU_CNTL_LINEHALFWIDTH__SHIFT) & A6XX_GRAS_SU_CNTL_LINEHALFWIDTH__MASK;
+}
+#define A6XX_GRAS_SU_CNTL_POLY_OFFSET				0x00000800
+#define A6XX_GRAS_SU_CNTL_MSAA_ENABLE				0x00002000
+
+#define REG_A6XX_GRAS_SU_POINT_MINMAX				0x00008091
+#define A6XX_GRAS_SU_POINT_MINMAX_MIN__MASK			0x0000ffff
+#define A6XX_GRAS_SU_POINT_MINMAX_MIN__SHIFT			0
+static inline uint32_t A6XX_GRAS_SU_POINT_MINMAX_MIN(float val)
+{
+	return ((((uint32_t)(val * 16.0))) << A6XX_GRAS_SU_POINT_MINMAX_MIN__SHIFT) & A6XX_GRAS_SU_POINT_MINMAX_MIN__MASK;
+}
+#define A6XX_GRAS_SU_POINT_MINMAX_MAX__MASK			0xffff0000
+#define A6XX_GRAS_SU_POINT_MINMAX_MAX__SHIFT			16
+static inline uint32_t A6XX_GRAS_SU_POINT_MINMAX_MAX(float val)
+{
+	return ((((uint32_t)(val * 16.0))) << A6XX_GRAS_SU_POINT_MINMAX_MAX__SHIFT) & A6XX_GRAS_SU_POINT_MINMAX_MAX__MASK;
+}
+
+#define REG_A6XX_GRAS_SU_POINT_SIZE				0x00008092
+#define A6XX_GRAS_SU_POINT_SIZE__MASK				0xffffffff
+#define A6XX_GRAS_SU_POINT_SIZE__SHIFT				0
+static inline uint32_t A6XX_GRAS_SU_POINT_SIZE(float val)
+{
+	return ((((int32_t)(val * 16.0))) << A6XX_GRAS_SU_POINT_SIZE__SHIFT) & A6XX_GRAS_SU_POINT_SIZE__MASK;
+}
+
+#define REG_A6XX_GRAS_SU_POLY_OFFSET_SCALE			0x00008095
+#define A6XX_GRAS_SU_POLY_OFFSET_SCALE__MASK			0xffffffff
+#define A6XX_GRAS_SU_POLY_OFFSET_SCALE__SHIFT			0
+static inline uint32_t A6XX_GRAS_SU_POLY_OFFSET_SCALE(float val)
+{
+	return ((fui(val)) << A6XX_GRAS_SU_POLY_OFFSET_SCALE__SHIFT) & A6XX_GRAS_SU_POLY_OFFSET_SCALE__MASK;
+}
+
+#define REG_A6XX_GRAS_SU_POLY_OFFSET_OFFSET			0x00008096
+#define A6XX_GRAS_SU_POLY_OFFSET_OFFSET__MASK			0xffffffff
+#define A6XX_GRAS_SU_POLY_OFFSET_OFFSET__SHIFT			0
+static inline uint32_t A6XX_GRAS_SU_POLY_OFFSET_OFFSET(float val)
+{
+	return ((fui(val)) << A6XX_GRAS_SU_POLY_OFFSET_OFFSET__SHIFT) & A6XX_GRAS_SU_POLY_OFFSET_OFFSET__MASK;
+}
+
+#define REG_A6XX_GRAS_SU_POLY_OFFSET_OFFSET_CLAMP		0x00008097
+#define A6XX_GRAS_SU_POLY_OFFSET_OFFSET_CLAMP__MASK		0xffffffff
+#define A6XX_GRAS_SU_POLY_OFFSET_OFFSET_CLAMP__SHIFT		0
+static inline uint32_t A6XX_GRAS_SU_POLY_OFFSET_OFFSET_CLAMP(float val)
+{
+	return ((fui(val)) << A6XX_GRAS_SU_POLY_OFFSET_OFFSET_CLAMP__SHIFT) & A6XX_GRAS_SU_POLY_OFFSET_OFFSET_CLAMP__MASK;
+}
+
+#define REG_A6XX_GRAS_SU_DEPTH_BUFFER_INFO			0x00008098
+#define A6XX_GRAS_SU_DEPTH_BUFFER_INFO_DEPTH_FORMAT__MASK	0x00000007
+#define A6XX_GRAS_SU_DEPTH_BUFFER_INFO_DEPTH_FORMAT__SHIFT	0
+static inline uint32_t A6XX_GRAS_SU_DEPTH_BUFFER_INFO_DEPTH_FORMAT(enum a6xx_depth_format val)
+{
+	return ((val) << A6XX_GRAS_SU_DEPTH_BUFFER_INFO_DEPTH_FORMAT__SHIFT) & A6XX_GRAS_SU_DEPTH_BUFFER_INFO_DEPTH_FORMAT__MASK;
+}
+
+#define REG_A6XX_GRAS_UNKNOWN_8099				0x00008099
+
+#define REG_A6XX_GRAS_UNKNOWN_809B				0x0000809b
+
+#define REG_A6XX_GRAS_RAS_MSAA_CNTL				0x000080a2
+#define A6XX_GRAS_RAS_MSAA_CNTL_SAMPLES__MASK			0x00000003
+#define A6XX_GRAS_RAS_MSAA_CNTL_SAMPLES__SHIFT			0
+static inline uint32_t A6XX_GRAS_RAS_MSAA_CNTL_SAMPLES(enum a3xx_msaa_samples val)
+{
+	return ((val) << A6XX_GRAS_RAS_MSAA_CNTL_SAMPLES__SHIFT) & A6XX_GRAS_RAS_MSAA_CNTL_SAMPLES__MASK;
+}
+
+#define REG_A6XX_GRAS_DEST_MSAA_CNTL				0x000080a3
+#define A6XX_GRAS_DEST_MSAA_CNTL_SAMPLES__MASK			0x00000003
+#define A6XX_GRAS_DEST_MSAA_CNTL_SAMPLES__SHIFT			0
+static inline uint32_t A6XX_GRAS_DEST_MSAA_CNTL_SAMPLES(enum a3xx_msaa_samples val)
+{
+	return ((val) << A6XX_GRAS_DEST_MSAA_CNTL_SAMPLES__SHIFT) & A6XX_GRAS_DEST_MSAA_CNTL_SAMPLES__MASK;
+}
+#define A6XX_GRAS_DEST_MSAA_CNTL_MSAA_DISABLE			0x00000004
+
+#define REG_A6XX_GRAS_UNKNOWN_80A4				0x000080a4
+
+#define REG_A6XX_GRAS_UNKNOWN_80A5				0x000080a5
+
+#define REG_A6XX_GRAS_UNKNOWN_80A6				0x000080a6
+
+#define REG_A6XX_GRAS_UNKNOWN_80AF				0x000080af
+
+#define REG_A6XX_GRAS_SC_SCREEN_SCISSOR_TL_0			0x000080b0
+#define A6XX_GRAS_SC_SCREEN_SCISSOR_TL_0_WINDOW_OFFSET_DISABLE	0x80000000
+#define A6XX_GRAS_SC_SCREEN_SCISSOR_TL_0_X__MASK		0x00007fff
+#define A6XX_GRAS_SC_SCREEN_SCISSOR_TL_0_X__SHIFT		0
+static inline uint32_t A6XX_GRAS_SC_SCREEN_SCISSOR_TL_0_X(uint32_t val)
+{
+	return ((val) << A6XX_GRAS_SC_SCREEN_SCISSOR_TL_0_X__SHIFT) & A6XX_GRAS_SC_SCREEN_SCISSOR_TL_0_X__MASK;
+}
+#define A6XX_GRAS_SC_SCREEN_SCISSOR_TL_0_Y__MASK		0x7fff0000
+#define A6XX_GRAS_SC_SCREEN_SCISSOR_TL_0_Y__SHIFT		16
+static inline uint32_t A6XX_GRAS_SC_SCREEN_SCISSOR_TL_0_Y(uint32_t val)
+{
+	return ((val) << A6XX_GRAS_SC_SCREEN_SCISSOR_TL_0_Y__SHIFT) & A6XX_GRAS_SC_SCREEN_SCISSOR_TL_0_Y__MASK;
+}
+
+#define REG_A6XX_GRAS_SC_SCREEN_SCISSOR_BR_0			0x000080b1
+#define A6XX_GRAS_SC_SCREEN_SCISSOR_BR_0_WINDOW_OFFSET_DISABLE	0x80000000
+#define A6XX_GRAS_SC_SCREEN_SCISSOR_BR_0_X__MASK		0x00007fff
+#define A6XX_GRAS_SC_SCREEN_SCISSOR_BR_0_X__SHIFT		0
+static inline uint32_t A6XX_GRAS_SC_SCREEN_SCISSOR_BR_0_X(uint32_t val)
+{
+	return ((val) << A6XX_GRAS_SC_SCREEN_SCISSOR_BR_0_X__SHIFT) & A6XX_GRAS_SC_SCREEN_SCISSOR_BR_0_X__MASK;
+}
+#define A6XX_GRAS_SC_SCREEN_SCISSOR_BR_0_Y__MASK		0x7fff0000
+#define A6XX_GRAS_SC_SCREEN_SCISSOR_BR_0_Y__SHIFT		16
+static inline uint32_t A6XX_GRAS_SC_SCREEN_SCISSOR_BR_0_Y(uint32_t val)
+{
+	return ((val) << A6XX_GRAS_SC_SCREEN_SCISSOR_BR_0_Y__SHIFT) & A6XX_GRAS_SC_SCREEN_SCISSOR_BR_0_Y__MASK;
+}
+
+#define REG_A6XX_GRAS_SC_VIEWPORT_SCISSOR_TL_0			0x000080d0
+#define A6XX_GRAS_SC_VIEWPORT_SCISSOR_TL_0_WINDOW_OFFSET_DISABLE	0x80000000
+#define A6XX_GRAS_SC_VIEWPORT_SCISSOR_TL_0_X__MASK		0x00007fff
+#define A6XX_GRAS_SC_VIEWPORT_SCISSOR_TL_0_X__SHIFT		0
+static inline uint32_t A6XX_GRAS_SC_VIEWPORT_SCISSOR_TL_0_X(uint32_t val)
+{
+	return ((val) << A6XX_GRAS_SC_VIEWPORT_SCISSOR_TL_0_X__SHIFT) & A6XX_GRAS_SC_VIEWPORT_SCISSOR_TL_0_X__MASK;
+}
+#define A6XX_GRAS_SC_VIEWPORT_SCISSOR_TL_0_Y__MASK		0x7fff0000
+#define A6XX_GRAS_SC_VIEWPORT_SCISSOR_TL_0_Y__SHIFT		16
+static inline uint32_t A6XX_GRAS_SC_VIEWPORT_SCISSOR_TL_0_Y(uint32_t val)
+{
+	return ((val) << A6XX_GRAS_SC_VIEWPORT_SCISSOR_TL_0_Y__SHIFT) & A6XX_GRAS_SC_VIEWPORT_SCISSOR_TL_0_Y__MASK;
+}
+
+#define REG_A6XX_GRAS_SC_VIEWPORT_SCISSOR_BR_0			0x000080d1
+#define A6XX_GRAS_SC_VIEWPORT_SCISSOR_BR_0_WINDOW_OFFSET_DISABLE	0x80000000
+#define A6XX_GRAS_SC_VIEWPORT_SCISSOR_BR_0_X__MASK		0x00007fff
+#define A6XX_GRAS_SC_VIEWPORT_SCISSOR_BR_0_X__SHIFT		0
+static inline uint32_t A6XX_GRAS_SC_VIEWPORT_SCISSOR_BR_0_X(uint32_t val)
+{
+	return ((val) << A6XX_GRAS_SC_VIEWPORT_SCISSOR_BR_0_X__SHIFT) & A6XX_GRAS_SC_VIEWPORT_SCISSOR_BR_0_X__MASK;
+}
+#define A6XX_GRAS_SC_VIEWPORT_SCISSOR_BR_0_Y__MASK		0x7fff0000
+#define A6XX_GRAS_SC_VIEWPORT_SCISSOR_BR_0_Y__SHIFT		16
+static inline uint32_t A6XX_GRAS_SC_VIEWPORT_SCISSOR_BR_0_Y(uint32_t val)
+{
+	return ((val) << A6XX_GRAS_SC_VIEWPORT_SCISSOR_BR_0_Y__SHIFT) & A6XX_GRAS_SC_VIEWPORT_SCISSOR_BR_0_Y__MASK;
+}
+
+#define REG_A6XX_GRAS_SC_WINDOW_SCISSOR_TL			0x000080f0
+#define A6XX_GRAS_SC_WINDOW_SCISSOR_TL_WINDOW_OFFSET_DISABLE	0x80000000
+#define A6XX_GRAS_SC_WINDOW_SCISSOR_TL_X__MASK			0x00007fff
+#define A6XX_GRAS_SC_WINDOW_SCISSOR_TL_X__SHIFT			0
+static inline uint32_t A6XX_GRAS_SC_WINDOW_SCISSOR_TL_X(uint32_t val)
+{
+	return ((val) << A6XX_GRAS_SC_WINDOW_SCISSOR_TL_X__SHIFT) & A6XX_GRAS_SC_WINDOW_SCISSOR_TL_X__MASK;
+}
+#define A6XX_GRAS_SC_WINDOW_SCISSOR_TL_Y__MASK			0x7fff0000
+#define A6XX_GRAS_SC_WINDOW_SCISSOR_TL_Y__SHIFT			16
+static inline uint32_t A6XX_GRAS_SC_WINDOW_SCISSOR_TL_Y(uint32_t val)
+{
+	return ((val) << A6XX_GRAS_SC_WINDOW_SCISSOR_TL_Y__SHIFT) & A6XX_GRAS_SC_WINDOW_SCISSOR_TL_Y__MASK;
+}
+
+#define REG_A6XX_GRAS_SC_WINDOW_SCISSOR_BR			0x000080f1
+#define A6XX_GRAS_SC_WINDOW_SCISSOR_BR_WINDOW_OFFSET_DISABLE	0x80000000
+#define A6XX_GRAS_SC_WINDOW_SCISSOR_BR_X__MASK			0x00007fff
+#define A6XX_GRAS_SC_WINDOW_SCISSOR_BR_X__SHIFT			0
+static inline uint32_t A6XX_GRAS_SC_WINDOW_SCISSOR_BR_X(uint32_t val)
+{
+	return ((val) << A6XX_GRAS_SC_WINDOW_SCISSOR_BR_X__SHIFT) & A6XX_GRAS_SC_WINDOW_SCISSOR_BR_X__MASK;
+}
+#define A6XX_GRAS_SC_WINDOW_SCISSOR_BR_Y__MASK			0x7fff0000
+#define A6XX_GRAS_SC_WINDOW_SCISSOR_BR_Y__SHIFT			16
+static inline uint32_t A6XX_GRAS_SC_WINDOW_SCISSOR_BR_Y(uint32_t val)
+{
+	return ((val) << A6XX_GRAS_SC_WINDOW_SCISSOR_BR_Y__SHIFT) & A6XX_GRAS_SC_WINDOW_SCISSOR_BR_Y__MASK;
+}
+
+#define REG_A6XX_GRAS_LRZ_CNTL					0x00008100
+#define A6XX_GRAS_LRZ_CNTL_ENABLE				0x00000001
+#define A6XX_GRAS_LRZ_CNTL_LRZ_WRITE				0x00000002
+#define A6XX_GRAS_LRZ_CNTL_GREATER				0x00000004
+
+#define REG_A6XX_GRAS_2D_BLIT_INFO				0x00008102
+#define A6XX_GRAS_2D_BLIT_INFO_COLOR_FORMAT__MASK		0x000000ff
+#define A6XX_GRAS_2D_BLIT_INFO_COLOR_FORMAT__SHIFT		0
+static inline uint32_t A6XX_GRAS_2D_BLIT_INFO_COLOR_FORMAT(enum a6xx_color_fmt val)
+{
+	return ((val) << A6XX_GRAS_2D_BLIT_INFO_COLOR_FORMAT__SHIFT) & A6XX_GRAS_2D_BLIT_INFO_COLOR_FORMAT__MASK;
+}
+
+#define REG_A6XX_GRAS_LRZ_BUFFER_BASE_LO			0x00008103
+
+#define REG_A6XX_GRAS_LRZ_BUFFER_BASE_HI			0x00008104
+
+#define REG_A6XX_GRAS_LRZ_BUFFER_PITCH				0x00008105
+#define A6XX_GRAS_LRZ_BUFFER_PITCH_PITCH__MASK			0x000007ff
+#define A6XX_GRAS_LRZ_BUFFER_PITCH_PITCH__SHIFT			0
+static inline uint32_t A6XX_GRAS_LRZ_BUFFER_PITCH_PITCH(uint32_t val)
+{
+	return ((val >> 5) << A6XX_GRAS_LRZ_BUFFER_PITCH_PITCH__SHIFT) & A6XX_GRAS_LRZ_BUFFER_PITCH_PITCH__MASK;
+}
+#define A6XX_GRAS_LRZ_BUFFER_PITCH_ARRAY_PITCH__MASK		0x003ff800
+#define A6XX_GRAS_LRZ_BUFFER_PITCH_ARRAY_PITCH__SHIFT		11
+static inline uint32_t A6XX_GRAS_LRZ_BUFFER_PITCH_ARRAY_PITCH(uint32_t val)
+{
+	return ((val >> 5) << A6XX_GRAS_LRZ_BUFFER_PITCH_ARRAY_PITCH__SHIFT) & A6XX_GRAS_LRZ_BUFFER_PITCH_ARRAY_PITCH__MASK;
+}
+
+#define REG_A6XX_GRAS_LRZ_FAST_CLEAR_BUFFER_BASE_LO		0x00008106
+
+#define REG_A6XX_GRAS_LRZ_FAST_CLEAR_BUFFER_BASE_HI		0x00008107
+
+#define REG_A6XX_GRAS_2D_BLIT_CNTL				0x00008400
+
+#define REG_A6XX_GRAS_2D_SRC_TL_X				0x00008401
+#define A6XX_GRAS_2D_SRC_TL_X_X__MASK				0x00ffff00
+#define A6XX_GRAS_2D_SRC_TL_X_X__SHIFT				8
+static inline uint32_t A6XX_GRAS_2D_SRC_TL_X_X(uint32_t val)
+{
+	return ((val) << A6XX_GRAS_2D_SRC_TL_X_X__SHIFT) & A6XX_GRAS_2D_SRC_TL_X_X__MASK;
+}
+
+#define REG_A6XX_GRAS_2D_SRC_BR_X				0x00008402
+#define A6XX_GRAS_2D_SRC_BR_X_X__MASK				0x00ffff00
+#define A6XX_GRAS_2D_SRC_BR_X_X__SHIFT				8
+static inline uint32_t A6XX_GRAS_2D_SRC_BR_X_X(uint32_t val)
+{
+	return ((val) << A6XX_GRAS_2D_SRC_BR_X_X__SHIFT) & A6XX_GRAS_2D_SRC_BR_X_X__MASK;
+}
+
+#define REG_A6XX_GRAS_2D_SRC_TL_Y				0x00008403
+#define A6XX_GRAS_2D_SRC_TL_Y_Y__MASK				0x00ffff00
+#define A6XX_GRAS_2D_SRC_TL_Y_Y__SHIFT				8
+static inline uint32_t A6XX_GRAS_2D_SRC_TL_Y_Y(uint32_t val)
+{
+	return ((val) << A6XX_GRAS_2D_SRC_TL_Y_Y__SHIFT) & A6XX_GRAS_2D_SRC_TL_Y_Y__MASK;
+}
+
+#define REG_A6XX_GRAS_2D_SRC_BR_Y				0x00008404
+#define A6XX_GRAS_2D_SRC_BR_Y_Y__MASK				0x00ffff00
+#define A6XX_GRAS_2D_SRC_BR_Y_Y__SHIFT				8
+static inline uint32_t A6XX_GRAS_2D_SRC_BR_Y_Y(uint32_t val)
+{
+	return ((val) << A6XX_GRAS_2D_SRC_BR_Y_Y__SHIFT) & A6XX_GRAS_2D_SRC_BR_Y_Y__MASK;
+}
+
+#define REG_A6XX_GRAS_2D_DST_TL					0x00008405
+#define A6XX_GRAS_2D_DST_TL_WINDOW_OFFSET_DISABLE		0x80000000
+#define A6XX_GRAS_2D_DST_TL_X__MASK				0x00007fff
+#define A6XX_GRAS_2D_DST_TL_X__SHIFT				0
+static inline uint32_t A6XX_GRAS_2D_DST_TL_X(uint32_t val)
+{
+	return ((val) << A6XX_GRAS_2D_DST_TL_X__SHIFT) & A6XX_GRAS_2D_DST_TL_X__MASK;
+}
+#define A6XX_GRAS_2D_DST_TL_Y__MASK				0x7fff0000
+#define A6XX_GRAS_2D_DST_TL_Y__SHIFT				16
+static inline uint32_t A6XX_GRAS_2D_DST_TL_Y(uint32_t val)
+{
+	return ((val) << A6XX_GRAS_2D_DST_TL_Y__SHIFT) & A6XX_GRAS_2D_DST_TL_Y__MASK;
+}
+
+#define REG_A6XX_GRAS_2D_DST_BR					0x00008406
+#define A6XX_GRAS_2D_DST_BR_WINDOW_OFFSET_DISABLE		0x80000000
+#define A6XX_GRAS_2D_DST_BR_X__MASK				0x00007fff
+#define A6XX_GRAS_2D_DST_BR_X__SHIFT				0
+static inline uint32_t A6XX_GRAS_2D_DST_BR_X(uint32_t val)
+{
+	return ((val) << A6XX_GRAS_2D_DST_BR_X__SHIFT) & A6XX_GRAS_2D_DST_BR_X__MASK;
+}
+#define A6XX_GRAS_2D_DST_BR_Y__MASK				0x7fff0000
+#define A6XX_GRAS_2D_DST_BR_Y__SHIFT				16
+static inline uint32_t A6XX_GRAS_2D_DST_BR_Y(uint32_t val)
+{
+	return ((val) << A6XX_GRAS_2D_DST_BR_Y__SHIFT) & A6XX_GRAS_2D_DST_BR_Y__MASK;
+}
+
+#define REG_A6XX_GRAS_RESOLVE_CNTL_1				0x0000840a
+#define A6XX_GRAS_RESOLVE_CNTL_1_WINDOW_OFFSET_DISABLE		0x80000000
+#define A6XX_GRAS_RESOLVE_CNTL_1_X__MASK			0x00007fff
+#define A6XX_GRAS_RESOLVE_CNTL_1_X__SHIFT			0
+static inline uint32_t A6XX_GRAS_RESOLVE_CNTL_1_X(uint32_t val)
+{
+	return ((val) << A6XX_GRAS_RESOLVE_CNTL_1_X__SHIFT) & A6XX_GRAS_RESOLVE_CNTL_1_X__MASK;
+}
+#define A6XX_GRAS_RESOLVE_CNTL_1_Y__MASK			0x7fff0000
+#define A6XX_GRAS_RESOLVE_CNTL_1_Y__SHIFT			16
+static inline uint32_t A6XX_GRAS_RESOLVE_CNTL_1_Y(uint32_t val)
+{
+	return ((val) << A6XX_GRAS_RESOLVE_CNTL_1_Y__SHIFT) & A6XX_GRAS_RESOLVE_CNTL_1_Y__MASK;
+}
+
+#define REG_A6XX_GRAS_RESOLVE_CNTL_2				0x0000840b
+#define A6XX_GRAS_RESOLVE_CNTL_2_WINDOW_OFFSET_DISABLE		0x80000000
+#define A6XX_GRAS_RESOLVE_CNTL_2_X__MASK			0x00007fff
+#define A6XX_GRAS_RESOLVE_CNTL_2_X__SHIFT			0
+static inline uint32_t A6XX_GRAS_RESOLVE_CNTL_2_X(uint32_t val)
+{
+	return ((val) << A6XX_GRAS_RESOLVE_CNTL_2_X__SHIFT) & A6XX_GRAS_RESOLVE_CNTL_2_X__MASK;
+}
+#define A6XX_GRAS_RESOLVE_CNTL_2_Y__MASK			0x7fff0000
+#define A6XX_GRAS_RESOLVE_CNTL_2_Y__SHIFT			16
+static inline uint32_t A6XX_GRAS_RESOLVE_CNTL_2_Y(uint32_t val)
+{
+	return ((val) << A6XX_GRAS_RESOLVE_CNTL_2_Y__SHIFT) & A6XX_GRAS_RESOLVE_CNTL_2_Y__MASK;
+}
+
+#define REG_A6XX_GRAS_UNKNOWN_8600				0x00008600
+
+#define REG_A6XX_RB_RAS_MSAA_CNTL				0x00008802
+#define A6XX_RB_RAS_MSAA_CNTL_SAMPLES__MASK			0x00000003
+#define A6XX_RB_RAS_MSAA_CNTL_SAMPLES__SHIFT			0
+static inline uint32_t A6XX_RB_RAS_MSAA_CNTL_SAMPLES(enum a3xx_msaa_samples val)
+{
+	return ((val) << A6XX_RB_RAS_MSAA_CNTL_SAMPLES__SHIFT) & A6XX_RB_RAS_MSAA_CNTL_SAMPLES__MASK;
+}
+
+#define REG_A6XX_RB_DEST_MSAA_CNTL				0x00008803
+#define A6XX_RB_DEST_MSAA_CNTL_SAMPLES__MASK			0x00000003
+#define A6XX_RB_DEST_MSAA_CNTL_SAMPLES__SHIFT			0
+static inline uint32_t A6XX_RB_DEST_MSAA_CNTL_SAMPLES(enum a3xx_msaa_samples val)
+{
+	return ((val) << A6XX_RB_DEST_MSAA_CNTL_SAMPLES__SHIFT) & A6XX_RB_DEST_MSAA_CNTL_SAMPLES__MASK;
+}
+#define A6XX_RB_DEST_MSAA_CNTL_MSAA_DISABLE			0x00000004
+
+#define REG_A6XX_RB_UNKNOWN_8804				0x00008804
+
+#define REG_A6XX_RB_UNKNOWN_8805				0x00008805
+
+#define REG_A6XX_RB_UNKNOWN_8806				0x00008806
+
+#define REG_A6XX_RB_RENDER_CONTROL0				0x00008809
+#define A6XX_RB_RENDER_CONTROL0_VARYING				0x00000001
+#define A6XX_RB_RENDER_CONTROL0_XCOORD				0x00000040
+#define A6XX_RB_RENDER_CONTROL0_YCOORD				0x00000080
+#define A6XX_RB_RENDER_CONTROL0_ZCOORD				0x00000100
+#define A6XX_RB_RENDER_CONTROL0_WCOORD				0x00000200
+#define A6XX_RB_RENDER_CONTROL0_UNK10				0x00000400
+
+#define REG_A6XX_RB_RENDER_CONTROL1				0x0000880a
+#define A6XX_RB_RENDER_CONTROL1_SAMPLEMASK			0x00000001
+#define A6XX_RB_RENDER_CONTROL1_FACENESS			0x00000002
+#define A6XX_RB_RENDER_CONTROL1_SAMPLEID			0x00000008
+
+#define REG_A6XX_RB_FS_OUTPUT_CNTL0				0x0000880b
+#define A6XX_RB_FS_OUTPUT_CNTL0_FRAG_WRITES_Z			0x00000002
+
+#define REG_A6XX_RB_FS_OUTPUT_CNTL1				0x0000880c
+#define A6XX_RB_FS_OUTPUT_CNTL1_MRT__MASK			0x0000000f
+#define A6XX_RB_FS_OUTPUT_CNTL1_MRT__SHIFT			0
+static inline uint32_t A6XX_RB_FS_OUTPUT_CNTL1_MRT(uint32_t val)
+{
+	return ((val) << A6XX_RB_FS_OUTPUT_CNTL1_MRT__SHIFT) & A6XX_RB_FS_OUTPUT_CNTL1_MRT__MASK;
+}
+
+#define REG_A6XX_RB_RENDER_COMPONENTS				0x0000880d
+#define A6XX_RB_RENDER_COMPONENTS_RT0__MASK			0x0000000f
+#define A6XX_RB_RENDER_COMPONENTS_RT0__SHIFT			0
+static inline uint32_t A6XX_RB_RENDER_COMPONENTS_RT0(uint32_t val)
+{
+	return ((val) << A6XX_RB_RENDER_COMPONENTS_RT0__SHIFT) & A6XX_RB_RENDER_COMPONENTS_RT0__MASK;
+}
+#define A6XX_RB_RENDER_COMPONENTS_RT1__MASK			0x000000f0
+#define A6XX_RB_RENDER_COMPONENTS_RT1__SHIFT			4
+static inline uint32_t A6XX_RB_RENDER_COMPONENTS_RT1(uint32_t val)
+{
+	return ((val) << A6XX_RB_RENDER_COMPONENTS_RT1__SHIFT) & A6XX_RB_RENDER_COMPONENTS_RT1__MASK;
+}
+#define A6XX_RB_RENDER_COMPONENTS_RT2__MASK			0x00000f00
+#define A6XX_RB_RENDER_COMPONENTS_RT2__SHIFT			8
+static inline uint32_t A6XX_RB_RENDER_COMPONENTS_RT2(uint32_t val)
+{
+	return ((val) << A6XX_RB_RENDER_COMPONENTS_RT2__SHIFT) & A6XX_RB_RENDER_COMPONENTS_RT2__MASK;
+}
+#define A6XX_RB_RENDER_COMPONENTS_RT3__MASK			0x0000f000
+#define A6XX_RB_RENDER_COMPONENTS_RT3__SHIFT			12
+static inline uint32_t A6XX_RB_RENDER_COMPONENTS_RT3(uint32_t val)
+{
+	return ((val) << A6XX_RB_RENDER_COMPONENTS_RT3__SHIFT) & A6XX_RB_RENDER_COMPONENTS_RT3__MASK;
+}
+#define A6XX_RB_RENDER_COMPONENTS_RT4__MASK			0x000f0000
+#define A6XX_RB_RENDER_COMPONENTS_RT4__SHIFT			16
+static inline uint32_t A6XX_RB_RENDER_COMPONENTS_RT4(uint32_t val)
+{
+	return ((val) << A6XX_RB_RENDER_COMPONENTS_RT4__SHIFT) & A6XX_RB_RENDER_COMPONENTS_RT4__MASK;
+}
+#define A6XX_RB_RENDER_COMPONENTS_RT5__MASK			0x00f00000
+#define A6XX_RB_RENDER_COMPONENTS_RT5__SHIFT			20
+static inline uint32_t A6XX_RB_RENDER_COMPONENTS_RT5(uint32_t val)
+{
+	return ((val) << A6XX_RB_RENDER_COMPONENTS_RT5__SHIFT) & A6XX_RB_RENDER_COMPONENTS_RT5__MASK;
+}
+#define A6XX_RB_RENDER_COMPONENTS_RT6__MASK			0x0f000000
+#define A6XX_RB_RENDER_COMPONENTS_RT6__SHIFT			24
+static inline uint32_t A6XX_RB_RENDER_COMPONENTS_RT6(uint32_t val)
+{
+	return ((val) << A6XX_RB_RENDER_COMPONENTS_RT6__SHIFT) & A6XX_RB_RENDER_COMPONENTS_RT6__MASK;
+}
+#define A6XX_RB_RENDER_COMPONENTS_RT7__MASK			0xf0000000
+#define A6XX_RB_RENDER_COMPONENTS_RT7__SHIFT			28
+static inline uint32_t A6XX_RB_RENDER_COMPONENTS_RT7(uint32_t val)
+{
+	return ((val) << A6XX_RB_RENDER_COMPONENTS_RT7__SHIFT) & A6XX_RB_RENDER_COMPONENTS_RT7__MASK;
+}
+
+#define REG_A6XX_RB_DITHER_CNTL					0x0000880e
+#define A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT0__MASK		0x00000003
+#define A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT0__SHIFT		0
+static inline uint32_t A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT0(enum adreno_rb_dither_mode val)
+{
+	return ((val) << A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT0__SHIFT) & A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT0__MASK;
+}
+#define A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT1__MASK		0x0000000c
+#define A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT1__SHIFT		2
+static inline uint32_t A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT1(enum adreno_rb_dither_mode val)
+{
+	return ((val) << A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT1__SHIFT) & A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT1__MASK;
+}
+#define A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT2__MASK		0x00000030
+#define A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT2__SHIFT		4
+static inline uint32_t A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT2(enum adreno_rb_dither_mode val)
+{
+	return ((val) << A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT2__SHIFT) & A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT2__MASK;
+}
+#define A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT3__MASK		0x000000c0
+#define A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT3__SHIFT		6
+static inline uint32_t A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT3(enum adreno_rb_dither_mode val)
+{
+	return ((val) << A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT3__SHIFT) & A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT3__MASK;
+}
+#define A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT4__MASK		0x00000300
+#define A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT4__SHIFT		8
+static inline uint32_t A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT4(enum adreno_rb_dither_mode val)
+{
+	return ((val) << A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT4__SHIFT) & A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT4__MASK;
+}
+#define A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT5__MASK		0x00000c00
+#define A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT5__SHIFT		10
+static inline uint32_t A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT5(enum adreno_rb_dither_mode val)
+{
+	return ((val) << A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT5__SHIFT) & A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT5__MASK;
+}
+#define A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT6__MASK		0x00001000
+#define A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT6__SHIFT		12
+static inline uint32_t A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT6(enum adreno_rb_dither_mode val)
+{
+	return ((val) << A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT6__SHIFT) & A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT6__MASK;
+}
+#define A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT7__MASK		0x0000c000
+#define A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT7__SHIFT		14
+static inline uint32_t A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT7(enum adreno_rb_dither_mode val)
+{
+	return ((val) << A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT7__SHIFT) & A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT7__MASK;
+}
+
+#define REG_A6XX_RB_SRGB_CNTL					0x0000880f
+#define A6XX_RB_SRGB_CNTL_SRGB_MRT0				0x00000001
+#define A6XX_RB_SRGB_CNTL_SRGB_MRT1				0x00000002
+#define A6XX_RB_SRGB_CNTL_SRGB_MRT2				0x00000004
+#define A6XX_RB_SRGB_CNTL_SRGB_MRT3				0x00000008
+#define A6XX_RB_SRGB_CNTL_SRGB_MRT4				0x00000010
+#define A6XX_RB_SRGB_CNTL_SRGB_MRT5				0x00000020
+#define A6XX_RB_SRGB_CNTL_SRGB_MRT6				0x00000040
+#define A6XX_RB_SRGB_CNTL_SRGB_MRT7				0x00000080
+
+#define REG_A6XX_RB_UNKNOWN_8818				0x00008818
+
+#define REG_A6XX_RB_UNKNOWN_8819				0x00008819
+
+#define REG_A6XX_RB_UNKNOWN_881A				0x0000881a
+
+#define REG_A6XX_RB_UNKNOWN_881B				0x0000881b
+
+#define REG_A6XX_RB_UNKNOWN_881C				0x0000881c
+
+#define REG_A6XX_RB_UNKNOWN_881D				0x0000881d
+
+#define REG_A6XX_RB_UNKNOWN_881E				0x0000881e
+
+static inline uint32_t REG_A6XX_RB_MRT(uint32_t i0) { return 0x00008820 + 0x8*i0; }
+
+static inline uint32_t REG_A6XX_RB_MRT_CONTROL(uint32_t i0) { return 0x00008820 + 0x8*i0; }
+#define A6XX_RB_MRT_CONTROL_BLEND				0x00000001
+#define A6XX_RB_MRT_CONTROL_BLEND2				0x00000002
+#define A6XX_RB_MRT_CONTROL_ROP_ENABLE				0x00000004
+#define A6XX_RB_MRT_CONTROL_ROP_CODE__MASK			0x00000078
+#define A6XX_RB_MRT_CONTROL_ROP_CODE__SHIFT			3
+static inline uint32_t A6XX_RB_MRT_CONTROL_ROP_CODE(enum a3xx_rop_code val)
+{
+	return ((val) << A6XX_RB_MRT_CONTROL_ROP_CODE__SHIFT) & A6XX_RB_MRT_CONTROL_ROP_CODE__MASK;
+}
+#define A6XX_RB_MRT_CONTROL_COMPONENT_ENABLE__MASK		0x00000780
+#define A6XX_RB_MRT_CONTROL_COMPONENT_ENABLE__SHIFT		7
+static inline uint32_t A6XX_RB_MRT_CONTROL_COMPONENT_ENABLE(uint32_t val)
+{
+	return ((val) << A6XX_RB_MRT_CONTROL_COMPONENT_ENABLE__SHIFT) & A6XX_RB_MRT_CONTROL_COMPONENT_ENABLE__MASK;
+}
+
+static inline uint32_t REG_A6XX_RB_MRT_BLEND_CONTROL(uint32_t i0) { return 0x00008821 + 0x8*i0; }
+#define A6XX_RB_MRT_BLEND_CONTROL_RGB_SRC_FACTOR__MASK		0x0000001f
+#define A6XX_RB_MRT_BLEND_CONTROL_RGB_SRC_FACTOR__SHIFT		0
+static inline uint32_t A6XX_RB_MRT_BLEND_CONTROL_RGB_SRC_FACTOR(enum adreno_rb_blend_factor val)
+{
+	return ((val) << A6XX_RB_MRT_BLEND_CONTROL_RGB_SRC_FACTOR__SHIFT) & A6XX_RB_MRT_BLEND_CONTROL_RGB_SRC_FACTOR__MASK;
+}
+#define A6XX_RB_MRT_BLEND_CONTROL_RGB_BLEND_OPCODE__MASK	0x000000e0
+#define A6XX_RB_MRT_BLEND_CONTROL_RGB_BLEND_OPCODE__SHIFT	5
+static inline uint32_t A6XX_RB_MRT_BLEND_CONTROL_RGB_BLEND_OPCODE(enum a3xx_rb_blend_opcode val)
+{
+	return ((val) << A6XX_RB_MRT_BLEND_CONTROL_RGB_BLEND_OPCODE__SHIFT) & A6XX_RB_MRT_BLEND_CONTROL_RGB_BLEND_OPCODE__MASK;
+}
+#define A6XX_RB_MRT_BLEND_CONTROL_RGB_DEST_FACTOR__MASK		0x00001f00
+#define A6XX_RB_MRT_BLEND_CONTROL_RGB_DEST_FACTOR__SHIFT	8
+static inline uint32_t A6XX_RB_MRT_BLEND_CONTROL_RGB_DEST_FACTOR(enum adreno_rb_blend_factor val)
+{
+	return ((val) << A6XX_RB_MRT_BLEND_CONTROL_RGB_DEST_FACTOR__SHIFT) & A6XX_RB_MRT_BLEND_CONTROL_RGB_DEST_FACTOR__MASK;
+}
+#define A6XX_RB_MRT_BLEND_CONTROL_ALPHA_SRC_FACTOR__MASK	0x001f0000
+#define A6XX_RB_MRT_BLEND_CONTROL_ALPHA_SRC_FACTOR__SHIFT	16
+static inline uint32_t A6XX_RB_MRT_BLEND_CONTROL_ALPHA_SRC_FACTOR(enum adreno_rb_blend_factor val)
+{
+	return ((val) << A6XX_RB_MRT_BLEND_CONTROL_ALPHA_SRC_FACTOR__SHIFT) & A6XX_RB_MRT_BLEND_CONTROL_ALPHA_SRC_FACTOR__MASK;
+}
+#define A6XX_RB_MRT_BLEND_CONTROL_ALPHA_BLEND_OPCODE__MASK	0x00e00000
+#define A6XX_RB_MRT_BLEND_CONTROL_ALPHA_BLEND_OPCODE__SHIFT	21
+static inline uint32_t A6XX_RB_MRT_BLEND_CONTROL_ALPHA_BLEND_OPCODE(enum a3xx_rb_blend_opcode val)
+{
+	return ((val) << A6XX_RB_MRT_BLEND_CONTROL_ALPHA_BLEND_OPCODE__SHIFT) & A6XX_RB_MRT_BLEND_CONTROL_ALPHA_BLEND_OPCODE__MASK;
+}
+#define A6XX_RB_MRT_BLEND_CONTROL_ALPHA_DEST_FACTOR__MASK	0x1f000000
+#define A6XX_RB_MRT_BLEND_CONTROL_ALPHA_DEST_FACTOR__SHIFT	24
+static inline uint32_t A6XX_RB_MRT_BLEND_CONTROL_ALPHA_DEST_FACTOR(enum adreno_rb_blend_factor val)
+{
+	return ((val) << A6XX_RB_MRT_BLEND_CONTROL_ALPHA_DEST_FACTOR__SHIFT) & A6XX_RB_MRT_BLEND_CONTROL_ALPHA_DEST_FACTOR__MASK;
+}
+
+static inline uint32_t REG_A6XX_RB_MRT_BUF_INFO(uint32_t i0) { return 0x00008822 + 0x8*i0; }
+#define A6XX_RB_MRT_BUF_INFO_COLOR_FORMAT__MASK			0x000000ff
+#define A6XX_RB_MRT_BUF_INFO_COLOR_FORMAT__SHIFT		0
+static inline uint32_t A6XX_RB_MRT_BUF_INFO_COLOR_FORMAT(enum a6xx_color_fmt val)
+{
+	return ((val) << A6XX_RB_MRT_BUF_INFO_COLOR_FORMAT__SHIFT) & A6XX_RB_MRT_BUF_INFO_COLOR_FORMAT__MASK;
+}
+#define A6XX_RB_MRT_BUF_INFO_COLOR_TILE_MODE__MASK		0x00000300
+#define A6XX_RB_MRT_BUF_INFO_COLOR_TILE_MODE__SHIFT		8
+static inline uint32_t A6XX_RB_MRT_BUF_INFO_COLOR_TILE_MODE(enum a6xx_tile_mode val)
+{
+	return ((val) << A6XX_RB_MRT_BUF_INFO_COLOR_TILE_MODE__SHIFT) & A6XX_RB_MRT_BUF_INFO_COLOR_TILE_MODE__MASK;
+}
+#define A6XX_RB_MRT_BUF_INFO_COLOR_SWAP__MASK			0x00006000
+#define A6XX_RB_MRT_BUF_INFO_COLOR_SWAP__SHIFT			13
+static inline uint32_t A6XX_RB_MRT_BUF_INFO_COLOR_SWAP(enum a3xx_color_swap val)
+{
+	return ((val) << A6XX_RB_MRT_BUF_INFO_COLOR_SWAP__SHIFT) & A6XX_RB_MRT_BUF_INFO_COLOR_SWAP__MASK;
+}
+#define A6XX_RB_MRT_BUF_INFO_COLOR_SRGB				0x00008000
+
+static inline uint32_t REG_A6XX_RB_MRT_PITCH(uint32_t i0) { return 0x00008823 + 0x8*i0; }
+#define A6XX_RB_MRT_PITCH__MASK					0xffffffff
+#define A6XX_RB_MRT_PITCH__SHIFT				0
+static inline uint32_t A6XX_RB_MRT_PITCH(uint32_t val)
+{
+	return ((val >> 6) << A6XX_RB_MRT_PITCH__SHIFT) & A6XX_RB_MRT_PITCH__MASK;
+}
+
+static inline uint32_t REG_A6XX_RB_MRT_ARRAY_PITCH(uint32_t i0) { return 0x00008824 + 0x8*i0; }
+#define A6XX_RB_MRT_ARRAY_PITCH__MASK				0xffffffff
+#define A6XX_RB_MRT_ARRAY_PITCH__SHIFT				0
+static inline uint32_t A6XX_RB_MRT_ARRAY_PITCH(uint32_t val)
+{
+	return ((val >> 6) << A6XX_RB_MRT_ARRAY_PITCH__SHIFT) & A6XX_RB_MRT_ARRAY_PITCH__MASK;
+}
+
+static inline uint32_t REG_A6XX_RB_MRT_BASE_LO(uint32_t i0) { return 0x00008825 + 0x8*i0; }
+
+static inline uint32_t REG_A6XX_RB_MRT_BASE_HI(uint32_t i0) { return 0x00008826 + 0x8*i0; }
+
+static inline uint32_t REG_A6XX_RB_MRT_BASE_GMEM(uint32_t i0) { return 0x00008827 + 0x8*i0; }
+
+#define REG_A6XX_RB_BLEND_RED_F32				0x00008860
+#define A6XX_RB_BLEND_RED_F32__MASK				0xffffffff
+#define A6XX_RB_BLEND_RED_F32__SHIFT				0
+static inline uint32_t A6XX_RB_BLEND_RED_F32(float val)
+{
+	return ((fui(val)) << A6XX_RB_BLEND_RED_F32__SHIFT) & A6XX_RB_BLEND_RED_F32__MASK;
+}
+
+#define REG_A6XX_RB_BLEND_GREEN_F32				0x00008861
+#define A6XX_RB_BLEND_GREEN_F32__MASK				0xffffffff
+#define A6XX_RB_BLEND_GREEN_F32__SHIFT				0
+static inline uint32_t A6XX_RB_BLEND_GREEN_F32(float val)
+{
+	return ((fui(val)) << A6XX_RB_BLEND_GREEN_F32__SHIFT) & A6XX_RB_BLEND_GREEN_F32__MASK;
+}
+
+#define REG_A6XX_RB_BLEND_BLUE_F32				0x00008862
+#define A6XX_RB_BLEND_BLUE_F32__MASK				0xffffffff
+#define A6XX_RB_BLEND_BLUE_F32__SHIFT				0
+static inline uint32_t A6XX_RB_BLEND_BLUE_F32(float val)
+{
+	return ((fui(val)) << A6XX_RB_BLEND_BLUE_F32__SHIFT) & A6XX_RB_BLEND_BLUE_F32__MASK;
+}
+
+#define REG_A6XX_RB_BLEND_ALPHA_F32				0x00008863
+#define A6XX_RB_BLEND_ALPHA_F32__MASK				0xffffffff
+#define A6XX_RB_BLEND_ALPHA_F32__SHIFT				0
+static inline uint32_t A6XX_RB_BLEND_ALPHA_F32(float val)
+{
+	return ((fui(val)) << A6XX_RB_BLEND_ALPHA_F32__SHIFT) & A6XX_RB_BLEND_ALPHA_F32__MASK;
+}
+
+#define REG_A6XX_RB_ALPHA_CONTROL				0x00008864
+#define A6XX_RB_ALPHA_CONTROL_ALPHA_REF__MASK			0x000000ff
+#define A6XX_RB_ALPHA_CONTROL_ALPHA_REF__SHIFT			0
+static inline uint32_t A6XX_RB_ALPHA_CONTROL_ALPHA_REF(uint32_t val)
+{
+	return ((val) << A6XX_RB_ALPHA_CONTROL_ALPHA_REF__SHIFT) & A6XX_RB_ALPHA_CONTROL_ALPHA_REF__MASK;
+}
+#define A6XX_RB_ALPHA_CONTROL_ALPHA_TEST			0x00000100
+#define A6XX_RB_ALPHA_CONTROL_ALPHA_TEST_FUNC__MASK		0x00000e00
+#define A6XX_RB_ALPHA_CONTROL_ALPHA_TEST_FUNC__SHIFT		9
+static inline uint32_t A6XX_RB_ALPHA_CONTROL_ALPHA_TEST_FUNC(enum adreno_compare_func val)
+{
+	return ((val) << A6XX_RB_ALPHA_CONTROL_ALPHA_TEST_FUNC__SHIFT) & A6XX_RB_ALPHA_CONTROL_ALPHA_TEST_FUNC__MASK;
+}
+
+#define REG_A6XX_RB_BLEND_CNTL					0x00008865
+#define A6XX_RB_BLEND_CNTL_ENABLE_BLEND__MASK			0x000000ff
+#define A6XX_RB_BLEND_CNTL_ENABLE_BLEND__SHIFT			0
+static inline uint32_t A6XX_RB_BLEND_CNTL_ENABLE_BLEND(uint32_t val)
+{
+	return ((val) << A6XX_RB_BLEND_CNTL_ENABLE_BLEND__SHIFT) & A6XX_RB_BLEND_CNTL_ENABLE_BLEND__MASK;
+}
+#define A6XX_RB_BLEND_CNTL_INDEPENDENT_BLEND			0x00000100
+#define A6XX_RB_BLEND_CNTL_SAMPLE_MASK__MASK			0xffff0000
+#define A6XX_RB_BLEND_CNTL_SAMPLE_MASK__SHIFT			16
+static inline uint32_t A6XX_RB_BLEND_CNTL_SAMPLE_MASK(uint32_t val)
+{
+	return ((val) << A6XX_RB_BLEND_CNTL_SAMPLE_MASK__SHIFT) & A6XX_RB_BLEND_CNTL_SAMPLE_MASK__MASK;
+}
+
+#define REG_A6XX_RB_DEPTH_CNTL					0x00008871
+#define A6XX_RB_DEPTH_CNTL_Z_ENABLE				0x00000001
+#define A6XX_RB_DEPTH_CNTL_Z_WRITE_ENABLE			0x00000002
+#define A6XX_RB_DEPTH_CNTL_ZFUNC__MASK				0x0000001c
+#define A6XX_RB_DEPTH_CNTL_ZFUNC__SHIFT				2
+static inline uint32_t A6XX_RB_DEPTH_CNTL_ZFUNC(enum adreno_compare_func val)
+{
+	return ((val) << A6XX_RB_DEPTH_CNTL_ZFUNC__SHIFT) & A6XX_RB_DEPTH_CNTL_ZFUNC__MASK;
+}
+#define A6XX_RB_DEPTH_CNTL_Z_TEST_ENABLE			0x00000040
+
+#define REG_A6XX_RB_DEPTH_BUFFER_INFO				0x00008872
+#define A6XX_RB_DEPTH_BUFFER_INFO_DEPTH_FORMAT__MASK		0x00000007
+#define A6XX_RB_DEPTH_BUFFER_INFO_DEPTH_FORMAT__SHIFT		0
+static inline uint32_t A6XX_RB_DEPTH_BUFFER_INFO_DEPTH_FORMAT(enum a6xx_depth_format val)
+{
+	return ((val) << A6XX_RB_DEPTH_BUFFER_INFO_DEPTH_FORMAT__SHIFT) & A6XX_RB_DEPTH_BUFFER_INFO_DEPTH_FORMAT__MASK;
+}
+
+#define REG_A6XX_RB_DEPTH_BUFFER_PITCH				0x00008873
+#define A6XX_RB_DEPTH_BUFFER_PITCH__MASK			0xffffffff
+#define A6XX_RB_DEPTH_BUFFER_PITCH__SHIFT			0
+static inline uint32_t A6XX_RB_DEPTH_BUFFER_PITCH(uint32_t val)
+{
+	return ((val >> 6) << A6XX_RB_DEPTH_BUFFER_PITCH__SHIFT) & A6XX_RB_DEPTH_BUFFER_PITCH__MASK;
+}
+
+#define REG_A6XX_RB_DEPTH_BUFFER_ARRAY_PITCH			0x00008874
+#define A6XX_RB_DEPTH_BUFFER_ARRAY_PITCH__MASK			0xffffffff
+#define A6XX_RB_DEPTH_BUFFER_ARRAY_PITCH__SHIFT			0
+static inline uint32_t A6XX_RB_DEPTH_BUFFER_ARRAY_PITCH(uint32_t val)
+{
+	return ((val >> 6) << A6XX_RB_DEPTH_BUFFER_ARRAY_PITCH__SHIFT) & A6XX_RB_DEPTH_BUFFER_ARRAY_PITCH__MASK;
+}
+
+#define REG_A6XX_RB_DEPTH_BUFFER_BASE_LO			0x00008875
+
+#define REG_A6XX_RB_DEPTH_BUFFER_BASE_HI			0x00008876
+
+#define REG_A6XX_RB_DEPTH_BUFFER_BASE_GMEM			0x00008877
+
+#define REG_A6XX_RB_UNKNOWN_8878				0x00008878
+
+#define REG_A6XX_RB_UNKNOWN_8879				0x00008879
+
+#define REG_A6XX_RB_STENCIL_CONTROL				0x00008880
+#define A6XX_RB_STENCIL_CONTROL_STENCIL_ENABLE			0x00000001
+#define A6XX_RB_STENCIL_CONTROL_STENCIL_ENABLE_BF		0x00000002
+#define A6XX_RB_STENCIL_CONTROL_STENCIL_READ			0x00000004
+#define A6XX_RB_STENCIL_CONTROL_FUNC__MASK			0x00000700
+#define A6XX_RB_STENCIL_CONTROL_FUNC__SHIFT			8
+static inline uint32_t A6XX_RB_STENCIL_CONTROL_FUNC(enum adreno_compare_func val)
+{
+	return ((val) << A6XX_RB_STENCIL_CONTROL_FUNC__SHIFT) & A6XX_RB_STENCIL_CONTROL_FUNC__MASK;
+}
+#define A6XX_RB_STENCIL_CONTROL_FAIL__MASK			0x00003800
+#define A6XX_RB_STENCIL_CONTROL_FAIL__SHIFT			11
+static inline uint32_t A6XX_RB_STENCIL_CONTROL_FAIL(enum adreno_stencil_op val)
+{
+	return ((val) << A6XX_RB_STENCIL_CONTROL_FAIL__SHIFT) & A6XX_RB_STENCIL_CONTROL_FAIL__MASK;
+}
+#define A6XX_RB_STENCIL_CONTROL_ZPASS__MASK			0x0001c000
+#define A6XX_RB_STENCIL_CONTROL_ZPASS__SHIFT			14
+static inline uint32_t A6XX_RB_STENCIL_CONTROL_ZPASS(enum adreno_stencil_op val)
+{
+	return ((val) << A6XX_RB_STENCIL_CONTROL_ZPASS__SHIFT) & A6XX_RB_STENCIL_CONTROL_ZPASS__MASK;
+}
+#define A6XX_RB_STENCIL_CONTROL_ZFAIL__MASK			0x000e0000
+#define A6XX_RB_STENCIL_CONTROL_ZFAIL__SHIFT			17
+static inline uint32_t A6XX_RB_STENCIL_CONTROL_ZFAIL(enum adreno_stencil_op val)
+{
+	return ((val) << A6XX_RB_STENCIL_CONTROL_ZFAIL__SHIFT) & A6XX_RB_STENCIL_CONTROL_ZFAIL__MASK;
+}
+#define A6XX_RB_STENCIL_CONTROL_FUNC_BF__MASK			0x00700000
+#define A6XX_RB_STENCIL_CONTROL_FUNC_BF__SHIFT			20
+static inline uint32_t A6XX_RB_STENCIL_CONTROL_FUNC_BF(enum adreno_compare_func val)
+{
+	return ((val) << A6XX_RB_STENCIL_CONTROL_FUNC_BF__SHIFT) & A6XX_RB_STENCIL_CONTROL_FUNC_BF__MASK;
+}
+#define A6XX_RB_STENCIL_CONTROL_FAIL_BF__MASK			0x03800000
+#define A6XX_RB_STENCIL_CONTROL_FAIL_BF__SHIFT			23
+static inline uint32_t A6XX_RB_STENCIL_CONTROL_FAIL_BF(enum adreno_stencil_op val)
+{
+	return ((val) << A6XX_RB_STENCIL_CONTROL_FAIL_BF__SHIFT) & A6XX_RB_STENCIL_CONTROL_FAIL_BF__MASK;
+}
+#define A6XX_RB_STENCIL_CONTROL_ZPASS_BF__MASK			0x1c000000
+#define A6XX_RB_STENCIL_CONTROL_ZPASS_BF__SHIFT			26
+static inline uint32_t A6XX_RB_STENCIL_CONTROL_ZPASS_BF(enum adreno_stencil_op val)
+{
+	return ((val) << A6XX_RB_STENCIL_CONTROL_ZPASS_BF__SHIFT) & A6XX_RB_STENCIL_CONTROL_ZPASS_BF__MASK;
+}
+#define A6XX_RB_STENCIL_CONTROL_ZFAIL_BF__MASK			0xe0000000
+#define A6XX_RB_STENCIL_CONTROL_ZFAIL_BF__SHIFT			29
+static inline uint32_t A6XX_RB_STENCIL_CONTROL_ZFAIL_BF(enum adreno_stencil_op val)
+{
+	return ((val) << A6XX_RB_STENCIL_CONTROL_ZFAIL_BF__SHIFT) & A6XX_RB_STENCIL_CONTROL_ZFAIL_BF__MASK;
+}
+
+#define REG_A6XX_RB_STENCIL_INFO				0x00008881
+#define A6XX_RB_STENCIL_INFO_SEPARATE_STENCIL			0x00000001
+
+#define REG_A6XX_RB_STENCIL_BUFFER_PITCH			0x00008882
+#define A6XX_RB_STENCIL_BUFFER_PITCH__MASK			0xffffffff
+#define A6XX_RB_STENCIL_BUFFER_PITCH__SHIFT			0
+static inline uint32_t A6XX_RB_STENCIL_BUFFER_PITCH(uint32_t val)
+{
+	return ((val >> 6) << A6XX_RB_STENCIL_BUFFER_PITCH__SHIFT) & A6XX_RB_STENCIL_BUFFER_PITCH__MASK;
+}
+
+#define REG_A6XX_RB_STENCIL_BUFFER_ARRAY_PITCH			0x00008883
+#define A6XX_RB_STENCIL_BUFFER_ARRAY_PITCH__MASK		0xffffffff
+#define A6XX_RB_STENCIL_BUFFER_ARRAY_PITCH__SHIFT		0
+static inline uint32_t A6XX_RB_STENCIL_BUFFER_ARRAY_PITCH(uint32_t val)
+{
+	return ((val >> 6) << A6XX_RB_STENCIL_BUFFER_ARRAY_PITCH__SHIFT) & A6XX_RB_STENCIL_BUFFER_ARRAY_PITCH__MASK;
+}
+
+#define REG_A6XX_RB_STENCIL_BUFFER_BASE_LO			0x00008884
+
+#define REG_A6XX_RB_STENCIL_BUFFER_BASE_HI			0x00008885
+
+#define REG_A6XX_RB_STENCIL_BUFFER_BASE_GMEM			0x00008886
+
+#define REG_A6XX_RB_STENCILREF					0x00008887
+#define A6XX_RB_STENCILREF_REF__MASK				0x000000ff
+#define A6XX_RB_STENCILREF_REF__SHIFT				0
+static inline uint32_t A6XX_RB_STENCILREF_REF(uint32_t val)
+{
+	return ((val) << A6XX_RB_STENCILREF_REF__SHIFT) & A6XX_RB_STENCILREF_REF__MASK;
+}
+
+#define REG_A6XX_RB_STENCILMASK					0x00008888
+#define A6XX_RB_STENCILMASK_MASK__MASK				0x000000ff
+#define A6XX_RB_STENCILMASK_MASK__SHIFT				0
+static inline uint32_t A6XX_RB_STENCILMASK_MASK(uint32_t val)
+{
+	return ((val) << A6XX_RB_STENCILMASK_MASK__SHIFT) & A6XX_RB_STENCILMASK_MASK__MASK;
+}
+
+#define REG_A6XX_RB_STENCILWRMASK				0x00008889
+#define A6XX_RB_STENCILWRMASK_WRMASK__MASK			0x000000ff
+#define A6XX_RB_STENCILWRMASK_WRMASK__SHIFT			0
+static inline uint32_t A6XX_RB_STENCILWRMASK_WRMASK(uint32_t val)
+{
+	return ((val) << A6XX_RB_STENCILWRMASK_WRMASK__SHIFT) & A6XX_RB_STENCILWRMASK_WRMASK__MASK;
+}
+
+#define REG_A6XX_RB_WINDOW_OFFSET				0x00008890
+#define A6XX_RB_WINDOW_OFFSET_WINDOW_OFFSET_DISABLE		0x80000000
+#define A6XX_RB_WINDOW_OFFSET_X__MASK				0x00007fff
+#define A6XX_RB_WINDOW_OFFSET_X__SHIFT				0
+static inline uint32_t A6XX_RB_WINDOW_OFFSET_X(uint32_t val)
+{
+	return ((val) << A6XX_RB_WINDOW_OFFSET_X__SHIFT) & A6XX_RB_WINDOW_OFFSET_X__MASK;
+}
+#define A6XX_RB_WINDOW_OFFSET_Y__MASK				0x7fff0000
+#define A6XX_RB_WINDOW_OFFSET_Y__SHIFT				16
+static inline uint32_t A6XX_RB_WINDOW_OFFSET_Y(uint32_t val)
+{
+	return ((val) << A6XX_RB_WINDOW_OFFSET_Y__SHIFT) & A6XX_RB_WINDOW_OFFSET_Y__MASK;
+}
+
+#define REG_A6XX_RB_SAMPLE_COUNT_CONTROL			0x00008891
+#define A6XX_RB_SAMPLE_COUNT_CONTROL_COPY			0x00000002
+
+#define REG_A6XX_RB_UNKNOWN_88D0				0x000088d0
+
+#define REG_A6XX_RB_BLIT_SCISSOR_TL				0x000088d1
+#define A6XX_RB_BLIT_SCISSOR_TL_WINDOW_OFFSET_DISABLE		0x80000000
+#define A6XX_RB_BLIT_SCISSOR_TL_X__MASK				0x00007fff
+#define A6XX_RB_BLIT_SCISSOR_TL_X__SHIFT			0
+static inline uint32_t A6XX_RB_BLIT_SCISSOR_TL_X(uint32_t val)
+{
+	return ((val) << A6XX_RB_BLIT_SCISSOR_TL_X__SHIFT) & A6XX_RB_BLIT_SCISSOR_TL_X__MASK;
+}
+#define A6XX_RB_BLIT_SCISSOR_TL_Y__MASK				0x7fff0000
+#define A6XX_RB_BLIT_SCISSOR_TL_Y__SHIFT			16
+static inline uint32_t A6XX_RB_BLIT_SCISSOR_TL_Y(uint32_t val)
+{
+	return ((val) << A6XX_RB_BLIT_SCISSOR_TL_Y__SHIFT) & A6XX_RB_BLIT_SCISSOR_TL_Y__MASK;
+}
+
+#define REG_A6XX_RB_BLIT_SCISSOR_BR				0x000088d2
+#define A6XX_RB_BLIT_SCISSOR_BR_WINDOW_OFFSET_DISABLE		0x80000000
+#define A6XX_RB_BLIT_SCISSOR_BR_X__MASK				0x00007fff
+#define A6XX_RB_BLIT_SCISSOR_BR_X__SHIFT			0
+static inline uint32_t A6XX_RB_BLIT_SCISSOR_BR_X(uint32_t val)
+{
+	return ((val) << A6XX_RB_BLIT_SCISSOR_BR_X__SHIFT) & A6XX_RB_BLIT_SCISSOR_BR_X__MASK;
+}
+#define A6XX_RB_BLIT_SCISSOR_BR_Y__MASK				0x7fff0000
+#define A6XX_RB_BLIT_SCISSOR_BR_Y__SHIFT			16
+static inline uint32_t A6XX_RB_BLIT_SCISSOR_BR_Y(uint32_t val)
+{
+	return ((val) << A6XX_RB_BLIT_SCISSOR_BR_Y__SHIFT) & A6XX_RB_BLIT_SCISSOR_BR_Y__MASK;
+}
+
+#define REG_A6XX_RB_BLIT_BASE_GMEM				0x000088d6
+
+#define REG_A6XX_RB_BLIT_DST_INFO				0x000088d7
+#define A6XX_RB_BLIT_DST_INFO_TILE_MODE__MASK			0x00000003
+#define A6XX_RB_BLIT_DST_INFO_TILE_MODE__SHIFT			0
+static inline uint32_t A6XX_RB_BLIT_DST_INFO_TILE_MODE(enum a6xx_tile_mode val)
+{
+	return ((val) << A6XX_RB_BLIT_DST_INFO_TILE_MODE__SHIFT) & A6XX_RB_BLIT_DST_INFO_TILE_MODE__MASK;
+}
+#define A6XX_RB_BLIT_DST_INFO_FLAGS				0x00000004
+#define A6XX_RB_BLIT_DST_INFO_COLOR_FORMAT__MASK		0x00007f80
+#define A6XX_RB_BLIT_DST_INFO_COLOR_FORMAT__SHIFT		7
+static inline uint32_t A6XX_RB_BLIT_DST_INFO_COLOR_FORMAT(enum a6xx_color_fmt val)
+{
+	return ((val) << A6XX_RB_BLIT_DST_INFO_COLOR_FORMAT__SHIFT) & A6XX_RB_BLIT_DST_INFO_COLOR_FORMAT__MASK;
+}
+#define A6XX_RB_BLIT_DST_INFO_COLOR_SWAP__MASK			0x00000060
+#define A6XX_RB_BLIT_DST_INFO_COLOR_SWAP__SHIFT			5
+static inline uint32_t A6XX_RB_BLIT_DST_INFO_COLOR_SWAP(enum a3xx_color_swap val)
+{
+	return ((val) << A6XX_RB_BLIT_DST_INFO_COLOR_SWAP__SHIFT) & A6XX_RB_BLIT_DST_INFO_COLOR_SWAP__MASK;
+}
+
+#define REG_A6XX_RB_BLIT_DST_LO					0x000088d8
+
+#define REG_A6XX_RB_BLIT_DST_HI					0x000088d9
+
+#define REG_A6XX_RB_BLIT_DST_PITCH				0x000088da
+#define A6XX_RB_BLIT_DST_PITCH__MASK				0xffffffff
+#define A6XX_RB_BLIT_DST_PITCH__SHIFT				0
+static inline uint32_t A6XX_RB_BLIT_DST_PITCH(uint32_t val)
+{
+	return ((val >> 6) << A6XX_RB_BLIT_DST_PITCH__SHIFT) & A6XX_RB_BLIT_DST_PITCH__MASK;
+}
+
+#define REG_A6XX_RB_BLIT_DST_ARRAY_PITCH			0x000088db
+#define A6XX_RB_BLIT_DST_ARRAY_PITCH__MASK			0xffffffff
+#define A6XX_RB_BLIT_DST_ARRAY_PITCH__SHIFT			0
+static inline uint32_t A6XX_RB_BLIT_DST_ARRAY_PITCH(uint32_t val)
+{
+	return ((val >> 6) << A6XX_RB_BLIT_DST_ARRAY_PITCH__SHIFT) & A6XX_RB_BLIT_DST_ARRAY_PITCH__MASK;
+}
+
+#define REG_A6XX_RB_BLIT_FLAG_DST_LO				0x000088dc
+
+#define REG_A6XX_RB_BLIT_FLAG_DST_HI				0x000088dd
+
+#define REG_A6XX_RB_BLIT_CLEAR_COLOR_DW0			0x000088df
+
+#define REG_A6XX_RB_BLIT_CLEAR_COLOR_DW1			0x000088e0
+
+#define REG_A6XX_RB_BLIT_CLEAR_COLOR_DW2			0x000088e1
+
+#define REG_A6XX_RB_BLIT_CLEAR_COLOR_DW3			0x000088e2
+
+#define REG_A6XX_RB_BLIT_INFO					0x000088e3
+#define A6XX_RB_BLIT_INFO_UNK0					0x00000001
+#define A6XX_RB_BLIT_INFO_FAST_CLEAR				0x00000002
+#define A6XX_RB_BLIT_INFO_INTEGER				0x00000004
+#define A6XX_RB_BLIT_INFO_UNK3					0x00000008
+#define A6XX_RB_BLIT_INFO_MASK__MASK				0x000000f0
+#define A6XX_RB_BLIT_INFO_MASK__SHIFT				4
+static inline uint32_t A6XX_RB_BLIT_INFO_MASK(uint32_t val)
+{
+	return ((val) << A6XX_RB_BLIT_INFO_MASK__SHIFT) & A6XX_RB_BLIT_INFO_MASK__MASK;
+}
+
+#define REG_A6XX_RB_UNKNOWN_88F0				0x000088f0
+
+#define REG_A6XX_RB_DEPTH_FLAG_BUFFER_BASE_LO			0x00008900
+
+#define REG_A6XX_RB_DEPTH_FLAG_BUFFER_BASE_HI			0x00008901
+
+#define REG_A6XX_RB_DEPTH_FLAG_BUFFER_PITCH			0x00008902
+
+static inline uint32_t REG_A6XX_RB_MRT_FLAG_BUFFER(uint32_t i0) { return 0x00008903 + 0x3*i0; }
+
+static inline uint32_t REG_A6XX_RB_MRT_FLAG_BUFFER_ADDR_LO(uint32_t i0) { return 0x00008903 + 0x3*i0; }
+
+static inline uint32_t REG_A6XX_RB_MRT_FLAG_BUFFER_ADDR_HI(uint32_t i0) { return 0x00008904 + 0x3*i0; }
+
+static inline uint32_t REG_A6XX_RB_MRT_FLAG_BUFFER_PITCH(uint32_t i0) { return 0x00008905 + 0x3*i0; }
+#define A6XX_RB_MRT_FLAG_BUFFER_PITCH_PITCH__MASK		0x000007ff
+#define A6XX_RB_MRT_FLAG_BUFFER_PITCH_PITCH__SHIFT		0
+static inline uint32_t A6XX_RB_MRT_FLAG_BUFFER_PITCH_PITCH(uint32_t val)
+{
+	return ((val >> 5) << A6XX_RB_MRT_FLAG_BUFFER_PITCH_PITCH__SHIFT) & A6XX_RB_MRT_FLAG_BUFFER_PITCH_PITCH__MASK;
+}
+#define A6XX_RB_MRT_FLAG_BUFFER_PITCH_ARRAY_PITCH__MASK		0x003ff800
+#define A6XX_RB_MRT_FLAG_BUFFER_PITCH_ARRAY_PITCH__SHIFT	11
+static inline uint32_t A6XX_RB_MRT_FLAG_BUFFER_PITCH_ARRAY_PITCH(uint32_t val)
+{
+	return ((val >> 5) << A6XX_RB_MRT_FLAG_BUFFER_PITCH_ARRAY_PITCH__SHIFT) & A6XX_RB_MRT_FLAG_BUFFER_PITCH_ARRAY_PITCH__MASK;
+}
+
+#define REG_A6XX_RB_SAMPLE_COUNT_ADDR_LO			0x00008927
+
+#define REG_A6XX_RB_SAMPLE_COUNT_ADDR_HI			0x00008928
+
+#define REG_A6XX_RB_2D_BLIT_CNTL				0x00008c00
+#define A6XX_RB_2D_BLIT_CNTL_COLOR_FORMAT__MASK			0x0000ff00
+#define A6XX_RB_2D_BLIT_CNTL_COLOR_FORMAT__SHIFT		8
+static inline uint32_t A6XX_RB_2D_BLIT_CNTL_COLOR_FORMAT(enum a6xx_color_fmt val)
+{
+	return ((val) << A6XX_RB_2D_BLIT_CNTL_COLOR_FORMAT__SHIFT) & A6XX_RB_2D_BLIT_CNTL_COLOR_FORMAT__MASK;
+}
+
+#define REG_A6XX_RB_2D_DST_INFO					0x00008c17
+#define A6XX_RB_2D_DST_INFO_COLOR_FORMAT__MASK			0x000000ff
+#define A6XX_RB_2D_DST_INFO_COLOR_FORMAT__SHIFT			0
+static inline uint32_t A6XX_RB_2D_DST_INFO_COLOR_FORMAT(enum a6xx_color_fmt val)
+{
+	return ((val) << A6XX_RB_2D_DST_INFO_COLOR_FORMAT__SHIFT) & A6XX_RB_2D_DST_INFO_COLOR_FORMAT__MASK;
+}
+#define A6XX_RB_2D_DST_INFO_TILE_MODE__MASK			0x00000300
+#define A6XX_RB_2D_DST_INFO_TILE_MODE__SHIFT			8
+static inline uint32_t A6XX_RB_2D_DST_INFO_TILE_MODE(enum a6xx_tile_mode val)
+{
+	return ((val) << A6XX_RB_2D_DST_INFO_TILE_MODE__SHIFT) & A6XX_RB_2D_DST_INFO_TILE_MODE__MASK;
+}
+#define A6XX_RB_2D_DST_INFO_COLOR_SWAP__MASK			0x00000c00
+#define A6XX_RB_2D_DST_INFO_COLOR_SWAP__SHIFT			10
+static inline uint32_t A6XX_RB_2D_DST_INFO_COLOR_SWAP(enum a3xx_color_swap val)
+{
+	return ((val) << A6XX_RB_2D_DST_INFO_COLOR_SWAP__SHIFT) & A6XX_RB_2D_DST_INFO_COLOR_SWAP__MASK;
+}
+#define A6XX_RB_2D_DST_INFO_FLAGS				0x00001000
+
+#define REG_A6XX_RB_2D_DST_LO					0x00008c18
+
+#define REG_A6XX_RB_2D_DST_HI					0x00008c19
+
+#define REG_A6XX_RB_2D_DST_SIZE					0x00008c1a
+#define A6XX_RB_2D_DST_SIZE_PITCH__MASK				0x0000ffff
+#define A6XX_RB_2D_DST_SIZE_PITCH__SHIFT			0
+static inline uint32_t A6XX_RB_2D_DST_SIZE_PITCH(uint32_t val)
+{
+	return ((val >> 6) << A6XX_RB_2D_DST_SIZE_PITCH__SHIFT) & A6XX_RB_2D_DST_SIZE_PITCH__MASK;
+}
+
+#define REG_A6XX_RB_2D_DST_FLAGS_LO				0x00008c20
+
+#define REG_A6XX_RB_2D_DST_FLAGS_HI				0x00008c21
+
+#define REG_A6XX_RB_2D_SRC_SOLID_C0				0x00008c2c
+
+#define REG_A6XX_RB_2D_SRC_SOLID_C1				0x00008c2d
+
+#define REG_A6XX_RB_2D_SRC_SOLID_C2				0x00008c2e
+
+#define REG_A6XX_RB_2D_SRC_SOLID_C3				0x00008c2f
+
+#define REG_A6XX_RB_UNKNOWN_8E01				0x00008e01
+
+#define REG_A6XX_RB_CCU_CNTL					0x00008e07
+
+#define REG_A6XX_VPC_UNKNOWN_9101				0x00009101
+
+#define REG_A6XX_VPC_GS_SIV_CNTL				0x00009104
+
+#define REG_A6XX_VPC_UNKNOWN_9108				0x00009108
+
+static inline uint32_t REG_A6XX_VPC_VARYING_INTERP(uint32_t i0) { return 0x00009200 + 0x1*i0; }
+
+static inline uint32_t REG_A6XX_VPC_VARYING_INTERP_MODE(uint32_t i0) { return 0x00009200 + 0x1*i0; }
+
+static inline uint32_t REG_A6XX_VPC_VARYING_PS_REPL(uint32_t i0) { return 0x00009208 + 0x1*i0; }
+
+static inline uint32_t REG_A6XX_VPC_VARYING_PS_REPL_MODE(uint32_t i0) { return 0x00009208 + 0x1*i0; }
+
+#define REG_A6XX_VPC_UNKNOWN_9210				0x00009210
+
+#define REG_A6XX_VPC_UNKNOWN_9211				0x00009211
+
+static inline uint32_t REG_A6XX_VPC_VAR(uint32_t i0) { return 0x00009212 + 0x1*i0; }
+
+static inline uint32_t REG_A6XX_VPC_VAR_DISABLE(uint32_t i0) { return 0x00009212 + 0x1*i0; }
+
+#define REG_A6XX_VPC_SO_CNTL					0x00009216
+#define A6XX_VPC_SO_CNTL_ENABLE					0x00010000
+
+#define REG_A6XX_VPC_SO_PROG					0x00009217
+#define A6XX_VPC_SO_PROG_A_BUF__MASK				0x00000003
+#define A6XX_VPC_SO_PROG_A_BUF__SHIFT				0
+static inline uint32_t A6XX_VPC_SO_PROG_A_BUF(uint32_t val)
+{
+	return ((val) << A6XX_VPC_SO_PROG_A_BUF__SHIFT) & A6XX_VPC_SO_PROG_A_BUF__MASK;
+}
+#define A6XX_VPC_SO_PROG_A_OFF__MASK				0x000007fc
+#define A6XX_VPC_SO_PROG_A_OFF__SHIFT				2
+static inline uint32_t A6XX_VPC_SO_PROG_A_OFF(uint32_t val)
+{
+	return ((val >> 2) << A6XX_VPC_SO_PROG_A_OFF__SHIFT) & A6XX_VPC_SO_PROG_A_OFF__MASK;
+}
+#define A6XX_VPC_SO_PROG_A_EN					0x00000800
+#define A6XX_VPC_SO_PROG_B_BUF__MASK				0x00003000
+#define A6XX_VPC_SO_PROG_B_BUF__SHIFT				12
+static inline uint32_t A6XX_VPC_SO_PROG_B_BUF(uint32_t val)
+{
+	return ((val) << A6XX_VPC_SO_PROG_B_BUF__SHIFT) & A6XX_VPC_SO_PROG_B_BUF__MASK;
+}
+#define A6XX_VPC_SO_PROG_B_OFF__MASK				0x007fc000
+#define A6XX_VPC_SO_PROG_B_OFF__SHIFT				14
+static inline uint32_t A6XX_VPC_SO_PROG_B_OFF(uint32_t val)
+{
+	return ((val >> 2) << A6XX_VPC_SO_PROG_B_OFF__SHIFT) & A6XX_VPC_SO_PROG_B_OFF__MASK;
+}
+#define A6XX_VPC_SO_PROG_B_EN					0x00800000
+
+static inline uint32_t REG_A6XX_VPC_SO(uint32_t i0) { return 0x0000921a + 0x7*i0; }
+
+static inline uint32_t REG_A6XX_VPC_SO_BUFFER_BASE_LO(uint32_t i0) { return 0x0000921a + 0x7*i0; }
+
+static inline uint32_t REG_A6XX_VPC_SO_BUFFER_BASE_HI(uint32_t i0) { return 0x0000921b + 0x7*i0; }
+
+static inline uint32_t REG_A6XX_VPC_SO_BUFFER_SIZE(uint32_t i0) { return 0x0000921c + 0x7*i0; }
+
+static inline uint32_t REG_A6XX_VPC_SO_NCOMP(uint32_t i0) { return 0x0000921d + 0x7*i0; }
+
+static inline uint32_t REG_A6XX_VPC_SO_BUFFER_OFFSET(uint32_t i0) { return 0x0000921e + 0x7*i0; }
+
+static inline uint32_t REG_A6XX_VPC_SO_FLUSH_BASE_LO(uint32_t i0) { return 0x0000921f + 0x7*i0; }
+
+static inline uint32_t REG_A6XX_VPC_SO_FLUSH_BASE_HI(uint32_t i0) { return 0x00009220 + 0x7*i0; }
+
+#define REG_A6XX_VPC_UNKNOWN_9236				0x00009236
+
+#define REG_A6XX_VPC_UNKNOWN_9300				0x00009300
+
+#define REG_A6XX_VPC_PACK					0x00009301
+#define A6XX_VPC_PACK_STRIDE_IN_VPC__MASK			0x000000ff
+#define A6XX_VPC_PACK_STRIDE_IN_VPC__SHIFT			0
+static inline uint32_t A6XX_VPC_PACK_STRIDE_IN_VPC(uint32_t val)
+{
+	return ((val) << A6XX_VPC_PACK_STRIDE_IN_VPC__SHIFT) & A6XX_VPC_PACK_STRIDE_IN_VPC__MASK;
+}
+#define A6XX_VPC_PACK_NUMNONPOSVAR__MASK			0x0000ff00
+#define A6XX_VPC_PACK_NUMNONPOSVAR__SHIFT			8
+static inline uint32_t A6XX_VPC_PACK_NUMNONPOSVAR(uint32_t val)
+{
+	return ((val) << A6XX_VPC_PACK_NUMNONPOSVAR__SHIFT) & A6XX_VPC_PACK_NUMNONPOSVAR__MASK;
+}
+#define A6XX_VPC_PACK_PSIZELOC__MASK				0x00ff0000
+#define A6XX_VPC_PACK_PSIZELOC__SHIFT				16
+static inline uint32_t A6XX_VPC_PACK_PSIZELOC(uint32_t val)
+{
+	return ((val) << A6XX_VPC_PACK_PSIZELOC__SHIFT) & A6XX_VPC_PACK_PSIZELOC__MASK;
+}
+
+#define REG_A6XX_VPC_CNTL_0					0x00009304
+#define A6XX_VPC_CNTL_0_NUMNONPOSVAR__MASK			0x000000ff
+#define A6XX_VPC_CNTL_0_NUMNONPOSVAR__SHIFT			0
+static inline uint32_t A6XX_VPC_CNTL_0_NUMNONPOSVAR(uint32_t val)
+{
+	return ((val) << A6XX_VPC_CNTL_0_NUMNONPOSVAR__SHIFT) & A6XX_VPC_CNTL_0_NUMNONPOSVAR__MASK;
+}
+#define A6XX_VPC_CNTL_0_VARYING					0x00010000
+
+#define REG_A6XX_VPC_SO_BUF_CNTL				0x00009305
+#define A6XX_VPC_SO_BUF_CNTL_BUF0				0x00000001
+#define A6XX_VPC_SO_BUF_CNTL_BUF1				0x00000008
+#define A6XX_VPC_SO_BUF_CNTL_BUF2				0x00000040
+#define A6XX_VPC_SO_BUF_CNTL_BUF3				0x00000200
+#define A6XX_VPC_SO_BUF_CNTL_ENABLE				0x00008000
+
+#define REG_A6XX_VPC_UNKNOWN_9600				0x00009600
+
+#define REG_A6XX_VPC_UNKNOWN_9602				0x00009602
+
+#define REG_A6XX_PC_UNKNOWN_9801				0x00009801
+
+#define REG_A6XX_PC_RESTART_INDEX				0x00009803
+
+#define REG_A6XX_PC_MODE_CNTL					0x00009804
+
+#define REG_A6XX_PC_UNKNOWN_9805				0x00009805
+
+#define REG_A6XX_PC_UNKNOWN_9981				0x00009981
+
+#define REG_A6XX_PC_PRIMITIVE_CNTL_0				0x00009b00
+#define A6XX_PC_PRIMITIVE_CNTL_0_PRIMITIVE_RESTART		0x00000001
+#define A6XX_PC_PRIMITIVE_CNTL_0_PROVOKING_VTX_LAST		0x00000002
+
+#define REG_A6XX_PC_PRIMITIVE_CNTL_1				0x00009b01
+#define A6XX_PC_PRIMITIVE_CNTL_1_STRIDE_IN_VPC__MASK		0x0000007f
+#define A6XX_PC_PRIMITIVE_CNTL_1_STRIDE_IN_VPC__SHIFT		0
+static inline uint32_t A6XX_PC_PRIMITIVE_CNTL_1_STRIDE_IN_VPC(uint32_t val)
+{
+	return ((val) << A6XX_PC_PRIMITIVE_CNTL_1_STRIDE_IN_VPC__SHIFT) & A6XX_PC_PRIMITIVE_CNTL_1_STRIDE_IN_VPC__MASK;
+}
+
+#define REG_A6XX_PC_UNKNOWN_9B06				0x00009b06
+
+#define REG_A6XX_PC_UNKNOWN_9B07				0x00009b07
+
+#define REG_A6XX_PC_TESSFACTOR_ADDR_LO				0x00009e08
+
+#define REG_A6XX_PC_TESSFACTOR_ADDR_HI				0x00009e09
+
+#define REG_A6XX_PC_UNKNOWN_9E72				0x00009e72
+
+#define REG_A6XX_VFD_CONTROL_0					0x0000a000
+#define A6XX_VFD_CONTROL_0_VTXCNT__MASK				0x0000003f
+#define A6XX_VFD_CONTROL_0_VTXCNT__SHIFT			0
+static inline uint32_t A6XX_VFD_CONTROL_0_VTXCNT(uint32_t val)
+{
+	return ((val) << A6XX_VFD_CONTROL_0_VTXCNT__SHIFT) & A6XX_VFD_CONTROL_0_VTXCNT__MASK;
+}
+
+#define REG_A6XX_VFD_CONTROL_1					0x0000a001
+#define A6XX_VFD_CONTROL_1_REGID4VTX__MASK			0x000000ff
+#define A6XX_VFD_CONTROL_1_REGID4VTX__SHIFT			0
+static inline uint32_t A6XX_VFD_CONTROL_1_REGID4VTX(uint32_t val)
+{
+	return ((val) << A6XX_VFD_CONTROL_1_REGID4VTX__SHIFT) & A6XX_VFD_CONTROL_1_REGID4VTX__MASK;
+}
+#define A6XX_VFD_CONTROL_1_REGID4INST__MASK			0x0000ff00
+#define A6XX_VFD_CONTROL_1_REGID4INST__SHIFT			8
+static inline uint32_t A6XX_VFD_CONTROL_1_REGID4INST(uint32_t val)
+{
+	return ((val) << A6XX_VFD_CONTROL_1_REGID4INST__SHIFT) & A6XX_VFD_CONTROL_1_REGID4INST__MASK;
+}
+#define A6XX_VFD_CONTROL_1_REGID4PRIMID__MASK			0x00ff0000
+#define A6XX_VFD_CONTROL_1_REGID4PRIMID__SHIFT			16
+static inline uint32_t A6XX_VFD_CONTROL_1_REGID4PRIMID(uint32_t val)
+{
+	return ((val) << A6XX_VFD_CONTROL_1_REGID4PRIMID__SHIFT) & A6XX_VFD_CONTROL_1_REGID4PRIMID__MASK;
+}
+
+#define REG_A6XX_VFD_CONTROL_2					0x0000a002
+#define A6XX_VFD_CONTROL_2_REGID_PATCHID__MASK			0x000000ff
+#define A6XX_VFD_CONTROL_2_REGID_PATCHID__SHIFT			0
+static inline uint32_t A6XX_VFD_CONTROL_2_REGID_PATCHID(uint32_t val)
+{
+	return ((val) << A6XX_VFD_CONTROL_2_REGID_PATCHID__SHIFT) & A6XX_VFD_CONTROL_2_REGID_PATCHID__MASK;
+}
+
+#define REG_A6XX_VFD_CONTROL_3					0x0000a003
+#define A6XX_VFD_CONTROL_3_REGID_PATCHID__MASK			0x0000ff00
+#define A6XX_VFD_CONTROL_3_REGID_PATCHID__SHIFT			8
+static inline uint32_t A6XX_VFD_CONTROL_3_REGID_PATCHID(uint32_t val)
+{
+	return ((val) << A6XX_VFD_CONTROL_3_REGID_PATCHID__SHIFT) & A6XX_VFD_CONTROL_3_REGID_PATCHID__MASK;
+}
+#define A6XX_VFD_CONTROL_3_REGID_TESSX__MASK			0x00ff0000
+#define A6XX_VFD_CONTROL_3_REGID_TESSX__SHIFT			16
+static inline uint32_t A6XX_VFD_CONTROL_3_REGID_TESSX(uint32_t val)
+{
+	return ((val) << A6XX_VFD_CONTROL_3_REGID_TESSX__SHIFT) & A6XX_VFD_CONTROL_3_REGID_TESSX__MASK;
+}
+#define A6XX_VFD_CONTROL_3_REGID_TESSY__MASK			0xff000000
+#define A6XX_VFD_CONTROL_3_REGID_TESSY__SHIFT			24
+static inline uint32_t A6XX_VFD_CONTROL_3_REGID_TESSY(uint32_t val)
+{
+	return ((val) << A6XX_VFD_CONTROL_3_REGID_TESSY__SHIFT) & A6XX_VFD_CONTROL_3_REGID_TESSY__MASK;
+}
+
+#define REG_A6XX_VFD_CONTROL_4					0x0000a004
+
+#define REG_A6XX_VFD_CONTROL_5					0x0000a005
+
+#define REG_A6XX_VFD_CONTROL_6					0x0000a006
+
+#define REG_A6XX_VFD_MODE_CNTL					0x0000a007
+#define A6XX_VFD_MODE_CNTL_BINNING_PASS				0x00000001
+
+#define REG_A6XX_VFD_UNKNOWN_A008				0x0000a008
+
+#define REG_A6XX_VFD_INDEX_OFFSET				0x0000a00e
+
+#define REG_A6XX_VFD_INSTANCE_START_OFFSET			0x0000a00f
+
+static inline uint32_t REG_A6XX_VFD_FETCH(uint32_t i0) { return 0x0000a010 + 0x4*i0; }
+
+static inline uint32_t REG_A6XX_VFD_FETCH_BASE_LO(uint32_t i0) { return 0x0000a010 + 0x4*i0; }
+
+static inline uint32_t REG_A6XX_VFD_FETCH_BASE_HI(uint32_t i0) { return 0x0000a011 + 0x4*i0; }
+
+static inline uint32_t REG_A6XX_VFD_FETCH_SIZE(uint32_t i0) { return 0x0000a012 + 0x4*i0; }
+
+static inline uint32_t REG_A6XX_VFD_FETCH_STRIDE(uint32_t i0) { return 0x0000a013 + 0x4*i0; }
+
+static inline uint32_t REG_A6XX_VFD_DECODE(uint32_t i0) { return 0x0000a090 + 0x2*i0; }
+
+static inline uint32_t REG_A6XX_VFD_DECODE_INSTR(uint32_t i0) { return 0x0000a090 + 0x2*i0; }
+#define A6XX_VFD_DECODE_INSTR_IDX__MASK				0x0000001f
+#define A6XX_VFD_DECODE_INSTR_IDX__SHIFT			0
+static inline uint32_t A6XX_VFD_DECODE_INSTR_IDX(uint32_t val)
+{
+	return ((val) << A6XX_VFD_DECODE_INSTR_IDX__SHIFT) & A6XX_VFD_DECODE_INSTR_IDX__MASK;
+}
+#define A6XX_VFD_DECODE_INSTR_INSTANCED				0x00020000
+#define A6XX_VFD_DECODE_INSTR_FORMAT__MASK			0x0ff00000
+#define A6XX_VFD_DECODE_INSTR_FORMAT__SHIFT			20
+static inline uint32_t A6XX_VFD_DECODE_INSTR_FORMAT(enum a6xx_vtx_fmt val)
+{
+	return ((val) << A6XX_VFD_DECODE_INSTR_FORMAT__SHIFT) & A6XX_VFD_DECODE_INSTR_FORMAT__MASK;
+}
+#define A6XX_VFD_DECODE_INSTR_SWAP__MASK			0x30000000
+#define A6XX_VFD_DECODE_INSTR_SWAP__SHIFT			28
+static inline uint32_t A6XX_VFD_DECODE_INSTR_SWAP(enum a3xx_color_swap val)
+{
+	return ((val) << A6XX_VFD_DECODE_INSTR_SWAP__SHIFT) & A6XX_VFD_DECODE_INSTR_SWAP__MASK;
+}
+#define A6XX_VFD_DECODE_INSTR_UNK30				0x40000000
+#define A6XX_VFD_DECODE_INSTR_FLOAT				0x80000000
+
+static inline uint32_t REG_A6XX_VFD_DECODE_STEP_RATE(uint32_t i0) { return 0x0000a091 + 0x2*i0; }
+
+static inline uint32_t REG_A6XX_VFD_DEST_CNTL(uint32_t i0) { return 0x0000a0d0 + 0x1*i0; }
+
+static inline uint32_t REG_A6XX_VFD_DEST_CNTL_INSTR(uint32_t i0) { return 0x0000a0d0 + 0x1*i0; }
+#define A6XX_VFD_DEST_CNTL_INSTR_WRITEMASK__MASK		0x0000000f
+#define A6XX_VFD_DEST_CNTL_INSTR_WRITEMASK__SHIFT		0
+static inline uint32_t A6XX_VFD_DEST_CNTL_INSTR_WRITEMASK(uint32_t val)
+{
+	return ((val) << A6XX_VFD_DEST_CNTL_INSTR_WRITEMASK__SHIFT) & A6XX_VFD_DEST_CNTL_INSTR_WRITEMASK__MASK;
+}
+#define A6XX_VFD_DEST_CNTL_INSTR_REGID__MASK			0x00000ff0
+#define A6XX_VFD_DEST_CNTL_INSTR_REGID__SHIFT			4
+static inline uint32_t A6XX_VFD_DEST_CNTL_INSTR_REGID(uint32_t val)
+{
+	return ((val) << A6XX_VFD_DEST_CNTL_INSTR_REGID__SHIFT) & A6XX_VFD_DEST_CNTL_INSTR_REGID__MASK;
+}
+
+#define REG_A6XX_SP_UNKNOWN_A0F8				0x0000a0f8
+
+#define REG_A6XX_SP_PRIMITIVE_CNTL				0x0000a802
+#define A6XX_SP_PRIMITIVE_CNTL_VSOUT__MASK			0x0000001f
+#define A6XX_SP_PRIMITIVE_CNTL_VSOUT__SHIFT			0
+static inline uint32_t A6XX_SP_PRIMITIVE_CNTL_VSOUT(uint32_t val)
+{
+	return ((val) << A6XX_SP_PRIMITIVE_CNTL_VSOUT__SHIFT) & A6XX_SP_PRIMITIVE_CNTL_VSOUT__MASK;
+}
+
+static inline uint32_t REG_A6XX_SP_VS_OUT(uint32_t i0) { return 0x0000a803 + 0x1*i0; }
+
+static inline uint32_t REG_A6XX_SP_VS_OUT_REG(uint32_t i0) { return 0x0000a803 + 0x1*i0; }
+#define A6XX_SP_VS_OUT_REG_A_REGID__MASK			0x000000ff
+#define A6XX_SP_VS_OUT_REG_A_REGID__SHIFT			0
+static inline uint32_t A6XX_SP_VS_OUT_REG_A_REGID(uint32_t val)
+{
+	return ((val) << A6XX_SP_VS_OUT_REG_A_REGID__SHIFT) & A6XX_SP_VS_OUT_REG_A_REGID__MASK;
+}
+#define A6XX_SP_VS_OUT_REG_A_COMPMASK__MASK			0x00000f00
+#define A6XX_SP_VS_OUT_REG_A_COMPMASK__SHIFT			8
+static inline uint32_t A6XX_SP_VS_OUT_REG_A_COMPMASK(uint32_t val)
+{
+	return ((val) << A6XX_SP_VS_OUT_REG_A_COMPMASK__SHIFT) & A6XX_SP_VS_OUT_REG_A_COMPMASK__MASK;
+}
+#define A6XX_SP_VS_OUT_REG_B_REGID__MASK			0x00ff0000
+#define A6XX_SP_VS_OUT_REG_B_REGID__SHIFT			16
+static inline uint32_t A6XX_SP_VS_OUT_REG_B_REGID(uint32_t val)
+{
+	return ((val) << A6XX_SP_VS_OUT_REG_B_REGID__SHIFT) & A6XX_SP_VS_OUT_REG_B_REGID__MASK;
+}
+#define A6XX_SP_VS_OUT_REG_B_COMPMASK__MASK			0x0f000000
+#define A6XX_SP_VS_OUT_REG_B_COMPMASK__SHIFT			24
+static inline uint32_t A6XX_SP_VS_OUT_REG_B_COMPMASK(uint32_t val)
+{
+	return ((val) << A6XX_SP_VS_OUT_REG_B_COMPMASK__SHIFT) & A6XX_SP_VS_OUT_REG_B_COMPMASK__MASK;
+}
+
+static inline uint32_t REG_A6XX_SP_VS_VPC_DST(uint32_t i0) { return 0x0000a813 + 0x1*i0; }
+
+static inline uint32_t REG_A6XX_SP_VS_VPC_DST_REG(uint32_t i0) { return 0x0000a813 + 0x1*i0; }
+#define A6XX_SP_VS_VPC_DST_REG_OUTLOC0__MASK			0x000000ff
+#define A6XX_SP_VS_VPC_DST_REG_OUTLOC0__SHIFT			0
+static inline uint32_t A6XX_SP_VS_VPC_DST_REG_OUTLOC0(uint32_t val)
+{
+	return ((val) << A6XX_SP_VS_VPC_DST_REG_OUTLOC0__SHIFT) & A6XX_SP_VS_VPC_DST_REG_OUTLOC0__MASK;
+}
+#define A6XX_SP_VS_VPC_DST_REG_OUTLOC1__MASK			0x0000ff00
+#define A6XX_SP_VS_VPC_DST_REG_OUTLOC1__SHIFT			8
+static inline uint32_t A6XX_SP_VS_VPC_DST_REG_OUTLOC1(uint32_t val)
+{
+	return ((val) << A6XX_SP_VS_VPC_DST_REG_OUTLOC1__SHIFT) & A6XX_SP_VS_VPC_DST_REG_OUTLOC1__MASK;
+}
+#define A6XX_SP_VS_VPC_DST_REG_OUTLOC2__MASK			0x00ff0000
+#define A6XX_SP_VS_VPC_DST_REG_OUTLOC2__SHIFT			16
+static inline uint32_t A6XX_SP_VS_VPC_DST_REG_OUTLOC2(uint32_t val)
+{
+	return ((val) << A6XX_SP_VS_VPC_DST_REG_OUTLOC2__SHIFT) & A6XX_SP_VS_VPC_DST_REG_OUTLOC2__MASK;
+}
+#define A6XX_SP_VS_VPC_DST_REG_OUTLOC3__MASK			0xff000000
+#define A6XX_SP_VS_VPC_DST_REG_OUTLOC3__SHIFT			24
+static inline uint32_t A6XX_SP_VS_VPC_DST_REG_OUTLOC3(uint32_t val)
+{
+	return ((val) << A6XX_SP_VS_VPC_DST_REG_OUTLOC3__SHIFT) & A6XX_SP_VS_VPC_DST_REG_OUTLOC3__MASK;
+}
+
+#define REG_A6XX_SP_VS_CTRL_REG0				0x0000a800
+#define A6XX_SP_VS_CTRL_REG0_HALFREGFOOTPRINT__MASK		0x0000007e
+#define A6XX_SP_VS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT		1
+static inline uint32_t A6XX_SP_VS_CTRL_REG0_HALFREGFOOTPRINT(uint32_t val)
+{
+	return ((val) << A6XX_SP_VS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT) & A6XX_SP_VS_CTRL_REG0_HALFREGFOOTPRINT__MASK;
+}
+#define A6XX_SP_VS_CTRL_REG0_FULLREGFOOTPRINT__MASK		0x00001f80
+#define A6XX_SP_VS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT		7
+static inline uint32_t A6XX_SP_VS_CTRL_REG0_FULLREGFOOTPRINT(uint32_t val)
+{
+	return ((val) << A6XX_SP_VS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT) & A6XX_SP_VS_CTRL_REG0_FULLREGFOOTPRINT__MASK;
+}
+#define A6XX_SP_VS_CTRL_REG0_BRANCHSTACK__MASK			0x000fc000
+#define A6XX_SP_VS_CTRL_REG0_BRANCHSTACK__SHIFT			14
+static inline uint32_t A6XX_SP_VS_CTRL_REG0_BRANCHSTACK(uint32_t val)
+{
+	return ((val) << A6XX_SP_VS_CTRL_REG0_BRANCHSTACK__SHIFT) & A6XX_SP_VS_CTRL_REG0_BRANCHSTACK__MASK;
+}
+#define A6XX_SP_VS_CTRL_REG0_THREADSIZE__MASK			0x00100000
+#define A6XX_SP_VS_CTRL_REG0_THREADSIZE__SHIFT			20
+static inline uint32_t A6XX_SP_VS_CTRL_REG0_THREADSIZE(enum a3xx_threadsize val)
+{
+	return ((val) << A6XX_SP_VS_CTRL_REG0_THREADSIZE__SHIFT) & A6XX_SP_VS_CTRL_REG0_THREADSIZE__MASK;
+}
+#define A6XX_SP_VS_CTRL_REG0_VARYING				0x00400000
+#define A6XX_SP_VS_CTRL_REG0_PIXLODENABLE			0x04000000
+#define A6XX_SP_VS_CTRL_REG0_MERGEDREGS				0x80000000
+
+#define REG_A6XX_SP_VS_OBJ_START_LO				0x0000a81c
+
+#define REG_A6XX_SP_VS_OBJ_START_HI				0x0000a81d
+
+#define REG_A6XX_SP_VS_TEX_COUNT				0x0000a822
+
+#define REG_A6XX_SP_VS_CONFIG					0x0000a823
+#define A6XX_SP_VS_CONFIG_ENABLED				0x00000100
+#define A6XX_SP_VS_CONFIG_NTEX__MASK				0x0001fe00
+#define A6XX_SP_VS_CONFIG_NTEX__SHIFT				9
+static inline uint32_t A6XX_SP_VS_CONFIG_NTEX(uint32_t val)
+{
+	return ((val) << A6XX_SP_VS_CONFIG_NTEX__SHIFT) & A6XX_SP_VS_CONFIG_NTEX__MASK;
+}
+#define A6XX_SP_VS_CONFIG_NSAMP__MASK				0x01fe0000
+#define A6XX_SP_VS_CONFIG_NSAMP__SHIFT				17
+static inline uint32_t A6XX_SP_VS_CONFIG_NSAMP(uint32_t val)
+{
+	return ((val) << A6XX_SP_VS_CONFIG_NSAMP__SHIFT) & A6XX_SP_VS_CONFIG_NSAMP__MASK;
+}
+
+#define REG_A6XX_SP_VS_INSTRLEN					0x0000a824
+
+#define REG_A6XX_SP_HS_CTRL_REG0				0x0000a830
+#define A6XX_SP_HS_CTRL_REG0_HALFREGFOOTPRINT__MASK		0x0000007e
+#define A6XX_SP_HS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT		1
+static inline uint32_t A6XX_SP_HS_CTRL_REG0_HALFREGFOOTPRINT(uint32_t val)
+{
+	return ((val) << A6XX_SP_HS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT) & A6XX_SP_HS_CTRL_REG0_HALFREGFOOTPRINT__MASK;
+}
+#define A6XX_SP_HS_CTRL_REG0_FULLREGFOOTPRINT__MASK		0x00001f80
+#define A6XX_SP_HS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT		7
+static inline uint32_t A6XX_SP_HS_CTRL_REG0_FULLREGFOOTPRINT(uint32_t val)
+{
+	return ((val) << A6XX_SP_HS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT) & A6XX_SP_HS_CTRL_REG0_FULLREGFOOTPRINT__MASK;
+}
+#define A6XX_SP_HS_CTRL_REG0_BRANCHSTACK__MASK			0x000fc000
+#define A6XX_SP_HS_CTRL_REG0_BRANCHSTACK__SHIFT			14
+static inline uint32_t A6XX_SP_HS_CTRL_REG0_BRANCHSTACK(uint32_t val)
+{
+	return ((val) << A6XX_SP_HS_CTRL_REG0_BRANCHSTACK__SHIFT) & A6XX_SP_HS_CTRL_REG0_BRANCHSTACK__MASK;
+}
+#define A6XX_SP_HS_CTRL_REG0_THREADSIZE__MASK			0x00100000
+#define A6XX_SP_HS_CTRL_REG0_THREADSIZE__SHIFT			20
+static inline uint32_t A6XX_SP_HS_CTRL_REG0_THREADSIZE(enum a3xx_threadsize val)
+{
+	return ((val) << A6XX_SP_HS_CTRL_REG0_THREADSIZE__SHIFT) & A6XX_SP_HS_CTRL_REG0_THREADSIZE__MASK;
+}
+#define A6XX_SP_HS_CTRL_REG0_VARYING				0x00400000
+#define A6XX_SP_HS_CTRL_REG0_PIXLODENABLE			0x04000000
+#define A6XX_SP_HS_CTRL_REG0_MERGEDREGS				0x80000000
+
+#define REG_A6XX_SP_HS_UNKNOWN_A831				0x0000a831
+
+#define REG_A6XX_SP_HS_OBJ_START_LO				0x0000a834
+
+#define REG_A6XX_SP_HS_OBJ_START_HI				0x0000a835
+
+#define REG_A6XX_SP_HS_TEX_COUNT				0x0000a83a
+
+#define REG_A6XX_SP_HS_CONFIG					0x0000a83b
+#define A6XX_SP_HS_CONFIG_ENABLED				0x00000100
+#define A6XX_SP_HS_CONFIG_NTEX__MASK				0x0001fe00
+#define A6XX_SP_HS_CONFIG_NTEX__SHIFT				9
+static inline uint32_t A6XX_SP_HS_CONFIG_NTEX(uint32_t val)
+{
+	return ((val) << A6XX_SP_HS_CONFIG_NTEX__SHIFT) & A6XX_SP_HS_CONFIG_NTEX__MASK;
+}
+#define A6XX_SP_HS_CONFIG_NSAMP__MASK				0x01fe0000
+#define A6XX_SP_HS_CONFIG_NSAMP__SHIFT				17
+static inline uint32_t A6XX_SP_HS_CONFIG_NSAMP(uint32_t val)
+{
+	return ((val) << A6XX_SP_HS_CONFIG_NSAMP__SHIFT) & A6XX_SP_HS_CONFIG_NSAMP__MASK;
+}
+
+#define REG_A6XX_SP_HS_INSTRLEN					0x0000a83c
+
+#define REG_A6XX_SP_DS_CTRL_REG0				0x0000a840
+#define A6XX_SP_DS_CTRL_REG0_HALFREGFOOTPRINT__MASK		0x0000007e
+#define A6XX_SP_DS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT		1
+static inline uint32_t A6XX_SP_DS_CTRL_REG0_HALFREGFOOTPRINT(uint32_t val)
+{
+	return ((val) << A6XX_SP_DS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT) & A6XX_SP_DS_CTRL_REG0_HALFREGFOOTPRINT__MASK;
+}
+#define A6XX_SP_DS_CTRL_REG0_FULLREGFOOTPRINT__MASK		0x00001f80
+#define A6XX_SP_DS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT		7
+static inline uint32_t A6XX_SP_DS_CTRL_REG0_FULLREGFOOTPRINT(uint32_t val)
+{
+	return ((val) << A6XX_SP_DS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT) & A6XX_SP_DS_CTRL_REG0_FULLREGFOOTPRINT__MASK;
+}
+#define A6XX_SP_DS_CTRL_REG0_BRANCHSTACK__MASK			0x000fc000
+#define A6XX_SP_DS_CTRL_REG0_BRANCHSTACK__SHIFT			14
+static inline uint32_t A6XX_SP_DS_CTRL_REG0_BRANCHSTACK(uint32_t val)
+{
+	return ((val) << A6XX_SP_DS_CTRL_REG0_BRANCHSTACK__SHIFT) & A6XX_SP_DS_CTRL_REG0_BRANCHSTACK__MASK;
+}
+#define A6XX_SP_DS_CTRL_REG0_THREADSIZE__MASK			0x00100000
+#define A6XX_SP_DS_CTRL_REG0_THREADSIZE__SHIFT			20
+static inline uint32_t A6XX_SP_DS_CTRL_REG0_THREADSIZE(enum a3xx_threadsize val)
+{
+	return ((val) << A6XX_SP_DS_CTRL_REG0_THREADSIZE__SHIFT) & A6XX_SP_DS_CTRL_REG0_THREADSIZE__MASK;
+}
+#define A6XX_SP_DS_CTRL_REG0_VARYING				0x00400000
+#define A6XX_SP_DS_CTRL_REG0_PIXLODENABLE			0x04000000
+#define A6XX_SP_DS_CTRL_REG0_MERGEDREGS				0x80000000
+
+#define REG_A6XX_SP_DS_OBJ_START_LO				0x0000a85c
+
+#define REG_A6XX_SP_DS_OBJ_START_HI				0x0000a85d
+
+#define REG_A6XX_SP_DS_TEX_COUNT				0x0000a862
+
+#define REG_A6XX_SP_DS_CONFIG					0x0000a863
+#define A6XX_SP_DS_CONFIG_ENABLED				0x00000100
+#define A6XX_SP_DS_CONFIG_NTEX__MASK				0x0001fe00
+#define A6XX_SP_DS_CONFIG_NTEX__SHIFT				9
+static inline uint32_t A6XX_SP_DS_CONFIG_NTEX(uint32_t val)
+{
+	return ((val) << A6XX_SP_DS_CONFIG_NTEX__SHIFT) & A6XX_SP_DS_CONFIG_NTEX__MASK;
+}
+#define A6XX_SP_DS_CONFIG_NSAMP__MASK				0x01fe0000
+#define A6XX_SP_DS_CONFIG_NSAMP__SHIFT				17
+static inline uint32_t A6XX_SP_DS_CONFIG_NSAMP(uint32_t val)
+{
+	return ((val) << A6XX_SP_DS_CONFIG_NSAMP__SHIFT) & A6XX_SP_DS_CONFIG_NSAMP__MASK;
+}
+
+#define REG_A6XX_SP_DS_INSTRLEN					0x0000a864
+
+#define REG_A6XX_SP_GS_CTRL_REG0				0x0000a870
+#define A6XX_SP_GS_CTRL_REG0_HALFREGFOOTPRINT__MASK		0x0000007e
+#define A6XX_SP_GS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT		1
+static inline uint32_t A6XX_SP_GS_CTRL_REG0_HALFREGFOOTPRINT(uint32_t val)
+{
+	return ((val) << A6XX_SP_GS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT) & A6XX_SP_GS_CTRL_REG0_HALFREGFOOTPRINT__MASK;
+}
+#define A6XX_SP_GS_CTRL_REG0_FULLREGFOOTPRINT__MASK		0x00001f80
+#define A6XX_SP_GS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT		7
+static inline uint32_t A6XX_SP_GS_CTRL_REG0_FULLREGFOOTPRINT(uint32_t val)
+{
+	return ((val) << A6XX_SP_GS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT) & A6XX_SP_GS_CTRL_REG0_FULLREGFOOTPRINT__MASK;
+}
+#define A6XX_SP_GS_CTRL_REG0_BRANCHSTACK__MASK			0x000fc000
+#define A6XX_SP_GS_CTRL_REG0_BRANCHSTACK__SHIFT			14
+static inline uint32_t A6XX_SP_GS_CTRL_REG0_BRANCHSTACK(uint32_t val)
+{
+	return ((val) << A6XX_SP_GS_CTRL_REG0_BRANCHSTACK__SHIFT) & A6XX_SP_GS_CTRL_REG0_BRANCHSTACK__MASK;
+}
+#define A6XX_SP_GS_CTRL_REG0_THREADSIZE__MASK			0x00100000
+#define A6XX_SP_GS_CTRL_REG0_THREADSIZE__SHIFT			20
+static inline uint32_t A6XX_SP_GS_CTRL_REG0_THREADSIZE(enum a3xx_threadsize val)
+{
+	return ((val) << A6XX_SP_GS_CTRL_REG0_THREADSIZE__SHIFT) & A6XX_SP_GS_CTRL_REG0_THREADSIZE__MASK;
+}
+#define A6XX_SP_GS_CTRL_REG0_VARYING				0x00400000
+#define A6XX_SP_GS_CTRL_REG0_PIXLODENABLE			0x04000000
+#define A6XX_SP_GS_CTRL_REG0_MERGEDREGS				0x80000000
+
+#define REG_A6XX_SP_GS_UNKNOWN_A871				0x0000a871
+
+#define REG_A6XX_SP_GS_OBJ_START_LO				0x0000a88d
+
+#define REG_A6XX_SP_GS_OBJ_START_HI				0x0000a88e
+
+#define REG_A6XX_SP_GS_TEX_COUNT				0x0000a893
+
+#define REG_A6XX_SP_GS_CONFIG					0x0000a894
+#define A6XX_SP_GS_CONFIG_ENABLED				0x00000100
+#define A6XX_SP_GS_CONFIG_NTEX__MASK				0x0001fe00
+#define A6XX_SP_GS_CONFIG_NTEX__SHIFT				9
+static inline uint32_t A6XX_SP_GS_CONFIG_NTEX(uint32_t val)
+{
+	return ((val) << A6XX_SP_GS_CONFIG_NTEX__SHIFT) & A6XX_SP_GS_CONFIG_NTEX__MASK;
+}
+#define A6XX_SP_GS_CONFIG_NSAMP__MASK				0x01fe0000
+#define A6XX_SP_GS_CONFIG_NSAMP__SHIFT				17
+static inline uint32_t A6XX_SP_GS_CONFIG_NSAMP(uint32_t val)
+{
+	return ((val) << A6XX_SP_GS_CONFIG_NSAMP__SHIFT) & A6XX_SP_GS_CONFIG_NSAMP__MASK;
+}
+
+#define REG_A6XX_SP_GS_INSTRLEN					0x0000a895
+
+#define REG_A6XX_SP_VS_TEX_SAMP_LO				0x0000a8a0
+
+#define REG_A6XX_SP_VS_TEX_SAMP_HI				0x0000a8a1
+
+#define REG_A6XX_SP_HS_TEX_SAMP_LO				0x0000a8a2
+
+#define REG_A6XX_SP_HS_TEX_SAMP_HI				0x0000a8a3
+
+#define REG_A6XX_SP_DS_TEX_SAMP_LO				0x0000a8a4
+
+#define REG_A6XX_SP_DS_TEX_SAMP_HI				0x0000a8a5
+
+#define REG_A6XX_SP_GS_TEX_SAMP_LO				0x0000a8a6
+
+#define REG_A6XX_SP_GS_TEX_SAMP_HI				0x0000a8a7
+
+#define REG_A6XX_SP_VS_TEX_CONST_LO				0x0000a8a8
+
+#define REG_A6XX_SP_VS_TEX_CONST_HI				0x0000a8a9
+
+#define REG_A6XX_SP_HS_TEX_CONST_LO				0x0000a8aa
+
+#define REG_A6XX_SP_HS_TEX_CONST_HI				0x0000a8ab
+
+#define REG_A6XX_SP_DS_TEX_CONST_LO				0x0000a8ac
+
+#define REG_A6XX_SP_DS_TEX_CONST_HI				0x0000a8ad
+
+#define REG_A6XX_SP_GS_TEX_CONST_LO				0x0000a8ae
+
+#define REG_A6XX_SP_GS_TEX_CONST_HI				0x0000a8af
+
+#define REG_A6XX_SP_FS_CTRL_REG0				0x0000a980
+#define A6XX_SP_FS_CTRL_REG0_HALFREGFOOTPRINT__MASK		0x0000007e
+#define A6XX_SP_FS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT		1
+static inline uint32_t A6XX_SP_FS_CTRL_REG0_HALFREGFOOTPRINT(uint32_t val)
+{
+	return ((val) << A6XX_SP_FS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT) & A6XX_SP_FS_CTRL_REG0_HALFREGFOOTPRINT__MASK;
+}
+#define A6XX_SP_FS_CTRL_REG0_FULLREGFOOTPRINT__MASK		0x00001f80
+#define A6XX_SP_FS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT		7
+static inline uint32_t A6XX_SP_FS_CTRL_REG0_FULLREGFOOTPRINT(uint32_t val)
+{
+	return ((val) << A6XX_SP_FS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT) & A6XX_SP_FS_CTRL_REG0_FULLREGFOOTPRINT__MASK;
+}
+#define A6XX_SP_FS_CTRL_REG0_BRANCHSTACK__MASK			0x000fc000
+#define A6XX_SP_FS_CTRL_REG0_BRANCHSTACK__SHIFT			14
+static inline uint32_t A6XX_SP_FS_CTRL_REG0_BRANCHSTACK(uint32_t val)
+{
+	return ((val) << A6XX_SP_FS_CTRL_REG0_BRANCHSTACK__SHIFT) & A6XX_SP_FS_CTRL_REG0_BRANCHSTACK__MASK;
+}
+#define A6XX_SP_FS_CTRL_REG0_THREADSIZE__MASK			0x00100000
+#define A6XX_SP_FS_CTRL_REG0_THREADSIZE__SHIFT			20
+static inline uint32_t A6XX_SP_FS_CTRL_REG0_THREADSIZE(enum a3xx_threadsize val)
+{
+	return ((val) << A6XX_SP_FS_CTRL_REG0_THREADSIZE__SHIFT) & A6XX_SP_FS_CTRL_REG0_THREADSIZE__MASK;
+}
+#define A6XX_SP_FS_CTRL_REG0_VARYING				0x00400000
+#define A6XX_SP_FS_CTRL_REG0_PIXLODENABLE			0x04000000
+#define A6XX_SP_FS_CTRL_REG0_MERGEDREGS				0x80000000
+
+#define REG_A6XX_SP_FS_OBJ_START_LO				0x0000a983
+
+#define REG_A6XX_SP_FS_OBJ_START_HI				0x0000a984
+
+#define REG_A6XX_SP_BLEND_CNTL					0x0000a989
+#define A6XX_SP_BLEND_CNTL_ENABLED				0x00000001
+#define A6XX_SP_BLEND_CNTL_UNK8					0x00000100
+
+#define REG_A6XX_SP_SRGB_CNTL					0x0000a98a
+#define A6XX_SP_SRGB_CNTL_SRGB_MRT0				0x00000001
+#define A6XX_SP_SRGB_CNTL_SRGB_MRT1				0x00000002
+#define A6XX_SP_SRGB_CNTL_SRGB_MRT2				0x00000004
+#define A6XX_SP_SRGB_CNTL_SRGB_MRT3				0x00000008
+#define A6XX_SP_SRGB_CNTL_SRGB_MRT4				0x00000010
+#define A6XX_SP_SRGB_CNTL_SRGB_MRT5				0x00000020
+#define A6XX_SP_SRGB_CNTL_SRGB_MRT6				0x00000040
+#define A6XX_SP_SRGB_CNTL_SRGB_MRT7				0x00000080
+
+#define REG_A6XX_SP_FS_RENDER_COMPONENTS			0x0000a98b
+#define A6XX_SP_FS_RENDER_COMPONENTS_RT0__MASK			0x0000000f
+#define A6XX_SP_FS_RENDER_COMPONENTS_RT0__SHIFT			0
+static inline uint32_t A6XX_SP_FS_RENDER_COMPONENTS_RT0(uint32_t val)
+{
+	return ((val) << A6XX_SP_FS_RENDER_COMPONENTS_RT0__SHIFT) & A6XX_SP_FS_RENDER_COMPONENTS_RT0__MASK;
+}
+#define A6XX_SP_FS_RENDER_COMPONENTS_RT1__MASK			0x000000f0
+#define A6XX_SP_FS_RENDER_COMPONENTS_RT1__SHIFT			4
+static inline uint32_t A6XX_SP_FS_RENDER_COMPONENTS_RT1(uint32_t val)
+{
+	return ((val) << A6XX_SP_FS_RENDER_COMPONENTS_RT1__SHIFT) & A6XX_SP_FS_RENDER_COMPONENTS_RT1__MASK;
+}
+#define A6XX_SP_FS_RENDER_COMPONENTS_RT2__MASK			0x00000f00
+#define A6XX_SP_FS_RENDER_COMPONENTS_RT2__SHIFT			8
+static inline uint32_t A6XX_SP_FS_RENDER_COMPONENTS_RT2(uint32_t val)
+{
+	return ((val) << A6XX_SP_FS_RENDER_COMPONENTS_RT2__SHIFT) & A6XX_SP_FS_RENDER_COMPONENTS_RT2__MASK;
+}
+#define A6XX_SP_FS_RENDER_COMPONENTS_RT3__MASK			0x0000f000
+#define A6XX_SP_FS_RENDER_COMPONENTS_RT3__SHIFT			12
+static inline uint32_t A6XX_SP_FS_RENDER_COMPONENTS_RT3(uint32_t val)
+{
+	return ((val) << A6XX_SP_FS_RENDER_COMPONENTS_RT3__SHIFT) & A6XX_SP_FS_RENDER_COMPONENTS_RT3__MASK;
+}
+#define A6XX_SP_FS_RENDER_COMPONENTS_RT4__MASK			0x000f0000
+#define A6XX_SP_FS_RENDER_COMPONENTS_RT4__SHIFT			16
+static inline uint32_t A6XX_SP_FS_RENDER_COMPONENTS_RT4(uint32_t val)
+{
+	return ((val) << A6XX_SP_FS_RENDER_COMPONENTS_RT4__SHIFT) & A6XX_SP_FS_RENDER_COMPONENTS_RT4__MASK;
+}
+#define A6XX_SP_FS_RENDER_COMPONENTS_RT5__MASK			0x00f00000
+#define A6XX_SP_FS_RENDER_COMPONENTS_RT5__SHIFT			20
+static inline uint32_t A6XX_SP_FS_RENDER_COMPONENTS_RT5(uint32_t val)
+{
+	return ((val) << A6XX_SP_FS_RENDER_COMPONENTS_RT5__SHIFT) & A6XX_SP_FS_RENDER_COMPONENTS_RT5__MASK;
+}
+#define A6XX_SP_FS_RENDER_COMPONENTS_RT6__MASK			0x0f000000
+#define A6XX_SP_FS_RENDER_COMPONENTS_RT6__SHIFT			24
+static inline uint32_t A6XX_SP_FS_RENDER_COMPONENTS_RT6(uint32_t val)
+{
+	return ((val) << A6XX_SP_FS_RENDER_COMPONENTS_RT6__SHIFT) & A6XX_SP_FS_RENDER_COMPONENTS_RT6__MASK;
+}
+#define A6XX_SP_FS_RENDER_COMPONENTS_RT7__MASK			0xf0000000
+#define A6XX_SP_FS_RENDER_COMPONENTS_RT7__SHIFT			28
+static inline uint32_t A6XX_SP_FS_RENDER_COMPONENTS_RT7(uint32_t val)
+{
+	return ((val) << A6XX_SP_FS_RENDER_COMPONENTS_RT7__SHIFT) & A6XX_SP_FS_RENDER_COMPONENTS_RT7__MASK;
+}
+
+#define REG_A6XX_SP_FS_OUTPUT_CNTL0				0x0000a98c
+#define A6XX_SP_FS_OUTPUT_CNTL0_DEPTH_REGID__MASK		0x0000ff00
+#define A6XX_SP_FS_OUTPUT_CNTL0_DEPTH_REGID__SHIFT		8
+static inline uint32_t A6XX_SP_FS_OUTPUT_CNTL0_DEPTH_REGID(uint32_t val)
+{
+	return ((val) << A6XX_SP_FS_OUTPUT_CNTL0_DEPTH_REGID__SHIFT) & A6XX_SP_FS_OUTPUT_CNTL0_DEPTH_REGID__MASK;
+}
+
+#define REG_A6XX_SP_FS_OUTPUT_CNTL1				0x0000a98d
+#define A6XX_SP_FS_OUTPUT_CNTL1_MRT__MASK			0x0000000f
+#define A6XX_SP_FS_OUTPUT_CNTL1_MRT__SHIFT			0
+static inline uint32_t A6XX_SP_FS_OUTPUT_CNTL1_MRT(uint32_t val)
+{
+	return ((val) << A6XX_SP_FS_OUTPUT_CNTL1_MRT__SHIFT) & A6XX_SP_FS_OUTPUT_CNTL1_MRT__MASK;
+}
+
+static inline uint32_t REG_A6XX_SP_FS_MRT(uint32_t i0) { return 0x0000a996 + 0x1*i0; }
+
+static inline uint32_t REG_A6XX_SP_FS_MRT_REG(uint32_t i0) { return 0x0000a996 + 0x1*i0; }
+#define A6XX_SP_FS_MRT_REG_COLOR_FORMAT__MASK			0x000000ff
+#define A6XX_SP_FS_MRT_REG_COLOR_FORMAT__SHIFT			0
+static inline uint32_t A6XX_SP_FS_MRT_REG_COLOR_FORMAT(enum a6xx_color_fmt val)
+{
+	return ((val) << A6XX_SP_FS_MRT_REG_COLOR_FORMAT__SHIFT) & A6XX_SP_FS_MRT_REG_COLOR_FORMAT__MASK;
+}
+#define A6XX_SP_FS_MRT_REG_COLOR_SINT				0x00000100
+#define A6XX_SP_FS_MRT_REG_COLOR_UINT				0x00000200
+#define A6XX_SP_FS_MRT_REG_COLOR_SRGB				0x00000400
+
+#define REG_A6XX_SP_FS_TEX_COUNT				0x0000a9a7
+
+#define REG_A6XX_SP_UNKNOWN_A9A8				0x0000a9a8
+
+#define REG_A6XX_SP_FS_TEX_SAMP_LO				0x0000a9e0
+
+#define REG_A6XX_SP_FS_TEX_SAMP_HI				0x0000a9e1
+
+#define REG_A6XX_SP_CS_TEX_SAMP_LO				0x0000a9e2
+
+#define REG_A6XX_SP_CS_TEX_SAMP_HI				0x0000a9e3
+
+#define REG_A6XX_SP_FS_TEX_CONST_LO				0x0000a9e4
+
+#define REG_A6XX_SP_FS_TEX_CONST_HI				0x0000a9e5
+
+#define REG_A6XX_SP_CS_TEX_CONST_LO				0x0000a9e6
+
+#define REG_A6XX_SP_CS_TEX_CONST_HI				0x0000a9e7
+
+static inline uint32_t REG_A6XX_SP_FS_OUTPUT(uint32_t i0) { return 0x0000a98e + 0x1*i0; }
+
+static inline uint32_t REG_A6XX_SP_FS_OUTPUT_REG(uint32_t i0) { return 0x0000a98e + 0x1*i0; }
+#define A6XX_SP_FS_OUTPUT_REG_REGID__MASK			0x000000ff
+#define A6XX_SP_FS_OUTPUT_REG_REGID__SHIFT			0
+static inline uint32_t A6XX_SP_FS_OUTPUT_REG_REGID(uint32_t val)
+{
+	return ((val) << A6XX_SP_FS_OUTPUT_REG_REGID__SHIFT) & A6XX_SP_FS_OUTPUT_REG_REGID__MASK;
+}
+#define A6XX_SP_FS_OUTPUT_REG_HALF_PRECISION			0x00000100
+
+#define REG_A6XX_SP_CS_CTRL_REG0				0x0000a9b0
+#define A6XX_SP_CS_CTRL_REG0_HALFREGFOOTPRINT__MASK		0x0000007e
+#define A6XX_SP_CS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT		1
+static inline uint32_t A6XX_SP_CS_CTRL_REG0_HALFREGFOOTPRINT(uint32_t val)
+{
+	return ((val) << A6XX_SP_CS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT) & A6XX_SP_CS_CTRL_REG0_HALFREGFOOTPRINT__MASK;
+}
+#define A6XX_SP_CS_CTRL_REG0_FULLREGFOOTPRINT__MASK		0x00001f80
+#define A6XX_SP_CS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT		7
+static inline uint32_t A6XX_SP_CS_CTRL_REG0_FULLREGFOOTPRINT(uint32_t val)
+{
+	return ((val) << A6XX_SP_CS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT) & A6XX_SP_CS_CTRL_REG0_FULLREGFOOTPRINT__MASK;
+}
+#define A6XX_SP_CS_CTRL_REG0_BRANCHSTACK__MASK			0x000fc000
+#define A6XX_SP_CS_CTRL_REG0_BRANCHSTACK__SHIFT			14
+static inline uint32_t A6XX_SP_CS_CTRL_REG0_BRANCHSTACK(uint32_t val)
+{
+	return ((val) << A6XX_SP_CS_CTRL_REG0_BRANCHSTACK__SHIFT) & A6XX_SP_CS_CTRL_REG0_BRANCHSTACK__MASK;
+}
+#define A6XX_SP_CS_CTRL_REG0_THREADSIZE__MASK			0x00100000
+#define A6XX_SP_CS_CTRL_REG0_THREADSIZE__SHIFT			20
+static inline uint32_t A6XX_SP_CS_CTRL_REG0_THREADSIZE(enum a3xx_threadsize val)
+{
+	return ((val) << A6XX_SP_CS_CTRL_REG0_THREADSIZE__SHIFT) & A6XX_SP_CS_CTRL_REG0_THREADSIZE__MASK;
+}
+#define A6XX_SP_CS_CTRL_REG0_VARYING				0x00400000
+#define A6XX_SP_CS_CTRL_REG0_PIXLODENABLE			0x04000000
+#define A6XX_SP_CS_CTRL_REG0_MERGEDREGS				0x80000000
+
+#define REG_A6XX_SP_CS_OBJ_START_LO				0x0000a9b4
+
+#define REG_A6XX_SP_CS_OBJ_START_HI				0x0000a9b5
+
+#define REG_A6XX_SP_CS_INSTRLEN					0x0000a9bc
+
+#define REG_A6XX_SP_UNKNOWN_AB00				0x0000ab00
+
+#define REG_A6XX_SP_FS_CONFIG					0x0000ab04
+#define A6XX_SP_FS_CONFIG_ENABLED				0x00000100
+#define A6XX_SP_FS_CONFIG_NTEX__MASK				0x0001fe00
+#define A6XX_SP_FS_CONFIG_NTEX__SHIFT				9
+static inline uint32_t A6XX_SP_FS_CONFIG_NTEX(uint32_t val)
+{
+	return ((val) << A6XX_SP_FS_CONFIG_NTEX__SHIFT) & A6XX_SP_FS_CONFIG_NTEX__MASK;
+}
+#define A6XX_SP_FS_CONFIG_NSAMP__MASK				0x01fe0000
+#define A6XX_SP_FS_CONFIG_NSAMP__SHIFT				17
+static inline uint32_t A6XX_SP_FS_CONFIG_NSAMP(uint32_t val)
+{
+	return ((val) << A6XX_SP_FS_CONFIG_NSAMP__SHIFT) & A6XX_SP_FS_CONFIG_NSAMP__MASK;
+}
+
+#define REG_A6XX_SP_FS_INSTRLEN					0x0000ab05
+
+#define REG_A6XX_SP_UNKNOWN_AE00				0x0000ae00
+
+#define REG_A6XX_SP_UNKNOWN_AE04				0x0000ae04
+
+#define REG_A6XX_SP_UNKNOWN_AE0F				0x0000ae0f
+
+#define REG_A6XX_SP_UNKNOWN_B182				0x0000b182
+
+#define REG_A6XX_SP_TP_RAS_MSAA_CNTL				0x0000b300
+#define A6XX_SP_TP_RAS_MSAA_CNTL_SAMPLES__MASK			0x00000003
+#define A6XX_SP_TP_RAS_MSAA_CNTL_SAMPLES__SHIFT			0
+static inline uint32_t A6XX_SP_TP_RAS_MSAA_CNTL_SAMPLES(enum a3xx_msaa_samples val)
+{
+	return ((val) << A6XX_SP_TP_RAS_MSAA_CNTL_SAMPLES__SHIFT) & A6XX_SP_TP_RAS_MSAA_CNTL_SAMPLES__MASK;
+}
+
+#define REG_A6XX_SP_TP_DEST_MSAA_CNTL				0x0000b301
+#define A6XX_SP_TP_DEST_MSAA_CNTL_SAMPLES__MASK			0x00000003
+#define A6XX_SP_TP_DEST_MSAA_CNTL_SAMPLES__SHIFT		0
+static inline uint32_t A6XX_SP_TP_DEST_MSAA_CNTL_SAMPLES(enum a3xx_msaa_samples val)
+{
+	return ((val) << A6XX_SP_TP_DEST_MSAA_CNTL_SAMPLES__SHIFT) & A6XX_SP_TP_DEST_MSAA_CNTL_SAMPLES__MASK;
+}
+#define A6XX_SP_TP_DEST_MSAA_CNTL_MSAA_DISABLE			0x00000004
+
+#define REG_A6XX_SP_TP_BORDER_COLOR_BASE_ADDR_LO		0x0000b302
+
+#define REG_A6XX_SP_TP_BORDER_COLOR_BASE_ADDR_HI		0x0000b303
+
+#define REG_A6XX_SP_TP_UNKNOWN_B304				0x0000b304
+
+#define REG_A6XX_SP_PS_2D_SRC_INFO				0x0000b4c0
+#define A6XX_SP_PS_2D_SRC_INFO_COLOR_FORMAT__MASK		0x000000ff
+#define A6XX_SP_PS_2D_SRC_INFO_COLOR_FORMAT__SHIFT		0
+static inline uint32_t A6XX_SP_PS_2D_SRC_INFO_COLOR_FORMAT(enum a6xx_color_fmt val)
+{
+	return ((val) << A6XX_SP_PS_2D_SRC_INFO_COLOR_FORMAT__SHIFT) & A6XX_SP_PS_2D_SRC_INFO_COLOR_FORMAT__MASK;
+}
+#define A6XX_SP_PS_2D_SRC_INFO_TILE_MODE__MASK			0x00000300
+#define A6XX_SP_PS_2D_SRC_INFO_TILE_MODE__SHIFT			8
+static inline uint32_t A6XX_SP_PS_2D_SRC_INFO_TILE_MODE(enum a6xx_tile_mode val)
+{
+	return ((val) << A6XX_SP_PS_2D_SRC_INFO_TILE_MODE__SHIFT) & A6XX_SP_PS_2D_SRC_INFO_TILE_MODE__MASK;
+}
+#define A6XX_SP_PS_2D_SRC_INFO_COLOR_SWAP__MASK			0x00000c00
+#define A6XX_SP_PS_2D_SRC_INFO_COLOR_SWAP__SHIFT		10
+static inline uint32_t A6XX_SP_PS_2D_SRC_INFO_COLOR_SWAP(enum a3xx_color_swap val)
+{
+	return ((val) << A6XX_SP_PS_2D_SRC_INFO_COLOR_SWAP__SHIFT) & A6XX_SP_PS_2D_SRC_INFO_COLOR_SWAP__MASK;
+}
+#define A6XX_SP_PS_2D_SRC_INFO_FLAGS				0x00001000
+
+#define REG_A6XX_SP_PS_2D_SRC_LO				0x0000b4c2
+
+#define REG_A6XX_SP_PS_2D_SRC_HI				0x0000b4c3
+
+#define REG_A6XX_SP_PS_2D_SRC_FLAGS_LO				0x0000b4ca
+
+#define REG_A6XX_SP_PS_2D_SRC_FLAGS_HI				0x0000b4cb
+
+#define REG_A6XX_SP_UNKNOWN_B600				0x0000b600
+
+#define REG_A6XX_SP_UNKNOWN_B605				0x0000b605
+
+#define REG_A6XX_HLSQ_VS_CNTL					0x0000b800
+#define A6XX_HLSQ_VS_CNTL_CONSTLEN__MASK			0x000000ff
+#define A6XX_HLSQ_VS_CNTL_CONSTLEN__SHIFT			0
+static inline uint32_t A6XX_HLSQ_VS_CNTL_CONSTLEN(uint32_t val)
+{
+	return ((val >> 2) << A6XX_HLSQ_VS_CNTL_CONSTLEN__SHIFT) & A6XX_HLSQ_VS_CNTL_CONSTLEN__MASK;
+}
+
+#define REG_A6XX_HLSQ_HS_CNTL					0x0000b801
+#define A6XX_HLSQ_HS_CNTL_CONSTLEN__MASK			0x000000ff
+#define A6XX_HLSQ_HS_CNTL_CONSTLEN__SHIFT			0
+static inline uint32_t A6XX_HLSQ_HS_CNTL_CONSTLEN(uint32_t val)
+{
+	return ((val >> 2) << A6XX_HLSQ_HS_CNTL_CONSTLEN__SHIFT) & A6XX_HLSQ_HS_CNTL_CONSTLEN__MASK;
+}
+
+#define REG_A6XX_HLSQ_DS_CNTL					0x0000b802
+#define A6XX_HLSQ_DS_CNTL_CONSTLEN__MASK			0x000000ff
+#define A6XX_HLSQ_DS_CNTL_CONSTLEN__SHIFT			0
+static inline uint32_t A6XX_HLSQ_DS_CNTL_CONSTLEN(uint32_t val)
+{
+	return ((val >> 2) << A6XX_HLSQ_DS_CNTL_CONSTLEN__SHIFT) & A6XX_HLSQ_DS_CNTL_CONSTLEN__MASK;
+}
+
+#define REG_A6XX_HLSQ_GS_CNTL					0x0000b803
+#define A6XX_HLSQ_GS_CNTL_CONSTLEN__MASK			0x000000ff
+#define A6XX_HLSQ_GS_CNTL_CONSTLEN__SHIFT			0
+static inline uint32_t A6XX_HLSQ_GS_CNTL_CONSTLEN(uint32_t val)
+{
+	return ((val >> 2) << A6XX_HLSQ_GS_CNTL_CONSTLEN__SHIFT) & A6XX_HLSQ_GS_CNTL_CONSTLEN__MASK;
+}
+
+#define REG_A6XX_HLSQ_CONTROL_1_REG				0x0000b982
+
+#define REG_A6XX_HLSQ_CONTROL_2_REG				0x0000b983
+#define A6XX_HLSQ_CONTROL_2_REG_FACEREGID__MASK			0x000000ff
+#define A6XX_HLSQ_CONTROL_2_REG_FACEREGID__SHIFT		0
+static inline uint32_t A6XX_HLSQ_CONTROL_2_REG_FACEREGID(uint32_t val)
+{
+	return ((val) << A6XX_HLSQ_CONTROL_2_REG_FACEREGID__SHIFT) & A6XX_HLSQ_CONTROL_2_REG_FACEREGID__MASK;
+}
+#define A6XX_HLSQ_CONTROL_2_REG_SAMPLEID__MASK			0x0000ff00
+#define A6XX_HLSQ_CONTROL_2_REG_SAMPLEID__SHIFT			8
+static inline uint32_t A6XX_HLSQ_CONTROL_2_REG_SAMPLEID(uint32_t val)
+{
+	return ((val) << A6XX_HLSQ_CONTROL_2_REG_SAMPLEID__SHIFT) & A6XX_HLSQ_CONTROL_2_REG_SAMPLEID__MASK;
+}
+#define A6XX_HLSQ_CONTROL_2_REG_SAMPLEMASK__MASK		0x00ff0000
+#define A6XX_HLSQ_CONTROL_2_REG_SAMPLEMASK__SHIFT		16
+static inline uint32_t A6XX_HLSQ_CONTROL_2_REG_SAMPLEMASK(uint32_t val)
+{
+	return ((val) << A6XX_HLSQ_CONTROL_2_REG_SAMPLEMASK__SHIFT) & A6XX_HLSQ_CONTROL_2_REG_SAMPLEMASK__MASK;
+}
+
+#define REG_A6XX_HLSQ_CONTROL_3_REG				0x0000b984
+#define A6XX_HLSQ_CONTROL_3_REG_FRAGCOORDXYREGID__MASK		0x000000ff
+#define A6XX_HLSQ_CONTROL_3_REG_FRAGCOORDXYREGID__SHIFT		0
+static inline uint32_t A6XX_HLSQ_CONTROL_3_REG_FRAGCOORDXYREGID(uint32_t val)
+{
+	return ((val) << A6XX_HLSQ_CONTROL_3_REG_FRAGCOORDXYREGID__SHIFT) & A6XX_HLSQ_CONTROL_3_REG_FRAGCOORDXYREGID__MASK;
+}
+
+#define REG_A6XX_HLSQ_CONTROL_4_REG				0x0000b985
+#define A6XX_HLSQ_CONTROL_4_REG_XYCOORDREGID__MASK		0x00ff0000
+#define A6XX_HLSQ_CONTROL_4_REG_XYCOORDREGID__SHIFT		16
+static inline uint32_t A6XX_HLSQ_CONTROL_4_REG_XYCOORDREGID(uint32_t val)
+{
+	return ((val) << A6XX_HLSQ_CONTROL_4_REG_XYCOORDREGID__SHIFT) & A6XX_HLSQ_CONTROL_4_REG_XYCOORDREGID__MASK;
+}
+#define A6XX_HLSQ_CONTROL_4_REG_ZWCOORDREGID__MASK		0xff000000
+#define A6XX_HLSQ_CONTROL_4_REG_ZWCOORDREGID__SHIFT		24
+static inline uint32_t A6XX_HLSQ_CONTROL_4_REG_ZWCOORDREGID(uint32_t val)
+{
+	return ((val) << A6XX_HLSQ_CONTROL_4_REG_ZWCOORDREGID__SHIFT) & A6XX_HLSQ_CONTROL_4_REG_ZWCOORDREGID__MASK;
+}
+
+#define REG_A6XX_HLSQ_CONTROL_5_REG				0x0000b986
+
+#define REG_A6XX_HLSQ_CS_NDRANGE_0				0x0000b990
+#define A6XX_HLSQ_CS_NDRANGE_0_KERNELDIM__MASK			0x00000003
+#define A6XX_HLSQ_CS_NDRANGE_0_KERNELDIM__SHIFT			0
+static inline uint32_t A6XX_HLSQ_CS_NDRANGE_0_KERNELDIM(uint32_t val)
+{
+	return ((val) << A6XX_HLSQ_CS_NDRANGE_0_KERNELDIM__SHIFT) & A6XX_HLSQ_CS_NDRANGE_0_KERNELDIM__MASK;
+}
+#define A6XX_HLSQ_CS_NDRANGE_0_LOCALSIZEX__MASK			0x00000ffc
+#define A6XX_HLSQ_CS_NDRANGE_0_LOCALSIZEX__SHIFT		2
+static inline uint32_t A6XX_HLSQ_CS_NDRANGE_0_LOCALSIZEX(uint32_t val)
+{
+	return ((val) << A6XX_HLSQ_CS_NDRANGE_0_LOCALSIZEX__SHIFT) & A6XX_HLSQ_CS_NDRANGE_0_LOCALSIZEX__MASK;
+}
+#define A6XX_HLSQ_CS_NDRANGE_0_LOCALSIZEY__MASK			0x003ff000
+#define A6XX_HLSQ_CS_NDRANGE_0_LOCALSIZEY__SHIFT		12
+static inline uint32_t A6XX_HLSQ_CS_NDRANGE_0_LOCALSIZEY(uint32_t val)
+{
+	return ((val) << A6XX_HLSQ_CS_NDRANGE_0_LOCALSIZEY__SHIFT) & A6XX_HLSQ_CS_NDRANGE_0_LOCALSIZEY__MASK;
+}
+#define A6XX_HLSQ_CS_NDRANGE_0_LOCALSIZEZ__MASK			0xffc00000
+#define A6XX_HLSQ_CS_NDRANGE_0_LOCALSIZEZ__SHIFT		22
+static inline uint32_t A6XX_HLSQ_CS_NDRANGE_0_LOCALSIZEZ(uint32_t val)
+{
+	return ((val) << A6XX_HLSQ_CS_NDRANGE_0_LOCALSIZEZ__SHIFT) & A6XX_HLSQ_CS_NDRANGE_0_LOCALSIZEZ__MASK;
+}
+
+#define REG_A6XX_HLSQ_CS_NDRANGE_1				0x0000b991
+#define A6XX_HLSQ_CS_NDRANGE_1_GLOBALSIZE_X__MASK		0xffffffff
+#define A6XX_HLSQ_CS_NDRANGE_1_GLOBALSIZE_X__SHIFT		0
+static inline uint32_t A6XX_HLSQ_CS_NDRANGE_1_GLOBALSIZE_X(uint32_t val)
+{
+	return ((val) << A6XX_HLSQ_CS_NDRANGE_1_GLOBALSIZE_X__SHIFT) & A6XX_HLSQ_CS_NDRANGE_1_GLOBALSIZE_X__MASK;
+}
+
+#define REG_A6XX_HLSQ_CS_NDRANGE_2				0x0000b992
+#define A6XX_HLSQ_CS_NDRANGE_2_GLOBALOFF_X__MASK		0xffffffff
+#define A6XX_HLSQ_CS_NDRANGE_2_GLOBALOFF_X__SHIFT		0
+static inline uint32_t A6XX_HLSQ_CS_NDRANGE_2_GLOBALOFF_X(uint32_t val)
+{
+	return ((val) << A6XX_HLSQ_CS_NDRANGE_2_GLOBALOFF_X__SHIFT) & A6XX_HLSQ_CS_NDRANGE_2_GLOBALOFF_X__MASK;
+}
+
+#define REG_A6XX_HLSQ_CS_NDRANGE_3				0x0000b993
+#define A6XX_HLSQ_CS_NDRANGE_3_GLOBALSIZE_Y__MASK		0xffffffff
+#define A6XX_HLSQ_CS_NDRANGE_3_GLOBALSIZE_Y__SHIFT		0
+static inline uint32_t A6XX_HLSQ_CS_NDRANGE_3_GLOBALSIZE_Y(uint32_t val)
+{
+	return ((val) << A6XX_HLSQ_CS_NDRANGE_3_GLOBALSIZE_Y__SHIFT) & A6XX_HLSQ_CS_NDRANGE_3_GLOBALSIZE_Y__MASK;
+}
+
+#define REG_A6XX_HLSQ_CS_NDRANGE_4				0x0000b994
+#define A6XX_HLSQ_CS_NDRANGE_4_GLOBALOFF_Y__MASK		0xffffffff
+#define A6XX_HLSQ_CS_NDRANGE_4_GLOBALOFF_Y__SHIFT		0
+static inline uint32_t A6XX_HLSQ_CS_NDRANGE_4_GLOBALOFF_Y(uint32_t val)
+{
+	return ((val) << A6XX_HLSQ_CS_NDRANGE_4_GLOBALOFF_Y__SHIFT) & A6XX_HLSQ_CS_NDRANGE_4_GLOBALOFF_Y__MASK;
+}
+
+#define REG_A6XX_HLSQ_CS_NDRANGE_5				0x0000b995
+#define A6XX_HLSQ_CS_NDRANGE_5_GLOBALSIZE_Z__MASK		0xffffffff
+#define A6XX_HLSQ_CS_NDRANGE_5_GLOBALSIZE_Z__SHIFT		0
+static inline uint32_t A6XX_HLSQ_CS_NDRANGE_5_GLOBALSIZE_Z(uint32_t val)
+{
+	return ((val) << A6XX_HLSQ_CS_NDRANGE_5_GLOBALSIZE_Z__SHIFT) & A6XX_HLSQ_CS_NDRANGE_5_GLOBALSIZE_Z__MASK;
+}
+
+#define REG_A6XX_HLSQ_CS_NDRANGE_6				0x0000b996
+#define A6XX_HLSQ_CS_NDRANGE_6_GLOBALOFF_Z__MASK		0xffffffff
+#define A6XX_HLSQ_CS_NDRANGE_6_GLOBALOFF_Z__SHIFT		0
+static inline uint32_t A6XX_HLSQ_CS_NDRANGE_6_GLOBALOFF_Z(uint32_t val)
+{
+	return ((val) << A6XX_HLSQ_CS_NDRANGE_6_GLOBALOFF_Z__SHIFT) & A6XX_HLSQ_CS_NDRANGE_6_GLOBALOFF_Z__MASK;
+}
+
+#define REG_A6XX_HLSQ_CS_CNTL_0					0x0000b997
+#define A6XX_HLSQ_CS_CNTL_0_WGIDCONSTID__MASK			0x000000ff
+#define A6XX_HLSQ_CS_CNTL_0_WGIDCONSTID__SHIFT			0
+static inline uint32_t A6XX_HLSQ_CS_CNTL_0_WGIDCONSTID(uint32_t val)
+{
+	return ((val) << A6XX_HLSQ_CS_CNTL_0_WGIDCONSTID__SHIFT) & A6XX_HLSQ_CS_CNTL_0_WGIDCONSTID__MASK;
+}
+#define A6XX_HLSQ_CS_CNTL_0_UNK0__MASK				0x0000ff00
+#define A6XX_HLSQ_CS_CNTL_0_UNK0__SHIFT				8
+static inline uint32_t A6XX_HLSQ_CS_CNTL_0_UNK0(uint32_t val)
+{
+	return ((val) << A6XX_HLSQ_CS_CNTL_0_UNK0__SHIFT) & A6XX_HLSQ_CS_CNTL_0_UNK0__MASK;
+}
+#define A6XX_HLSQ_CS_CNTL_0_UNK1__MASK				0x00ff0000
+#define A6XX_HLSQ_CS_CNTL_0_UNK1__SHIFT				16
+static inline uint32_t A6XX_HLSQ_CS_CNTL_0_UNK1(uint32_t val)
+{
+	return ((val) << A6XX_HLSQ_CS_CNTL_0_UNK1__SHIFT) & A6XX_HLSQ_CS_CNTL_0_UNK1__MASK;
+}
+#define A6XX_HLSQ_CS_CNTL_0_LOCALIDREGID__MASK			0xff000000
+#define A6XX_HLSQ_CS_CNTL_0_LOCALIDREGID__SHIFT			24
+static inline uint32_t A6XX_HLSQ_CS_CNTL_0_LOCALIDREGID(uint32_t val)
+{
+	return ((val) << A6XX_HLSQ_CS_CNTL_0_LOCALIDREGID__SHIFT) & A6XX_HLSQ_CS_CNTL_0_LOCALIDREGID__MASK;
+}
+
+#define REG_A6XX_HLSQ_CS_KERNEL_GROUP_X				0x0000b999
+
+#define REG_A6XX_HLSQ_CS_KERNEL_GROUP_Y				0x0000b99a
+
+#define REG_A6XX_HLSQ_CS_KERNEL_GROUP_Z				0x0000b99b
+
+#define REG_A6XX_HLSQ_UPDATE_CNTL				0x0000bb08
+
+#define REG_A6XX_HLSQ_FS_CNTL					0x0000bb10
+#define A6XX_HLSQ_FS_CNTL_CONSTLEN__MASK			0x000000ff
+#define A6XX_HLSQ_FS_CNTL_CONSTLEN__SHIFT			0
+static inline uint32_t A6XX_HLSQ_FS_CNTL_CONSTLEN(uint32_t val)
+{
+	return ((val >> 2) << A6XX_HLSQ_FS_CNTL_CONSTLEN__SHIFT) & A6XX_HLSQ_FS_CNTL_CONSTLEN__MASK;
+}
+
+#define REG_A6XX_HLSQ_UNKNOWN_BB11				0x0000bb11
+
+#define REG_A6XX_HLSQ_UNKNOWN_BE00				0x0000be00
+
+#define REG_A6XX_HLSQ_UNKNOWN_BE01				0x0000be01
+
+#define REG_A6XX_HLSQ_UNKNOWN_BE04				0x0000be04
+
+#define REG_A6XX_TEX_SAMP_0					0x00000000
+#define A6XX_TEX_SAMP_0_MIPFILTER_LINEAR_NEAR			0x00000001
+#define A6XX_TEX_SAMP_0_XY_MAG__MASK				0x00000006
+#define A6XX_TEX_SAMP_0_XY_MAG__SHIFT				1
+static inline uint32_t A6XX_TEX_SAMP_0_XY_MAG(enum a6xx_tex_filter val)
+{
+	return ((val) << A6XX_TEX_SAMP_0_XY_MAG__SHIFT) & A6XX_TEX_SAMP_0_XY_MAG__MASK;
+}
+#define A6XX_TEX_SAMP_0_XY_MIN__MASK				0x00000018
+#define A6XX_TEX_SAMP_0_XY_MIN__SHIFT				3
+static inline uint32_t A6XX_TEX_SAMP_0_XY_MIN(enum a6xx_tex_filter val)
+{
+	return ((val) << A6XX_TEX_SAMP_0_XY_MIN__SHIFT) & A6XX_TEX_SAMP_0_XY_MIN__MASK;
+}
+#define A6XX_TEX_SAMP_0_WRAP_S__MASK				0x000000e0
+#define A6XX_TEX_SAMP_0_WRAP_S__SHIFT				5
+static inline uint32_t A6XX_TEX_SAMP_0_WRAP_S(enum a6xx_tex_clamp val)
+{
+	return ((val) << A6XX_TEX_SAMP_0_WRAP_S__SHIFT) & A6XX_TEX_SAMP_0_WRAP_S__MASK;
+}
+#define A6XX_TEX_SAMP_0_WRAP_T__MASK				0x00000700
+#define A6XX_TEX_SAMP_0_WRAP_T__SHIFT				8
+static inline uint32_t A6XX_TEX_SAMP_0_WRAP_T(enum a6xx_tex_clamp val)
+{
+	return ((val) << A6XX_TEX_SAMP_0_WRAP_T__SHIFT) & A6XX_TEX_SAMP_0_WRAP_T__MASK;
+}
+#define A6XX_TEX_SAMP_0_WRAP_R__MASK				0x00003800
+#define A6XX_TEX_SAMP_0_WRAP_R__SHIFT				11
+static inline uint32_t A6XX_TEX_SAMP_0_WRAP_R(enum a6xx_tex_clamp val)
+{
+	return ((val) << A6XX_TEX_SAMP_0_WRAP_R__SHIFT) & A6XX_TEX_SAMP_0_WRAP_R__MASK;
+}
+#define A6XX_TEX_SAMP_0_ANISO__MASK				0x0001c000
+#define A6XX_TEX_SAMP_0_ANISO__SHIFT				14
+static inline uint32_t A6XX_TEX_SAMP_0_ANISO(enum a6xx_tex_aniso val)
+{
+	return ((val) << A6XX_TEX_SAMP_0_ANISO__SHIFT) & A6XX_TEX_SAMP_0_ANISO__MASK;
+}
+#define A6XX_TEX_SAMP_0_LOD_BIAS__MASK				0xfff80000
+#define A6XX_TEX_SAMP_0_LOD_BIAS__SHIFT				19
+static inline uint32_t A6XX_TEX_SAMP_0_LOD_BIAS(float val)
+{
+	return ((((int32_t)(val * 256.0))) << A6XX_TEX_SAMP_0_LOD_BIAS__SHIFT) & A6XX_TEX_SAMP_0_LOD_BIAS__MASK;
+}
+
+#define REG_A6XX_TEX_SAMP_1					0x00000001
+#define A6XX_TEX_SAMP_1_COMPARE_FUNC__MASK			0x0000000e
+#define A6XX_TEX_SAMP_1_COMPARE_FUNC__SHIFT			1
+static inline uint32_t A6XX_TEX_SAMP_1_COMPARE_FUNC(enum adreno_compare_func val)
+{
+	return ((val) << A6XX_TEX_SAMP_1_COMPARE_FUNC__SHIFT) & A6XX_TEX_SAMP_1_COMPARE_FUNC__MASK;
+}
+#define A6XX_TEX_SAMP_1_CUBEMAPSEAMLESSFILTOFF			0x00000010
+#define A6XX_TEX_SAMP_1_UNNORM_COORDS				0x00000020
+#define A6XX_TEX_SAMP_1_MIPFILTER_LINEAR_FAR			0x00000040
+#define A6XX_TEX_SAMP_1_MAX_LOD__MASK				0x000fff00
+#define A6XX_TEX_SAMP_1_MAX_LOD__SHIFT				8
+static inline uint32_t A6XX_TEX_SAMP_1_MAX_LOD(float val)
+{
+	return ((((uint32_t)(val * 256.0))) << A6XX_TEX_SAMP_1_MAX_LOD__SHIFT) & A6XX_TEX_SAMP_1_MAX_LOD__MASK;
+}
+#define A6XX_TEX_SAMP_1_MIN_LOD__MASK				0xfff00000
+#define A6XX_TEX_SAMP_1_MIN_LOD__SHIFT				20
+static inline uint32_t A6XX_TEX_SAMP_1_MIN_LOD(float val)
+{
+	return ((((uint32_t)(val * 256.0))) << A6XX_TEX_SAMP_1_MIN_LOD__SHIFT) & A6XX_TEX_SAMP_1_MIN_LOD__MASK;
+}
+
+#define REG_A6XX_TEX_SAMP_2					0x00000002
+#define A6XX_TEX_SAMP_2_BCOLOR_OFFSET__MASK			0xfffffff0
+#define A6XX_TEX_SAMP_2_BCOLOR_OFFSET__SHIFT			4
+static inline uint32_t A6XX_TEX_SAMP_2_BCOLOR_OFFSET(uint32_t val)
+{
+	return ((val) << A6XX_TEX_SAMP_2_BCOLOR_OFFSET__SHIFT) & A6XX_TEX_SAMP_2_BCOLOR_OFFSET__MASK;
+}
+
+#define REG_A6XX_TEX_SAMP_3					0x00000003
+
+#define REG_A6XX_TEX_CONST_0					0x00000000
+#define A6XX_TEX_CONST_0_TILE_MODE__MASK			0x00000003
+#define A6XX_TEX_CONST_0_TILE_MODE__SHIFT			0
+static inline uint32_t A6XX_TEX_CONST_0_TILE_MODE(enum a6xx_tile_mode val)
+{
+	return ((val) << A6XX_TEX_CONST_0_TILE_MODE__SHIFT) & A6XX_TEX_CONST_0_TILE_MODE__MASK;
+}
+#define A6XX_TEX_CONST_0_SRGB					0x00000004
+#define A6XX_TEX_CONST_0_SWIZ_X__MASK				0x00000070
+#define A6XX_TEX_CONST_0_SWIZ_X__SHIFT				4
+static inline uint32_t A6XX_TEX_CONST_0_SWIZ_X(enum a6xx_tex_swiz val)
+{
+	return ((val) << A6XX_TEX_CONST_0_SWIZ_X__SHIFT) & A6XX_TEX_CONST_0_SWIZ_X__MASK;
+}
+#define A6XX_TEX_CONST_0_SWIZ_Y__MASK				0x00000380
+#define A6XX_TEX_CONST_0_SWIZ_Y__SHIFT				7
+static inline uint32_t A6XX_TEX_CONST_0_SWIZ_Y(enum a6xx_tex_swiz val)
+{
+	return ((val) << A6XX_TEX_CONST_0_SWIZ_Y__SHIFT) & A6XX_TEX_CONST_0_SWIZ_Y__MASK;
+}
+#define A6XX_TEX_CONST_0_SWIZ_Z__MASK				0x00001c00
+#define A6XX_TEX_CONST_0_SWIZ_Z__SHIFT				10
+static inline uint32_t A6XX_TEX_CONST_0_SWIZ_Z(enum a6xx_tex_swiz val)
+{
+	return ((val) << A6XX_TEX_CONST_0_SWIZ_Z__SHIFT) & A6XX_TEX_CONST_0_SWIZ_Z__MASK;
+}
+#define A6XX_TEX_CONST_0_SWIZ_W__MASK				0x0000e000
+#define A6XX_TEX_CONST_0_SWIZ_W__SHIFT				13
+static inline uint32_t A6XX_TEX_CONST_0_SWIZ_W(enum a6xx_tex_swiz val)
+{
+	return ((val) << A6XX_TEX_CONST_0_SWIZ_W__SHIFT) & A6XX_TEX_CONST_0_SWIZ_W__MASK;
+}
+#define A6XX_TEX_CONST_0_MIPLVLS__MASK				0x000f0000
+#define A6XX_TEX_CONST_0_MIPLVLS__SHIFT				16
+static inline uint32_t A6XX_TEX_CONST_0_MIPLVLS(uint32_t val)
+{
+	return ((val) << A6XX_TEX_CONST_0_MIPLVLS__SHIFT) & A6XX_TEX_CONST_0_MIPLVLS__MASK;
+}
+#define A6XX_TEX_CONST_0_FMT__MASK				0x3fc00000
+#define A6XX_TEX_CONST_0_FMT__SHIFT				22
+static inline uint32_t A6XX_TEX_CONST_0_FMT(enum a6xx_tex_fmt val)
+{
+	return ((val) << A6XX_TEX_CONST_0_FMT__SHIFT) & A6XX_TEX_CONST_0_FMT__MASK;
+}
+#define A6XX_TEX_CONST_0_SWAP__MASK				0xc0000000
+#define A6XX_TEX_CONST_0_SWAP__SHIFT				30
+static inline uint32_t A6XX_TEX_CONST_0_SWAP(enum a3xx_color_swap val)
+{
+	return ((val) << A6XX_TEX_CONST_0_SWAP__SHIFT) & A6XX_TEX_CONST_0_SWAP__MASK;
+}
+
+#define REG_A6XX_TEX_CONST_1					0x00000001
+#define A6XX_TEX_CONST_1_WIDTH__MASK				0x00007fff
+#define A6XX_TEX_CONST_1_WIDTH__SHIFT				0
+static inline uint32_t A6XX_TEX_CONST_1_WIDTH(uint32_t val)
+{
+	return ((val) << A6XX_TEX_CONST_1_WIDTH__SHIFT) & A6XX_TEX_CONST_1_WIDTH__MASK;
+}
+#define A6XX_TEX_CONST_1_HEIGHT__MASK				0x3fff8000
+#define A6XX_TEX_CONST_1_HEIGHT__SHIFT				15
+static inline uint32_t A6XX_TEX_CONST_1_HEIGHT(uint32_t val)
+{
+	return ((val) << A6XX_TEX_CONST_1_HEIGHT__SHIFT) & A6XX_TEX_CONST_1_HEIGHT__MASK;
+}
+
+#define REG_A6XX_TEX_CONST_2					0x00000002
+#define A6XX_TEX_CONST_2_FETCHSIZE__MASK			0x0000000f
+#define A6XX_TEX_CONST_2_FETCHSIZE__SHIFT			0
+static inline uint32_t A6XX_TEX_CONST_2_FETCHSIZE(enum a6xx_tex_fetchsize val)
+{
+	return ((val) << A6XX_TEX_CONST_2_FETCHSIZE__SHIFT) & A6XX_TEX_CONST_2_FETCHSIZE__MASK;
+}
+#define A6XX_TEX_CONST_2_PITCH__MASK				0x1fffff80
+#define A6XX_TEX_CONST_2_PITCH__SHIFT				7
+static inline uint32_t A6XX_TEX_CONST_2_PITCH(uint32_t val)
+{
+	return ((val) << A6XX_TEX_CONST_2_PITCH__SHIFT) & A6XX_TEX_CONST_2_PITCH__MASK;
+}
+#define A6XX_TEX_CONST_2_TYPE__MASK				0x60000000
+#define A6XX_TEX_CONST_2_TYPE__SHIFT				29
+static inline uint32_t A6XX_TEX_CONST_2_TYPE(enum a6xx_tex_type val)
+{
+	return ((val) << A6XX_TEX_CONST_2_TYPE__SHIFT) & A6XX_TEX_CONST_2_TYPE__MASK;
+}
+
+#define REG_A6XX_TEX_CONST_3					0x00000003
+#define A6XX_TEX_CONST_3_ARRAY_PITCH__MASK			0x00003fff
+#define A6XX_TEX_CONST_3_ARRAY_PITCH__SHIFT			0
+static inline uint32_t A6XX_TEX_CONST_3_ARRAY_PITCH(uint32_t val)
+{
+	return ((val >> 12) << A6XX_TEX_CONST_3_ARRAY_PITCH__SHIFT) & A6XX_TEX_CONST_3_ARRAY_PITCH__MASK;
+}
+#define A6XX_TEX_CONST_3_FLAG					0x10000000
+
+#define REG_A6XX_TEX_CONST_4					0x00000004
+#define A6XX_TEX_CONST_4_BASE_LO__MASK				0xffffffe0
+#define A6XX_TEX_CONST_4_BASE_LO__SHIFT				5
+static inline uint32_t A6XX_TEX_CONST_4_BASE_LO(uint32_t val)
+{
+	return ((val >> 5) << A6XX_TEX_CONST_4_BASE_LO__SHIFT) & A6XX_TEX_CONST_4_BASE_LO__MASK;
+}
+
+#define REG_A6XX_TEX_CONST_5					0x00000005
+#define A6XX_TEX_CONST_5_BASE_HI__MASK				0x0001ffff
+#define A6XX_TEX_CONST_5_BASE_HI__SHIFT				0
+static inline uint32_t A6XX_TEX_CONST_5_BASE_HI(uint32_t val)
+{
+	return ((val) << A6XX_TEX_CONST_5_BASE_HI__SHIFT) & A6XX_TEX_CONST_5_BASE_HI__MASK;
+}
+#define A6XX_TEX_CONST_5_DEPTH__MASK				0x3ffe0000
+#define A6XX_TEX_CONST_5_DEPTH__SHIFT				17
+static inline uint32_t A6XX_TEX_CONST_5_DEPTH(uint32_t val)
+{
+	return ((val) << A6XX_TEX_CONST_5_DEPTH__SHIFT) & A6XX_TEX_CONST_5_DEPTH__MASK;
+}
+
+#define REG_A6XX_TEX_CONST_6					0x00000006
+
+#define REG_A6XX_TEX_CONST_7					0x00000007
+#define A6XX_TEX_CONST_7_FLAG_LO__MASK				0xffffffe0
+#define A6XX_TEX_CONST_7_FLAG_LO__SHIFT				5
+static inline uint32_t A6XX_TEX_CONST_7_FLAG_LO(uint32_t val)
+{
+	return ((val >> 5) << A6XX_TEX_CONST_7_FLAG_LO__SHIFT) & A6XX_TEX_CONST_7_FLAG_LO__MASK;
+}
+
+#define REG_A6XX_TEX_CONST_8					0x00000008
+#define A6XX_TEX_CONST_8_BASE_HI__MASK				0x0001ffff
+#define A6XX_TEX_CONST_8_BASE_HI__SHIFT				0
+static inline uint32_t A6XX_TEX_CONST_8_BASE_HI(uint32_t val)
+{
+	return ((val) << A6XX_TEX_CONST_8_BASE_HI__SHIFT) & A6XX_TEX_CONST_8_BASE_HI__MASK;
+}
+
+#define REG_A6XX_TEX_CONST_9					0x00000009
+
+#define REG_A6XX_TEX_CONST_10					0x0000000a
+
+#define REG_A6XX_TEX_CONST_11					0x0000000b
+
+#define REG_A6XX_TEX_CONST_12					0x0000000c
+
+#define REG_A6XX_TEX_CONST_13					0x0000000d
+
+#define REG_A6XX_TEX_CONST_14					0x0000000e
+
+#define REG_A6XX_TEX_CONST_15					0x0000000f
+
+
+#endif /* A6XX_XML */
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c
new file mode 100644
index 000000000000..bbb8126ec5c5
--- /dev/null
+++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c
@@ -0,0 +1,1207 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2017-2018 The Linux Foundation. All rights reserved. */
+
+#include <linux/clk.h>
+#include <linux/iopoll.h>
+#include <linux/pm_opp.h>
+#include <soc/qcom/cmd-db.h>
+
+#include "a6xx_gpu.h"
+#include "a6xx_gmu.xml.h"
+
+static irqreturn_t a6xx_gmu_irq(int irq, void *data)
+{
+	struct a6xx_gmu *gmu = data;
+	u32 status;
+
+	status = gmu_read(gmu, REG_A6XX_GMU_AO_HOST_INTERRUPT_STATUS);
+	gmu_write(gmu, REG_A6XX_GMU_AO_HOST_INTERRUPT_CLR, status);
+
+	if (status & A6XX_GMU_AO_HOST_INTERRUPT_STATUS_WDOG_BITE) {
+		dev_err_ratelimited(gmu->dev, "GMU watchdog expired\n");
+
+		/* Temporary until we can recover safely */
+		BUG();
+	}
+
+	if (status &  A6XX_GMU_AO_HOST_INTERRUPT_STATUS_HOST_AHB_BUS_ERROR)
+		dev_err_ratelimited(gmu->dev, "GMU AHB bus error\n");
+
+	if (status & A6XX_GMU_AO_HOST_INTERRUPT_STATUS_FENCE_ERR)
+		dev_err_ratelimited(gmu->dev, "GMU fence error: 0x%x\n",
+			gmu_read(gmu, REG_A6XX_GMU_AHB_FENCE_STATUS));
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t a6xx_hfi_irq(int irq, void *data)
+{
+	struct a6xx_gmu *gmu = data;
+	u32 status;
+
+	status = gmu_read(gmu, REG_A6XX_GMU_GMU2HOST_INTR_INFO);
+	gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_CLR, status);
+
+	if (status & A6XX_GMU_GMU2HOST_INTR_INFO_MSGQ)
+		tasklet_schedule(&gmu->hfi_tasklet);
+
+	if (status & A6XX_GMU_GMU2HOST_INTR_INFO_CM3_FAULT) {
+		dev_err_ratelimited(gmu->dev, "GMU firmware fault\n");
+
+		/* Temporary until we can recover safely */
+		BUG();
+	}
+
+	return IRQ_HANDLED;
+}
+
+/* Check to see if the GX rail is still powered */
+static bool a6xx_gmu_gx_is_on(struct a6xx_gmu *gmu)
+{
+	u32 val = gmu_read(gmu, REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS);
+
+	return !(val &
+		(A6XX_GMU_SPTPRAC_PWR_CLK_STATUS_GX_HM_GDSC_POWER_OFF |
+		A6XX_GMU_SPTPRAC_PWR_CLK_STATUS_GX_HM_CLK_OFF));
+}
+
+static int a6xx_gmu_set_freq(struct a6xx_gmu *gmu, int index)
+{
+	gmu_write(gmu, REG_A6XX_GMU_DCVS_ACK_OPTION, 0);
+
+	gmu_write(gmu, REG_A6XX_GMU_DCVS_PERF_SETTING,
+		((index << 24) & 0xff) | (3 & 0xf));
+
+	/*
+	 * Send an invalid index as a vote for the bus bandwidth and let the
+	 * firmware decide on the right vote
+	 */
+	gmu_write(gmu, REG_A6XX_GMU_DCVS_BW_SETTING, 0xff);
+
+	/* Set and clear the OOB for DCVS to trigger the GMU */
+	a6xx_gmu_set_oob(gmu, GMU_OOB_DCVS_SET);
+	a6xx_gmu_clear_oob(gmu, GMU_OOB_DCVS_SET);
+
+	return gmu_read(gmu, REG_A6XX_GMU_DCVS_RETURN);
+}
+
+static bool a6xx_gmu_check_idle_level(struct a6xx_gmu *gmu)
+{
+	u32 val;
+	int local = gmu->idle_level;
+
+	/* SPTP and IFPC both report as IFPC */
+	if (gmu->idle_level == GMU_IDLE_STATE_SPTP)
+		local = GMU_IDLE_STATE_IFPC;
+
+	val = gmu_read(gmu, REG_A6XX_GPU_GMU_CX_GMU_RPMH_POWER_STATE);
+
+	if (val == local) {
+		if (gmu->idle_level != GMU_IDLE_STATE_IFPC ||
+			!a6xx_gmu_gx_is_on(gmu))
+			return true;
+	}
+
+	return false;
+}
+
+/* Wait for the GMU to get to its most idle state */
+int a6xx_gmu_wait_for_idle(struct a6xx_gpu *a6xx_gpu)
+{
+	struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
+
+	return spin_until(a6xx_gmu_check_idle_level(gmu));
+}
+
+static int a6xx_gmu_start(struct a6xx_gmu *gmu)
+{
+	int ret;
+	u32 val;
+
+	gmu_write(gmu, REG_A6XX_GMU_CM3_SYSRESET, 1);
+	gmu_write(gmu, REG_A6XX_GMU_CM3_SYSRESET, 0);
+
+	ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_CM3_FW_INIT_RESULT, val,
+		val == 0xbabeface, 100, 10000);
+
+	if (ret)
+		dev_err(gmu->dev, "GMU firmware initialization timed out\n");
+
+	return ret;
+}
+
+static int a6xx_gmu_hfi_start(struct a6xx_gmu *gmu)
+{
+	u32 val;
+	int ret;
+
+	gmu_rmw(gmu, REG_A6XX_GMU_GMU2HOST_INTR_MASK,
+		A6XX_GMU_GMU2HOST_INTR_INFO_MSGQ, 0);
+
+	gmu_write(gmu, REG_A6XX_GMU_HFI_CTRL_INIT, 1);
+
+	ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_HFI_CTRL_STATUS, val,
+		val & 1, 100, 10000);
+	if (ret)
+		dev_err(gmu->dev, "Unable to start the HFI queues\n");
+
+	return ret;
+}
+
+/* Trigger a OOB (out of band) request to the GMU */
+int a6xx_gmu_set_oob(struct a6xx_gmu *gmu, enum a6xx_gmu_oob_state state)
+{
+	int ret;
+	u32 val;
+	int request, ack;
+	const char *name;
+
+	switch (state) {
+	case GMU_OOB_GPU_SET:
+		request = GMU_OOB_GPU_SET_REQUEST;
+		ack = GMU_OOB_GPU_SET_ACK;
+		name = "GPU_SET";
+		break;
+	case GMU_OOB_BOOT_SLUMBER:
+		request = GMU_OOB_BOOT_SLUMBER_REQUEST;
+		ack = GMU_OOB_BOOT_SLUMBER_ACK;
+		name = "BOOT_SLUMBER";
+		break;
+	case GMU_OOB_DCVS_SET:
+		request = GMU_OOB_DCVS_REQUEST;
+		ack = GMU_OOB_DCVS_ACK;
+		name = "GPU_DCVS";
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	/* Trigger the equested OOB operation */
+	gmu_write(gmu, REG_A6XX_GMU_HOST2GMU_INTR_SET, 1 << request);
+
+	/* Wait for the acknowledge interrupt */
+	ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_GMU2HOST_INTR_INFO, val,
+		val & (1 << ack), 100, 10000);
+
+	if (ret)
+		dev_err(gmu->dev,
+			"Timeout waiting for GMU OOB set %s: 0x%x\n",
+				name,
+				gmu_read(gmu, REG_A6XX_GMU_GMU2HOST_INTR_INFO));
+
+	/* Clear the acknowledge interrupt */
+	gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_CLR, 1 << ack);
+
+	return ret;
+}
+
+/* Clear a pending OOB state in the GMU */
+void a6xx_gmu_clear_oob(struct a6xx_gmu *gmu, enum a6xx_gmu_oob_state state)
+{
+	switch (state) {
+	case GMU_OOB_GPU_SET:
+		gmu_write(gmu, REG_A6XX_GMU_HOST2GMU_INTR_SET,
+			1 << GMU_OOB_GPU_SET_CLEAR);
+		break;
+	case GMU_OOB_BOOT_SLUMBER:
+		gmu_write(gmu, REG_A6XX_GMU_HOST2GMU_INTR_SET,
+			1 << GMU_OOB_BOOT_SLUMBER_CLEAR);
+		break;
+	case GMU_OOB_DCVS_SET:
+		gmu_write(gmu, REG_A6XX_GMU_HOST2GMU_INTR_SET,
+			1 << GMU_OOB_DCVS_CLEAR);
+		break;
+	}
+}
+
+/* Enable CPU control of SPTP power power collapse */
+static int a6xx_sptprac_enable(struct a6xx_gmu *gmu)
+{
+	int ret;
+	u32 val;
+
+	gmu_write(gmu, REG_A6XX_GMU_GX_SPTPRAC_POWER_CONTROL, 0x778000);
+
+	ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS, val,
+		(val & 0x38) == 0x28, 1, 100);
+
+	if (ret) {
+		dev_err(gmu->dev, "Unable to power on SPTPRAC: 0x%x\n",
+			gmu_read(gmu, REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS));
+	}
+
+	return 0;
+}
+
+/* Disable CPU control of SPTP power power collapse */
+static void a6xx_sptprac_disable(struct a6xx_gmu *gmu)
+{
+	u32 val;
+	int ret;
+
+	/* Make sure retention is on */
+	gmu_rmw(gmu, REG_A6XX_GPU_CC_GX_GDSCR, 0, (1 << 11));
+
+	gmu_write(gmu, REG_A6XX_GMU_GX_SPTPRAC_POWER_CONTROL, 0x778001);
+
+	ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS, val,
+		(val & 0x04), 100, 10000);
+
+	if (ret)
+		dev_err(gmu->dev, "failed to power off SPTPRAC: 0x%x\n",
+			gmu_read(gmu, REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS));
+}
+
+/* Let the GMU know we are starting a boot sequence */
+static int a6xx_gmu_gfx_rail_on(struct a6xx_gmu *gmu)
+{
+	u32 vote;
+
+	/* Let the GMU know we are getting ready for boot */
+	gmu_write(gmu, REG_A6XX_GMU_BOOT_SLUMBER_OPTION, 0);
+
+	/* Choose the "default" power level as the highest available */
+	vote = gmu->gx_arc_votes[gmu->nr_gpu_freqs - 1];
+
+	gmu_write(gmu, REG_A6XX_GMU_GX_VOTE_IDX, vote & 0xff);
+	gmu_write(gmu, REG_A6XX_GMU_MX_VOTE_IDX, (vote >> 8) & 0xff);
+
+	/* Let the GMU know the boot sequence has started */
+	return a6xx_gmu_set_oob(gmu, GMU_OOB_BOOT_SLUMBER);
+}
+
+/* Let the GMU know that we are about to go into slumber */
+static int a6xx_gmu_notify_slumber(struct a6xx_gmu *gmu)
+{
+	int ret;
+
+	/* Disable the power counter so the GMU isn't busy */
+	gmu_write(gmu, REG_A6XX_GMU_CX_GMU_POWER_COUNTER_ENABLE, 0);
+
+	/* Disable SPTP_PC if the CPU is responsible for it */
+	if (gmu->idle_level < GMU_IDLE_STATE_SPTP)
+		a6xx_sptprac_disable(gmu);
+
+	/* Tell the GMU to get ready to slumber */
+	gmu_write(gmu, REG_A6XX_GMU_BOOT_SLUMBER_OPTION, 1);
+
+	ret = a6xx_gmu_set_oob(gmu, GMU_OOB_BOOT_SLUMBER);
+	a6xx_gmu_clear_oob(gmu, GMU_OOB_BOOT_SLUMBER);
+
+	if (!ret) {
+		/* Check to see if the GMU really did slumber */
+		if (gmu_read(gmu, REG_A6XX_GPU_GMU_CX_GMU_RPMH_POWER_STATE)
+			!= 0x0f) {
+			dev_err(gmu->dev, "The GMU did not go into slumber\n");
+			ret = -ETIMEDOUT;
+		}
+	}
+
+	/* Put fence into allow mode */
+	gmu_write(gmu, REG_A6XX_GMU_AO_AHB_FENCE_CTRL, 0);
+	return ret;
+}
+
+static int a6xx_rpmh_start(struct a6xx_gmu *gmu)
+{
+	int ret;
+	u32 val;
+
+	gmu_write(gmu, REG_A6XX_GMU_RSCC_CONTROL_REQ, 1 << 1);
+	/* Wait for the register to finish posting */
+	wmb();
+
+	ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_RSCC_CONTROL_ACK, val,
+		val & (1 << 1), 100, 10000);
+	if (ret) {
+		dev_err(gmu->dev, "Unable to power on the GPU RSC\n");
+		return ret;
+	}
+
+	ret = gmu_poll_timeout(gmu, REG_A6XX_RSCC_SEQ_BUSY_DRV0, val,
+		!val, 100, 10000);
+
+	if (!ret) {
+		gmu_write(gmu, REG_A6XX_GMU_RSCC_CONTROL_REQ, 0);
+
+		/* Re-enable the power counter */
+		gmu_write(gmu, REG_A6XX_GMU_CX_GMU_POWER_COUNTER_ENABLE, 1);
+		return 0;
+	}
+
+	dev_err(gmu->dev, "GPU RSC sequence stuck while waking up the GPU\n");
+	return ret;
+}
+
+static void a6xx_rpmh_stop(struct a6xx_gmu *gmu)
+{
+	int ret;
+	u32 val;
+
+	gmu_write(gmu, REG_A6XX_GMU_RSCC_CONTROL_REQ, 1);
+
+	ret = gmu_poll_timeout(gmu, REG_A6XX_GPU_RSCC_RSC_STATUS0_DRV0,
+		val, val & (1 << 16), 100, 10000);
+	if (ret)
+		dev_err(gmu->dev, "Unable to power off the GPU RSC\n");
+
+	gmu_write(gmu, REG_A6XX_GMU_RSCC_CONTROL_REQ, 0);
+}
+
+static void a6xx_gmu_rpmh_init(struct a6xx_gmu *gmu)
+{
+	/* Disable SDE clock gating */
+	gmu_write(gmu, REG_A6XX_GPU_RSCC_RSC_STATUS0_DRV0, BIT(24));
+
+	/* Setup RSC PDC handshake for sleep and wakeup */
+	gmu_write(gmu, REG_A6XX_RSCC_PDC_SLAVE_ID_DRV0, 1);
+	gmu_write(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_DATA, 0);
+	gmu_write(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_ADDR, 0);
+	gmu_write(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_DATA + 2, 0);
+	gmu_write(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_ADDR + 2, 0);
+	gmu_write(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_DATA + 4, 0x80000000);
+	gmu_write(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_ADDR + 4, 0);
+	gmu_write(gmu, REG_A6XX_RSCC_OVERRIDE_START_ADDR, 0);
+	gmu_write(gmu, REG_A6XX_RSCC_PDC_SEQ_START_ADDR, 0x4520);
+	gmu_write(gmu, REG_A6XX_RSCC_PDC_MATCH_VALUE_LO, 0x4510);
+	gmu_write(gmu, REG_A6XX_RSCC_PDC_MATCH_VALUE_HI, 0x4514);
+
+	/* Load RSC sequencer uCode for sleep and wakeup */
+	gmu_write(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0, 0xa7a506a0);
+	gmu_write(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 1, 0xa1e6a6e7);
+	gmu_write(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 2, 0xa2e081e1);
+	gmu_write(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 3, 0xe9a982e2);
+	gmu_write(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 4, 0x0020e8a8);
+
+	/* Load PDC sequencer uCode for power up and power down sequence */
+	pdc_write(gmu, REG_A6XX_PDC_GPU_SEQ_MEM_0, 0xfebea1e1);
+	pdc_write(gmu, REG_A6XX_PDC_GPU_SEQ_MEM_0 + 1, 0xa5a4a3a2);
+	pdc_write(gmu, REG_A6XX_PDC_GPU_SEQ_MEM_0 + 2, 0x8382a6e0);
+	pdc_write(gmu, REG_A6XX_PDC_GPU_SEQ_MEM_0 + 3, 0xbce3e284);
+	pdc_write(gmu, REG_A6XX_PDC_GPU_SEQ_MEM_0 + 4, 0x002081fc);
+
+	/* Set TCS commands used by PDC sequence for low power modes */
+	pdc_write(gmu, REG_A6XX_PDC_GPU_TCS1_CMD_ENABLE_BANK, 7);
+	pdc_write(gmu, REG_A6XX_PDC_GPU_TCS1_CMD_WAIT_FOR_CMPL_BANK, 0);
+	pdc_write(gmu, REG_A6XX_PDC_GPU_TCS1_CONTROL, 0);
+	pdc_write(gmu, REG_A6XX_PDC_GPU_TCS1_CMD0_MSGID, 0x10108);
+	pdc_write(gmu, REG_A6XX_PDC_GPU_TCS1_CMD0_ADDR, 0x30010);
+	pdc_write(gmu, REG_A6XX_PDC_GPU_TCS1_CMD0_DATA, 1);
+	pdc_write(gmu, REG_A6XX_PDC_GPU_TCS1_CMD0_MSGID + 4, 0x10108);
+	pdc_write(gmu, REG_A6XX_PDC_GPU_TCS1_CMD0_ADDR + 4, 0x30000);
+	pdc_write(gmu, REG_A6XX_PDC_GPU_TCS1_CMD0_DATA + 4, 0x0);
+	pdc_write(gmu, REG_A6XX_PDC_GPU_TCS1_CMD0_MSGID + 8, 0x10108);
+	pdc_write(gmu, REG_A6XX_PDC_GPU_TCS1_CMD0_ADDR + 8, 0x30080);
+	pdc_write(gmu, REG_A6XX_PDC_GPU_TCS1_CMD0_DATA + 8, 0x0);
+	pdc_write(gmu, REG_A6XX_PDC_GPU_TCS3_CMD_ENABLE_BANK, 7);
+	pdc_write(gmu, REG_A6XX_PDC_GPU_TCS3_CMD_WAIT_FOR_CMPL_BANK, 0);
+	pdc_write(gmu, REG_A6XX_PDC_GPU_TCS3_CONTROL, 0);
+	pdc_write(gmu, REG_A6XX_PDC_GPU_TCS3_CMD0_MSGID, 0x10108);
+	pdc_write(gmu, REG_A6XX_PDC_GPU_TCS3_CMD0_ADDR, 0x30010);
+	pdc_write(gmu, REG_A6XX_PDC_GPU_TCS3_CMD0_DATA, 2);
+	pdc_write(gmu, REG_A6XX_PDC_GPU_TCS3_CMD0_MSGID + 4, 0x10108);
+	pdc_write(gmu, REG_A6XX_PDC_GPU_TCS3_CMD0_ADDR + 4, 0x30000);
+	pdc_write(gmu, REG_A6XX_PDC_GPU_TCS3_CMD0_DATA + 4, 0x3);
+	pdc_write(gmu, REG_A6XX_PDC_GPU_TCS3_CMD0_MSGID + 8, 0x10108);
+	pdc_write(gmu, REG_A6XX_PDC_GPU_TCS3_CMD0_ADDR + 8, 0x30080);
+	pdc_write(gmu, REG_A6XX_PDC_GPU_TCS3_CMD0_DATA + 8, 0x3);
+
+	/* Setup GPU PDC */
+	pdc_write(gmu, REG_A6XX_PDC_GPU_SEQ_START_ADDR, 0);
+	pdc_write(gmu, REG_A6XX_PDC_GPU_ENABLE_PDC, 0x80000001);
+
+	/* ensure no writes happen before the uCode is fully written */
+	wmb();
+}
+
+/*
+ * The lowest 16 bits of this value are the number of XO clock cycles for main
+ * hysteresis which is set at 0x1680 cycles (300 us).  The higher 16 bits are
+ * for the shorter hysteresis that happens after main - this is 0xa (.5 us)
+ */
+
+#define GMU_PWR_COL_HYST 0x000a1680
+
+/* Set up the idle state for the GMU */
+static void a6xx_gmu_power_config(struct a6xx_gmu *gmu)
+{
+	/* Disable GMU WB/RB buffer */
+	gmu_write(gmu, REG_A6XX_GMU_SYS_BUS_CONFIG, 0x1);
+
+	gmu_write(gmu, REG_A6XX_GMU_PWR_COL_INTER_FRAME_CTRL, 0x9c40400);
+
+	switch (gmu->idle_level) {
+	case GMU_IDLE_STATE_IFPC:
+		gmu_write(gmu, REG_A6XX_GMU_PWR_COL_INTER_FRAME_HYST,
+			GMU_PWR_COL_HYST);
+		gmu_rmw(gmu, REG_A6XX_GMU_PWR_COL_INTER_FRAME_CTRL, 0,
+			A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_IFPC_ENABLE |
+			A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_HM_POWER_COLLAPSE_ENABLE);
+		/* Fall through */
+	case GMU_IDLE_STATE_SPTP:
+		gmu_write(gmu, REG_A6XX_GMU_PWR_COL_SPTPRAC_HYST,
+			GMU_PWR_COL_HYST);
+		gmu_rmw(gmu, REG_A6XX_GMU_PWR_COL_INTER_FRAME_CTRL, 0,
+			A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_IFPC_ENABLE |
+			A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_SPTPRAC_POWER_CONTROL_ENABLE);
+	}
+
+	/* Enable RPMh GPU client */
+	gmu_rmw(gmu, REG_A6XX_GMU_RPMH_CTRL, 0,
+		A6XX_GMU_RPMH_CTRL_RPMH_INTERFACE_ENABLE |
+		A6XX_GMU_RPMH_CTRL_LLC_VOTE_ENABLE |
+		A6XX_GMU_RPMH_CTRL_DDR_VOTE_ENABLE |
+		A6XX_GMU_RPMH_CTRL_MX_VOTE_ENABLE |
+		A6XX_GMU_RPMH_CTRL_CX_VOTE_ENABLE |
+		A6XX_GMU_RPMH_CTRL_GFX_VOTE_ENABLE);
+}
+
+static int a6xx_gmu_fw_start(struct a6xx_gmu *gmu, unsigned int state)
+{
+	static bool rpmh_init;
+	struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
+	struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
+	int i, ret;
+	u32 chipid;
+	u32 *image;
+
+	if (state == GMU_WARM_BOOT) {
+		ret = a6xx_rpmh_start(gmu);
+		if (ret)
+			return ret;
+	} else {
+		if (WARN(!adreno_gpu->fw[ADRENO_FW_GMU],
+			"GMU firmware is not loaded\n"))
+			return -ENOENT;
+
+		/* Sanity check the size of the firmware that was loaded */
+		if (adreno_gpu->fw[ADRENO_FW_GMU]->size > 0x8000) {
+			dev_err(gmu->dev,
+				"GMU firmware is bigger than the available region\n");
+			return -EINVAL;
+		}
+
+		/* Turn on register retention */
+		gmu_write(gmu, REG_A6XX_GMU_GENERAL_7, 1);
+
+		/* We only need to load the RPMh microcode once */
+		if (!rpmh_init) {
+			a6xx_gmu_rpmh_init(gmu);
+			rpmh_init = true;
+		} else if (state != GMU_RESET) {
+			ret = a6xx_rpmh_start(gmu);
+			if (ret)
+				return ret;
+		}
+
+		image = (u32 *) adreno_gpu->fw[ADRENO_FW_GMU]->data;
+
+		for (i = 0; i < adreno_gpu->fw[ADRENO_FW_GMU]->size >> 2; i++)
+			gmu_write(gmu, REG_A6XX_GMU_CM3_ITCM_START + i,
+				image[i]);
+	}
+
+	gmu_write(gmu, REG_A6XX_GMU_CM3_FW_INIT_RESULT, 0);
+	gmu_write(gmu, REG_A6XX_GMU_CM3_BOOT_CONFIG, 0x02);
+
+	/* Write the iova of the HFI table */
+	gmu_write(gmu, REG_A6XX_GMU_HFI_QTBL_ADDR, gmu->hfi->iova);
+	gmu_write(gmu, REG_A6XX_GMU_HFI_QTBL_INFO, 1);
+
+	gmu_write(gmu, REG_A6XX_GMU_AHB_FENCE_RANGE_0,
+		(1 << 31) | (0xa << 18) | (0xa0));
+
+	chipid = adreno_gpu->rev.core << 24;
+	chipid |= adreno_gpu->rev.major << 16;
+	chipid |= adreno_gpu->rev.minor << 12;
+	chipid |= adreno_gpu->rev.patchid << 8;
+
+	gmu_write(gmu, REG_A6XX_GMU_HFI_SFR_ADDR, chipid);
+
+	/* Set up the lowest idle level on the GMU */
+	a6xx_gmu_power_config(gmu);
+
+	ret = a6xx_gmu_start(gmu);
+	if (ret)
+		return ret;
+
+	ret = a6xx_gmu_gfx_rail_on(gmu);
+	if (ret)
+		return ret;
+
+	/* Enable SPTP_PC if the CPU is responsible for it */
+	if (gmu->idle_level < GMU_IDLE_STATE_SPTP) {
+		ret = a6xx_sptprac_enable(gmu);
+		if (ret)
+			return ret;
+	}
+
+	ret = a6xx_gmu_hfi_start(gmu);
+	if (ret)
+		return ret;
+
+	/* FIXME: Do we need this wmb() here? */
+	wmb();
+
+	return 0;
+}
+
+#define A6XX_HFI_IRQ_MASK \
+	(A6XX_GMU_GMU2HOST_INTR_INFO_MSGQ | \
+	 A6XX_GMU_GMU2HOST_INTR_INFO_CM3_FAULT)
+
+#define A6XX_GMU_IRQ_MASK \
+	(A6XX_GMU_AO_HOST_INTERRUPT_STATUS_WDOG_BITE | \
+	 A6XX_GMU_AO_HOST_INTERRUPT_STATUS_HOST_AHB_BUS_ERROR | \
+	 A6XX_GMU_AO_HOST_INTERRUPT_STATUS_FENCE_ERR)
+
+static void a6xx_gmu_irq_enable(struct a6xx_gmu *gmu)
+{
+	gmu_write(gmu, REG_A6XX_GMU_AO_HOST_INTERRUPT_CLR, ~0);
+	gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_CLR, ~0);
+
+	gmu_write(gmu, REG_A6XX_GMU_AO_HOST_INTERRUPT_MASK,
+		~A6XX_GMU_IRQ_MASK);
+	gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_MASK,
+		~A6XX_HFI_IRQ_MASK);
+
+	enable_irq(gmu->gmu_irq);
+	enable_irq(gmu->hfi_irq);
+}
+
+static void a6xx_gmu_irq_disable(struct a6xx_gmu *gmu)
+{
+	disable_irq(gmu->gmu_irq);
+	disable_irq(gmu->hfi_irq);
+
+	gmu_write(gmu, REG_A6XX_GMU_AO_HOST_INTERRUPT_MASK, ~0);
+	gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_MASK, ~0);
+}
+
+int a6xx_gmu_reset(struct a6xx_gpu *a6xx_gpu)
+{
+	struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
+	int ret;
+	u32 val;
+
+	/* Flush all the queues */
+	a6xx_hfi_stop(gmu);
+
+	/* Stop the interrupts */
+	a6xx_gmu_irq_disable(gmu);
+
+	/* Force off SPTP in case the GMU is managing it */
+	a6xx_sptprac_disable(gmu);
+
+	/* Make sure there are no outstanding RPMh votes */
+	gmu_poll_timeout(gmu, REG_A6XX_RSCC_TCS0_DRV0_STATUS, val,
+		(val & 1), 100, 10000);
+	gmu_poll_timeout(gmu, REG_A6XX_RSCC_TCS1_DRV0_STATUS, val,
+		(val & 1), 100, 10000);
+	gmu_poll_timeout(gmu, REG_A6XX_RSCC_TCS2_DRV0_STATUS, val,
+		(val & 1), 100, 10000);
+	gmu_poll_timeout(gmu, REG_A6XX_RSCC_TCS3_DRV0_STATUS, val,
+		(val & 1), 100, 1000);
+
+	/* Force off the GX GSDC */
+	regulator_force_disable(gmu->gx);
+
+	/* Disable the resources */
+	clk_bulk_disable_unprepare(gmu->nr_clocks, gmu->clocks);
+	pm_runtime_put_sync(gmu->dev);
+
+	/* Re-enable the resources */
+	pm_runtime_get_sync(gmu->dev);
+
+	/* Use a known rate to bring up the GMU */
+	clk_set_rate(gmu->core_clk, 200000000);
+	ret = clk_bulk_prepare_enable(gmu->nr_clocks, gmu->clocks);
+	if (ret)
+		goto out;
+
+	a6xx_gmu_irq_enable(gmu);
+
+	ret = a6xx_gmu_fw_start(gmu, GMU_RESET);
+	if (!ret)
+		ret = a6xx_hfi_start(gmu, GMU_COLD_BOOT);
+
+	/* Set the GPU back to the highest power frequency */
+	a6xx_gmu_set_freq(gmu, gmu->nr_gpu_freqs - 1);
+
+out:
+	if (ret)
+		a6xx_gmu_clear_oob(gmu, GMU_OOB_BOOT_SLUMBER);
+
+	return ret;
+}
+
+int a6xx_gmu_resume(struct a6xx_gpu *a6xx_gpu)
+{
+	struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
+	int status, ret;
+
+	if (WARN(!gmu->mmio, "The GMU is not set up yet\n"))
+		return 0;
+
+	/* Turn on the resources */
+	pm_runtime_get_sync(gmu->dev);
+
+	/* Use a known rate to bring up the GMU */
+	clk_set_rate(gmu->core_clk, 200000000);
+	ret = clk_bulk_prepare_enable(gmu->nr_clocks, gmu->clocks);
+	if (ret)
+		goto out;
+
+	a6xx_gmu_irq_enable(gmu);
+
+	/* Check to see if we are doing a cold or warm boot */
+	status = gmu_read(gmu, REG_A6XX_GMU_GENERAL_7) == 1 ?
+		GMU_WARM_BOOT : GMU_COLD_BOOT;
+
+	ret = a6xx_gmu_fw_start(gmu, status);
+	if (ret)
+		goto out;
+
+	ret = a6xx_hfi_start(gmu, status);
+
+	/* Set the GPU to the highest power frequency */
+	a6xx_gmu_set_freq(gmu, gmu->nr_gpu_freqs - 1);
+
+out:
+	/* Make sure to turn off the boot OOB request on error */
+	if (ret)
+		a6xx_gmu_clear_oob(gmu, GMU_OOB_BOOT_SLUMBER);
+
+	return ret;
+}
+
+bool a6xx_gmu_isidle(struct a6xx_gmu *gmu)
+{
+	u32 reg;
+
+	if (!gmu->mmio)
+		return true;
+
+	reg = gmu_read(gmu, REG_A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS);
+
+	if (reg &  A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS_GPUBUSYIGNAHB)
+		return false;
+
+	return true;
+}
+
+int a6xx_gmu_stop(struct a6xx_gpu *a6xx_gpu)
+{
+	struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
+	u32 val;
+
+	/*
+	 * The GMU may still be in slumber unless the GPU started so check and
+	 * skip putting it back into slumber if so
+	 */
+	val = gmu_read(gmu, REG_A6XX_GPU_GMU_CX_GMU_RPMH_POWER_STATE);
+
+	if (val != 0xf) {
+		int ret = a6xx_gmu_wait_for_idle(a6xx_gpu);
+
+		/* Temporary until we can recover safely */
+		BUG_ON(ret);
+
+		/* tell the GMU we want to slumber */
+		a6xx_gmu_notify_slumber(gmu);
+
+		ret = gmu_poll_timeout(gmu,
+			REG_A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS, val,
+			!(val & A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS_GPUBUSYIGNAHB),
+			100, 10000);
+
+		/*
+		 * Let the user know we failed to slumber but don't worry too
+		 * much because we are powering down anyway
+		 */
+
+		if (ret)
+			dev_err(gmu->dev,
+				"Unable to slumber GMU: status = 0%x/0%x\n",
+				gmu_read(gmu,
+					REG_A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS),
+				gmu_read(gmu,
+					REG_A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS2));
+	}
+
+	/* Turn off HFI */
+	a6xx_hfi_stop(gmu);
+
+	/* Stop the interrupts and mask the hardware */
+	a6xx_gmu_irq_disable(gmu);
+
+	/* Tell RPMh to power off the GPU */
+	a6xx_rpmh_stop(gmu);
+
+	clk_bulk_disable_unprepare(gmu->nr_clocks, gmu->clocks);
+
+	pm_runtime_put_sync(gmu->dev);
+
+	return 0;
+}
+
+static void a6xx_gmu_memory_free(struct a6xx_gmu *gmu, struct a6xx_gmu_bo *bo)
+{
+	int count, i;
+	u64 iova;
+
+	if (IS_ERR_OR_NULL(bo))
+		return;
+
+	count = bo->size >> PAGE_SHIFT;
+	iova = bo->iova;
+
+	for (i = 0; i < count; i++, iova += PAGE_SIZE) {
+		iommu_unmap(gmu->domain, iova, PAGE_SIZE);
+		__free_pages(bo->pages[i], 0);
+	}
+
+	kfree(bo->pages);
+	kfree(bo);
+}
+
+static struct a6xx_gmu_bo *a6xx_gmu_memory_alloc(struct a6xx_gmu *gmu,
+		size_t size)
+{
+	struct a6xx_gmu_bo *bo;
+	int ret, count, i;
+
+	bo = kzalloc(sizeof(*bo), GFP_KERNEL);
+	if (!bo)
+		return ERR_PTR(-ENOMEM);
+
+	bo->size = PAGE_ALIGN(size);
+
+	count = bo->size >> PAGE_SHIFT;
+
+	bo->pages = kcalloc(count, sizeof(struct page *), GFP_KERNEL);
+	if (!bo->pages) {
+		kfree(bo);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	for (i = 0; i < count; i++) {
+		bo->pages[i] = alloc_page(GFP_KERNEL);
+		if (!bo->pages[i])
+			goto err;
+	}
+
+	bo->iova = gmu->uncached_iova_base;
+
+	for (i = 0; i < count; i++) {
+		ret = iommu_map(gmu->domain,
+			bo->iova + (PAGE_SIZE * i),
+			page_to_phys(bo->pages[i]), PAGE_SIZE,
+			IOMMU_READ | IOMMU_WRITE);
+
+		if (ret) {
+			dev_err(gmu->dev, "Unable to map GMU buffer object\n");
+
+			for (i = i - 1 ; i >= 0; i--)
+				iommu_unmap(gmu->domain,
+					bo->iova + (PAGE_SIZE * i),
+					PAGE_SIZE);
+
+			goto err;
+		}
+	}
+
+	bo->virt = vmap(bo->pages, count, VM_IOREMAP,
+		pgprot_writecombine(PAGE_KERNEL));
+	if (!bo->virt)
+		goto err;
+
+	/* Align future IOVA addresses on 1MB boundaries */
+	gmu->uncached_iova_base += ALIGN(size, SZ_1M);
+
+	return bo;
+
+err:
+	for (i = 0; i < count; i++) {
+		if (bo->pages[i])
+			__free_pages(bo->pages[i], 0);
+	}
+
+	kfree(bo->pages);
+	kfree(bo);
+
+	return ERR_PTR(-ENOMEM);
+}
+
+static int a6xx_gmu_memory_probe(struct a6xx_gmu *gmu)
+{
+	int ret;
+
+	/*
+	 * The GMU address space is hardcoded to treat the range
+	 * 0x60000000 - 0x80000000 as un-cached memory. All buffers shared
+	 * between the GMU and the CPU will live in this space
+	 */
+	gmu->uncached_iova_base = 0x60000000;
+
+
+	gmu->domain = iommu_domain_alloc(&platform_bus_type);
+	if (!gmu->domain)
+		return -ENODEV;
+
+	ret = iommu_attach_device(gmu->domain, gmu->dev);
+
+	if (ret) {
+		iommu_domain_free(gmu->domain);
+		gmu->domain = NULL;
+	}
+
+	return ret;
+}
+
+/* Get the list of RPMh voltage levels from cmd-db */
+static int a6xx_gmu_rpmh_arc_cmds(const char *id, void *vals, int size)
+{
+	u32 len = cmd_db_read_aux_data_len(id);
+
+	if (!len)
+		return 0;
+
+	if (WARN_ON(len > size))
+		return -EINVAL;
+
+	cmd_db_read_aux_data(id, vals, len);
+
+	/*
+	 * The data comes back as an array of unsigned shorts so adjust the
+	 * count accordingly
+	 */
+	return len >> 1;
+}
+
+/* Return the 'arc-level' for the given frequency */
+static u32 a6xx_gmu_get_arc_level(struct device *dev, unsigned long freq)
+{
+	struct dev_pm_opp *opp;
+	struct device_node *np;
+	u32 val = 0;
+
+	if (!freq)
+		return 0;
+
+	opp  = dev_pm_opp_find_freq_exact(dev, freq, true);
+	if (IS_ERR(opp))
+		return 0;
+
+	np = dev_pm_opp_get_of_node(opp);
+
+	if (np) {
+		of_property_read_u32(np, "qcom,level", &val);
+		of_node_put(np);
+	}
+
+	dev_pm_opp_put(opp);
+
+	return val;
+}
+
+static int a6xx_gmu_rpmh_arc_votes_init(struct device *dev, u32 *votes,
+		unsigned long *freqs, int freqs_count,
+		u16 *pri, int pri_count,
+		u16 *sec, int sec_count)
+{
+	int i, j;
+
+	/* Construct a vote for each frequency */
+	for (i = 0; i < freqs_count; i++) {
+		u8 pindex = 0, sindex = 0;
+		u32 level = a6xx_gmu_get_arc_level(dev, freqs[i]);
+
+		/* Get the primary index that matches the arc level */
+		for (j = 0; j < pri_count; j++) {
+			if (pri[j] >= level) {
+				pindex = j;
+				break;
+			}
+		}
+
+		if (j == pri_count) {
+			dev_err(dev,
+				"Level %u not found in in the RPMh list\n",
+					level);
+			dev_err(dev, "Available levels:\n");
+			for (j = 0; j < pri_count; j++)
+				dev_err(dev, "  %u\n", pri[j]);
+
+			return -EINVAL;
+		}
+
+		/*
+		 * Look for a level in in the secondary list that matches. If
+		 * nothing fits, use the maximum non zero vote
+		 */
+
+		for (j = 0; j < sec_count; j++) {
+			if (sec[j] >= level) {
+				sindex = j;
+				break;
+			} else if (sec[j]) {
+				sindex = j;
+			}
+		}
+
+		/* Construct the vote */
+		votes[i] = ((pri[pindex] & 0xffff) << 16) |
+			(sindex << 8) | pindex;
+	}
+
+	return 0;
+}
+
+/*
+ * The GMU votes with the RPMh for itself and on behalf of the GPU but we need
+ * to construct the list of votes on the CPU and send it over. Query the RPMh
+ * voltage levels and build the votes
+ */
+
+static int a6xx_gmu_rpmh_votes_init(struct a6xx_gmu *gmu)
+{
+	struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
+	struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
+	struct msm_gpu *gpu = &adreno_gpu->base;
+
+	u16 gx[16], cx[16], mx[16];
+	u32 gxcount, cxcount, mxcount;
+	int ret;
+
+	/* Get the list of available voltage levels for each component */
+	gxcount = a6xx_gmu_rpmh_arc_cmds("gfx.lvl", gx, sizeof(gx));
+	cxcount = a6xx_gmu_rpmh_arc_cmds("cx.lvl", cx, sizeof(cx));
+	mxcount = a6xx_gmu_rpmh_arc_cmds("mx.lvl", mx, sizeof(mx));
+
+	/* Build the GX votes */
+	ret = a6xx_gmu_rpmh_arc_votes_init(&gpu->pdev->dev, gmu->gx_arc_votes,
+		gmu->gpu_freqs, gmu->nr_gpu_freqs,
+		gx, gxcount, mx, mxcount);
+
+	/* Build the CX votes */
+	ret |= a6xx_gmu_rpmh_arc_votes_init(gmu->dev, gmu->cx_arc_votes,
+		gmu->gmu_freqs, gmu->nr_gmu_freqs,
+		cx, cxcount, mx, mxcount);
+
+	return ret;
+}
+
+static int a6xx_gmu_build_freq_table(struct device *dev, unsigned long *freqs,
+		u32 size)
+{
+	int count = dev_pm_opp_get_opp_count(dev);
+	struct dev_pm_opp *opp;
+	int i, index = 0;
+	unsigned long freq = 1;
+
+	/*
+	 * The OPP table doesn't contain the "off" frequency level so we need to
+	 * add 1 to the table size to account for it
+	 */
+
+	if (WARN(count + 1 > size,
+		"The GMU frequency table is being truncated\n"))
+		count = size - 1;
+
+	/* Set the "off" frequency */
+	freqs[index++] = 0;
+
+	for (i = 0; i < count; i++) {
+		opp = dev_pm_opp_find_freq_ceil(dev, &freq);
+		if (IS_ERR(opp))
+			break;
+
+		dev_pm_opp_put(opp);
+		freqs[index++] = freq++;
+	}
+
+	return index;
+}
+
+static int a6xx_gmu_pwrlevels_probe(struct a6xx_gmu *gmu)
+{
+	struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
+	struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
+	struct msm_gpu *gpu = &adreno_gpu->base;
+
+	int ret = 0;
+
+	/*
+	 * The GMU handles its own frequency switching so build a list of
+	 * available frequencies to send during initialization
+	 */
+	ret = dev_pm_opp_of_add_table(gmu->dev);
+	if (ret) {
+		dev_err(gmu->dev, "Unable to set the OPP table for the GMU\n");
+		return ret;
+	}
+
+	gmu->nr_gmu_freqs = a6xx_gmu_build_freq_table(gmu->dev,
+		gmu->gmu_freqs, ARRAY_SIZE(gmu->gmu_freqs));
+
+	/*
+	 * The GMU also handles GPU frequency switching so build a list
+	 * from the GPU OPP table
+	 */
+	gmu->nr_gpu_freqs = a6xx_gmu_build_freq_table(&gpu->pdev->dev,
+		gmu->gpu_freqs, ARRAY_SIZE(gmu->gpu_freqs));
+
+	/* Build the list of RPMh votes that we'll send to the GMU */
+	return a6xx_gmu_rpmh_votes_init(gmu);
+}
+
+static int a6xx_gmu_clocks_probe(struct a6xx_gmu *gmu)
+{
+	int ret = msm_clk_bulk_get(gmu->dev, &gmu->clocks);
+
+	if (ret < 1)
+		return ret;
+
+	gmu->nr_clocks = ret;
+
+	gmu->core_clk = msm_clk_bulk_get_clock(gmu->clocks,
+		gmu->nr_clocks, "gmu");
+
+	return 0;
+}
+
+static void __iomem *a6xx_gmu_get_mmio(struct platform_device *pdev,
+		const char *name)
+{
+	void __iomem *ret;
+	struct resource *res = platform_get_resource_byname(pdev,
+			IORESOURCE_MEM, name);
+
+	if (!res) {
+		dev_err(&pdev->dev, "Unable to find the %s registers\n", name);
+		return ERR_PTR(-EINVAL);
+	}
+
+	ret = devm_ioremap(&pdev->dev, res->start, resource_size(res));
+	if (!ret) {
+		dev_err(&pdev->dev, "Unable to map the %s registers\n", name);
+		return ERR_PTR(-EINVAL);
+	}
+
+	return ret;
+}
+
+static int a6xx_gmu_get_irq(struct a6xx_gmu *gmu, struct platform_device *pdev,
+		const char *name, irq_handler_t handler)
+{
+	int irq, ret;
+
+	irq = platform_get_irq_byname(pdev, name);
+
+	ret = devm_request_irq(&pdev->dev, irq, handler, IRQF_TRIGGER_HIGH,
+		name, gmu);
+	if (ret) {
+		dev_err(&pdev->dev, "Unable to get interrupt %s\n", name);
+		return ret;
+	}
+
+	disable_irq(irq);
+
+	return irq;
+}
+
+void a6xx_gmu_remove(struct a6xx_gpu *a6xx_gpu)
+{
+	struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
+
+	if (IS_ERR_OR_NULL(gmu->mmio))
+		return;
+
+	pm_runtime_disable(gmu->dev);
+	a6xx_gmu_stop(a6xx_gpu);
+
+	a6xx_gmu_irq_disable(gmu);
+	a6xx_gmu_memory_free(gmu, gmu->hfi);
+
+	iommu_detach_device(gmu->domain, gmu->dev);
+
+	iommu_domain_free(gmu->domain);
+}
+
+int a6xx_gmu_probe(struct a6xx_gpu *a6xx_gpu, struct device_node *node)
+{
+	struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
+	struct platform_device *pdev = of_find_device_by_node(node);
+	int ret;
+
+	if (!pdev)
+		return -ENODEV;
+
+	gmu->dev = &pdev->dev;
+
+	of_dma_configure(gmu->dev, node, false);
+
+	/* Fow now, don't do anything fancy until we get our feet under us */
+	gmu->idle_level = GMU_IDLE_STATE_ACTIVE;
+
+	pm_runtime_enable(gmu->dev);
+	gmu->gx = devm_regulator_get(gmu->dev, "vdd");
+
+	/* Get the list of clocks */
+	ret = a6xx_gmu_clocks_probe(gmu);
+	if (ret)
+		return ret;
+
+	/* Set up the IOMMU context bank */
+	ret = a6xx_gmu_memory_probe(gmu);
+	if (ret)
+		return ret;
+
+	/* Allocate memory for for the HFI queues */
+	gmu->hfi = a6xx_gmu_memory_alloc(gmu, SZ_16K);
+	if (IS_ERR(gmu->hfi))
+		goto err;
+
+	/* Allocate memory for the GMU debug region */
+	gmu->debug = a6xx_gmu_memory_alloc(gmu, SZ_16K);
+	if (IS_ERR(gmu->debug))
+		goto err;
+
+	/* Map the GMU registers */
+	gmu->mmio = a6xx_gmu_get_mmio(pdev, "gmu");
+
+	/* Map the GPU power domain controller registers */
+	gmu->pdc_mmio = a6xx_gmu_get_mmio(pdev, "gmu_pdc");
+
+	if (IS_ERR(gmu->mmio) || IS_ERR(gmu->pdc_mmio))
+		goto err;
+
+	/* Get the HFI and GMU interrupts */
+	gmu->hfi_irq = a6xx_gmu_get_irq(gmu, pdev, "hfi", a6xx_hfi_irq);
+	gmu->gmu_irq = a6xx_gmu_get_irq(gmu, pdev, "gmu", a6xx_gmu_irq);
+
+	if (gmu->hfi_irq < 0 || gmu->gmu_irq < 0)
+		goto err;
+
+	/* Set up a tasklet to handle GMU HFI responses */
+	tasklet_init(&gmu->hfi_tasklet, a6xx_hfi_task, (unsigned long) gmu);
+
+	/* Get the power levels for the GMU and GPU */
+	a6xx_gmu_pwrlevels_probe(gmu);
+
+	/* Set up the HFI queues */
+	a6xx_hfi_init(gmu);
+
+	return 0;
+err:
+	a6xx_gmu_memory_free(gmu, gmu->hfi);
+
+	if (gmu->domain) {
+		iommu_detach_device(gmu->domain, gmu->dev);
+
+		iommu_domain_free(gmu->domain);
+	}
+
+	return -ENODEV;
+}
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.h b/drivers/gpu/drm/msm/adreno/a6xx_gmu.h
new file mode 100644
index 000000000000..d9a386c18799
--- /dev/null
+++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.h
@@ -0,0 +1,162 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (c) 2017 The Linux Foundation. All rights reserved. */
+
+#ifndef _A6XX_GMU_H_
+#define _A6XX_GMU_H_
+
+#include <linux/interrupt.h>
+#include "msm_drv.h"
+#include "a6xx_hfi.h"
+
+struct a6xx_gmu_bo {
+	void *virt;
+	size_t size;
+	u64 iova;
+	struct page **pages;
+};
+
+/*
+ * These define the different GMU wake up options - these define how both the
+ * CPU and the GMU bring up the hardware
+ */
+
+/* THe GMU has already been booted and the rentention registers are active */
+#define GMU_WARM_BOOT 0
+
+/* the GMU is coming up for the first time or back from a power collapse */
+#define GMU_COLD_BOOT 1
+
+/* The GMU is being soft reset after a fault */
+#define GMU_RESET 2
+
+/*
+ * These define the level of control that the GMU has - the higher the number
+ * the more things that the GMU hardware controls on its own.
+ */
+
+/* The GMU does not do any idle state management */
+#define GMU_IDLE_STATE_ACTIVE 0
+
+/* The GMU manages SPTP power collapse */
+#define GMU_IDLE_STATE_SPTP 2
+
+/* The GMU does automatic IFPC (intra-frame power collapse) */
+#define GMU_IDLE_STATE_IFPC 3
+
+struct a6xx_gmu {
+	struct device *dev;
+
+	void * __iomem mmio;
+	void * __iomem pdc_mmio;
+
+	int hfi_irq;
+	int gmu_irq;
+
+	struct regulator *gx;
+
+	struct iommu_domain *domain;
+	u64 uncached_iova_base;
+
+	int idle_level;
+
+	struct a6xx_gmu_bo *hfi;
+	struct a6xx_gmu_bo *debug;
+
+	int nr_clocks;
+	struct clk_bulk_data *clocks;
+	struct clk *core_clk;
+
+	int nr_gpu_freqs;
+	unsigned long gpu_freqs[16];
+	u32 gx_arc_votes[16];
+
+	int nr_gmu_freqs;
+	unsigned long gmu_freqs[4];
+	u32 cx_arc_votes[4];
+
+	struct a6xx_hfi_queue queues[2];
+
+	struct tasklet_struct hfi_tasklet;
+};
+
+static inline u32 gmu_read(struct a6xx_gmu *gmu, u32 offset)
+{
+	return msm_readl(gmu->mmio + (offset << 2));
+}
+
+static inline void gmu_write(struct a6xx_gmu *gmu, u32 offset, u32 value)
+{
+	return msm_writel(value, gmu->mmio + (offset << 2));
+}
+
+static inline void pdc_write(struct a6xx_gmu *gmu, u32 offset, u32 value)
+{
+	return msm_writel(value, gmu->pdc_mmio + (offset << 2));
+}
+
+static inline void gmu_rmw(struct a6xx_gmu *gmu, u32 reg, u32 mask, u32 or)
+{
+	u32 val = gmu_read(gmu, reg);
+
+	val &= ~mask;
+
+	gmu_write(gmu, reg, val | or);
+}
+
+#define gmu_poll_timeout(gmu, addr, val, cond, interval, timeout) \
+	readl_poll_timeout((gmu)->mmio + ((addr) << 2), val, cond, \
+		interval, timeout)
+
+/*
+ * These are the available OOB (out of band requests) to the GMU where "out of
+ * band" means that the CPU talks to the GMU directly and not through HFI.
+ * Normally this works by writing a ITCM/DTCM register and then triggering a
+ * interrupt (the "request" bit) and waiting for an acknowledgment (the "ack"
+ * bit). The state is cleared by writing the "clear' bit to the GMU interrupt.
+ *
+ * These are used to force the GMU/GPU to stay on during a critical sequence or
+ * for hardware workarounds.
+ */
+
+enum a6xx_gmu_oob_state {
+	GMU_OOB_BOOT_SLUMBER = 0,
+	GMU_OOB_GPU_SET,
+	GMU_OOB_DCVS_SET,
+};
+
+/* These are the interrupt / ack bits for each OOB request that are set
+ * in a6xx_gmu_set_oob and a6xx_clear_oob
+ */
+
+/*
+ * Let the GMU know that a boot or slumber operation has started. The value in
+ * REG_A6XX_GMU_BOOT_SLUMBER_OPTION lets the GMU know which operation we are
+ * doing
+ */
+#define GMU_OOB_BOOT_SLUMBER_REQUEST	22
+#define GMU_OOB_BOOT_SLUMBER_ACK	30
+#define GMU_OOB_BOOT_SLUMBER_CLEAR	30
+
+/*
+ * Set a new power level for the GPU when the CPU is doing frequency scaling
+ */
+#define GMU_OOB_DCVS_REQUEST	23
+#define GMU_OOB_DCVS_ACK	31
+#define GMU_OOB_DCVS_CLEAR	31
+
+/*
+ * Let the GMU know to not turn off any GPU registers while the CPU is in a
+ * critical section
+ */
+#define GMU_OOB_GPU_SET_REQUEST	16
+#define GMU_OOB_GPU_SET_ACK	24
+#define GMU_OOB_GPU_SET_CLEAR	24
+
+
+void a6xx_hfi_init(struct a6xx_gmu *gmu);
+int a6xx_hfi_start(struct a6xx_gmu *gmu, int boot_state);
+void a6xx_hfi_stop(struct a6xx_gmu *gmu);
+
+void a6xx_hfi_task(unsigned long data);
+
+#endif
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.xml.h b/drivers/gpu/drm/msm/adreno/a6xx_gmu.xml.h
new file mode 100644
index 000000000000..ef68098d2adc
--- /dev/null
+++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.xml.h
@@ -0,0 +1,382 @@
+#ifndef A6XX_GMU_XML
+#define A6XX_GMU_XML
+
+/* Autogenerated file, DO NOT EDIT manually!
+
+This file was generated by the rules-ng-ng headergen tool in this git repository:
+http://github.com/freedreno/envytools/
+git clone https://github.com/freedreno/envytools.git
+
+The rules-ng-ng source files this header was generated from are:
+- /home/robclark/src/envytools/rnndb/adreno.xml               (    501 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/freedreno_copyright.xml  (   1572 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/adreno/a2xx.xml          (  36805 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/adreno/adreno_common.xml (  13634 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/adreno/adreno_pm4.xml    (  42393 bytes, from 2018-08-06 18:45:45)
+- /home/robclark/src/envytools/rnndb/adreno/a3xx.xml          (  83840 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/adreno/a4xx.xml          ( 112086 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/adreno/a5xx.xml          ( 147240 bytes, from 2018-08-06 18:45:45)
+- /home/robclark/src/envytools/rnndb/adreno/a6xx.xml          ( 101627 bytes, from 2018-08-06 18:45:45)
+- /home/robclark/src/envytools/rnndb/adreno/a6xx_gmu.xml      (  10431 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/adreno/ocmem.xml         (   1773 bytes, from 2018-07-03 19:37:13)
+
+Copyright (C) 2013-2018 by the following authors:
+- Rob Clark <robdclark@gmail.com> (robclark)
+- Ilia Mirkin <imirkin@alum.mit.edu> (imirkin)
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice (including the
+next paragraph) shall be included in all copies or substantial
+portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/
+
+
+#define A6XX_GMU_GPU_IDLE_STATUS_BUSY_IGN_AHB			0x00800000
+#define A6XX_GMU_GPU_IDLE_STATUS_CX_GX_CPU_BUSY_IGN_AHB		0x40000000
+#define A6XX_GMU_OOB_BOOT_SLUMBER_SET_MASK			0x00400000
+#define A6XX_GMU_OOB_BOOT_SLUMBER_CHECK_MASK			0x40000000
+#define A6XX_GMU_OOB_BOOT_SLUMBER_CLEAR_MASK			0x40000000
+#define A6XX_GMU_OOB_DCVS_SET_MASK				0x00800000
+#define A6XX_GMU_OOB_DCVS_CHECK_MASK				0x80000000
+#define A6XX_GMU_OOB_DCVS_CLEAR_MASK				0x80000000
+#define A6XX_GMU_OOB_GPU_SET_MASK				0x00040000
+#define A6XX_GMU_OOB_GPU_CHECK_MASK				0x04000000
+#define A6XX_GMU_OOB_GPU_CLEAR_MASK				0x04000000
+#define A6XX_GMU_OOB_PERFCNTR_SET_MASK				0x00020000
+#define A6XX_GMU_OOB_PERFCNTR_CHECK_MASK			0x02000000
+#define A6XX_GMU_OOB_PERFCNTR_CLEAR_MASK			0x02000000
+#define A6XX_HFI_IRQ_MSGQ_MASK					0x00000001
+#define A6XX_HFI_IRQ_DSGQ_MASK					0x00000002
+#define A6XX_HFI_IRQ_BLOCKED_MSG_MASK				0x00000004
+#define A6XX_HFI_IRQ_CM3_FAULT_MASK				0x00800000
+#define A6XX_HFI_IRQ_GMU_ERR_MASK__MASK				0x007f0000
+#define A6XX_HFI_IRQ_GMU_ERR_MASK__SHIFT			16
+static inline uint32_t A6XX_HFI_IRQ_GMU_ERR_MASK(uint32_t val)
+{
+	return ((val) << A6XX_HFI_IRQ_GMU_ERR_MASK__SHIFT) & A6XX_HFI_IRQ_GMU_ERR_MASK__MASK;
+}
+#define A6XX_HFI_IRQ_OOB_MASK__MASK				0xff000000
+#define A6XX_HFI_IRQ_OOB_MASK__SHIFT				24
+static inline uint32_t A6XX_HFI_IRQ_OOB_MASK(uint32_t val)
+{
+	return ((val) << A6XX_HFI_IRQ_OOB_MASK__SHIFT) & A6XX_HFI_IRQ_OOB_MASK__MASK;
+}
+#define A6XX_HFI_H2F_IRQ_MASK_BIT				0x00000001
+#define REG_A6XX_GPU_GMU_GX_SPTPRAC_CLOCK_CONTROL		0x00000080
+
+#define REG_A6XX_GMU_GX_SPTPRAC_POWER_CONTROL			0x00000081
+
+#define REG_A6XX_GMU_CM3_ITCM_START				0x00000c00
+
+#define REG_A6XX_GMU_CM3_DTCM_START				0x00001c00
+
+#define REG_A6XX_GMU_NMI_CONTROL_STATUS				0x000023f0
+
+#define REG_A6XX_GMU_BOOT_SLUMBER_OPTION			0x000023f8
+
+#define REG_A6XX_GMU_GX_VOTE_IDX				0x000023f9
+
+#define REG_A6XX_GMU_MX_VOTE_IDX				0x000023fa
+
+#define REG_A6XX_GMU_DCVS_ACK_OPTION				0x000023fc
+
+#define REG_A6XX_GMU_DCVS_PERF_SETTING				0x000023fd
+
+#define REG_A6XX_GMU_DCVS_BW_SETTING				0x000023fe
+
+#define REG_A6XX_GMU_DCVS_RETURN				0x000023ff
+
+#define REG_A6XX_GMU_SYS_BUS_CONFIG				0x00004c0f
+
+#define REG_A6XX_GMU_CM3_SYSRESET				0x00005000
+
+#define REG_A6XX_GMU_CM3_BOOT_CONFIG				0x00005001
+
+#define REG_A6XX_GMU_CM3_FW_BUSY				0x0000501a
+
+#define REG_A6XX_GMU_CM3_FW_INIT_RESULT				0x0000501c
+
+#define REG_A6XX_GMU_CM3_CFG					0x0000502d
+
+#define REG_A6XX_GMU_CX_GMU_POWER_COUNTER_ENABLE		0x00005040
+
+#define REG_A6XX_GMU_CX_GMU_POWER_COUNTER_SELECT_0		0x00005041
+
+#define REG_A6XX_GMU_CX_GMU_POWER_COUNTER_SELECT_1		0x00005042
+
+#define REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_0_L		0x00005044
+
+#define REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_0_H		0x00005045
+
+#define REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_1_L		0x00005046
+
+#define REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_1_H		0x00005047
+
+#define REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_2_L		0x00005048
+
+#define REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_2_H		0x00005049
+
+#define REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_3_L		0x0000504a
+
+#define REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_3_H		0x0000504b
+
+#define REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_4_L		0x0000504c
+
+#define REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_4_H		0x0000504d
+
+#define REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_5_L		0x0000504e
+
+#define REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_5_H		0x0000504f
+
+#define REG_A6XX_GMU_PWR_COL_INTER_FRAME_CTRL			0x000050c0
+#define A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_IFPC_ENABLE		0x00000001
+#define A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_HM_POWER_COLLAPSE_ENABLE	0x00000002
+#define A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_SPTPRAC_POWER_CONTROL_ENABLE	0x00000004
+#define A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_NUM_PASS_SKIPS__MASK	0x00003c00
+#define A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_NUM_PASS_SKIPS__SHIFT	10
+static inline uint32_t A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_NUM_PASS_SKIPS(uint32_t val)
+{
+	return ((val) << A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_NUM_PASS_SKIPS__SHIFT) & A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_NUM_PASS_SKIPS__MASK;
+}
+#define A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_MIN_PASS_LENGTH__MASK	0xffffc000
+#define A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_MIN_PASS_LENGTH__SHIFT	14
+static inline uint32_t A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_MIN_PASS_LENGTH(uint32_t val)
+{
+	return ((val) << A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_MIN_PASS_LENGTH__SHIFT) & A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_MIN_PASS_LENGTH__MASK;
+}
+
+#define REG_A6XX_GMU_PWR_COL_INTER_FRAME_HYST			0x000050c1
+
+#define REG_A6XX_GMU_PWR_COL_SPTPRAC_HYST			0x000050c2
+
+#define REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS			0x000050d0
+#define A6XX_GMU_SPTPRAC_PWR_CLK_STATUS_SPTPRAC_GDSC_POWERING_OFF	0x00000001
+#define A6XX_GMU_SPTPRAC_PWR_CLK_STATUS_SPTPRAC_GDSC_POWERING_ON	0x00000002
+#define A6XX_GMU_SPTPRAC_PWR_CLK_STATUS_SPTPRAC_GDSC_POWER_ON	0x00000004
+#define A6XX_GMU_SPTPRAC_PWR_CLK_STATUS_SPTPRAC_GDSC_POWER_OFF	0x00000008
+#define A6XX_GMU_SPTPRAC_PWR_CLK_STATUS_SP_CLOCK_OFF		0x00000010
+#define A6XX_GMU_SPTPRAC_PWR_CLK_STATUS_GMU_UP_POWER_STATE	0x00000020
+#define A6XX_GMU_SPTPRAC_PWR_CLK_STATUS_GX_HM_GDSC_POWER_OFF	0x00000040
+#define A6XX_GMU_SPTPRAC_PWR_CLK_STATUS_GX_HM_CLK_OFF		0x00000080
+
+#define REG_A6XX_GMU_GPU_NAP_CTRL				0x000050e4
+#define A6XX_GMU_GPU_NAP_CTRL_HW_NAP_ENABLE			0x00000001
+#define A6XX_GMU_GPU_NAP_CTRL_SID__MASK				0x000001f0
+#define A6XX_GMU_GPU_NAP_CTRL_SID__SHIFT			4
+static inline uint32_t A6XX_GMU_GPU_NAP_CTRL_SID(uint32_t val)
+{
+	return ((val) << A6XX_GMU_GPU_NAP_CTRL_SID__SHIFT) & A6XX_GMU_GPU_NAP_CTRL_SID__MASK;
+}
+
+#define REG_A6XX_GMU_RPMH_CTRL					0x000050e8
+#define A6XX_GMU_RPMH_CTRL_RPMH_INTERFACE_ENABLE		0x00000001
+#define A6XX_GMU_RPMH_CTRL_LLC_VOTE_ENABLE			0x00000010
+#define A6XX_GMU_RPMH_CTRL_DDR_VOTE_ENABLE			0x00000100
+#define A6XX_GMU_RPMH_CTRL_MX_VOTE_ENABLE			0x00000200
+#define A6XX_GMU_RPMH_CTRL_CX_VOTE_ENABLE			0x00000400
+#define A6XX_GMU_RPMH_CTRL_GFX_VOTE_ENABLE			0x00000800
+#define A6XX_GMU_RPMH_CTRL_DDR_MIN_VOTE_ENABLE			0x00001000
+#define A6XX_GMU_RPMH_CTRL_MX_MIN_VOTE_ENABLE			0x00002000
+#define A6XX_GMU_RPMH_CTRL_CX_MIN_VOTE_ENABLE			0x00004000
+#define A6XX_GMU_RPMH_CTRL_GFX_MIN_VOTE_ENABLE			0x00008000
+
+#define REG_A6XX_GMU_RPMH_HYST_CTRL				0x000050e9
+
+#define REG_A6XX_GPU_GMU_CX_GMU_RPMH_POWER_STATE		0x000050ec
+
+#define REG_A6XX_GMU_BOOT_KMD_LM_HANDSHAKE			0x000051f0
+
+#define REG_A6XX_GMU_LLM_GLM_SLEEP_CTRL				0x00005157
+
+#define REG_A6XX_GMU_LLM_GLM_SLEEP_STATUS			0x00005158
+
+#define REG_A6XX_GMU_ALWAYS_ON_COUNTER_L			0x00005088
+
+#define REG_A6XX_GMU_ALWAYS_ON_COUNTER_H			0x00005089
+
+#define REG_A6XX_GMU_GMU_PWR_COL_KEEPALIVE			0x000050c3
+
+#define REG_A6XX_GMU_HFI_CTRL_STATUS				0x00005180
+
+#define REG_A6XX_GMU_HFI_VERSION_INFO				0x00005181
+
+#define REG_A6XX_GMU_HFI_SFR_ADDR				0x00005182
+
+#define REG_A6XX_GMU_HFI_MMAP_ADDR				0x00005183
+
+#define REG_A6XX_GMU_HFI_QTBL_INFO				0x00005184
+
+#define REG_A6XX_GMU_HFI_QTBL_ADDR				0x00005185
+
+#define REG_A6XX_GMU_HFI_CTRL_INIT				0x00005186
+
+#define REG_A6XX_GMU_GMU2HOST_INTR_SET				0x00005190
+
+#define REG_A6XX_GMU_GMU2HOST_INTR_CLR				0x00005191
+
+#define REG_A6XX_GMU_GMU2HOST_INTR_INFO				0x00005192
+#define A6XX_GMU_GMU2HOST_INTR_INFO_MSGQ			0x00000001
+#define A6XX_GMU_GMU2HOST_INTR_INFO_CM3_FAULT			0x00800000
+
+#define REG_A6XX_GMU_GMU2HOST_INTR_MASK				0x00005193
+
+#define REG_A6XX_GMU_HOST2GMU_INTR_SET				0x00005194
+
+#define REG_A6XX_GMU_HOST2GMU_INTR_CLR				0x00005195
+
+#define REG_A6XX_GMU_HOST2GMU_INTR_RAW_INFO			0x00005196
+
+#define REG_A6XX_GMU_HOST2GMU_INTR_EN_0				0x00005197
+
+#define REG_A6XX_GMU_HOST2GMU_INTR_EN_1				0x00005198
+
+#define REG_A6XX_GMU_HOST2GMU_INTR_EN_2				0x00005199
+
+#define REG_A6XX_GMU_HOST2GMU_INTR_EN_3				0x0000519a
+
+#define REG_A6XX_GMU_HOST2GMU_INTR_INFO_0			0x0000519b
+
+#define REG_A6XX_GMU_HOST2GMU_INTR_INFO_1			0x0000519c
+
+#define REG_A6XX_GMU_HOST2GMU_INTR_INFO_2			0x0000519d
+
+#define REG_A6XX_GMU_HOST2GMU_INTR_INFO_3			0x0000519e
+
+#define REG_A6XX_GMU_GENERAL_1					0x000051c6
+
+#define REG_A6XX_GMU_GENERAL_7					0x000051cc
+
+#define REG_A6XX_GMU_ISENSE_CTRL				0x0000515d
+
+#define REG_A6XX_GPU_CS_ENABLE_REG				0x00008920
+
+#define REG_A6XX_GPU_GMU_CX_GMU_ISENSE_CTRL			0x0000515d
+
+#define REG_A6XX_GPU_CS_AMP_CALIBRATION_CONTROL3		0x00008578
+
+#define REG_A6XX_GPU_CS_AMP_CALIBRATION_CONTROL2		0x00008558
+
+#define REG_A6XX_GPU_CS_A_SENSOR_CTRL_0				0x00008580
+
+#define REG_A6XX_GPU_CS_A_SENSOR_CTRL_2				0x00027ada
+
+#define REG_A6XX_GPU_CS_SENSOR_GENERAL_STATUS			0x0000881a
+
+#define REG_A6XX_GPU_CS_AMP_CALIBRATION_CONTROL1		0x00008957
+
+#define REG_A6XX_GPU_CS_SENSOR_GENERAL_STATUS			0x0000881a
+
+#define REG_A6XX_GPU_CS_AMP_CALIBRATION_STATUS1_0		0x0000881d
+
+#define REG_A6XX_GPU_CS_AMP_CALIBRATION_STATUS1_2		0x0000881f
+
+#define REG_A6XX_GPU_CS_AMP_CALIBRATION_STATUS1_4		0x00008821
+
+#define REG_A6XX_GPU_CS_AMP_CALIBRATION_DONE			0x00008965
+
+#define REG_A6XX_GPU_CS_AMP_PERIOD_CTRL				0x0000896d
+
+#define REG_A6XX_GPU_CS_AMP_CALIBRATION_DONE			0x00008965
+
+#define REG_A6XX_GPU_GMU_CX_GMU_PWR_THRESHOLD			0x0000514d
+
+#define REG_A6XX_GMU_AO_INTERRUPT_EN				0x00009303
+
+#define REG_A6XX_GMU_AO_HOST_INTERRUPT_CLR			0x00009304
+
+#define REG_A6XX_GMU_AO_HOST_INTERRUPT_STATUS			0x00009305
+#define A6XX_GMU_AO_HOST_INTERRUPT_STATUS_WDOG_BITE		0x00000001
+#define A6XX_GMU_AO_HOST_INTERRUPT_STATUS_RSCC_COMP		0x00000002
+#define A6XX_GMU_AO_HOST_INTERRUPT_STATUS_VDROOP		0x00000004
+#define A6XX_GMU_AO_HOST_INTERRUPT_STATUS_FENCE_ERR		0x00000008
+#define A6XX_GMU_AO_HOST_INTERRUPT_STATUS_DBD_WAKEUP		0x00000010
+#define A6XX_GMU_AO_HOST_INTERRUPT_STATUS_HOST_AHB_BUS_ERROR	0x00000020
+
+#define REG_A6XX_GMU_AO_HOST_INTERRUPT_MASK			0x00009306
+
+#define REG_A6XX_GPU_GMU_AO_GMU_CGC_MODE_CNTL			0x00009309
+
+#define REG_A6XX_GPU_GMU_AO_GMU_CGC_DELAY_CNTL			0x0000930a
+
+#define REG_A6XX_GPU_GMU_AO_GMU_CGC_HYST_CNTL			0x0000930b
+
+#define REG_A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS			0x0000930c
+#define A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS_GPUBUSYIGNAHB	0x00800000
+
+#define REG_A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS2			0x0000930d
+
+#define REG_A6XX_GPU_GMU_AO_GPU_CX_BUSY_MASK			0x0000930e
+
+#define REG_A6XX_GMU_AO_AHB_FENCE_CTRL				0x00009310
+
+#define REG_A6XX_GMU_AHB_FENCE_STATUS				0x00009313
+
+#define REG_A6XX_GMU_RBBM_INT_UNMASKED_STATUS			0x00009315
+
+#define REG_A6XX_GMU_AO_SPARE_CNTL				0x00009316
+
+#define REG_A6XX_GPU_RSCC_RSC_STATUS0_DRV0			0x00008c04
+
+#define REG_A6XX_GMU_RSCC_CONTROL_REQ				0x00009307
+
+#define REG_A6XX_GMU_RSCC_CONTROL_ACK				0x00009308
+
+#define REG_A6XX_GMU_AHB_FENCE_RANGE_0				0x00009311
+
+#define REG_A6XX_GMU_AHB_FENCE_RANGE_1				0x00009312
+
+#define REG_A6XX_GPU_CC_GX_GDSCR				0x00009c03
+
+#define REG_A6XX_GPU_CC_GX_DOMAIN_MISC				0x00009d42
+
+#define REG_A6XX_RSCC_PDC_SEQ_START_ADDR			0x00008c08
+
+#define REG_A6XX_RSCC_PDC_MATCH_VALUE_LO			0x00008c09
+
+#define REG_A6XX_RSCC_PDC_MATCH_VALUE_HI			0x00008c0a
+
+#define REG_A6XX_RSCC_PDC_SLAVE_ID_DRV0				0x00008c0b
+
+#define REG_A6XX_RSCC_HIDDEN_TCS_CMD0_ADDR			0x00008c0d
+
+#define REG_A6XX_RSCC_HIDDEN_TCS_CMD0_DATA			0x00008c0e
+
+#define REG_A6XX_RSCC_TIMESTAMP_UNIT0_TIMESTAMP_L_DRV0		0x00008c82
+
+#define REG_A6XX_RSCC_TIMESTAMP_UNIT0_TIMESTAMP_H_DRV0		0x00008c83
+
+#define REG_A6XX_RSCC_TIMESTAMP_UNIT1_EN_DRV0			0x00008c89
+
+#define REG_A6XX_RSCC_TIMESTAMP_UNIT1_OUTPUT_DRV0		0x00008c8c
+
+#define REG_A6XX_RSCC_OVERRIDE_START_ADDR			0x00008d00
+
+#define REG_A6XX_RSCC_SEQ_BUSY_DRV0				0x00008d01
+
+#define REG_A6XX_RSCC_SEQ_MEM_0_DRV0				0x00008d80
+
+#define REG_A6XX_RSCC_TCS0_DRV0_STATUS				0x00008f46
+
+#define REG_A6XX_RSCC_TCS1_DRV0_STATUS				0x000090ae
+
+#define REG_A6XX_RSCC_TCS2_DRV0_STATUS				0x00009216
+
+#define REG_A6XX_RSCC_TCS3_DRV0_STATUS				0x0000937e
+
+
+#endif /* A6XX_GMU_XML */
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
new file mode 100644
index 000000000000..c629f742a1d1
--- /dev/null
+++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
@@ -0,0 +1,818 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2017-2018 The Linux Foundation. All rights reserved. */
+
+
+#include "msm_gem.h"
+#include "msm_mmu.h"
+#include "a6xx_gpu.h"
+#include "a6xx_gmu.xml.h"
+
+static inline bool _a6xx_check_idle(struct msm_gpu *gpu)
+{
+	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
+	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
+
+	/* Check that the GMU is idle */
+	if (!a6xx_gmu_isidle(&a6xx_gpu->gmu))
+		return false;
+
+	/* Check tha the CX master is idle */
+	if (gpu_read(gpu, REG_A6XX_RBBM_STATUS) &
+			~A6XX_RBBM_STATUS_CP_AHB_BUSY_CX_MASTER)
+		return false;
+
+	return !(gpu_read(gpu, REG_A6XX_RBBM_INT_0_STATUS) &
+		A6XX_RBBM_INT_0_MASK_RBBM_HANG_DETECT);
+}
+
+bool a6xx_idle(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
+{
+	/* wait for CP to drain ringbuffer: */
+	if (!adreno_idle(gpu, ring))
+		return false;
+
+	if (spin_until(_a6xx_check_idle(gpu))) {
+		DRM_ERROR("%s: %ps: timeout waiting for GPU to idle: status %8.8X irq %8.8X rptr/wptr %d/%d\n",
+			gpu->name, __builtin_return_address(0),
+			gpu_read(gpu, REG_A6XX_RBBM_STATUS),
+			gpu_read(gpu, REG_A6XX_RBBM_INT_0_STATUS),
+			gpu_read(gpu, REG_A6XX_CP_RB_RPTR),
+			gpu_read(gpu, REG_A6XX_CP_RB_WPTR));
+		return false;
+	}
+
+	return true;
+}
+
+static void a6xx_flush(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
+{
+	uint32_t wptr;
+	unsigned long flags;
+
+	spin_lock_irqsave(&ring->lock, flags);
+
+	/* Copy the shadow to the actual register */
+	ring->cur = ring->next;
+
+	/* Make sure to wrap wptr if we need to */
+	wptr = get_wptr(ring);
+
+	spin_unlock_irqrestore(&ring->lock, flags);
+
+	/* Make sure everything is posted before making a decision */
+	mb();
+
+	gpu_write(gpu, REG_A6XX_CP_RB_WPTR, wptr);
+}
+
+static void a6xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit,
+	struct msm_file_private *ctx)
+{
+	struct msm_drm_private *priv = gpu->dev->dev_private;
+	struct msm_ringbuffer *ring = submit->ring;
+	unsigned int i;
+
+	/* Invalidate CCU depth and color */
+	OUT_PKT7(ring, CP_EVENT_WRITE, 1);
+	OUT_RING(ring, PC_CCU_INVALIDATE_DEPTH);
+
+	OUT_PKT7(ring, CP_EVENT_WRITE, 1);
+	OUT_RING(ring, PC_CCU_INVALIDATE_COLOR);
+
+	/* Submit the commands */
+	for (i = 0; i < submit->nr_cmds; i++) {
+		switch (submit->cmd[i].type) {
+		case MSM_SUBMIT_CMD_IB_TARGET_BUF:
+			break;
+		case MSM_SUBMIT_CMD_CTX_RESTORE_BUF:
+			if (priv->lastctx == ctx)
+				break;
+		case MSM_SUBMIT_CMD_BUF:
+			OUT_PKT7(ring, CP_INDIRECT_BUFFER_PFE, 3);
+			OUT_RING(ring, lower_32_bits(submit->cmd[i].iova));
+			OUT_RING(ring, upper_32_bits(submit->cmd[i].iova));
+			OUT_RING(ring, submit->cmd[i].size);
+			break;
+		}
+	}
+
+	/* Write the fence to the scratch register */
+	OUT_PKT4(ring, REG_A6XX_CP_SCRATCH_REG(2), 1);
+	OUT_RING(ring, submit->seqno);
+
+	/*
+	 * Execute a CACHE_FLUSH_TS event. This will ensure that the
+	 * timestamp is written to the memory and then triggers the interrupt
+	 */
+	OUT_PKT7(ring, CP_EVENT_WRITE, 4);
+	OUT_RING(ring, CACHE_FLUSH_TS | (1 << 31));
+	OUT_RING(ring, lower_32_bits(rbmemptr(ring, fence)));
+	OUT_RING(ring, upper_32_bits(rbmemptr(ring, fence)));
+	OUT_RING(ring, submit->seqno);
+
+	a6xx_flush(gpu, ring);
+}
+
+static const struct {
+	u32 offset;
+	u32 value;
+} a6xx_hwcg[] = {
+	{REG_A6XX_RBBM_CLOCK_CNTL_SP0, 0x22222222},
+	{REG_A6XX_RBBM_CLOCK_CNTL_SP1, 0x22222222},
+	{REG_A6XX_RBBM_CLOCK_CNTL_SP2, 0x22222222},
+	{REG_A6XX_RBBM_CLOCK_CNTL_SP3, 0x22222222},
+	{REG_A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02022220},
+	{REG_A6XX_RBBM_CLOCK_CNTL2_SP1, 0x02022220},
+	{REG_A6XX_RBBM_CLOCK_CNTL2_SP2, 0x02022220},
+	{REG_A6XX_RBBM_CLOCK_CNTL2_SP3, 0x02022220},
+	{REG_A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080},
+	{REG_A6XX_RBBM_CLOCK_DELAY_SP1, 0x00000080},
+	{REG_A6XX_RBBM_CLOCK_DELAY_SP2, 0x00000080},
+	{REG_A6XX_RBBM_CLOCK_DELAY_SP3, 0x00000080},
+	{REG_A6XX_RBBM_CLOCK_HYST_SP0, 0x0000f3cf},
+	{REG_A6XX_RBBM_CLOCK_HYST_SP1, 0x0000f3cf},
+	{REG_A6XX_RBBM_CLOCK_HYST_SP2, 0x0000f3cf},
+	{REG_A6XX_RBBM_CLOCK_HYST_SP3, 0x0000f3cf},
+	{REG_A6XX_RBBM_CLOCK_CNTL_TP0, 0x02222222},
+	{REG_A6XX_RBBM_CLOCK_CNTL_TP1, 0x02222222},
+	{REG_A6XX_RBBM_CLOCK_CNTL_TP2, 0x02222222},
+	{REG_A6XX_RBBM_CLOCK_CNTL_TP3, 0x02222222},
+	{REG_A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222},
+	{REG_A6XX_RBBM_CLOCK_CNTL2_TP1, 0x22222222},
+	{REG_A6XX_RBBM_CLOCK_CNTL2_TP2, 0x22222222},
+	{REG_A6XX_RBBM_CLOCK_CNTL2_TP3, 0x22222222},
+	{REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222},
+	{REG_A6XX_RBBM_CLOCK_CNTL3_TP1, 0x22222222},
+	{REG_A6XX_RBBM_CLOCK_CNTL3_TP2, 0x22222222},
+	{REG_A6XX_RBBM_CLOCK_CNTL3_TP3, 0x22222222},
+	{REG_A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222},
+	{REG_A6XX_RBBM_CLOCK_CNTL4_TP1, 0x00022222},
+	{REG_A6XX_RBBM_CLOCK_CNTL4_TP2, 0x00022222},
+	{REG_A6XX_RBBM_CLOCK_CNTL4_TP3, 0x00022222},
+	{REG_A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777},
+	{REG_A6XX_RBBM_CLOCK_HYST_TP1, 0x77777777},
+	{REG_A6XX_RBBM_CLOCK_HYST_TP2, 0x77777777},
+	{REG_A6XX_RBBM_CLOCK_HYST_TP3, 0x77777777},
+	{REG_A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777},
+	{REG_A6XX_RBBM_CLOCK_HYST2_TP1, 0x77777777},
+	{REG_A6XX_RBBM_CLOCK_HYST2_TP2, 0x77777777},
+	{REG_A6XX_RBBM_CLOCK_HYST2_TP3, 0x77777777},
+	{REG_A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777},
+	{REG_A6XX_RBBM_CLOCK_HYST3_TP1, 0x77777777},
+	{REG_A6XX_RBBM_CLOCK_HYST3_TP2, 0x77777777},
+	{REG_A6XX_RBBM_CLOCK_HYST3_TP3, 0x77777777},
+	{REG_A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777},
+	{REG_A6XX_RBBM_CLOCK_HYST4_TP1, 0x00077777},
+	{REG_A6XX_RBBM_CLOCK_HYST4_TP2, 0x00077777},
+	{REG_A6XX_RBBM_CLOCK_HYST4_TP3, 0x00077777},
+	{REG_A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111},
+	{REG_A6XX_RBBM_CLOCK_DELAY_TP1, 0x11111111},
+	{REG_A6XX_RBBM_CLOCK_DELAY_TP2, 0x11111111},
+	{REG_A6XX_RBBM_CLOCK_DELAY_TP3, 0x11111111},
+	{REG_A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111},
+	{REG_A6XX_RBBM_CLOCK_DELAY2_TP1, 0x11111111},
+	{REG_A6XX_RBBM_CLOCK_DELAY2_TP2, 0x11111111},
+	{REG_A6XX_RBBM_CLOCK_DELAY2_TP3, 0x11111111},
+	{REG_A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111},
+	{REG_A6XX_RBBM_CLOCK_DELAY3_TP1, 0x11111111},
+	{REG_A6XX_RBBM_CLOCK_DELAY3_TP2, 0x11111111},
+	{REG_A6XX_RBBM_CLOCK_DELAY3_TP3, 0x11111111},
+	{REG_A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111},
+	{REG_A6XX_RBBM_CLOCK_DELAY4_TP1, 0x00011111},
+	{REG_A6XX_RBBM_CLOCK_DELAY4_TP2, 0x00011111},
+	{REG_A6XX_RBBM_CLOCK_DELAY4_TP3, 0x00011111},
+	{REG_A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222},
+	{REG_A6XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222},
+	{REG_A6XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222},
+	{REG_A6XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222},
+	{REG_A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004},
+	{REG_A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002},
+	{REG_A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222},
+	{REG_A6XX_RBBM_CLOCK_CNTL_RB1, 0x22222222},
+	{REG_A6XX_RBBM_CLOCK_CNTL_RB2, 0x22222222},
+	{REG_A6XX_RBBM_CLOCK_CNTL_RB3, 0x22222222},
+	{REG_A6XX_RBBM_CLOCK_CNTL2_RB0, 0x00002222},
+	{REG_A6XX_RBBM_CLOCK_CNTL2_RB1, 0x00002222},
+	{REG_A6XX_RBBM_CLOCK_CNTL2_RB2, 0x00002222},
+	{REG_A6XX_RBBM_CLOCK_CNTL2_RB3, 0x00002222},
+	{REG_A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220},
+	{REG_A6XX_RBBM_CLOCK_CNTL_CCU1, 0x00002220},
+	{REG_A6XX_RBBM_CLOCK_CNTL_CCU2, 0x00002220},
+	{REG_A6XX_RBBM_CLOCK_CNTL_CCU3, 0x00002220},
+	{REG_A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040f00},
+	{REG_A6XX_RBBM_CLOCK_HYST_RB_CCU1, 0x00040f00},
+	{REG_A6XX_RBBM_CLOCK_HYST_RB_CCU2, 0x00040f00},
+	{REG_A6XX_RBBM_CLOCK_HYST_RB_CCU3, 0x00040f00},
+	{REG_A6XX_RBBM_CLOCK_CNTL_RAC, 0x05022022},
+	{REG_A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555},
+	{REG_A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011},
+	{REG_A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044},
+	{REG_A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222},
+	{REG_A6XX_RBBM_CLOCK_MODE_GPC, 0x00222222},
+	{REG_A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222},
+	{REG_A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000},
+	{REG_A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004},
+	{REG_A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000},
+	{REG_A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000},
+	{REG_A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000},
+	{REG_A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200},
+	{REG_A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222},
+	{REG_A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002},
+	{REG_A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222},
+	{REG_A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222},
+	{REG_A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111},
+	{REG_A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555}
+};
+
+static void a6xx_set_hwcg(struct msm_gpu *gpu, bool state)
+{
+	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
+	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
+	struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
+	unsigned int i;
+	u32 val;
+
+	val = gpu_read(gpu, REG_A6XX_RBBM_CLOCK_CNTL);
+
+	/* Don't re-program the registers if they are already correct */
+	if ((!state && !val) || (state && (val == 0x8aa8aa02)))
+		return;
+
+	/* Disable SP clock before programming HWCG registers */
+	gmu_rmw(gmu, REG_A6XX_GPU_GMU_GX_SPTPRAC_CLOCK_CONTROL, 1, 0);
+
+	for (i = 0; i < ARRAY_SIZE(a6xx_hwcg); i++)
+		gpu_write(gpu, a6xx_hwcg[i].offset,
+			state ? a6xx_hwcg[i].value : 0);
+
+	/* Enable SP clock */
+	gmu_rmw(gmu, REG_A6XX_GPU_GMU_GX_SPTPRAC_CLOCK_CONTROL, 0, 1);
+
+	gpu_write(gpu, REG_A6XX_RBBM_CLOCK_CNTL, state ? 0x8aa8aa02 : 0);
+}
+
+static int a6xx_cp_init(struct msm_gpu *gpu)
+{
+	struct msm_ringbuffer *ring = gpu->rb[0];
+
+	OUT_PKT7(ring, CP_ME_INIT, 8);
+
+	OUT_RING(ring, 0x0000002f);
+
+	/* Enable multiple hardware contexts */
+	OUT_RING(ring, 0x00000003);
+
+	/* Enable error detection */
+	OUT_RING(ring, 0x20000000);
+
+	/* Don't enable header dump */
+	OUT_RING(ring, 0x00000000);
+	OUT_RING(ring, 0x00000000);
+
+	/* No workarounds enabled */
+	OUT_RING(ring, 0x00000000);
+
+	/* Pad rest of the cmds with 0's */
+	OUT_RING(ring, 0x00000000);
+	OUT_RING(ring, 0x00000000);
+
+	a6xx_flush(gpu, ring);
+	return a6xx_idle(gpu, ring) ? 0 : -EINVAL;
+}
+
+static int a6xx_ucode_init(struct msm_gpu *gpu)
+{
+	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
+	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
+
+	if (!a6xx_gpu->sqe_bo) {
+		a6xx_gpu->sqe_bo = adreno_fw_create_bo(gpu,
+			adreno_gpu->fw[ADRENO_FW_SQE], &a6xx_gpu->sqe_iova);
+
+		if (IS_ERR(a6xx_gpu->sqe_bo)) {
+			int ret = PTR_ERR(a6xx_gpu->sqe_bo);
+
+			a6xx_gpu->sqe_bo = NULL;
+			DRM_DEV_ERROR(&gpu->pdev->dev,
+				"Could not allocate SQE ucode: %d\n", ret);
+
+			return ret;
+		}
+	}
+
+	gpu_write64(gpu, REG_A6XX_CP_SQE_INSTR_BASE_LO,
+		REG_A6XX_CP_SQE_INSTR_BASE_HI, a6xx_gpu->sqe_iova);
+
+	return 0;
+}
+
+#define A6XX_INT_MASK (A6XX_RBBM_INT_0_MASK_CP_AHB_ERROR | \
+	  A6XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNCFIFO_OVERFLOW | \
+	  A6XX_RBBM_INT_0_MASK_CP_HW_ERROR | \
+	  A6XX_RBBM_INT_0_MASK_CP_IB2 | \
+	  A6XX_RBBM_INT_0_MASK_CP_IB1 | \
+	  A6XX_RBBM_INT_0_MASK_CP_RB | \
+	  A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS | \
+	  A6XX_RBBM_INT_0_MASK_RBBM_ATB_BUS_OVERFLOW | \
+	  A6XX_RBBM_INT_0_MASK_RBBM_HANG_DETECT | \
+	  A6XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS | \
+	  A6XX_RBBM_INT_0_MASK_UCHE_TRAP_INTR)
+
+static int a6xx_hw_init(struct msm_gpu *gpu)
+{
+	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
+	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
+	int ret;
+
+	/* Make sure the GMU keeps the GPU on while we set it up */
+	a6xx_gmu_set_oob(&a6xx_gpu->gmu, GMU_OOB_GPU_SET);
+
+	gpu_write(gpu, REG_A6XX_RBBM_SECVID_TSB_CNTL, 0);
+
+	/*
+	 * Disable the trusted memory range - we don't actually supported secure
+	 * memory rendering at this point in time and we don't want to block off
+	 * part of the virtual memory space.
+	 */
+	gpu_write64(gpu, REG_A6XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO,
+		REG_A6XX_RBBM_SECVID_TSB_TRUSTED_BASE_HI, 0x00000000);
+	gpu_write(gpu, REG_A6XX_RBBM_SECVID_TSB_TRUSTED_SIZE, 0x00000000);
+
+	/* enable hardware clockgating */
+	a6xx_set_hwcg(gpu, true);
+
+	/* VBIF start */
+	gpu_write(gpu, REG_A6XX_VBIF_GATE_OFF_WRREQ_EN, 0x00000009);
+	gpu_write(gpu, REG_A6XX_RBBM_VBIF_CLIENT_QOS_CNTL, 0x3);
+
+	/* Make all blocks contribute to the GPU BUSY perf counter */
+	gpu_write(gpu, REG_A6XX_RBBM_PERFCTR_GPU_BUSY_MASKED, 0xffffffff);
+
+	/* Disable L2 bypass in the UCHE */
+	gpu_write(gpu, REG_A6XX_UCHE_WRITE_RANGE_MAX_LO, 0xffffffc0);
+	gpu_write(gpu, REG_A6XX_UCHE_WRITE_RANGE_MAX_HI, 0x0001ffff);
+	gpu_write(gpu, REG_A6XX_UCHE_TRAP_BASE_LO, 0xfffff000);
+	gpu_write(gpu, REG_A6XX_UCHE_TRAP_BASE_HI, 0x0001ffff);
+	gpu_write(gpu, REG_A6XX_UCHE_WRITE_THRU_BASE_LO, 0xfffff000);
+	gpu_write(gpu, REG_A6XX_UCHE_WRITE_THRU_BASE_HI, 0x0001ffff);
+
+	/* Set the GMEM VA range [0x100000:0x100000 + gpu->gmem - 1] */
+	gpu_write64(gpu, REG_A6XX_UCHE_GMEM_RANGE_MIN_LO,
+		REG_A6XX_UCHE_GMEM_RANGE_MIN_HI, 0x00100000);
+
+	gpu_write64(gpu, REG_A6XX_UCHE_GMEM_RANGE_MAX_LO,
+		REG_A6XX_UCHE_GMEM_RANGE_MAX_HI,
+		0x00100000 + adreno_gpu->gmem - 1);
+
+	gpu_write(gpu, REG_A6XX_UCHE_FILTER_CNTL, 0x804);
+	gpu_write(gpu, REG_A6XX_UCHE_CACHE_WAYS, 0x4);
+
+	gpu_write(gpu, REG_A6XX_CP_ROQ_THRESHOLDS_2, 0x010000c0);
+	gpu_write(gpu, REG_A6XX_CP_ROQ_THRESHOLDS_1, 0x8040362c);
+
+	/* Setting the mem pool size */
+	gpu_write(gpu, REG_A6XX_CP_MEM_POOL_SIZE, 128);
+
+	/* Setting the primFifo thresholds default values */
+	gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, (0x300 << 11));
+
+	/* Set the AHB default slave response to "ERROR" */
+	gpu_write(gpu, REG_A6XX_CP_AHB_CNTL, 0x1);
+
+	/* Turn on performance counters */
+	gpu_write(gpu, REG_A6XX_RBBM_PERFCTR_CNTL, 0x1);
+
+	/* Select CP0 to always count cycles */
+	gpu_write(gpu, REG_A6XX_CP_PERFCTR_CP_SEL_0, PERF_CP_ALWAYS_COUNT);
+
+	/* FIXME: not sure if this should live here or in a6xx_gmu.c */
+	gmu_write(&a6xx_gpu->gmu,  REG_A6XX_GPU_GMU_AO_GPU_CX_BUSY_MASK,
+		0xff000000);
+	gmu_rmw(&a6xx_gpu->gmu, REG_A6XX_GMU_CX_GMU_POWER_COUNTER_SELECT_0,
+		0xff, 0x20);
+	gmu_write(&a6xx_gpu->gmu, REG_A6XX_GMU_CX_GMU_POWER_COUNTER_ENABLE,
+		0x01);
+
+	gpu_write(gpu, REG_A6XX_RB_NC_MODE_CNTL, 2 << 1);
+	gpu_write(gpu, REG_A6XX_TPL1_NC_MODE_CNTL, 2 << 1);
+	gpu_write(gpu, REG_A6XX_SP_NC_MODE_CNTL, 2 << 1);
+	gpu_write(gpu, REG_A6XX_UCHE_MODE_CNTL, 2 << 21);
+
+	/* Enable fault detection */
+	gpu_write(gpu, REG_A6XX_RBBM_INTERFACE_HANG_INT_CNTL,
+		(1 << 30) | 0x1fffff);
+
+	gpu_write(gpu, REG_A6XX_UCHE_CLIENT_PF, 1);
+
+	/* Protect registers from the CP */
+	gpu_write(gpu, REG_A6XX_CP_PROTECT_CNTL, 0x00000003);
+
+	gpu_write(gpu, REG_A6XX_CP_PROTECT(0),
+		A6XX_PROTECT_RDONLY(0x600, 0x51));
+	gpu_write(gpu, REG_A6XX_CP_PROTECT(1), A6XX_PROTECT_RW(0xae50, 0x2));
+	gpu_write(gpu, REG_A6XX_CP_PROTECT(2), A6XX_PROTECT_RW(0x9624, 0x13));
+	gpu_write(gpu, REG_A6XX_CP_PROTECT(3), A6XX_PROTECT_RW(0x8630, 0x8));
+	gpu_write(gpu, REG_A6XX_CP_PROTECT(4), A6XX_PROTECT_RW(0x9e70, 0x1));
+	gpu_write(gpu, REG_A6XX_CP_PROTECT(5), A6XX_PROTECT_RW(0x9e78, 0x187));
+	gpu_write(gpu, REG_A6XX_CP_PROTECT(6), A6XX_PROTECT_RW(0xf000, 0x810));
+	gpu_write(gpu, REG_A6XX_CP_PROTECT(7),
+		A6XX_PROTECT_RDONLY(0xfc00, 0x3));
+	gpu_write(gpu, REG_A6XX_CP_PROTECT(8), A6XX_PROTECT_RW(0x50e, 0x0));
+	gpu_write(gpu, REG_A6XX_CP_PROTECT(9), A6XX_PROTECT_RDONLY(0x50f, 0x0));
+	gpu_write(gpu, REG_A6XX_CP_PROTECT(10), A6XX_PROTECT_RW(0x510, 0x0));
+	gpu_write(gpu, REG_A6XX_CP_PROTECT(11),
+		A6XX_PROTECT_RDONLY(0x0, 0x4f9));
+	gpu_write(gpu, REG_A6XX_CP_PROTECT(12),
+		A6XX_PROTECT_RDONLY(0x501, 0xa));
+	gpu_write(gpu, REG_A6XX_CP_PROTECT(13),
+		A6XX_PROTECT_RDONLY(0x511, 0x44));
+	gpu_write(gpu, REG_A6XX_CP_PROTECT(14), A6XX_PROTECT_RW(0xe00, 0xe));
+	gpu_write(gpu, REG_A6XX_CP_PROTECT(15), A6XX_PROTECT_RW(0x8e00, 0x0));
+	gpu_write(gpu, REG_A6XX_CP_PROTECT(16), A6XX_PROTECT_RW(0x8e50, 0xf));
+	gpu_write(gpu, REG_A6XX_CP_PROTECT(17), A6XX_PROTECT_RW(0xbe02, 0x0));
+	gpu_write(gpu, REG_A6XX_CP_PROTECT(18),
+		A6XX_PROTECT_RW(0xbe20, 0x11f3));
+	gpu_write(gpu, REG_A6XX_CP_PROTECT(19), A6XX_PROTECT_RW(0x800, 0x82));
+	gpu_write(gpu, REG_A6XX_CP_PROTECT(20), A6XX_PROTECT_RW(0x8a0, 0x8));
+	gpu_write(gpu, REG_A6XX_CP_PROTECT(21), A6XX_PROTECT_RW(0x8ab, 0x19));
+	gpu_write(gpu, REG_A6XX_CP_PROTECT(22), A6XX_PROTECT_RW(0x900, 0x4d));
+	gpu_write(gpu, REG_A6XX_CP_PROTECT(23), A6XX_PROTECT_RW(0x98d, 0x76));
+	gpu_write(gpu, REG_A6XX_CP_PROTECT(24),
+			A6XX_PROTECT_RDONLY(0x8d0, 0x23));
+	gpu_write(gpu, REG_A6XX_CP_PROTECT(25),
+			A6XX_PROTECT_RDONLY(0x980, 0x4));
+	gpu_write(gpu, REG_A6XX_CP_PROTECT(26), A6XX_PROTECT_RW(0xa630, 0x0));
+
+	/* Enable interrupts */
+	gpu_write(gpu, REG_A6XX_RBBM_INT_0_MASK, A6XX_INT_MASK);
+
+	ret = adreno_hw_init(gpu);
+	if (ret)
+		goto out;
+
+	ret = a6xx_ucode_init(gpu);
+	if (ret)
+		goto out;
+
+	/* Always come up on rb 0 */
+	a6xx_gpu->cur_ring = gpu->rb[0];
+
+	/* Enable the SQE_to start the CP engine */
+	gpu_write(gpu, REG_A6XX_CP_SQE_CNTL, 1);
+
+	ret = a6xx_cp_init(gpu);
+	if (ret)
+		goto out;
+
+	gpu_write(gpu, REG_A6XX_RBBM_SECVID_TRUST_CNTL, 0x0);
+
+out:
+	/*
+	 * Tell the GMU that we are done touching the GPU and it can start power
+	 * management
+	 */
+	a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_GPU_SET);
+
+	/* Take the GMU out of its special boot mode */
+	a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_BOOT_SLUMBER);
+
+	return ret;
+}
+
+static void a6xx_dump(struct msm_gpu *gpu)
+{
+	dev_info(&gpu->pdev->dev, "status:   %08x\n",
+			gpu_read(gpu, REG_A6XX_RBBM_STATUS));
+	adreno_dump(gpu);
+}
+
+#define VBIF_RESET_ACK_TIMEOUT	100
+#define VBIF_RESET_ACK_MASK	0x00f0
+
+static void a6xx_recover(struct msm_gpu *gpu)
+{
+	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
+	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
+	int i;
+
+	adreno_dump_info(gpu);
+
+	for (i = 0; i < 8; i++)
+		dev_info(&gpu->pdev->dev, "CP_SCRATCH_REG%d: %u\n", i,
+			gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(i)));
+
+	if (hang_debug)
+		a6xx_dump(gpu);
+
+	/*
+	 * Turn off keep alive that might have been enabled by the hang
+	 * interrupt
+	 */
+	gmu_write(&a6xx_gpu->gmu, REG_A6XX_GMU_GMU_PWR_COL_KEEPALIVE, 0);
+
+	gpu->funcs->pm_suspend(gpu);
+	gpu->funcs->pm_resume(gpu);
+
+	msm_gpu_hw_init(gpu);
+}
+
+static int a6xx_fault_handler(void *arg, unsigned long iova, int flags)
+{
+	struct msm_gpu *gpu = arg;
+
+	pr_warn_ratelimited("*** gpu fault: iova=%08lx, flags=%d (%u,%u,%u,%u)\n",
+			iova, flags,
+			gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(4)),
+			gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(5)),
+			gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(6)),
+			gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(7)));
+
+	return -EFAULT;
+}
+
+static void a6xx_cp_hw_err_irq(struct msm_gpu *gpu)
+{
+	u32 status = gpu_read(gpu, REG_A6XX_CP_INTERRUPT_STATUS);
+
+	if (status & A6XX_CP_INT_CP_OPCODE_ERROR) {
+		u32 val;
+
+		gpu_write(gpu, REG_A6XX_CP_SQE_STAT_ADDR, 1);
+		val = gpu_read(gpu, REG_A6XX_CP_SQE_STAT_DATA);
+		dev_err_ratelimited(&gpu->pdev->dev,
+			"CP | opcode error | possible opcode=0x%8.8X\n",
+			val);
+	}
+
+	if (status & A6XX_CP_INT_CP_UCODE_ERROR)
+		dev_err_ratelimited(&gpu->pdev->dev,
+			"CP ucode error interrupt\n");
+
+	if (status & A6XX_CP_INT_CP_HW_FAULT_ERROR)
+		dev_err_ratelimited(&gpu->pdev->dev, "CP | HW fault | status=0x%8.8X\n",
+			gpu_read(gpu, REG_A6XX_CP_HW_FAULT));
+
+	if (status & A6XX_CP_INT_CP_REGISTER_PROTECTION_ERROR) {
+		u32 val = gpu_read(gpu, REG_A6XX_CP_PROTECT_STATUS);
+
+		dev_err_ratelimited(&gpu->pdev->dev,
+			"CP | protected mode error | %s | addr=0x%8.8X | status=0x%8.8X\n",
+			val & (1 << 20) ? "READ" : "WRITE",
+			(val & 0x3ffff), val);
+	}
+
+	if (status & A6XX_CP_INT_CP_AHB_ERROR)
+		dev_err_ratelimited(&gpu->pdev->dev, "CP AHB error interrupt\n");
+
+	if (status & A6XX_CP_INT_CP_VSD_PARITY_ERROR)
+		dev_err_ratelimited(&gpu->pdev->dev, "CP VSD decoder parity error\n");
+
+	if (status & A6XX_CP_INT_CP_ILLEGAL_INSTR_ERROR)
+		dev_err_ratelimited(&gpu->pdev->dev, "CP illegal instruction error\n");
+
+}
+
+static void a6xx_fault_detect_irq(struct msm_gpu *gpu)
+{
+	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
+	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
+	struct drm_device *dev = gpu->dev;
+	struct msm_drm_private *priv = dev->dev_private;
+	struct msm_ringbuffer *ring = gpu->funcs->active_ring(gpu);
+
+	/*
+	 * Force the GPU to stay on until after we finish
+	 * collecting information
+	 */
+	gmu_write(&a6xx_gpu->gmu, REG_A6XX_GMU_GMU_PWR_COL_KEEPALIVE, 1);
+
+	DRM_DEV_ERROR(&gpu->pdev->dev,
+		"gpu fault ring %d fence %x status %8.8X rb %4.4x/%4.4x ib1 %16.16llX/%4.4x ib2 %16.16llX/%4.4x\n",
+		ring ? ring->id : -1, ring ? ring->seqno : 0,
+		gpu_read(gpu, REG_A6XX_RBBM_STATUS),
+		gpu_read(gpu, REG_A6XX_CP_RB_RPTR),
+		gpu_read(gpu, REG_A6XX_CP_RB_WPTR),
+		gpu_read64(gpu, REG_A6XX_CP_IB1_BASE, REG_A6XX_CP_IB1_BASE_HI),
+		gpu_read(gpu, REG_A6XX_CP_IB1_REM_SIZE),
+		gpu_read64(gpu, REG_A6XX_CP_IB2_BASE, REG_A6XX_CP_IB2_BASE_HI),
+		gpu_read(gpu, REG_A6XX_CP_IB2_REM_SIZE));
+
+	/* Turn off the hangcheck timer to keep it from bothering us */
+	del_timer(&gpu->hangcheck_timer);
+
+	queue_work(priv->wq, &gpu->recover_work);
+}
+
+static irqreturn_t a6xx_irq(struct msm_gpu *gpu)
+{
+	u32 status = gpu_read(gpu, REG_A6XX_RBBM_INT_0_STATUS);
+
+	gpu_write(gpu, REG_A6XX_RBBM_INT_CLEAR_CMD, status);
+
+	if (status & A6XX_RBBM_INT_0_MASK_RBBM_HANG_DETECT)
+		a6xx_fault_detect_irq(gpu);
+
+	if (status & A6XX_RBBM_INT_0_MASK_CP_AHB_ERROR)
+		dev_err_ratelimited(&gpu->pdev->dev, "CP | AHB bus error\n");
+
+	if (status & A6XX_RBBM_INT_0_MASK_CP_HW_ERROR)
+		a6xx_cp_hw_err_irq(gpu);
+
+	if (status & A6XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNCFIFO_OVERFLOW)
+		dev_err_ratelimited(&gpu->pdev->dev, "RBBM | ATB ASYNC overflow\n");
+
+	if (status & A6XX_RBBM_INT_0_MASK_RBBM_ATB_BUS_OVERFLOW)
+		dev_err_ratelimited(&gpu->pdev->dev, "RBBM | ATB bus overflow\n");
+
+	if (status & A6XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS)
+		dev_err_ratelimited(&gpu->pdev->dev, "UCHE | Out of bounds access\n");
+
+	if (status & A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS)
+		msm_gpu_retire(gpu);
+
+	return IRQ_HANDLED;
+}
+
+static const u32 a6xx_register_offsets[REG_ADRENO_REGISTER_MAX] = {
+	REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_BASE, REG_A6XX_CP_RB_BASE),
+	REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_BASE_HI, REG_A6XX_CP_RB_BASE_HI),
+	REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_RPTR_ADDR,
+		REG_A6XX_CP_RB_RPTR_ADDR_LO),
+	REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_RPTR_ADDR_HI,
+		REG_A6XX_CP_RB_RPTR_ADDR_HI),
+	REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_RPTR, REG_A6XX_CP_RB_RPTR),
+	REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_WPTR, REG_A6XX_CP_RB_WPTR),
+	REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_CNTL, REG_A6XX_CP_RB_CNTL),
+};
+
+static const u32 a6xx_registers[] = {
+	0x0000, 0x0002, 0x0010, 0x0010, 0x0012, 0x0012, 0x0018, 0x001b,
+	0x001e, 0x0032, 0x0038, 0x003c, 0x0042, 0x0042, 0x0044, 0x0044,
+	0x0047, 0x0047, 0x0056, 0x0056, 0x00ad, 0x00ae, 0x00b0, 0x00fb,
+	0x0100, 0x011d, 0x0200, 0x020d, 0x0210, 0x0213, 0x0218, 0x023d,
+	0x0400, 0x04f9, 0x0500, 0x0500, 0x0505, 0x050b, 0x050e, 0x0511,
+	0x0533, 0x0533, 0x0540, 0x0555, 0x0800, 0x0808, 0x0810, 0x0813,
+	0x0820, 0x0821, 0x0823, 0x0827, 0x0830, 0x0833, 0x0840, 0x0843,
+	0x084f, 0x086f, 0x0880, 0x088a, 0x08a0, 0x08ab, 0x08c0, 0x08c4,
+	0x08d0, 0x08dd, 0x08f0, 0x08f3, 0x0900, 0x0903, 0x0908, 0x0911,
+	0x0928, 0x093e, 0x0942, 0x094d, 0x0980, 0x0984, 0x098d, 0x0996,
+	0x0998, 0x099e, 0x09a0, 0x09a6, 0x09a8, 0x09ae, 0x09b0, 0x09b1,
+	0x09c2, 0x09c8, 0x0a00, 0x0a03, 0x0c00, 0x0c04, 0x0c06, 0x0c06,
+	0x0c10, 0x0cd9, 0x0e00, 0x0e0e, 0x0e10, 0x0e13, 0x0e17, 0x0e19,
+	0x0e1c, 0x0e2b, 0x0e30, 0x0e32, 0x0e38, 0x0e39, 0x8600, 0x8601,
+	0x8610, 0x861b, 0x8620, 0x8620, 0x8628, 0x862b, 0x8630, 0x8637,
+	0x8e01, 0x8e01, 0x8e04, 0x8e05, 0x8e07, 0x8e08, 0x8e0c, 0x8e0c,
+	0x8e10, 0x8e1c, 0x8e20, 0x8e25, 0x8e28, 0x8e28, 0x8e2c, 0x8e2f,
+	0x8e3b, 0x8e3e, 0x8e40, 0x8e43, 0x8e50, 0x8e5e, 0x8e70, 0x8e77,
+	0x9600, 0x9604, 0x9624, 0x9637, 0x9e00, 0x9e01, 0x9e03, 0x9e0e,
+	0x9e11, 0x9e16, 0x9e19, 0x9e19, 0x9e1c, 0x9e1c, 0x9e20, 0x9e23,
+	0x9e30, 0x9e31, 0x9e34, 0x9e34, 0x9e70, 0x9e72, 0x9e78, 0x9e79,
+	0x9e80, 0x9fff, 0xa600, 0xa601, 0xa603, 0xa603, 0xa60a, 0xa60a,
+	0xa610, 0xa617, 0xa630, 0xa630,
+	~0
+};
+
+static int a6xx_pm_resume(struct msm_gpu *gpu)
+{
+	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
+	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
+	int ret;
+
+	ret = a6xx_gmu_resume(a6xx_gpu);
+
+	gpu->needs_hw_init = true;
+
+	return ret;
+}
+
+static int a6xx_pm_suspend(struct msm_gpu *gpu)
+{
+	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
+	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
+
+	/*
+	 * Make sure the GMU is idle before continuing (because some transitions
+	 * may use VBIF
+	 */
+	a6xx_gmu_wait_for_idle(a6xx_gpu);
+
+	/* Clear the VBIF pipe before shutting down */
+	/* FIXME: This accesses the GPU - do we need to make sure it is on? */
+	gpu_write(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL0, 0xf);
+	spin_until((gpu_read(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL1) & 0xf) == 0xf);
+	gpu_write(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL0, 0);
+
+	return a6xx_gmu_stop(a6xx_gpu);
+}
+
+static int a6xx_get_timestamp(struct msm_gpu *gpu, uint64_t *value)
+{
+	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
+	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
+
+	/* Force the GPU power on so we can read this register */
+	a6xx_gmu_set_oob(&a6xx_gpu->gmu, GMU_OOB_GPU_SET);
+
+	*value = gpu_read64(gpu, REG_A6XX_RBBM_PERFCTR_CP_0_LO,
+		REG_A6XX_RBBM_PERFCTR_CP_0_HI);
+
+	a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_GPU_SET);
+	return 0;
+}
+
+#if defined(CONFIG_DEBUG_FS) || defined(CONFIG_DEV_COREDUMP)
+static void a6xx_show(struct msm_gpu *gpu, struct msm_gpu_state *state,
+		struct drm_printer *p)
+{
+	adreno_show(gpu, state, p);
+}
+#endif
+
+static struct msm_ringbuffer *a6xx_active_ring(struct msm_gpu *gpu)
+{
+	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
+	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
+
+	return a6xx_gpu->cur_ring;
+}
+
+static void a6xx_destroy(struct msm_gpu *gpu)
+{
+	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
+	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
+
+	if (a6xx_gpu->sqe_bo) {
+		if (a6xx_gpu->sqe_iova)
+			msm_gem_put_iova(a6xx_gpu->sqe_bo, gpu->aspace);
+		drm_gem_object_unreference_unlocked(a6xx_gpu->sqe_bo);
+	}
+
+	a6xx_gmu_remove(a6xx_gpu);
+
+	adreno_gpu_cleanup(adreno_gpu);
+	kfree(a6xx_gpu);
+}
+
+static const struct adreno_gpu_funcs funcs = {
+	.base = {
+		.get_param = adreno_get_param,
+		.hw_init = a6xx_hw_init,
+		.pm_suspend = a6xx_pm_suspend,
+		.pm_resume = a6xx_pm_resume,
+		.recover = a6xx_recover,
+		.submit = a6xx_submit,
+		.flush = a6xx_flush,
+		.active_ring = a6xx_active_ring,
+		.irq = a6xx_irq,
+		.destroy = a6xx_destroy,
+#if defined(CONFIG_DEBUG_FS) || defined(CONFIG_DEV_COREDUMP)
+		.show = a6xx_show,
+#endif
+	},
+	.get_timestamp = a6xx_get_timestamp,
+};
+
+struct msm_gpu *a6xx_gpu_init(struct drm_device *dev)
+{
+	struct msm_drm_private *priv = dev->dev_private;
+	struct platform_device *pdev = priv->gpu_pdev;
+	struct device_node *node;
+	struct a6xx_gpu *a6xx_gpu;
+	struct adreno_gpu *adreno_gpu;
+	struct msm_gpu *gpu;
+	int ret;
+
+	a6xx_gpu = kzalloc(sizeof(*a6xx_gpu), GFP_KERNEL);
+	if (!a6xx_gpu)
+		return ERR_PTR(-ENOMEM);
+
+	adreno_gpu = &a6xx_gpu->base;
+	gpu = &adreno_gpu->base;
+
+	adreno_gpu->registers = a6xx_registers;
+	adreno_gpu->reg_offsets = a6xx_register_offsets;
+
+	ret = adreno_gpu_init(dev, pdev, adreno_gpu, &funcs, 1);
+	if (ret) {
+		a6xx_destroy(&(a6xx_gpu->base.base));
+		return ERR_PTR(ret);
+	}
+
+	/* Check if there is a GMU phandle and set it up */
+	node = of_parse_phandle(pdev->dev.of_node, "gmu", 0);
+
+	/* FIXME: How do we gracefully handle this? */
+	BUG_ON(!node);
+
+	ret = a6xx_gmu_probe(a6xx_gpu, node);
+	if (ret) {
+		a6xx_destroy(&(a6xx_gpu->base.base));
+		return ERR_PTR(ret);
+	}
+
+	if (gpu->aspace)
+		msm_mmu_set_fault_handler(gpu->aspace->mmu, gpu,
+				a6xx_fault_handler);
+
+	return gpu;
+}
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu.h b/drivers/gpu/drm/msm/adreno/a6xx_gpu.h
new file mode 100644
index 000000000000..dd69e5b0e692
--- /dev/null
+++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu.h
@@ -0,0 +1,60 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (c) 2017 The Linux Foundation. All rights reserved. */
+
+#ifndef __A6XX_GPU_H__
+#define __A6XX_GPU_H__
+
+
+#include "adreno_gpu.h"
+#include "a6xx.xml.h"
+
+#include "a6xx_gmu.h"
+
+extern bool hang_debug;
+
+struct a6xx_gpu {
+	struct adreno_gpu base;
+
+	struct drm_gem_object *sqe_bo;
+	uint64_t sqe_iova;
+
+	struct msm_ringbuffer *cur_ring;
+
+	struct a6xx_gmu gmu;
+};
+
+#define to_a6xx_gpu(x) container_of(x, struct a6xx_gpu, base)
+
+/*
+ * Given a register and a count, return a value to program into
+ * REG_CP_PROTECT_REG(n) - this will block both reads and writes for _len
+ * registers starting at _reg.
+ */
+#define A6XX_PROTECT_RW(_reg, _len) \
+	((1 << 31) | \
+	(((_len) & 0x3FFF) << 18) | ((_reg) & 0x3FFFF))
+
+/*
+ * Same as above, but allow reads over the range. For areas of mixed use (such
+ * as performance counters) this allows us to protect a much larger range with a
+ * single register
+ */
+#define A6XX_PROTECT_RDONLY(_reg, _len) \
+	((((_len) & 0x3FFF) << 18) | ((_reg) & 0x3FFFF))
+
+
+int a6xx_gmu_resume(struct a6xx_gpu *gpu);
+int a6xx_gmu_stop(struct a6xx_gpu *gpu);
+
+int a6xx_gmu_wait_for_idle(struct a6xx_gpu *gpu);
+
+int a6xx_gmu_reset(struct a6xx_gpu *a6xx_gpu);
+bool a6xx_gmu_isidle(struct a6xx_gmu *gmu);
+
+int a6xx_gmu_set_oob(struct a6xx_gmu *gmu, enum a6xx_gmu_oob_state state);
+void a6xx_gmu_clear_oob(struct a6xx_gmu *gmu, enum a6xx_gmu_oob_state state);
+
+int a6xx_gmu_probe(struct a6xx_gpu *a6xx_gpu, struct device_node *node);
+void a6xx_gmu_remove(struct a6xx_gpu *a6xx_gpu);
+
+#endif /* __A6XX_GPU_H__ */
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_hfi.c b/drivers/gpu/drm/msm/adreno/a6xx_hfi.c
new file mode 100644
index 000000000000..f19ef4cb6ea4
--- /dev/null
+++ b/drivers/gpu/drm/msm/adreno/a6xx_hfi.c
@@ -0,0 +1,435 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2017-2018 The Linux Foundation. All rights reserved. */
+
+#include <linux/completion.h>
+#include <linux/circ_buf.h>
+#include <linux/list.h>
+
+#include "a6xx_gmu.h"
+#include "a6xx_gmu.xml.h"
+
+#define HFI_MSG_ID(val) [val] = #val
+
+static const char * const a6xx_hfi_msg_id[] = {
+	HFI_MSG_ID(HFI_H2F_MSG_INIT),
+	HFI_MSG_ID(HFI_H2F_MSG_FW_VERSION),
+	HFI_MSG_ID(HFI_H2F_MSG_BW_TABLE),
+	HFI_MSG_ID(HFI_H2F_MSG_PERF_TABLE),
+	HFI_MSG_ID(HFI_H2F_MSG_TEST),
+};
+
+static int a6xx_hfi_queue_read(struct a6xx_hfi_queue *queue, u32 *data,
+		u32 dwords)
+{
+	struct a6xx_hfi_queue_header *header = queue->header;
+	u32 i, hdr, index = header->read_index;
+
+	if (header->read_index == header->write_index) {
+		header->rx_request = 1;
+		return 0;
+	}
+
+	hdr = queue->data[index];
+
+	/*
+	 * If we are to assume that the GMU firmware is in fact a rational actor
+	 * and is programmed to not send us a larger response than we expect
+	 * then we can also assume that if the header size is unexpectedly large
+	 * that it is due to memory corruption and/or hardware failure. In this
+	 * case the only reasonable course of action is to BUG() to help harden
+	 * the failure.
+	 */
+
+	BUG_ON(HFI_HEADER_SIZE(hdr) > dwords);
+
+	for (i = 0; i < HFI_HEADER_SIZE(hdr); i++) {
+		data[i] = queue->data[index];
+		index = (index + 1) % header->size;
+	}
+
+	header->read_index = index;
+	return HFI_HEADER_SIZE(hdr);
+}
+
+static int a6xx_hfi_queue_write(struct a6xx_gmu *gmu,
+	struct a6xx_hfi_queue *queue, u32 *data, u32 dwords)
+{
+	struct a6xx_hfi_queue_header *header = queue->header;
+	u32 i, space, index = header->write_index;
+
+	spin_lock(&queue->lock);
+
+	space = CIRC_SPACE(header->write_index, header->read_index,
+		header->size);
+	if (space < dwords) {
+		header->dropped++;
+		spin_unlock(&queue->lock);
+		return -ENOSPC;
+	}
+
+	for (i = 0; i < dwords; i++) {
+		queue->data[index] = data[i];
+		index = (index + 1) % header->size;
+	}
+
+	header->write_index = index;
+	spin_unlock(&queue->lock);
+
+	gmu_write(gmu, REG_A6XX_GMU_HOST2GMU_INTR_SET, 0x01);
+	return 0;
+}
+
+struct a6xx_hfi_response {
+	u32 id;
+	u32 seqnum;
+	struct list_head node;
+	struct completion complete;
+
+	u32 error;
+	u32 payload[16];
+};
+
+/*
+ * Incoming HFI ack messages can come in out of order so we need to store all
+ * the pending messages on a list until they are handled.
+ */
+static spinlock_t hfi_ack_lock = __SPIN_LOCK_UNLOCKED(message_lock);
+static LIST_HEAD(hfi_ack_list);
+
+static void a6xx_hfi_handle_ack(struct a6xx_gmu *gmu,
+		struct a6xx_hfi_msg_response *msg)
+{
+	struct a6xx_hfi_response *resp;
+	u32 id, seqnum;
+
+	/* msg->ret_header contains the header of the message being acked */
+	id = HFI_HEADER_ID(msg->ret_header);
+	seqnum = HFI_HEADER_SEQNUM(msg->ret_header);
+
+	spin_lock(&hfi_ack_lock);
+	list_for_each_entry(resp, &hfi_ack_list, node) {
+		if (resp->id == id && resp->seqnum == seqnum) {
+			resp->error = msg->error;
+			memcpy(resp->payload, msg->payload,
+				sizeof(resp->payload));
+
+			complete(&resp->complete);
+			spin_unlock(&hfi_ack_lock);
+			return;
+		}
+	}
+	spin_unlock(&hfi_ack_lock);
+
+	dev_err(gmu->dev, "Nobody was waiting for HFI message %d\n", seqnum);
+}
+
+static void a6xx_hfi_handle_error(struct a6xx_gmu *gmu,
+		struct a6xx_hfi_msg_response *msg)
+{
+	struct a6xx_hfi_msg_error *error = (struct a6xx_hfi_msg_error *) msg;
+
+	dev_err(gmu->dev, "GMU firmware error %d\n", error->code);
+}
+
+void a6xx_hfi_task(unsigned long data)
+{
+	struct a6xx_gmu *gmu = (struct a6xx_gmu *) data;
+	struct a6xx_hfi_queue *queue = &gmu->queues[HFI_RESPONSE_QUEUE];
+	struct a6xx_hfi_msg_response resp;
+
+	for (;;) {
+		u32 id;
+		int ret = a6xx_hfi_queue_read(queue, (u32 *) &resp,
+			sizeof(resp) >> 2);
+
+		/* Returns the number of bytes copied or negative on error */
+		if (ret <= 0) {
+			if (ret < 0)
+				dev_err(gmu->dev,
+					"Unable to read the HFI message queue\n");
+			break;
+		}
+
+		id = HFI_HEADER_ID(resp.header);
+
+		if (id == HFI_F2H_MSG_ACK)
+			a6xx_hfi_handle_ack(gmu, &resp);
+		else if (id == HFI_F2H_MSG_ERROR)
+			a6xx_hfi_handle_error(gmu, &resp);
+	}
+}
+
+static int a6xx_hfi_send_msg(struct a6xx_gmu *gmu, int id,
+		void *data, u32 size, u32 *payload, u32 payload_size)
+{
+	struct a6xx_hfi_queue *queue = &gmu->queues[HFI_COMMAND_QUEUE];
+	struct a6xx_hfi_response resp = { 0 };
+	int ret, dwords = size >> 2;
+	u32 seqnum;
+
+	seqnum = atomic_inc_return(&queue->seqnum) % 0xfff;
+
+	/* First dword of the message is the message header - fill it in */
+	*((u32 *) data) = (seqnum << 20) | (HFI_MSG_CMD << 16) |
+		(dwords << 8) | id;
+
+	init_completion(&resp.complete);
+	resp.id = id;
+	resp.seqnum = seqnum;
+
+	spin_lock_bh(&hfi_ack_lock);
+	list_add_tail(&resp.node, &hfi_ack_list);
+	spin_unlock_bh(&hfi_ack_lock);
+
+	ret = a6xx_hfi_queue_write(gmu, queue, data, dwords);
+	if (ret) {
+		dev_err(gmu->dev, "Unable to send message %s id %d\n",
+			a6xx_hfi_msg_id[id], seqnum);
+		goto out;
+	}
+
+	/* Wait up to 5 seconds for the response */
+	ret = wait_for_completion_timeout(&resp.complete,
+		msecs_to_jiffies(5000));
+	if (!ret) {
+		dev_err(gmu->dev,
+			"Message %s id %d timed out waiting for response\n",
+			a6xx_hfi_msg_id[id], seqnum);
+		ret = -ETIMEDOUT;
+	} else
+		ret = 0;
+
+out:
+	spin_lock_bh(&hfi_ack_lock);
+	list_del(&resp.node);
+	spin_unlock_bh(&hfi_ack_lock);
+
+	if (ret)
+		return ret;
+
+	if (resp.error) {
+		dev_err(gmu->dev, "Message %s id %d returned error %d\n",
+			a6xx_hfi_msg_id[id], seqnum, resp.error);
+		return -EINVAL;
+	}
+
+	if (payload && payload_size) {
+		int copy = min_t(u32, payload_size, sizeof(resp.payload));
+
+		memcpy(payload, resp.payload, copy);
+	}
+
+	return 0;
+}
+
+static int a6xx_hfi_send_gmu_init(struct a6xx_gmu *gmu, int boot_state)
+{
+	struct a6xx_hfi_msg_gmu_init_cmd msg = { 0 };
+
+	msg.dbg_buffer_addr = (u32) gmu->debug->iova;
+	msg.dbg_buffer_size = (u32) gmu->debug->size;
+	msg.boot_state = boot_state;
+
+	return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_INIT, &msg, sizeof(msg),
+		NULL, 0);
+}
+
+static int a6xx_hfi_get_fw_version(struct a6xx_gmu *gmu, u32 *version)
+{
+	struct a6xx_hfi_msg_fw_version msg = { 0 };
+
+	/* Currently supporting version 1.1 */
+	msg.supported_version = (1 << 28) | (1 << 16);
+
+	return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_FW_VERSION, &msg, sizeof(msg),
+		version, sizeof(*version));
+}
+
+static int a6xx_hfi_send_perf_table(struct a6xx_gmu *gmu)
+{
+	struct a6xx_hfi_msg_perf_table msg = { 0 };
+	int i;
+
+	msg.num_gpu_levels = gmu->nr_gpu_freqs;
+	msg.num_gmu_levels = gmu->nr_gmu_freqs;
+
+	for (i = 0; i < gmu->nr_gpu_freqs; i++) {
+		msg.gx_votes[i].vote = gmu->gx_arc_votes[i];
+		msg.gx_votes[i].freq = gmu->gpu_freqs[i] / 1000;
+	}
+
+	for (i = 0; i < gmu->nr_gmu_freqs; i++) {
+		msg.cx_votes[i].vote = gmu->cx_arc_votes[i];
+		msg.cx_votes[i].freq = gmu->gmu_freqs[i] / 1000;
+	}
+
+	return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_PERF_TABLE, &msg, sizeof(msg),
+		NULL, 0);
+}
+
+static int a6xx_hfi_send_bw_table(struct a6xx_gmu *gmu)
+{
+	struct a6xx_hfi_msg_bw_table msg = { 0 };
+
+	/*
+	 * The sdm845 GMU doesn't do bus frequency scaling on its own but it
+	 * does need at least one entry in the list because it might be accessed
+	 * when the GMU is shutting down. Send a single "off" entry.
+	 */
+
+	msg.bw_level_num = 1;
+
+	msg.ddr_cmds_num = 3;
+	msg.ddr_wait_bitmask = 0x07;
+
+	msg.ddr_cmds_addrs[0] = 0x50000;
+	msg.ddr_cmds_addrs[1] = 0x5005c;
+	msg.ddr_cmds_addrs[2] = 0x5000c;
+
+	msg.ddr_cmds_data[0][0] =  0x40000000;
+	msg.ddr_cmds_data[0][1] =  0x40000000;
+	msg.ddr_cmds_data[0][2] =  0x40000000;
+
+	/*
+	 * These are the CX (CNOC) votes.  This is used but the values for the
+	 * sdm845 GMU are known and fixed so we can hard code them.
+	 */
+
+	msg.cnoc_cmds_num = 3;
+	msg.cnoc_wait_bitmask = 0x05;
+
+	msg.cnoc_cmds_addrs[0] = 0x50034;
+	msg.cnoc_cmds_addrs[1] = 0x5007c;
+	msg.cnoc_cmds_addrs[2] = 0x5004c;
+
+	msg.cnoc_cmds_data[0][0] =  0x40000000;
+	msg.cnoc_cmds_data[0][1] =  0x00000000;
+	msg.cnoc_cmds_data[0][2] =  0x40000000;
+
+	msg.cnoc_cmds_data[1][0] =  0x60000001;
+	msg.cnoc_cmds_data[1][1] =  0x20000001;
+	msg.cnoc_cmds_data[1][2] =  0x60000001;
+
+	return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_BW_TABLE, &msg, sizeof(msg),
+		NULL, 0);
+}
+
+static int a6xx_hfi_send_test(struct a6xx_gmu *gmu)
+{
+	struct a6xx_hfi_msg_test msg = { 0 };
+
+	return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_TEST, &msg, sizeof(msg),
+		NULL, 0);
+}
+
+int a6xx_hfi_start(struct a6xx_gmu *gmu, int boot_state)
+{
+	int ret;
+
+	ret = a6xx_hfi_send_gmu_init(gmu, boot_state);
+	if (ret)
+		return ret;
+
+	ret = a6xx_hfi_get_fw_version(gmu, NULL);
+	if (ret)
+		return ret;
+
+	/*
+	 * We have to get exchange version numbers per the sequence but at this
+	 * point th kernel driver doesn't need to know the exact version of
+	 * the GMU firmware
+	 */
+
+	ret = a6xx_hfi_send_perf_table(gmu);
+	if (ret)
+		return ret;
+
+	ret = a6xx_hfi_send_bw_table(gmu);
+	if (ret)
+		return ret;
+
+	/*
+	 * Let the GMU know that there won't be any more HFI messages until next
+	 * boot
+	 */
+	a6xx_hfi_send_test(gmu);
+
+	return 0;
+}
+
+void a6xx_hfi_stop(struct a6xx_gmu *gmu)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(gmu->queues); i++) {
+		struct a6xx_hfi_queue *queue = &gmu->queues[i];
+
+		if (!queue->header)
+			continue;
+
+		if (queue->header->read_index != queue->header->write_index)
+			dev_err(gmu->dev, "HFI queue %d is not empty\n", i);
+
+		queue->header->read_index = 0;
+		queue->header->write_index = 0;
+	}
+}
+
+static void a6xx_hfi_queue_init(struct a6xx_hfi_queue *queue,
+		struct a6xx_hfi_queue_header *header, void *virt, u64 iova,
+		u32 id)
+{
+	spin_lock_init(&queue->lock);
+	queue->header = header;
+	queue->data = virt;
+	atomic_set(&queue->seqnum, 0);
+
+	/* Set up the shared memory header */
+	header->iova = iova;
+	header->type =  10 << 8 | id;
+	header->status = 1;
+	header->size = SZ_4K >> 2;
+	header->msg_size = 0;
+	header->dropped = 0;
+	header->rx_watermark = 1;
+	header->tx_watermark = 1;
+	header->rx_request = 1;
+	header->tx_request = 0;
+	header->read_index = 0;
+	header->write_index = 0;
+}
+
+void a6xx_hfi_init(struct a6xx_gmu *gmu)
+{
+	struct a6xx_gmu_bo *hfi = gmu->hfi;
+	struct a6xx_hfi_queue_table_header *table = hfi->virt;
+	struct a6xx_hfi_queue_header *headers = hfi->virt + sizeof(*table);
+	u64 offset;
+	int table_size;
+
+	/*
+	 * The table size is the size of the table header plus all of the queue
+	 * headers
+	 */
+	table_size = sizeof(*table);
+	table_size += (ARRAY_SIZE(gmu->queues) *
+		sizeof(struct a6xx_hfi_queue_header));
+
+	table->version = 0;
+	table->size = table_size;
+	/* First queue header is located immediately after the table header */
+	table->qhdr0_offset = sizeof(*table) >> 2;
+	table->qhdr_size = sizeof(struct a6xx_hfi_queue_header) >> 2;
+	table->num_queues = ARRAY_SIZE(gmu->queues);
+	table->active_queues = ARRAY_SIZE(gmu->queues);
+
+	/* Command queue */
+	offset = SZ_4K;
+	a6xx_hfi_queue_init(&gmu->queues[0], &headers[0], hfi->virt + offset,
+		hfi->iova + offset, 0);
+
+	/* GMU response queue */
+	offset += SZ_4K;
+	a6xx_hfi_queue_init(&gmu->queues[1], &headers[1], hfi->virt + offset,
+		hfi->iova + offset, 4);
+}
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_hfi.h b/drivers/gpu/drm/msm/adreno/a6xx_hfi.h
new file mode 100644
index 000000000000..60d1319fa44f
--- /dev/null
+++ b/drivers/gpu/drm/msm/adreno/a6xx_hfi.h
@@ -0,0 +1,127 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (c) 2017 The Linux Foundation. All rights reserved. */
+
+#ifndef _A6XX_HFI_H_
+#define _A6XX_HFI_H_
+
+struct a6xx_hfi_queue_table_header {
+	u32 version;
+	u32 size;		/* Size of the queue table in dwords */
+	u32 qhdr0_offset;	/* Offset of the first queue header */
+	u32 qhdr_size;		/* Size of the queue headers */
+	u32 num_queues;		/* Number of total queues */
+	u32 active_queues;	/* Number of active queues */
+};
+
+struct a6xx_hfi_queue_header {
+	u32 status;
+	u32 iova;
+	u32 type;
+	u32 size;
+	u32 msg_size;
+	u32 dropped;
+	u32 rx_watermark;
+	u32 tx_watermark;
+	u32 rx_request;
+	u32 tx_request;
+	u32 read_index;
+	u32 write_index;
+};
+
+struct a6xx_hfi_queue {
+	struct a6xx_hfi_queue_header *header;
+	spinlock_t lock;
+	u32 *data;
+	atomic_t seqnum;
+};
+
+/* This is the outgoing queue to the GMU */
+#define HFI_COMMAND_QUEUE 0
+
+/* THis is the incoming response queue from the GMU */
+#define HFI_RESPONSE_QUEUE 1
+
+#define HFI_HEADER_ID(msg) ((msg) & 0xff)
+#define HFI_HEADER_SIZE(msg) (((msg) >> 8) & 0xff)
+#define HFI_HEADER_SEQNUM(msg) (((msg) >> 20) & 0xfff)
+
+/* FIXME: Do we need this or can we use ARRAY_SIZE? */
+#define HFI_RESPONSE_PAYLOAD_SIZE 16
+
+/* HFI message types */
+
+#define HFI_MSG_CMD 0
+#define HFI_MSG_ACK 2
+
+#define HFI_F2H_MSG_ACK 126
+
+struct a6xx_hfi_msg_response {
+	u32 header;
+	u32 ret_header;
+	u32 error;
+	u32 payload[HFI_RESPONSE_PAYLOAD_SIZE];
+};
+
+#define HFI_F2H_MSG_ERROR 100
+
+struct a6xx_hfi_msg_error {
+	u32 header;
+	u32 code;
+	u32 payload[2];
+};
+
+#define HFI_H2F_MSG_INIT 0
+
+struct a6xx_hfi_msg_gmu_init_cmd {
+	u32 header;
+	u32 seg_id;
+	u32 dbg_buffer_addr;
+	u32 dbg_buffer_size;
+	u32 boot_state;
+};
+
+#define HFI_H2F_MSG_FW_VERSION 1
+
+struct a6xx_hfi_msg_fw_version {
+	u32 header;
+	u32 supported_version;
+};
+
+#define HFI_H2F_MSG_PERF_TABLE 4
+
+struct perf_level {
+	u32 vote;
+	u32 freq;
+};
+
+struct a6xx_hfi_msg_perf_table {
+	u32 header;
+	u32 num_gpu_levels;
+	u32 num_gmu_levels;
+
+	struct perf_level gx_votes[16];
+	struct perf_level cx_votes[4];
+};
+
+#define HFI_H2F_MSG_BW_TABLE 3
+
+struct a6xx_hfi_msg_bw_table {
+	u32 header;
+	u32 bw_level_num;
+	u32 cnoc_cmds_num;
+	u32 ddr_cmds_num;
+	u32 cnoc_wait_bitmask;
+	u32 ddr_wait_bitmask;
+	u32 cnoc_cmds_addrs[6];
+	u32 cnoc_cmds_data[2][6];
+	u32 ddr_cmds_addrs[8];
+	u32 ddr_cmds_data[16][8];
+};
+
+#define HFI_H2F_MSG_TEST 5
+
+struct a6xx_hfi_msg_test {
+	u32 header;
+};
+
+#endif
diff --git a/drivers/gpu/drm/msm/adreno/adreno_common.xml.h b/drivers/gpu/drm/msm/adreno/adreno_common.xml.h
index b634cf71352b..5dace1350810 100644
--- a/drivers/gpu/drm/msm/adreno/adreno_common.xml.h
+++ b/drivers/gpu/drm/msm/adreno/adreno_common.xml.h
@@ -8,17 +8,19 @@ http://github.com/freedreno/envytools/
 git clone https://github.com/freedreno/envytools.git
 
 The rules-ng-ng source files this header was generated from are:
-- /home/robclark/src/freedreno/envytools/rnndb/adreno.xml               (    431 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml  (   1572 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/a2xx.xml          (  37162 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_common.xml (  13324 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_pm4.xml    (  31866 bytes, from 2017-06-06 18:26:14)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/a3xx.xml          (  83840 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/a4xx.xml          ( 111898 bytes, from 2017-06-06 18:23:59)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/a5xx.xml          ( 139480 bytes, from 2017-06-16 12:44:39)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/ocmem.xml         (   1773 bytes, from 2017-05-17 13:21:27)
-
-Copyright (C) 2013-2017 by the following authors:
+- /home/robclark/src/envytools/rnndb/adreno.xml               (    501 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/freedreno_copyright.xml  (   1572 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/adreno/a2xx.xml          (  36805 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/adreno/adreno_common.xml (  13634 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/adreno/adreno_pm4.xml    (  42393 bytes, from 2018-08-06 18:45:45)
+- /home/robclark/src/envytools/rnndb/adreno/a3xx.xml          (  83840 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/adreno/a4xx.xml          ( 112086 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/adreno/a5xx.xml          ( 147240 bytes, from 2018-08-06 18:45:45)
+- /home/robclark/src/envytools/rnndb/adreno/a6xx.xml          ( 101627 bytes, from 2018-08-06 18:45:45)
+- /home/robclark/src/envytools/rnndb/adreno/a6xx_gmu.xml      (  10431 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/adreno/ocmem.xml         (   1773 bytes, from 2018-07-03 19:37:13)
+
+Copyright (C) 2013-2018 by the following authors:
 - Rob Clark <robdclark@gmail.com> (robclark)
 - Ilia Mirkin <imirkin@alum.mit.edu> (imirkin)
 
@@ -44,6 +46,14 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */
 
 
+enum chip {
+	A2XX = 0,
+	A3XX = 0,
+	A4XX = 0,
+	A5XX = 0,
+	A6XX = 0,
+};
+
 enum adreno_pa_su_sc_draw {
 	PC_DRAW_POINTS = 0,
 	PC_DRAW_LINES = 1,
@@ -181,6 +191,12 @@ enum a3xx_rb_blend_opcode {
 	BLEND_MAX_DST_SRC = 4,
 };
 
+enum a4xx_tess_spacing {
+	EQUAL_SPACING = 0,
+	ODD_SPACING = 2,
+	EVEN_SPACING = 3,
+};
+
 #define REG_AXXX_CP_RB_BASE					0x000001c0
 
 #define REG_AXXX_CP_RB_CNTL					0x000001c1
diff --git a/drivers/gpu/drm/msm/adreno/adreno_device.c b/drivers/gpu/drm/msm/adreno/adreno_device.c
index 0ae5ace65462..7d3e9a129ac7 100644
--- a/drivers/gpu/drm/msm/adreno/adreno_device.c
+++ b/drivers/gpu/drm/msm/adreno/adreno_device.c
@@ -35,6 +35,7 @@ static const struct adreno_info gpulist[] = {
 			[ADRENO_FW_PFP] = "a300_pfp.fw",
 		},
 		.gmem  = SZ_256K,
+		.inactive_period = DRM_MSM_INACTIVE_PERIOD,
 		.init  = a3xx_gpu_init,
 	}, {
 		.rev   = ADRENO_REV(3, 0, 6, 0),
@@ -45,6 +46,7 @@ static const struct adreno_info gpulist[] = {
 			[ADRENO_FW_PFP] = "a300_pfp.fw",
 		},
 		.gmem  = SZ_128K,
+		.inactive_period = DRM_MSM_INACTIVE_PERIOD,
 		.init  = a3xx_gpu_init,
 	}, {
 		.rev   = ADRENO_REV(3, 2, ANY_ID, ANY_ID),
@@ -55,6 +57,7 @@ static const struct adreno_info gpulist[] = {
 			[ADRENO_FW_PFP] = "a300_pfp.fw",
 		},
 		.gmem  = SZ_512K,
+		.inactive_period = DRM_MSM_INACTIVE_PERIOD,
 		.init  = a3xx_gpu_init,
 	}, {
 		.rev   = ADRENO_REV(3, 3, 0, ANY_ID),
@@ -65,6 +68,7 @@ static const struct adreno_info gpulist[] = {
 			[ADRENO_FW_PFP] = "a330_pfp.fw",
 		},
 		.gmem  = SZ_1M,
+		.inactive_period = DRM_MSM_INACTIVE_PERIOD,
 		.init  = a3xx_gpu_init,
 	}, {
 		.rev   = ADRENO_REV(4, 2, 0, ANY_ID),
@@ -75,6 +79,7 @@ static const struct adreno_info gpulist[] = {
 			[ADRENO_FW_PFP] = "a420_pfp.fw",
 		},
 		.gmem  = (SZ_1M + SZ_512K),
+		.inactive_period = DRM_MSM_INACTIVE_PERIOD,
 		.init  = a4xx_gpu_init,
 	}, {
 		.rev   = ADRENO_REV(4, 3, 0, ANY_ID),
@@ -85,6 +90,7 @@ static const struct adreno_info gpulist[] = {
 			[ADRENO_FW_PFP] = "a420_pfp.fw",
 		},
 		.gmem  = (SZ_1M + SZ_512K),
+		.inactive_period = DRM_MSM_INACTIVE_PERIOD,
 		.init  = a4xx_gpu_init,
 	}, {
 		.rev = ADRENO_REV(5, 3, 0, 2),
@@ -96,10 +102,25 @@ static const struct adreno_info gpulist[] = {
 			[ADRENO_FW_GPMU] = "a530v3_gpmu.fw2",
 		},
 		.gmem = SZ_1M,
+		/*
+		 * Increase inactive period to 250 to avoid bouncing
+		 * the GDSC which appears to make it grumpy
+		 */
+		.inactive_period = 250,
 		.quirks = ADRENO_QUIRK_TWO_PASS_USE_WFI |
 			ADRENO_QUIRK_FAULT_DETECT_MASK,
 		.init = a5xx_gpu_init,
 		.zapfw = "a530_zap.mdt",
+	}, {
+		.rev = ADRENO_REV(6, 3, 0, ANY_ID),
+		.revn = 630,
+		.name = "A630",
+		.fw = {
+			[ADRENO_FW_SQE] = "a630_sqe.fw",
+			[ADRENO_FW_GMU] = "a630_gmu.bin",
+		},
+		.gmem = SZ_1M,
+		.init = a6xx_gpu_init,
 	},
 };
 
@@ -116,6 +137,8 @@ MODULE_FIRMWARE("qcom/a530_zap.mdt");
 MODULE_FIRMWARE("qcom/a530_zap.b00");
 MODULE_FIRMWARE("qcom/a530_zap.b01");
 MODULE_FIRMWARE("qcom/a530_zap.b02");
+MODULE_FIRMWARE("qcom/a630_sqe.fw");
+MODULE_FIRMWARE("qcom/a630_gmu.bin");
 
 static inline bool _rev_match(uint8_t entry, uint8_t id)
 {
@@ -144,6 +167,7 @@ struct msm_gpu *adreno_load_gpu(struct drm_device *dev)
 	struct msm_drm_private *priv = dev->dev_private;
 	struct platform_device *pdev = priv->gpu_pdev;
 	struct msm_gpu *gpu = NULL;
+	struct adreno_gpu *adreno_gpu;
 	int ret;
 
 	if (pdev)
@@ -154,11 +178,31 @@ struct msm_gpu *adreno_load_gpu(struct drm_device *dev)
 		return NULL;
 	}
 
-	pm_runtime_get_sync(&pdev->dev);
+	adreno_gpu = to_adreno_gpu(gpu);
+
+	/*
+	 * The number one reason for HW init to fail is if the firmware isn't
+	 * loaded yet. Try that first and don't bother continuing on
+	 * otherwise
+	 */
+
+	ret = adreno_load_fw(adreno_gpu);
+	if (ret)
+		return NULL;
+
+	/* Make sure pm runtime is active and reset any previous errors */
+	pm_runtime_set_active(&pdev->dev);
+
+	ret = pm_runtime_get_sync(&pdev->dev);
+	if (ret < 0) {
+		dev_err(dev->dev, "Couldn't power up the GPU: %d\n", ret);
+		return NULL;
+	}
+
 	mutex_lock(&dev->struct_mutex);
 	ret = msm_gpu_hw_init(gpu);
 	mutex_unlock(&dev->struct_mutex);
-	pm_runtime_put_sync(&pdev->dev);
+	pm_runtime_put_autosuspend(&pdev->dev);
 	if (ret) {
 		dev_err(dev->dev, "gpu hw init failed: %d\n", ret);
 		return NULL;
@@ -316,6 +360,7 @@ static int adreno_suspend(struct device *dev)
 #endif
 
 static const struct dev_pm_ops adreno_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume)
 	SET_RUNTIME_PM_OPS(adreno_suspend, adreno_resume, NULL)
 };
 
diff --git a/drivers/gpu/drm/msm/adreno/adreno_gpu.c b/drivers/gpu/drm/msm/adreno/adreno_gpu.c
index 17d0506d058c..da1363a0c54d 100644
--- a/drivers/gpu/drm/msm/adreno/adreno_gpu.c
+++ b/drivers/gpu/drm/msm/adreno/adreno_gpu.c
@@ -17,7 +17,10 @@
  * this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include <linux/ascii85.h>
+#include <linux/kernel.h>
 #include <linux/pm_opp.h>
+#include <linux/slab.h>
 #include "adreno_gpu.h"
 #include "msm_gem.h"
 #include "msm_mmu.h"
@@ -70,10 +73,12 @@ adreno_request_fw(struct adreno_gpu *adreno_gpu, const char *fwname)
 {
 	struct drm_device *drm = adreno_gpu->base.dev;
 	const struct firmware *fw = NULL;
-	char newname[strlen("qcom/") + strlen(fwname) + 1];
+	char *newname;
 	int ret;
 
-	sprintf(newname, "qcom/%s", fwname);
+	newname = kasprintf(GFP_KERNEL, "qcom/%s", fwname);
+	if (!newname)
+		return ERR_PTR(-ENOMEM);
 
 	/*
 	 * Try first to load from qcom/$fwfile using a direct load (to avoid
@@ -87,11 +92,12 @@ adreno_request_fw(struct adreno_gpu *adreno_gpu, const char *fwname)
 			dev_info(drm->dev, "loaded %s from new location\n",
 				newname);
 			adreno_gpu->fwloc = FW_LOCATION_NEW;
-			return fw;
+			goto out;
 		} else if (adreno_gpu->fwloc != FW_LOCATION_UNKNOWN) {
 			dev_err(drm->dev, "failed to load %s: %d\n",
 				newname, ret);
-			return ERR_PTR(ret);
+			fw = ERR_PTR(ret);
+			goto out;
 		}
 	}
 
@@ -106,11 +112,12 @@ adreno_request_fw(struct adreno_gpu *adreno_gpu, const char *fwname)
 			dev_info(drm->dev, "loaded %s from legacy location\n",
 				newname);
 			adreno_gpu->fwloc = FW_LOCATION_LEGACY;
-			return fw;
+			goto out;
 		} else if (adreno_gpu->fwloc != FW_LOCATION_UNKNOWN) {
 			dev_err(drm->dev, "failed to load %s: %d\n",
 				fwname, ret);
-			return ERR_PTR(ret);
+			fw = ERR_PTR(ret);
+			goto out;
 		}
 	}
 
@@ -126,19 +133,23 @@ adreno_request_fw(struct adreno_gpu *adreno_gpu, const char *fwname)
 			dev_info(drm->dev, "loaded %s with helper\n",
 				newname);
 			adreno_gpu->fwloc = FW_LOCATION_HELPER;
-			return fw;
+			goto out;
 		} else if (adreno_gpu->fwloc != FW_LOCATION_UNKNOWN) {
 			dev_err(drm->dev, "failed to load %s: %d\n",
 				newname, ret);
-			return ERR_PTR(ret);
+			fw = ERR_PTR(ret);
+			goto out;
 		}
 	}
 
 	dev_err(drm->dev, "failed to load %s\n", fwname);
-	return ERR_PTR(-ENOENT);
+	fw = ERR_PTR(-ENOENT);
+out:
+	kfree(newname);
+	return fw;
 }
 
-static int adreno_load_fw(struct adreno_gpu *adreno_gpu)
+int adreno_load_fw(struct adreno_gpu *adreno_gpu)
 {
 	int i;
 
@@ -368,40 +379,185 @@ bool adreno_idle(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
 	return false;
 }
 
-#ifdef CONFIG_DEBUG_FS
-void adreno_show(struct msm_gpu *gpu, struct seq_file *m)
+int adreno_gpu_state_get(struct msm_gpu *gpu, struct msm_gpu_state *state)
 {
 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
+	int i, count = 0;
+
+	kref_init(&state->ref);
+
+	ktime_get_real_ts64(&state->time);
+
+	for (i = 0; i < gpu->nr_rings; i++) {
+		int size = 0, j;
+
+		state->ring[i].fence = gpu->rb[i]->memptrs->fence;
+		state->ring[i].iova = gpu->rb[i]->iova;
+		state->ring[i].seqno = gpu->rb[i]->seqno;
+		state->ring[i].rptr = get_rptr(adreno_gpu, gpu->rb[i]);
+		state->ring[i].wptr = get_wptr(gpu->rb[i]);
+
+		/* Copy at least 'wptr' dwords of the data */
+		size = state->ring[i].wptr;
+
+		/* After wptr find the last non zero dword to save space */
+		for (j = state->ring[i].wptr; j < MSM_GPU_RINGBUFFER_SZ >> 2; j++)
+			if (gpu->rb[i]->start[j])
+				size = j + 1;
+
+		if (size) {
+			state->ring[i].data = kmalloc(size << 2, GFP_KERNEL);
+			if (state->ring[i].data) {
+				memcpy(state->ring[i].data, gpu->rb[i]->start, size << 2);
+				state->ring[i].data_size = size << 2;
+			}
+		}
+	}
+
+	/* Count the number of registers */
+	for (i = 0; adreno_gpu->registers[i] != ~0; i += 2)
+		count += adreno_gpu->registers[i + 1] -
+			adreno_gpu->registers[i] + 1;
+
+	state->registers = kcalloc(count * 2, sizeof(u32), GFP_KERNEL);
+	if (state->registers) {
+		int pos = 0;
+
+		for (i = 0; adreno_gpu->registers[i] != ~0; i += 2) {
+			u32 start = adreno_gpu->registers[i];
+			u32 end   = adreno_gpu->registers[i + 1];
+			u32 addr;
+
+			for (addr = start; addr <= end; addr++) {
+				state->registers[pos++] = addr;
+				state->registers[pos++] = gpu_read(gpu, addr);
+			}
+		}
+
+		state->nr_registers = count;
+	}
+
+	return 0;
+}
+
+void adreno_gpu_state_destroy(struct msm_gpu_state *state)
+{
 	int i;
 
-	seq_printf(m, "revision: %d (%d.%d.%d.%d)\n",
+	for (i = 0; i < ARRAY_SIZE(state->ring); i++)
+		kfree(state->ring[i].data);
+
+	for (i = 0; state->bos && i < state->nr_bos; i++)
+		kvfree(state->bos[i].data);
+
+	kfree(state->bos);
+	kfree(state->comm);
+	kfree(state->cmd);
+	kfree(state->registers);
+}
+
+static void adreno_gpu_state_kref_destroy(struct kref *kref)
+{
+	struct msm_gpu_state *state = container_of(kref,
+		struct msm_gpu_state, ref);
+
+	adreno_gpu_state_destroy(state);
+	kfree(state);
+}
+
+int adreno_gpu_state_put(struct msm_gpu_state *state)
+{
+	if (IS_ERR_OR_NULL(state))
+		return 1;
+
+	return kref_put(&state->ref, adreno_gpu_state_kref_destroy);
+}
+
+#if defined(CONFIG_DEBUG_FS) || defined(CONFIG_DEV_COREDUMP)
+
+static void adreno_show_object(struct drm_printer *p, u32 *ptr, int len)
+{
+	char out[ASCII85_BUFSZ];
+	long l, datalen, i;
+
+	if (!ptr || !len)
+		return;
+
+	/*
+	 * Only dump the non-zero part of the buffer - rarely will any data
+	 * completely fill the entire allocated size of the buffer
+	 */
+	for (datalen = 0, i = 0; i < len >> 2; i++) {
+		if (ptr[i])
+			datalen = (i << 2) + 1;
+	}
+
+	/* Skip printing the object if it is empty */
+	if (datalen == 0)
+		return;
+
+	l = ascii85_encode_len(datalen);
+
+	drm_puts(p, "    data: !!ascii85 |\n");
+	drm_puts(p, "     ");
+
+	for (i = 0; i < l; i++)
+		drm_puts(p, ascii85_encode(ptr[i], out));
+
+	drm_puts(p, "\n");
+}
+
+void adreno_show(struct msm_gpu *gpu, struct msm_gpu_state *state,
+		struct drm_printer *p)
+{
+	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
+	int i;
+
+	if (IS_ERR_OR_NULL(state))
+		return;
+
+	drm_printf(p, "revision: %d (%d.%d.%d.%d)\n",
 			adreno_gpu->info->revn, adreno_gpu->rev.core,
 			adreno_gpu->rev.major, adreno_gpu->rev.minor,
 			adreno_gpu->rev.patchid);
 
-	for (i = 0; i < gpu->nr_rings; i++) {
-		struct msm_ringbuffer *ring = gpu->rb[i];
+	drm_printf(p, "rbbm-status: 0x%08x\n", state->rbbm_status);
 
-		seq_printf(m, "rb %d: fence:    %d/%d\n", i,
-			ring->memptrs->fence, ring->seqno);
+	drm_puts(p, "ringbuffer:\n");
 
-		seq_printf(m, "      rptr:     %d\n",
-			get_rptr(adreno_gpu, ring));
-		seq_printf(m, "rb wptr:  %d\n", get_wptr(ring));
+	for (i = 0; i < gpu->nr_rings; i++) {
+		drm_printf(p, "  - id: %d\n", i);
+		drm_printf(p, "    iova: 0x%016llx\n", state->ring[i].iova);
+		drm_printf(p, "    last-fence: %d\n", state->ring[i].seqno);
+		drm_printf(p, "    retired-fence: %d\n", state->ring[i].fence);
+		drm_printf(p, "    rptr: %d\n", state->ring[i].rptr);
+		drm_printf(p, "    wptr: %d\n", state->ring[i].wptr);
+		drm_printf(p, "    size: %d\n", MSM_GPU_RINGBUFFER_SZ);
+
+		adreno_show_object(p, state->ring[i].data,
+			state->ring[i].data_size);
 	}
 
-	/* dump these out in a form that can be parsed by demsm: */
-	seq_printf(m, "IO:region %s 00000000 00020000\n", gpu->name);
-	for (i = 0; adreno_gpu->registers[i] != ~0; i += 2) {
-		uint32_t start = adreno_gpu->registers[i];
-		uint32_t end   = adreno_gpu->registers[i+1];
-		uint32_t addr;
+	if (state->bos) {
+		drm_puts(p, "bos:\n");
 
-		for (addr = start; addr <= end; addr++) {
-			uint32_t val = gpu_read(gpu, addr);
-			seq_printf(m, "IO:R %08x %08x\n", addr<<2, val);
+		for (i = 0; i < state->nr_bos; i++) {
+			drm_printf(p, "  - iova: 0x%016llx\n",
+				state->bos[i].iova);
+			drm_printf(p, "    size: %zd\n", state->bos[i].size);
+
+			adreno_show_object(p, state->bos[i].data,
+				state->bos[i].size);
 		}
 	}
+
+	drm_puts(p, "registers:\n");
+
+	for (i = 0; i < state->nr_registers; i++) {
+		drm_printf(p, "  - { offset: 0x%04x, value: 0x%08x }\n",
+			state->registers[i * 2] << 2,
+			state->registers[(i * 2) + 1]);
+	}
 }
 #endif
 
@@ -565,7 +721,8 @@ int adreno_gpu_init(struct drm_device *drm, struct platform_device *pdev,
 
 	adreno_get_pwrlevels(&pdev->dev, gpu);
 
-	pm_runtime_set_autosuspend_delay(&pdev->dev, DRM_MSM_INACTIVE_PERIOD);
+	pm_runtime_set_autosuspend_delay(&pdev->dev,
+		adreno_gpu->info->inactive_period);
 	pm_runtime_use_autosuspend(&pdev->dev);
 	pm_runtime_enable(&pdev->dev);
 
diff --git a/drivers/gpu/drm/msm/adreno/adreno_gpu.h b/drivers/gpu/drm/msm/adreno/adreno_gpu.h
index d6b0e7b813f4..de6e6ee42fba 100644
--- a/drivers/gpu/drm/msm/adreno/adreno_gpu.h
+++ b/drivers/gpu/drm/msm/adreno/adreno_gpu.h
@@ -50,7 +50,9 @@ enum adreno_regs {
 
 enum {
 	ADRENO_FW_PM4 = 0,
+	ADRENO_FW_SQE = 0, /* a6xx */
 	ADRENO_FW_PFP = 1,
+	ADRENO_FW_GMU = 1, /* a6xx */
 	ADRENO_FW_GPMU = 2,
 	ADRENO_FW_MAX,
 };
@@ -84,6 +86,7 @@ struct adreno_info {
 	enum adreno_quirks quirks;
 	struct msm_gpu *(*init)(struct drm_device *dev);
 	const char *zapfw;
+	u32 inactive_period;
 };
 
 const struct adreno_info *adreno_info(struct adreno_rev rev);
@@ -214,8 +217,9 @@ void adreno_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit,
 		struct msm_file_private *ctx);
 void adreno_flush(struct msm_gpu *gpu, struct msm_ringbuffer *ring);
 bool adreno_idle(struct msm_gpu *gpu, struct msm_ringbuffer *ring);
-#ifdef CONFIG_DEBUG_FS
-void adreno_show(struct msm_gpu *gpu, struct seq_file *m);
+#if defined(CONFIG_DEBUG_FS) || defined(CONFIG_DEV_COREDUMP)
+void adreno_show(struct msm_gpu *gpu, struct msm_gpu_state *state,
+		struct drm_printer *p);
 #endif
 void adreno_dump_info(struct msm_gpu *gpu);
 void adreno_dump(struct msm_gpu *gpu);
@@ -226,7 +230,12 @@ int adreno_gpu_init(struct drm_device *drm, struct platform_device *pdev,
 		struct adreno_gpu *gpu, const struct adreno_gpu_funcs *funcs,
 		int nr_rings);
 void adreno_gpu_cleanup(struct adreno_gpu *gpu);
+int adreno_load_fw(struct adreno_gpu *adreno_gpu);
 
+void adreno_gpu_state_destroy(struct msm_gpu_state *state);
+
+int adreno_gpu_state_get(struct msm_gpu *gpu, struct msm_gpu_state *state);
+int adreno_gpu_state_put(struct msm_gpu_state *state);
 
 /* ringbuffer helpers (the parts that are adreno specific) */
 
@@ -328,6 +337,7 @@ static inline void adreno_gpu_write(struct adreno_gpu *gpu,
 struct msm_gpu *a3xx_gpu_init(struct drm_device *dev);
 struct msm_gpu *a4xx_gpu_init(struct drm_device *dev);
 struct msm_gpu *a5xx_gpu_init(struct drm_device *dev);
+struct msm_gpu *a6xx_gpu_init(struct drm_device *dev);
 
 static inline void adreno_gpu_write64(struct adreno_gpu *gpu,
 		enum adreno_regs lo, enum adreno_regs hi, u64 data)
diff --git a/drivers/gpu/drm/msm/adreno/adreno_pm4.xml.h b/drivers/gpu/drm/msm/adreno/adreno_pm4.xml.h
index fb605a3534cf..03a91e10b310 100644
--- a/drivers/gpu/drm/msm/adreno/adreno_pm4.xml.h
+++ b/drivers/gpu/drm/msm/adreno/adreno_pm4.xml.h
@@ -8,17 +8,19 @@ http://github.com/freedreno/envytools/
 git clone https://github.com/freedreno/envytools.git
 
 The rules-ng-ng source files this header was generated from are:
-- /home/robclark/src/freedreno/envytools/rnndb/adreno.xml               (    431 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml  (   1572 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/a2xx.xml          (  37162 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_common.xml (  13324 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_pm4.xml    (  31866 bytes, from 2017-06-06 18:26:14)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/a3xx.xml          (  83840 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/a4xx.xml          ( 111898 bytes, from 2017-06-06 18:23:59)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/a5xx.xml          ( 139480 bytes, from 2017-06-16 12:44:39)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/ocmem.xml         (   1773 bytes, from 2017-05-17 13:21:27)
-
-Copyright (C) 2013-2017 by the following authors:
+- /home/robclark/src/envytools/rnndb/adreno.xml               (    501 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/freedreno_copyright.xml  (   1572 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/adreno/a2xx.xml          (  36805 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/adreno/adreno_common.xml (  13634 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/adreno/adreno_pm4.xml    (  42393 bytes, from 2018-08-06 18:45:45)
+- /home/robclark/src/envytools/rnndb/adreno/a3xx.xml          (  83840 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/adreno/a4xx.xml          ( 112086 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/adreno/a5xx.xml          ( 147240 bytes, from 2018-08-06 18:45:45)
+- /home/robclark/src/envytools/rnndb/adreno/a6xx.xml          ( 101627 bytes, from 2018-08-06 18:45:45)
+- /home/robclark/src/envytools/rnndb/adreno/a6xx_gmu.xml      (  10431 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/adreno/ocmem.xml         (   1773 bytes, from 2018-07-03 19:37:13)
+
+Copyright (C) 2013-2018 by the following authors:
 - Rob Clark <robdclark@gmail.com> (robclark)
 - Ilia Mirkin <imirkin@alum.mit.edu> (imirkin)
 
@@ -71,7 +73,8 @@ enum vgt_event_type {
 	FLUSH_SO_1 = 18,
 	FLUSH_SO_2 = 19,
 	FLUSH_SO_3 = 20,
-	UNK_19 = 25,
+	PC_CCU_INVALIDATE_DEPTH = 24,
+	PC_CCU_INVALIDATE_COLOR = 25,
 	UNK_1C = 28,
 	UNK_1D = 29,
 	BLIT = 30,
@@ -199,9 +202,12 @@ enum adreno_pm4_type3_packets {
 	CP_WAIT_MEM_WRITES = 18,
 	CP_COND_REG_EXEC = 71,
 	CP_MEM_TO_REG = 66,
+	CP_EXEC_CS_INDIRECT = 65,
 	CP_EXEC_CS = 51,
 	CP_PERFCOUNTER_ACTION = 80,
 	CP_SMMU_TABLE_UPDATE = 83,
+	CP_SET_MARKER = 101,
+	CP_SET_PSEUDO_REG = 86,
 	CP_CONTEXT_REG_BUNCH = 92,
 	CP_YIELD_ENABLE = 28,
 	CP_SKIP_IB2_ENABLE_GLOBAL = 29,
@@ -215,7 +221,10 @@ enum adreno_pm4_type3_packets {
 	CP_COMPUTE_CHECKPOINT = 110,
 	CP_MEM_TO_MEM = 115,
 	CP_BLIT = 44,
-	CP_UNK_39 = 57,
+	CP_REG_TEST = 57,
+	CP_SET_MODE = 99,
+	CP_LOAD_STATE6_GEOM = 50,
+	CP_LOAD_STATE6_FRAG = 52,
 	IN_IB_PREFETCH_END = 23,
 	IN_SUBBLK_PREFETCH = 31,
 	IN_INSTR_PREFETCH = 32,
@@ -224,6 +233,11 @@ enum adreno_pm4_type3_packets {
 	IN_INCR_UPDT_STATE = 85,
 	IN_INCR_UPDT_CONST = 86,
 	IN_INCR_UPDT_INSTR = 87,
+	PKT4 = 4,
+	CP_UNK_A6XX_14 = 20,
+	CP_UNK_A6XX_36 = 54,
+	CP_UNK_A6XX_55 = 85,
+	UNK_A6XX_6D = 109,
 };
 
 enum adreno_state_block {
@@ -278,6 +292,33 @@ enum a4xx_state_src {
 	SS4_INDIRECT = 2,
 };
 
+enum a6xx_state_block {
+	SB6_VS_TEX = 0,
+	SB6_HS_TEX = 1,
+	SB6_DS_TEX = 2,
+	SB6_GS_TEX = 3,
+	SB6_FS_TEX = 4,
+	SB6_CS_TEX = 5,
+	SB6_VS_SHADER = 8,
+	SB6_HS_SHADER = 9,
+	SB6_DS_SHADER = 10,
+	SB6_GS_SHADER = 11,
+	SB6_FS_SHADER = 12,
+	SB6_CS_SHADER = 13,
+	SB6_SSBO = 14,
+	SB6_CS_SSBO = 15,
+};
+
+enum a6xx_state_type {
+	ST6_SHADER = 0,
+	ST6_CONSTANTS = 1,
+};
+
+enum a6xx_state_src {
+	SS6_DIRECT = 0,
+	SS6_INDIRECT = 2,
+};
+
 enum a4xx_index_size {
 	INDEX4_SIZE_8_BIT = 0,
 	INDEX4_SIZE_16_BIT = 1,
@@ -300,6 +341,7 @@ enum render_mode_cmd {
 	GMEM = 3,
 	BLIT2D = 5,
 	BLIT2DSCALE = 7,
+	END2D = 8,
 };
 
 enum cp_blit_cmd {
@@ -308,6 +350,22 @@ enum cp_blit_cmd {
 	BLIT_OP_SCALE = 3,
 };
 
+enum a6xx_render_mode {
+	RM6_BYPASS = 1,
+	RM6_BINNING = 2,
+	RM6_GMEM = 4,
+	RM6_BLIT2D = 5,
+	RM6_RESOLVE = 6,
+};
+
+enum pseudo_reg {
+	SMMU_INFO = 0,
+	NON_SECURE_SAVE_ADDR = 1,
+	SECURE_SAVE_ADDR = 2,
+	NON_PRIV_SAVE_ADDR = 3,
+	COUNTER = 4,
+};
+
 #define REG_CP_LOAD_STATE_0					0x00000000
 #define CP_LOAD_STATE_0_DST_OFF__MASK				0x0000ffff
 #define CP_LOAD_STATE_0_DST_OFF__SHIFT				0
@@ -349,7 +407,7 @@ static inline uint32_t CP_LOAD_STATE_1_EXT_SRC_ADDR(uint32_t val)
 }
 
 #define REG_CP_LOAD_STATE4_0					0x00000000
-#define CP_LOAD_STATE4_0_DST_OFF__MASK				0x0000ffff
+#define CP_LOAD_STATE4_0_DST_OFF__MASK				0x00003fff
 #define CP_LOAD_STATE4_0_DST_OFF__SHIFT				0
 static inline uint32_t CP_LOAD_STATE4_0_DST_OFF(uint32_t val)
 {
@@ -396,6 +454,54 @@ static inline uint32_t CP_LOAD_STATE4_2_EXT_SRC_ADDR_HI(uint32_t val)
 	return ((val) << CP_LOAD_STATE4_2_EXT_SRC_ADDR_HI__SHIFT) & CP_LOAD_STATE4_2_EXT_SRC_ADDR_HI__MASK;
 }
 
+#define REG_CP_LOAD_STATE6_0					0x00000000
+#define CP_LOAD_STATE6_0_DST_OFF__MASK				0x00003fff
+#define CP_LOAD_STATE6_0_DST_OFF__SHIFT				0
+static inline uint32_t CP_LOAD_STATE6_0_DST_OFF(uint32_t val)
+{
+	return ((val) << CP_LOAD_STATE6_0_DST_OFF__SHIFT) & CP_LOAD_STATE6_0_DST_OFF__MASK;
+}
+#define CP_LOAD_STATE6_0_STATE_TYPE__MASK			0x00004000
+#define CP_LOAD_STATE6_0_STATE_TYPE__SHIFT			14
+static inline uint32_t CP_LOAD_STATE6_0_STATE_TYPE(enum a6xx_state_type val)
+{
+	return ((val) << CP_LOAD_STATE6_0_STATE_TYPE__SHIFT) & CP_LOAD_STATE6_0_STATE_TYPE__MASK;
+}
+#define CP_LOAD_STATE6_0_STATE_SRC__MASK			0x00030000
+#define CP_LOAD_STATE6_0_STATE_SRC__SHIFT			16
+static inline uint32_t CP_LOAD_STATE6_0_STATE_SRC(enum a6xx_state_src val)
+{
+	return ((val) << CP_LOAD_STATE6_0_STATE_SRC__SHIFT) & CP_LOAD_STATE6_0_STATE_SRC__MASK;
+}
+#define CP_LOAD_STATE6_0_STATE_BLOCK__MASK			0x003c0000
+#define CP_LOAD_STATE6_0_STATE_BLOCK__SHIFT			18
+static inline uint32_t CP_LOAD_STATE6_0_STATE_BLOCK(enum a6xx_state_block val)
+{
+	return ((val) << CP_LOAD_STATE6_0_STATE_BLOCK__SHIFT) & CP_LOAD_STATE6_0_STATE_BLOCK__MASK;
+}
+#define CP_LOAD_STATE6_0_NUM_UNIT__MASK				0xffc00000
+#define CP_LOAD_STATE6_0_NUM_UNIT__SHIFT			22
+static inline uint32_t CP_LOAD_STATE6_0_NUM_UNIT(uint32_t val)
+{
+	return ((val) << CP_LOAD_STATE6_0_NUM_UNIT__SHIFT) & CP_LOAD_STATE6_0_NUM_UNIT__MASK;
+}
+
+#define REG_CP_LOAD_STATE6_1					0x00000001
+#define CP_LOAD_STATE6_1_EXT_SRC_ADDR__MASK			0xfffffffc
+#define CP_LOAD_STATE6_1_EXT_SRC_ADDR__SHIFT			2
+static inline uint32_t CP_LOAD_STATE6_1_EXT_SRC_ADDR(uint32_t val)
+{
+	return ((val >> 2) << CP_LOAD_STATE6_1_EXT_SRC_ADDR__SHIFT) & CP_LOAD_STATE6_1_EXT_SRC_ADDR__MASK;
+}
+
+#define REG_CP_LOAD_STATE6_2					0x00000002
+#define CP_LOAD_STATE6_2_EXT_SRC_ADDR_HI__MASK			0xffffffff
+#define CP_LOAD_STATE6_2_EXT_SRC_ADDR_HI__SHIFT			0
+static inline uint32_t CP_LOAD_STATE6_2_EXT_SRC_ADDR_HI(uint32_t val)
+{
+	return ((val) << CP_LOAD_STATE6_2_EXT_SRC_ADDR_HI__SHIFT) & CP_LOAD_STATE6_2_EXT_SRC_ADDR_HI__MASK;
+}
+
 #define REG_CP_DRAW_INDX_0					0x00000000
 #define CP_DRAW_INDX_0_VIZ_QUERY__MASK				0xffffffff
 #define CP_DRAW_INDX_0_VIZ_QUERY__SHIFT				0
@@ -580,6 +686,153 @@ static inline uint32_t CP_DRAW_INDX_OFFSET_5_INDX_SIZE(uint32_t val)
 	return ((val) << CP_DRAW_INDX_OFFSET_5_INDX_SIZE__SHIFT) & CP_DRAW_INDX_OFFSET_5_INDX_SIZE__MASK;
 }
 
+#define REG_A4XX_CP_DRAW_INDIRECT_0				0x00000000
+#define A4XX_CP_DRAW_INDIRECT_0_PRIM_TYPE__MASK			0x0000003f
+#define A4XX_CP_DRAW_INDIRECT_0_PRIM_TYPE__SHIFT		0
+static inline uint32_t A4XX_CP_DRAW_INDIRECT_0_PRIM_TYPE(enum pc_di_primtype val)
+{
+	return ((val) << A4XX_CP_DRAW_INDIRECT_0_PRIM_TYPE__SHIFT) & A4XX_CP_DRAW_INDIRECT_0_PRIM_TYPE__MASK;
+}
+#define A4XX_CP_DRAW_INDIRECT_0_SOURCE_SELECT__MASK		0x000000c0
+#define A4XX_CP_DRAW_INDIRECT_0_SOURCE_SELECT__SHIFT		6
+static inline uint32_t A4XX_CP_DRAW_INDIRECT_0_SOURCE_SELECT(enum pc_di_src_sel val)
+{
+	return ((val) << A4XX_CP_DRAW_INDIRECT_0_SOURCE_SELECT__SHIFT) & A4XX_CP_DRAW_INDIRECT_0_SOURCE_SELECT__MASK;
+}
+#define A4XX_CP_DRAW_INDIRECT_0_VIS_CULL__MASK			0x00000300
+#define A4XX_CP_DRAW_INDIRECT_0_VIS_CULL__SHIFT			8
+static inline uint32_t A4XX_CP_DRAW_INDIRECT_0_VIS_CULL(enum pc_di_vis_cull_mode val)
+{
+	return ((val) << A4XX_CP_DRAW_INDIRECT_0_VIS_CULL__SHIFT) & A4XX_CP_DRAW_INDIRECT_0_VIS_CULL__MASK;
+}
+#define A4XX_CP_DRAW_INDIRECT_0_INDEX_SIZE__MASK		0x00000c00
+#define A4XX_CP_DRAW_INDIRECT_0_INDEX_SIZE__SHIFT		10
+static inline uint32_t A4XX_CP_DRAW_INDIRECT_0_INDEX_SIZE(enum a4xx_index_size val)
+{
+	return ((val) << A4XX_CP_DRAW_INDIRECT_0_INDEX_SIZE__SHIFT) & A4XX_CP_DRAW_INDIRECT_0_INDEX_SIZE__MASK;
+}
+#define A4XX_CP_DRAW_INDIRECT_0_TESS_MODE__MASK			0x01f00000
+#define A4XX_CP_DRAW_INDIRECT_0_TESS_MODE__SHIFT		20
+static inline uint32_t A4XX_CP_DRAW_INDIRECT_0_TESS_MODE(uint32_t val)
+{
+	return ((val) << A4XX_CP_DRAW_INDIRECT_0_TESS_MODE__SHIFT) & A4XX_CP_DRAW_INDIRECT_0_TESS_MODE__MASK;
+}
+
+#define REG_A4XX_CP_DRAW_INDIRECT_1				0x00000001
+#define A4XX_CP_DRAW_INDIRECT_1_INDIRECT__MASK			0xffffffff
+#define A4XX_CP_DRAW_INDIRECT_1_INDIRECT__SHIFT			0
+static inline uint32_t A4XX_CP_DRAW_INDIRECT_1_INDIRECT(uint32_t val)
+{
+	return ((val) << A4XX_CP_DRAW_INDIRECT_1_INDIRECT__SHIFT) & A4XX_CP_DRAW_INDIRECT_1_INDIRECT__MASK;
+}
+
+
+#define REG_A5XX_CP_DRAW_INDIRECT_2				0x00000002
+#define A5XX_CP_DRAW_INDIRECT_2_INDIRECT_HI__MASK		0xffffffff
+#define A5XX_CP_DRAW_INDIRECT_2_INDIRECT_HI__SHIFT		0
+static inline uint32_t A5XX_CP_DRAW_INDIRECT_2_INDIRECT_HI(uint32_t val)
+{
+	return ((val) << A5XX_CP_DRAW_INDIRECT_2_INDIRECT_HI__SHIFT) & A5XX_CP_DRAW_INDIRECT_2_INDIRECT_HI__MASK;
+}
+
+#define REG_A4XX_CP_DRAW_INDX_INDIRECT_0			0x00000000
+#define A4XX_CP_DRAW_INDX_INDIRECT_0_PRIM_TYPE__MASK		0x0000003f
+#define A4XX_CP_DRAW_INDX_INDIRECT_0_PRIM_TYPE__SHIFT		0
+static inline uint32_t A4XX_CP_DRAW_INDX_INDIRECT_0_PRIM_TYPE(enum pc_di_primtype val)
+{
+	return ((val) << A4XX_CP_DRAW_INDX_INDIRECT_0_PRIM_TYPE__SHIFT) & A4XX_CP_DRAW_INDX_INDIRECT_0_PRIM_TYPE__MASK;
+}
+#define A4XX_CP_DRAW_INDX_INDIRECT_0_SOURCE_SELECT__MASK	0x000000c0
+#define A4XX_CP_DRAW_INDX_INDIRECT_0_SOURCE_SELECT__SHIFT	6
+static inline uint32_t A4XX_CP_DRAW_INDX_INDIRECT_0_SOURCE_SELECT(enum pc_di_src_sel val)
+{
+	return ((val) << A4XX_CP_DRAW_INDX_INDIRECT_0_SOURCE_SELECT__SHIFT) & A4XX_CP_DRAW_INDX_INDIRECT_0_SOURCE_SELECT__MASK;
+}
+#define A4XX_CP_DRAW_INDX_INDIRECT_0_VIS_CULL__MASK		0x00000300
+#define A4XX_CP_DRAW_INDX_INDIRECT_0_VIS_CULL__SHIFT		8
+static inline uint32_t A4XX_CP_DRAW_INDX_INDIRECT_0_VIS_CULL(enum pc_di_vis_cull_mode val)
+{
+	return ((val) << A4XX_CP_DRAW_INDX_INDIRECT_0_VIS_CULL__SHIFT) & A4XX_CP_DRAW_INDX_INDIRECT_0_VIS_CULL__MASK;
+}
+#define A4XX_CP_DRAW_INDX_INDIRECT_0_INDEX_SIZE__MASK		0x00000c00
+#define A4XX_CP_DRAW_INDX_INDIRECT_0_INDEX_SIZE__SHIFT		10
+static inline uint32_t A4XX_CP_DRAW_INDX_INDIRECT_0_INDEX_SIZE(enum a4xx_index_size val)
+{
+	return ((val) << A4XX_CP_DRAW_INDX_INDIRECT_0_INDEX_SIZE__SHIFT) & A4XX_CP_DRAW_INDX_INDIRECT_0_INDEX_SIZE__MASK;
+}
+#define A4XX_CP_DRAW_INDX_INDIRECT_0_TESS_MODE__MASK		0x01f00000
+#define A4XX_CP_DRAW_INDX_INDIRECT_0_TESS_MODE__SHIFT		20
+static inline uint32_t A4XX_CP_DRAW_INDX_INDIRECT_0_TESS_MODE(uint32_t val)
+{
+	return ((val) << A4XX_CP_DRAW_INDX_INDIRECT_0_TESS_MODE__SHIFT) & A4XX_CP_DRAW_INDX_INDIRECT_0_TESS_MODE__MASK;
+}
+
+
+#define REG_A4XX_CP_DRAW_INDX_INDIRECT_1			0x00000001
+#define A4XX_CP_DRAW_INDX_INDIRECT_1_INDX_BASE__MASK		0xffffffff
+#define A4XX_CP_DRAW_INDX_INDIRECT_1_INDX_BASE__SHIFT		0
+static inline uint32_t A4XX_CP_DRAW_INDX_INDIRECT_1_INDX_BASE(uint32_t val)
+{
+	return ((val) << A4XX_CP_DRAW_INDX_INDIRECT_1_INDX_BASE__SHIFT) & A4XX_CP_DRAW_INDX_INDIRECT_1_INDX_BASE__MASK;
+}
+
+#define REG_A4XX_CP_DRAW_INDX_INDIRECT_2			0x00000002
+#define A4XX_CP_DRAW_INDX_INDIRECT_2_INDX_SIZE__MASK		0xffffffff
+#define A4XX_CP_DRAW_INDX_INDIRECT_2_INDX_SIZE__SHIFT		0
+static inline uint32_t A4XX_CP_DRAW_INDX_INDIRECT_2_INDX_SIZE(uint32_t val)
+{
+	return ((val) << A4XX_CP_DRAW_INDX_INDIRECT_2_INDX_SIZE__SHIFT) & A4XX_CP_DRAW_INDX_INDIRECT_2_INDX_SIZE__MASK;
+}
+
+#define REG_A4XX_CP_DRAW_INDX_INDIRECT_3			0x00000003
+#define A4XX_CP_DRAW_INDX_INDIRECT_3_INDIRECT__MASK		0xffffffff
+#define A4XX_CP_DRAW_INDX_INDIRECT_3_INDIRECT__SHIFT		0
+static inline uint32_t A4XX_CP_DRAW_INDX_INDIRECT_3_INDIRECT(uint32_t val)
+{
+	return ((val) << A4XX_CP_DRAW_INDX_INDIRECT_3_INDIRECT__SHIFT) & A4XX_CP_DRAW_INDX_INDIRECT_3_INDIRECT__MASK;
+}
+
+
+#define REG_A5XX_CP_DRAW_INDX_INDIRECT_1			0x00000001
+#define A5XX_CP_DRAW_INDX_INDIRECT_1_INDX_BASE_LO__MASK		0xffffffff
+#define A5XX_CP_DRAW_INDX_INDIRECT_1_INDX_BASE_LO__SHIFT	0
+static inline uint32_t A5XX_CP_DRAW_INDX_INDIRECT_1_INDX_BASE_LO(uint32_t val)
+{
+	return ((val) << A5XX_CP_DRAW_INDX_INDIRECT_1_INDX_BASE_LO__SHIFT) & A5XX_CP_DRAW_INDX_INDIRECT_1_INDX_BASE_LO__MASK;
+}
+
+#define REG_A5XX_CP_DRAW_INDX_INDIRECT_2			0x00000002
+#define A5XX_CP_DRAW_INDX_INDIRECT_2_INDX_BASE_HI__MASK		0xffffffff
+#define A5XX_CP_DRAW_INDX_INDIRECT_2_INDX_BASE_HI__SHIFT	0
+static inline uint32_t A5XX_CP_DRAW_INDX_INDIRECT_2_INDX_BASE_HI(uint32_t val)
+{
+	return ((val) << A5XX_CP_DRAW_INDX_INDIRECT_2_INDX_BASE_HI__SHIFT) & A5XX_CP_DRAW_INDX_INDIRECT_2_INDX_BASE_HI__MASK;
+}
+
+#define REG_A5XX_CP_DRAW_INDX_INDIRECT_3			0x00000003
+#define A5XX_CP_DRAW_INDX_INDIRECT_3_MAX_INDICES__MASK		0xffffffff
+#define A5XX_CP_DRAW_INDX_INDIRECT_3_MAX_INDICES__SHIFT		0
+static inline uint32_t A5XX_CP_DRAW_INDX_INDIRECT_3_MAX_INDICES(uint32_t val)
+{
+	return ((val) << A5XX_CP_DRAW_INDX_INDIRECT_3_MAX_INDICES__SHIFT) & A5XX_CP_DRAW_INDX_INDIRECT_3_MAX_INDICES__MASK;
+}
+
+#define REG_A5XX_CP_DRAW_INDX_INDIRECT_4			0x00000004
+#define A5XX_CP_DRAW_INDX_INDIRECT_4_INDIRECT_LO__MASK		0xffffffff
+#define A5XX_CP_DRAW_INDX_INDIRECT_4_INDIRECT_LO__SHIFT		0
+static inline uint32_t A5XX_CP_DRAW_INDX_INDIRECT_4_INDIRECT_LO(uint32_t val)
+{
+	return ((val) << A5XX_CP_DRAW_INDX_INDIRECT_4_INDIRECT_LO__SHIFT) & A5XX_CP_DRAW_INDX_INDIRECT_4_INDIRECT_LO__MASK;
+}
+
+#define REG_A5XX_CP_DRAW_INDX_INDIRECT_5			0x00000005
+#define A5XX_CP_DRAW_INDX_INDIRECT_5_INDIRECT_HI__MASK		0xffffffff
+#define A5XX_CP_DRAW_INDX_INDIRECT_5_INDIRECT_HI__SHIFT		0
+static inline uint32_t A5XX_CP_DRAW_INDX_INDIRECT_5_INDIRECT_HI(uint32_t val)
+{
+	return ((val) << A5XX_CP_DRAW_INDX_INDIRECT_5_INDIRECT_HI__SHIFT) & A5XX_CP_DRAW_INDX_INDIRECT_5_INDIRECT_HI__MASK;
+}
+
 static inline uint32_t REG_CP_SET_DRAW_STATE_(uint32_t i0) { return 0x00000000 + 0x3*i0; }
 
 static inline uint32_t REG_CP_SET_DRAW_STATE__0(uint32_t i0) { return 0x00000000 + 0x3*i0; }
@@ -593,6 +846,12 @@ static inline uint32_t CP_SET_DRAW_STATE__0_COUNT(uint32_t val)
 #define CP_SET_DRAW_STATE__0_DISABLE				0x00020000
 #define CP_SET_DRAW_STATE__0_DISABLE_ALL_GROUPS			0x00040000
 #define CP_SET_DRAW_STATE__0_LOAD_IMMED				0x00080000
+#define CP_SET_DRAW_STATE__0_ENABLE_MASK__MASK			0x00f00000
+#define CP_SET_DRAW_STATE__0_ENABLE_MASK__SHIFT			20
+static inline uint32_t CP_SET_DRAW_STATE__0_ENABLE_MASK(uint32_t val)
+{
+	return ((val) << CP_SET_DRAW_STATE__0_ENABLE_MASK__SHIFT) & CP_SET_DRAW_STATE__0_ENABLE_MASK__MASK;
+}
 #define CP_SET_DRAW_STATE__0_GROUP_ID__MASK			0x1f000000
 #define CP_SET_DRAW_STATE__0_GROUP_ID__SHIFT			24
 static inline uint32_t CP_SET_DRAW_STATE__0_GROUP_ID(uint32_t val)
@@ -708,6 +967,22 @@ static inline uint32_t CP_SET_BIN_DATA5_4_BIN_SIZE_ADDRESS_HI(uint32_t val)
 	return ((val) << CP_SET_BIN_DATA5_4_BIN_SIZE_ADDRESS_HI__SHIFT) & CP_SET_BIN_DATA5_4_BIN_SIZE_ADDRESS_HI__MASK;
 }
 
+#define REG_CP_SET_BIN_DATA5_5					0x00000005
+#define CP_SET_BIN_DATA5_5_XXX_ADDRESS_LO__MASK			0xffffffff
+#define CP_SET_BIN_DATA5_5_XXX_ADDRESS_LO__SHIFT		0
+static inline uint32_t CP_SET_BIN_DATA5_5_XXX_ADDRESS_LO(uint32_t val)
+{
+	return ((val) << CP_SET_BIN_DATA5_5_XXX_ADDRESS_LO__SHIFT) & CP_SET_BIN_DATA5_5_XXX_ADDRESS_LO__MASK;
+}
+
+#define REG_CP_SET_BIN_DATA5_6					0x00000006
+#define CP_SET_BIN_DATA5_6_XXX_ADDRESS_HI__MASK			0xffffffff
+#define CP_SET_BIN_DATA5_6_XXX_ADDRESS_HI__SHIFT		0
+static inline uint32_t CP_SET_BIN_DATA5_6_XXX_ADDRESS_HI(uint32_t val)
+{
+	return ((val) << CP_SET_BIN_DATA5_6_XXX_ADDRESS_HI__SHIFT) & CP_SET_BIN_DATA5_6_XXX_ADDRESS_HI__MASK;
+}
+
 #define REG_CP_REG_TO_MEM_0					0x00000000
 #define CP_REG_TO_MEM_0_REG__MASK				0x0000ffff
 #define CP_REG_TO_MEM_0_REG__SHIFT				0
@@ -732,6 +1007,46 @@ static inline uint32_t CP_REG_TO_MEM_1_DEST(uint32_t val)
 	return ((val) << CP_REG_TO_MEM_1_DEST__SHIFT) & CP_REG_TO_MEM_1_DEST__MASK;
 }
 
+#define REG_CP_REG_TO_MEM_2					0x00000002
+#define CP_REG_TO_MEM_2_DEST_HI__MASK				0xffffffff
+#define CP_REG_TO_MEM_2_DEST_HI__SHIFT				0
+static inline uint32_t CP_REG_TO_MEM_2_DEST_HI(uint32_t val)
+{
+	return ((val) << CP_REG_TO_MEM_2_DEST_HI__SHIFT) & CP_REG_TO_MEM_2_DEST_HI__MASK;
+}
+
+#define REG_CP_MEM_TO_REG_0					0x00000000
+#define CP_MEM_TO_REG_0_REG__MASK				0x0000ffff
+#define CP_MEM_TO_REG_0_REG__SHIFT				0
+static inline uint32_t CP_MEM_TO_REG_0_REG(uint32_t val)
+{
+	return ((val) << CP_MEM_TO_REG_0_REG__SHIFT) & CP_MEM_TO_REG_0_REG__MASK;
+}
+#define CP_MEM_TO_REG_0_CNT__MASK				0x3ff80000
+#define CP_MEM_TO_REG_0_CNT__SHIFT				19
+static inline uint32_t CP_MEM_TO_REG_0_CNT(uint32_t val)
+{
+	return ((val) << CP_MEM_TO_REG_0_CNT__SHIFT) & CP_MEM_TO_REG_0_CNT__MASK;
+}
+#define CP_MEM_TO_REG_0_64B					0x40000000
+#define CP_MEM_TO_REG_0_ACCUMULATE				0x80000000
+
+#define REG_CP_MEM_TO_REG_1					0x00000001
+#define CP_MEM_TO_REG_1_SRC__MASK				0xffffffff
+#define CP_MEM_TO_REG_1_SRC__SHIFT				0
+static inline uint32_t CP_MEM_TO_REG_1_SRC(uint32_t val)
+{
+	return ((val) << CP_MEM_TO_REG_1_SRC__SHIFT) & CP_MEM_TO_REG_1_SRC__MASK;
+}
+
+#define REG_CP_MEM_TO_REG_2					0x00000002
+#define CP_MEM_TO_REG_2_SRC_HI__MASK				0xffffffff
+#define CP_MEM_TO_REG_2_SRC_HI__SHIFT				0
+static inline uint32_t CP_MEM_TO_REG_2_SRC_HI(uint32_t val)
+{
+	return ((val) << CP_MEM_TO_REG_2_SRC_HI__SHIFT) & CP_MEM_TO_REG_2_SRC_HI__MASK;
+}
+
 #define REG_CP_MEM_TO_MEM_0					0x00000000
 #define CP_MEM_TO_MEM_0_NEG_A					0x00000001
 #define CP_MEM_TO_MEM_0_NEG_B					0x00000002
@@ -953,15 +1268,15 @@ static inline uint32_t CP_COMPUTE_CHECKPOINT_1_ADDR_0_HI(uint32_t val)
 #define REG_CP_COMPUTE_CHECKPOINT_2				0x00000002
 
 #define REG_CP_COMPUTE_CHECKPOINT_3				0x00000003
-
-#define REG_CP_COMPUTE_CHECKPOINT_4				0x00000004
-#define CP_COMPUTE_CHECKPOINT_4_ADDR_1_LEN__MASK		0xffffffff
-#define CP_COMPUTE_CHECKPOINT_4_ADDR_1_LEN__SHIFT		0
-static inline uint32_t CP_COMPUTE_CHECKPOINT_4_ADDR_1_LEN(uint32_t val)
+#define CP_COMPUTE_CHECKPOINT_3_ADDR_1_LEN__MASK		0xffffffff
+#define CP_COMPUTE_CHECKPOINT_3_ADDR_1_LEN__SHIFT		0
+static inline uint32_t CP_COMPUTE_CHECKPOINT_3_ADDR_1_LEN(uint32_t val)
 {
-	return ((val) << CP_COMPUTE_CHECKPOINT_4_ADDR_1_LEN__SHIFT) & CP_COMPUTE_CHECKPOINT_4_ADDR_1_LEN__MASK;
+	return ((val) << CP_COMPUTE_CHECKPOINT_3_ADDR_1_LEN__SHIFT) & CP_COMPUTE_CHECKPOINT_3_ADDR_1_LEN__MASK;
 }
 
+#define REG_CP_COMPUTE_CHECKPOINT_4				0x00000004
+
 #define REG_CP_COMPUTE_CHECKPOINT_5				0x00000005
 #define CP_COMPUTE_CHECKPOINT_5_ADDR_1_LO__MASK			0xffffffff
 #define CP_COMPUTE_CHECKPOINT_5_ADDR_1_LO__SHIFT		0
@@ -978,6 +1293,8 @@ static inline uint32_t CP_COMPUTE_CHECKPOINT_6_ADDR_1_HI(uint32_t val)
 	return ((val) << CP_COMPUTE_CHECKPOINT_6_ADDR_1_HI__SHIFT) & CP_COMPUTE_CHECKPOINT_6_ADDR_1_HI__MASK;
 }
 
+#define REG_CP_COMPUTE_CHECKPOINT_7				0x00000007
+
 #define REG_CP_PERFCOUNTER_ACTION_0				0x00000000
 
 #define REG_CP_PERFCOUNTER_ACTION_1				0x00000001
@@ -1032,13 +1349,13 @@ static inline uint32_t CP_BLIT_0_OP(enum cp_blit_cmd val)
 }
 
 #define REG_CP_BLIT_1						0x00000001
-#define CP_BLIT_1_SRC_X1__MASK					0x0000ffff
+#define CP_BLIT_1_SRC_X1__MASK					0x00003fff
 #define CP_BLIT_1_SRC_X1__SHIFT					0
 static inline uint32_t CP_BLIT_1_SRC_X1(uint32_t val)
 {
 	return ((val) << CP_BLIT_1_SRC_X1__SHIFT) & CP_BLIT_1_SRC_X1__MASK;
 }
-#define CP_BLIT_1_SRC_Y1__MASK					0xffff0000
+#define CP_BLIT_1_SRC_Y1__MASK					0x3fff0000
 #define CP_BLIT_1_SRC_Y1__SHIFT					16
 static inline uint32_t CP_BLIT_1_SRC_Y1(uint32_t val)
 {
@@ -1046,13 +1363,13 @@ static inline uint32_t CP_BLIT_1_SRC_Y1(uint32_t val)
 }
 
 #define REG_CP_BLIT_2						0x00000002
-#define CP_BLIT_2_SRC_X2__MASK					0x0000ffff
+#define CP_BLIT_2_SRC_X2__MASK					0x00003fff
 #define CP_BLIT_2_SRC_X2__SHIFT					0
 static inline uint32_t CP_BLIT_2_SRC_X2(uint32_t val)
 {
 	return ((val) << CP_BLIT_2_SRC_X2__SHIFT) & CP_BLIT_2_SRC_X2__MASK;
 }
-#define CP_BLIT_2_SRC_Y2__MASK					0xffff0000
+#define CP_BLIT_2_SRC_Y2__MASK					0x3fff0000
 #define CP_BLIT_2_SRC_Y2__SHIFT					16
 static inline uint32_t CP_BLIT_2_SRC_Y2(uint32_t val)
 {
@@ -1060,13 +1377,13 @@ static inline uint32_t CP_BLIT_2_SRC_Y2(uint32_t val)
 }
 
 #define REG_CP_BLIT_3						0x00000003
-#define CP_BLIT_3_DST_X1__MASK					0x0000ffff
+#define CP_BLIT_3_DST_X1__MASK					0x00003fff
 #define CP_BLIT_3_DST_X1__SHIFT					0
 static inline uint32_t CP_BLIT_3_DST_X1(uint32_t val)
 {
 	return ((val) << CP_BLIT_3_DST_X1__SHIFT) & CP_BLIT_3_DST_X1__MASK;
 }
-#define CP_BLIT_3_DST_Y1__MASK					0xffff0000
+#define CP_BLIT_3_DST_Y1__MASK					0x3fff0000
 #define CP_BLIT_3_DST_Y1__SHIFT					16
 static inline uint32_t CP_BLIT_3_DST_Y1(uint32_t val)
 {
@@ -1074,13 +1391,13 @@ static inline uint32_t CP_BLIT_3_DST_Y1(uint32_t val)
 }
 
 #define REG_CP_BLIT_4						0x00000004
-#define CP_BLIT_4_DST_X2__MASK					0x0000ffff
+#define CP_BLIT_4_DST_X2__MASK					0x00003fff
 #define CP_BLIT_4_DST_X2__SHIFT					0
 static inline uint32_t CP_BLIT_4_DST_X2(uint32_t val)
 {
 	return ((val) << CP_BLIT_4_DST_X2__SHIFT) & CP_BLIT_4_DST_X2__MASK;
 }
-#define CP_BLIT_4_DST_Y2__MASK					0xffff0000
+#define CP_BLIT_4_DST_Y2__MASK					0x3fff0000
 #define CP_BLIT_4_DST_Y2__SHIFT					16
 static inline uint32_t CP_BLIT_4_DST_Y2(uint32_t val)
 {
@@ -1113,5 +1430,129 @@ static inline uint32_t CP_EXEC_CS_3_NGROUPS_Z(uint32_t val)
 	return ((val) << CP_EXEC_CS_3_NGROUPS_Z__SHIFT) & CP_EXEC_CS_3_NGROUPS_Z__MASK;
 }
 
+#define REG_A4XX_CP_EXEC_CS_INDIRECT_0				0x00000000
+
+
+#define REG_A4XX_CP_EXEC_CS_INDIRECT_1				0x00000001
+#define A4XX_CP_EXEC_CS_INDIRECT_1_ADDR__MASK			0xffffffff
+#define A4XX_CP_EXEC_CS_INDIRECT_1_ADDR__SHIFT			0
+static inline uint32_t A4XX_CP_EXEC_CS_INDIRECT_1_ADDR(uint32_t val)
+{
+	return ((val) << A4XX_CP_EXEC_CS_INDIRECT_1_ADDR__SHIFT) & A4XX_CP_EXEC_CS_INDIRECT_1_ADDR__MASK;
+}
+
+#define REG_A4XX_CP_EXEC_CS_INDIRECT_2				0x00000002
+#define A4XX_CP_EXEC_CS_INDIRECT_2_LOCALSIZEX__MASK		0x00000ffc
+#define A4XX_CP_EXEC_CS_INDIRECT_2_LOCALSIZEX__SHIFT		2
+static inline uint32_t A4XX_CP_EXEC_CS_INDIRECT_2_LOCALSIZEX(uint32_t val)
+{
+	return ((val) << A4XX_CP_EXEC_CS_INDIRECT_2_LOCALSIZEX__SHIFT) & A4XX_CP_EXEC_CS_INDIRECT_2_LOCALSIZEX__MASK;
+}
+#define A4XX_CP_EXEC_CS_INDIRECT_2_LOCALSIZEY__MASK		0x003ff000
+#define A4XX_CP_EXEC_CS_INDIRECT_2_LOCALSIZEY__SHIFT		12
+static inline uint32_t A4XX_CP_EXEC_CS_INDIRECT_2_LOCALSIZEY(uint32_t val)
+{
+	return ((val) << A4XX_CP_EXEC_CS_INDIRECT_2_LOCALSIZEY__SHIFT) & A4XX_CP_EXEC_CS_INDIRECT_2_LOCALSIZEY__MASK;
+}
+#define A4XX_CP_EXEC_CS_INDIRECT_2_LOCALSIZEZ__MASK		0xffc00000
+#define A4XX_CP_EXEC_CS_INDIRECT_2_LOCALSIZEZ__SHIFT		22
+static inline uint32_t A4XX_CP_EXEC_CS_INDIRECT_2_LOCALSIZEZ(uint32_t val)
+{
+	return ((val) << A4XX_CP_EXEC_CS_INDIRECT_2_LOCALSIZEZ__SHIFT) & A4XX_CP_EXEC_CS_INDIRECT_2_LOCALSIZEZ__MASK;
+}
+
+
+#define REG_A5XX_CP_EXEC_CS_INDIRECT_1				0x00000001
+#define A5XX_CP_EXEC_CS_INDIRECT_1_ADDR_LO__MASK		0xffffffff
+#define A5XX_CP_EXEC_CS_INDIRECT_1_ADDR_LO__SHIFT		0
+static inline uint32_t A5XX_CP_EXEC_CS_INDIRECT_1_ADDR_LO(uint32_t val)
+{
+	return ((val) << A5XX_CP_EXEC_CS_INDIRECT_1_ADDR_LO__SHIFT) & A5XX_CP_EXEC_CS_INDIRECT_1_ADDR_LO__MASK;
+}
+
+#define REG_A5XX_CP_EXEC_CS_INDIRECT_2				0x00000002
+#define A5XX_CP_EXEC_CS_INDIRECT_2_ADDR_HI__MASK		0xffffffff
+#define A5XX_CP_EXEC_CS_INDIRECT_2_ADDR_HI__SHIFT		0
+static inline uint32_t A5XX_CP_EXEC_CS_INDIRECT_2_ADDR_HI(uint32_t val)
+{
+	return ((val) << A5XX_CP_EXEC_CS_INDIRECT_2_ADDR_HI__SHIFT) & A5XX_CP_EXEC_CS_INDIRECT_2_ADDR_HI__MASK;
+}
+
+#define REG_A5XX_CP_EXEC_CS_INDIRECT_3				0x00000003
+#define A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEX__MASK		0x00000ffc
+#define A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEX__SHIFT		2
+static inline uint32_t A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEX(uint32_t val)
+{
+	return ((val) << A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEX__SHIFT) & A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEX__MASK;
+}
+#define A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEY__MASK		0x003ff000
+#define A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEY__SHIFT		12
+static inline uint32_t A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEY(uint32_t val)
+{
+	return ((val) << A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEY__SHIFT) & A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEY__MASK;
+}
+#define A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEZ__MASK		0xffc00000
+#define A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEZ__SHIFT		22
+static inline uint32_t A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEZ(uint32_t val)
+{
+	return ((val) << A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEZ__SHIFT) & A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEZ__MASK;
+}
+
+#define REG_A2XX_CP_SET_MARKER_0				0x00000000
+#define A2XX_CP_SET_MARKER_0_MARKER__MASK			0x0000000f
+#define A2XX_CP_SET_MARKER_0_MARKER__SHIFT			0
+static inline uint32_t A2XX_CP_SET_MARKER_0_MARKER(uint32_t val)
+{
+	return ((val) << A2XX_CP_SET_MARKER_0_MARKER__SHIFT) & A2XX_CP_SET_MARKER_0_MARKER__MASK;
+}
+#define A2XX_CP_SET_MARKER_0_MODE__MASK				0x0000000f
+#define A2XX_CP_SET_MARKER_0_MODE__SHIFT			0
+static inline uint32_t A2XX_CP_SET_MARKER_0_MODE(enum a6xx_render_mode val)
+{
+	return ((val) << A2XX_CP_SET_MARKER_0_MODE__SHIFT) & A2XX_CP_SET_MARKER_0_MODE__MASK;
+}
+#define A2XX_CP_SET_MARKER_0_IFPC				0x00000100
+
+static inline uint32_t REG_A2XX_CP_SET_PSEUDO_REG_(uint32_t i0) { return 0x00000000 + 0x3*i0; }
+
+static inline uint32_t REG_A2XX_CP_SET_PSEUDO_REG__0(uint32_t i0) { return 0x00000000 + 0x3*i0; }
+#define A2XX_CP_SET_PSEUDO_REG__0_PSEUDO_REG__MASK		0x00000007
+#define A2XX_CP_SET_PSEUDO_REG__0_PSEUDO_REG__SHIFT		0
+static inline uint32_t A2XX_CP_SET_PSEUDO_REG__0_PSEUDO_REG(enum pseudo_reg val)
+{
+	return ((val) << A2XX_CP_SET_PSEUDO_REG__0_PSEUDO_REG__SHIFT) & A2XX_CP_SET_PSEUDO_REG__0_PSEUDO_REG__MASK;
+}
+
+static inline uint32_t REG_A2XX_CP_SET_PSEUDO_REG__1(uint32_t i0) { return 0x00000001 + 0x3*i0; }
+#define A2XX_CP_SET_PSEUDO_REG__1_LO__MASK			0xffffffff
+#define A2XX_CP_SET_PSEUDO_REG__1_LO__SHIFT			0
+static inline uint32_t A2XX_CP_SET_PSEUDO_REG__1_LO(uint32_t val)
+{
+	return ((val) << A2XX_CP_SET_PSEUDO_REG__1_LO__SHIFT) & A2XX_CP_SET_PSEUDO_REG__1_LO__MASK;
+}
+
+static inline uint32_t REG_A2XX_CP_SET_PSEUDO_REG__2(uint32_t i0) { return 0x00000002 + 0x3*i0; }
+#define A2XX_CP_SET_PSEUDO_REG__2_HI__MASK			0xffffffff
+#define A2XX_CP_SET_PSEUDO_REG__2_HI__SHIFT			0
+static inline uint32_t A2XX_CP_SET_PSEUDO_REG__2_HI(uint32_t val)
+{
+	return ((val) << A2XX_CP_SET_PSEUDO_REG__2_HI__SHIFT) & A2XX_CP_SET_PSEUDO_REG__2_HI__MASK;
+}
+
+#define REG_A2XX_CP_REG_TEST_0					0x00000000
+#define A2XX_CP_REG_TEST_0_REG__MASK				0x00000fff
+#define A2XX_CP_REG_TEST_0_REG__SHIFT				0
+static inline uint32_t A2XX_CP_REG_TEST_0_REG(uint32_t val)
+{
+	return ((val) << A2XX_CP_REG_TEST_0_REG__SHIFT) & A2XX_CP_REG_TEST_0_REG__MASK;
+}
+#define A2XX_CP_REG_TEST_0_BIT__MASK				0x01f00000
+#define A2XX_CP_REG_TEST_0_BIT__SHIFT				20
+static inline uint32_t A2XX_CP_REG_TEST_0_BIT(uint32_t val)
+{
+	return ((val) << A2XX_CP_REG_TEST_0_BIT__SHIFT) & A2XX_CP_REG_TEST_0_BIT__MASK;
+}
+#define A2XX_CP_REG_TEST_0_UNK25				0x02000000
+
 
 #endif /* ADRENO_PM4_XML */
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_core_irq.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_core_irq.c
new file mode 100644
index 000000000000..879c13fe74e0
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_core_irq.c
@@ -0,0 +1,479 @@
+/* Copyright (c) 2015-2018, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#define pr_fmt(fmt)	"[drm:%s:%d] " fmt, __func__, __LINE__
+
+#include <linux/debugfs.h>
+#include <linux/irqdomain.h>
+#include <linux/irq.h>
+#include <linux/kthread.h>
+
+#include "dpu_core_irq.h"
+#include "dpu_trace.h"
+
+/**
+ * dpu_core_irq_callback_handler - dispatch core interrupts
+ * @arg:		private data of callback handler
+ * @irq_idx:		interrupt index
+ */
+static void dpu_core_irq_callback_handler(void *arg, int irq_idx)
+{
+	struct dpu_kms *dpu_kms = arg;
+	struct dpu_irq *irq_obj = &dpu_kms->irq_obj;
+	struct dpu_irq_callback *cb;
+	unsigned long irq_flags;
+
+	pr_debug("irq_idx=%d\n", irq_idx);
+
+	if (list_empty(&irq_obj->irq_cb_tbl[irq_idx])) {
+		DRM_ERROR("no registered cb, idx:%d enable_count:%d\n", irq_idx,
+			atomic_read(&dpu_kms->irq_obj.enable_counts[irq_idx]));
+	}
+
+	atomic_inc(&irq_obj->irq_counts[irq_idx]);
+
+	/*
+	 * Perform registered function callback
+	 */
+	spin_lock_irqsave(&dpu_kms->irq_obj.cb_lock, irq_flags);
+	list_for_each_entry(cb, &irq_obj->irq_cb_tbl[irq_idx], list)
+		if (cb->func)
+			cb->func(cb->arg, irq_idx);
+	spin_unlock_irqrestore(&dpu_kms->irq_obj.cb_lock, irq_flags);
+
+	/*
+	 * Clear pending interrupt status in HW.
+	 * NOTE: dpu_core_irq_callback_handler is protected by top-level
+	 *       spinlock, so it is safe to clear any interrupt status here.
+	 */
+	dpu_kms->hw_intr->ops.clear_intr_status_nolock(
+			dpu_kms->hw_intr,
+			irq_idx);
+}
+
+int dpu_core_irq_idx_lookup(struct dpu_kms *dpu_kms,
+		enum dpu_intr_type intr_type, u32 instance_idx)
+{
+	if (!dpu_kms || !dpu_kms->hw_intr ||
+			!dpu_kms->hw_intr->ops.irq_idx_lookup)
+		return -EINVAL;
+
+	return dpu_kms->hw_intr->ops.irq_idx_lookup(intr_type,
+			instance_idx);
+}
+
+/**
+ * _dpu_core_irq_enable - enable core interrupt given by the index
+ * @dpu_kms:		Pointer to dpu kms context
+ * @irq_idx:		interrupt index
+ */
+static int _dpu_core_irq_enable(struct dpu_kms *dpu_kms, int irq_idx)
+{
+	unsigned long irq_flags;
+	int ret = 0, enable_count;
+
+	if (!dpu_kms || !dpu_kms->hw_intr ||
+			!dpu_kms->irq_obj.enable_counts ||
+			!dpu_kms->irq_obj.irq_counts) {
+		DPU_ERROR("invalid params\n");
+		return -EINVAL;
+	}
+
+	if (irq_idx < 0 || irq_idx >= dpu_kms->hw_intr->irq_idx_tbl_size) {
+		DPU_ERROR("invalid IRQ index: [%d]\n", irq_idx);
+		return -EINVAL;
+	}
+
+	enable_count = atomic_read(&dpu_kms->irq_obj.enable_counts[irq_idx]);
+	DRM_DEBUG_KMS("irq_idx=%d enable_count=%d\n", irq_idx, enable_count);
+	trace_dpu_core_irq_enable_idx(irq_idx, enable_count);
+
+	if (atomic_inc_return(&dpu_kms->irq_obj.enable_counts[irq_idx]) == 1) {
+		ret = dpu_kms->hw_intr->ops.enable_irq(
+				dpu_kms->hw_intr,
+				irq_idx);
+		if (ret)
+			DPU_ERROR("Fail to enable IRQ for irq_idx:%d\n",
+					irq_idx);
+
+		DPU_DEBUG("irq_idx=%d ret=%d\n", irq_idx, ret);
+
+		spin_lock_irqsave(&dpu_kms->irq_obj.cb_lock, irq_flags);
+		/* empty callback list but interrupt is enabled */
+		if (list_empty(&dpu_kms->irq_obj.irq_cb_tbl[irq_idx]))
+			DPU_ERROR("irq_idx=%d enabled with no callback\n",
+					irq_idx);
+		spin_unlock_irqrestore(&dpu_kms->irq_obj.cb_lock, irq_flags);
+	}
+
+	return ret;
+}
+
+int dpu_core_irq_enable(struct dpu_kms *dpu_kms, int *irq_idxs, u32 irq_count)
+{
+	int i, ret = 0, counts;
+
+	if (!dpu_kms || !irq_idxs || !irq_count) {
+		DPU_ERROR("invalid params\n");
+		return -EINVAL;
+	}
+
+	counts = atomic_read(&dpu_kms->irq_obj.enable_counts[irq_idxs[0]]);
+	if (counts)
+		DRM_ERROR("irq_idx=%d enable_count=%d\n", irq_idxs[0], counts);
+
+	for (i = 0; (i < irq_count) && !ret; i++)
+		ret = _dpu_core_irq_enable(dpu_kms, irq_idxs[i]);
+
+	return ret;
+}
+
+/**
+ * _dpu_core_irq_disable - disable core interrupt given by the index
+ * @dpu_kms:		Pointer to dpu kms context
+ * @irq_idx:		interrupt index
+ */
+static int _dpu_core_irq_disable(struct dpu_kms *dpu_kms, int irq_idx)
+{
+	int ret = 0, enable_count;
+
+	if (!dpu_kms || !dpu_kms->hw_intr || !dpu_kms->irq_obj.enable_counts) {
+		DPU_ERROR("invalid params\n");
+		return -EINVAL;
+	}
+
+	if (irq_idx < 0 || irq_idx >= dpu_kms->hw_intr->irq_idx_tbl_size) {
+		DPU_ERROR("invalid IRQ index: [%d]\n", irq_idx);
+		return -EINVAL;
+	}
+
+	enable_count = atomic_read(&dpu_kms->irq_obj.enable_counts[irq_idx]);
+	DRM_DEBUG_KMS("irq_idx=%d enable_count=%d\n", irq_idx, enable_count);
+	trace_dpu_core_irq_disable_idx(irq_idx, enable_count);
+
+	if (atomic_dec_return(&dpu_kms->irq_obj.enable_counts[irq_idx]) == 0) {
+		ret = dpu_kms->hw_intr->ops.disable_irq(
+				dpu_kms->hw_intr,
+				irq_idx);
+		if (ret)
+			DPU_ERROR("Fail to disable IRQ for irq_idx:%d\n",
+					irq_idx);
+		DPU_DEBUG("irq_idx=%d ret=%d\n", irq_idx, ret);
+	}
+
+	return ret;
+}
+
+int dpu_core_irq_disable(struct dpu_kms *dpu_kms, int *irq_idxs, u32 irq_count)
+{
+	int i, ret = 0, counts;
+
+	if (!dpu_kms || !irq_idxs || !irq_count) {
+		DPU_ERROR("invalid params\n");
+		return -EINVAL;
+	}
+
+	counts = atomic_read(&dpu_kms->irq_obj.enable_counts[irq_idxs[0]]);
+	if (counts == 2)
+		DRM_ERROR("irq_idx=%d enable_count=%d\n", irq_idxs[0], counts);
+
+	for (i = 0; (i < irq_count) && !ret; i++)
+		ret = _dpu_core_irq_disable(dpu_kms, irq_idxs[i]);
+
+	return ret;
+}
+
+u32 dpu_core_irq_read(struct dpu_kms *dpu_kms, int irq_idx, bool clear)
+{
+	if (!dpu_kms || !dpu_kms->hw_intr ||
+			!dpu_kms->hw_intr->ops.get_interrupt_status)
+		return 0;
+
+	if (irq_idx < 0) {
+		DPU_ERROR("[%pS] invalid irq_idx=%d\n",
+				__builtin_return_address(0), irq_idx);
+		return 0;
+	}
+
+	return dpu_kms->hw_intr->ops.get_interrupt_status(dpu_kms->hw_intr,
+			irq_idx, clear);
+}
+
+int dpu_core_irq_register_callback(struct dpu_kms *dpu_kms, int irq_idx,
+		struct dpu_irq_callback *register_irq_cb)
+{
+	unsigned long irq_flags;
+
+	if (!dpu_kms || !dpu_kms->irq_obj.irq_cb_tbl) {
+		DPU_ERROR("invalid params\n");
+		return -EINVAL;
+	}
+
+	if (!register_irq_cb || !register_irq_cb->func) {
+		DPU_ERROR("invalid irq_cb:%d func:%d\n",
+				register_irq_cb != NULL,
+				register_irq_cb ?
+					register_irq_cb->func != NULL : -1);
+		return -EINVAL;
+	}
+
+	if (irq_idx < 0 || irq_idx >= dpu_kms->hw_intr->irq_idx_tbl_size) {
+		DPU_ERROR("invalid IRQ index: [%d]\n", irq_idx);
+		return -EINVAL;
+	}
+
+	DPU_DEBUG("[%pS] irq_idx=%d\n", __builtin_return_address(0), irq_idx);
+
+	spin_lock_irqsave(&dpu_kms->irq_obj.cb_lock, irq_flags);
+	trace_dpu_core_irq_register_callback(irq_idx, register_irq_cb);
+	list_del_init(&register_irq_cb->list);
+	list_add_tail(&register_irq_cb->list,
+			&dpu_kms->irq_obj.irq_cb_tbl[irq_idx]);
+	spin_unlock_irqrestore(&dpu_kms->irq_obj.cb_lock, irq_flags);
+
+	return 0;
+}
+
+int dpu_core_irq_unregister_callback(struct dpu_kms *dpu_kms, int irq_idx,
+		struct dpu_irq_callback *register_irq_cb)
+{
+	unsigned long irq_flags;
+
+	if (!dpu_kms || !dpu_kms->irq_obj.irq_cb_tbl) {
+		DPU_ERROR("invalid params\n");
+		return -EINVAL;
+	}
+
+	if (!register_irq_cb || !register_irq_cb->func) {
+		DPU_ERROR("invalid irq_cb:%d func:%d\n",
+				register_irq_cb != NULL,
+				register_irq_cb ?
+					register_irq_cb->func != NULL : -1);
+		return -EINVAL;
+	}
+
+	if (irq_idx < 0 || irq_idx >= dpu_kms->hw_intr->irq_idx_tbl_size) {
+		DPU_ERROR("invalid IRQ index: [%d]\n", irq_idx);
+		return -EINVAL;
+	}
+
+	DPU_DEBUG("[%pS] irq_idx=%d\n", __builtin_return_address(0), irq_idx);
+
+	spin_lock_irqsave(&dpu_kms->irq_obj.cb_lock, irq_flags);
+	trace_dpu_core_irq_unregister_callback(irq_idx, register_irq_cb);
+	list_del_init(&register_irq_cb->list);
+	/* empty callback list but interrupt is still enabled */
+	if (list_empty(&dpu_kms->irq_obj.irq_cb_tbl[irq_idx]) &&
+			atomic_read(&dpu_kms->irq_obj.enable_counts[irq_idx]))
+		DPU_ERROR("irq_idx=%d enabled with no callback\n", irq_idx);
+	spin_unlock_irqrestore(&dpu_kms->irq_obj.cb_lock, irq_flags);
+
+	return 0;
+}
+
+static void dpu_clear_all_irqs(struct dpu_kms *dpu_kms)
+{
+	if (!dpu_kms || !dpu_kms->hw_intr ||
+			!dpu_kms->hw_intr->ops.clear_all_irqs)
+		return;
+
+	dpu_kms->hw_intr->ops.clear_all_irqs(dpu_kms->hw_intr);
+}
+
+static void dpu_disable_all_irqs(struct dpu_kms *dpu_kms)
+{
+	if (!dpu_kms || !dpu_kms->hw_intr ||
+			!dpu_kms->hw_intr->ops.disable_all_irqs)
+		return;
+
+	dpu_kms->hw_intr->ops.disable_all_irqs(dpu_kms->hw_intr);
+}
+
+#ifdef CONFIG_DEBUG_FS
+#define DEFINE_DPU_DEBUGFS_SEQ_FOPS(__prefix)				\
+static int __prefix ## _open(struct inode *inode, struct file *file)	\
+{									\
+	return single_open(file, __prefix ## _show, inode->i_private);	\
+}									\
+static const struct file_operations __prefix ## _fops = {		\
+	.owner = THIS_MODULE,						\
+	.open = __prefix ## _open,					\
+	.release = single_release,					\
+	.read = seq_read,						\
+	.llseek = seq_lseek,						\
+}
+
+static int dpu_debugfs_core_irq_show(struct seq_file *s, void *v)
+{
+	struct dpu_irq *irq_obj = s->private;
+	struct dpu_irq_callback *cb;
+	unsigned long irq_flags;
+	int i, irq_count, enable_count, cb_count;
+
+	if (!irq_obj || !irq_obj->enable_counts || !irq_obj->irq_cb_tbl) {
+		DPU_ERROR("invalid parameters\n");
+		return 0;
+	}
+
+	for (i = 0; i < irq_obj->total_irqs; i++) {
+		spin_lock_irqsave(&irq_obj->cb_lock, irq_flags);
+		cb_count = 0;
+		irq_count = atomic_read(&irq_obj->irq_counts[i]);
+		enable_count = atomic_read(&irq_obj->enable_counts[i]);
+		list_for_each_entry(cb, &irq_obj->irq_cb_tbl[i], list)
+			cb_count++;
+		spin_unlock_irqrestore(&irq_obj->cb_lock, irq_flags);
+
+		if (irq_count || enable_count || cb_count)
+			seq_printf(s, "idx:%d irq:%d enable:%d cb:%d\n",
+					i, irq_count, enable_count, cb_count);
+	}
+
+	return 0;
+}
+
+DEFINE_DPU_DEBUGFS_SEQ_FOPS(dpu_debugfs_core_irq);
+
+int dpu_debugfs_core_irq_init(struct dpu_kms *dpu_kms,
+		struct dentry *parent)
+{
+	dpu_kms->irq_obj.debugfs_file = debugfs_create_file("core_irq", 0600,
+			parent, &dpu_kms->irq_obj,
+			&dpu_debugfs_core_irq_fops);
+
+	return 0;
+}
+
+void dpu_debugfs_core_irq_destroy(struct dpu_kms *dpu_kms)
+{
+	debugfs_remove(dpu_kms->irq_obj.debugfs_file);
+	dpu_kms->irq_obj.debugfs_file = NULL;
+}
+
+#else
+int dpu_debugfs_core_irq_init(struct dpu_kms *dpu_kms,
+		struct dentry *parent)
+{
+	return 0;
+}
+
+void dpu_debugfs_core_irq_destroy(struct dpu_kms *dpu_kms)
+{
+}
+#endif
+
+void dpu_core_irq_preinstall(struct dpu_kms *dpu_kms)
+{
+	struct msm_drm_private *priv;
+	int i;
+
+	if (!dpu_kms) {
+		DPU_ERROR("invalid dpu_kms\n");
+		return;
+	} else if (!dpu_kms->dev) {
+		DPU_ERROR("invalid drm device\n");
+		return;
+	} else if (!dpu_kms->dev->dev_private) {
+		DPU_ERROR("invalid device private\n");
+		return;
+	}
+	priv = dpu_kms->dev->dev_private;
+
+	pm_runtime_get_sync(&dpu_kms->pdev->dev);
+	dpu_clear_all_irqs(dpu_kms);
+	dpu_disable_all_irqs(dpu_kms);
+	pm_runtime_put_sync(&dpu_kms->pdev->dev);
+
+	spin_lock_init(&dpu_kms->irq_obj.cb_lock);
+
+	/* Create irq callbacks for all possible irq_idx */
+	dpu_kms->irq_obj.total_irqs = dpu_kms->hw_intr->irq_idx_tbl_size;
+	dpu_kms->irq_obj.irq_cb_tbl = kcalloc(dpu_kms->irq_obj.total_irqs,
+			sizeof(struct list_head), GFP_KERNEL);
+	dpu_kms->irq_obj.enable_counts = kcalloc(dpu_kms->irq_obj.total_irqs,
+			sizeof(atomic_t), GFP_KERNEL);
+	dpu_kms->irq_obj.irq_counts = kcalloc(dpu_kms->irq_obj.total_irqs,
+			sizeof(atomic_t), GFP_KERNEL);
+	for (i = 0; i < dpu_kms->irq_obj.total_irqs; i++) {
+		INIT_LIST_HEAD(&dpu_kms->irq_obj.irq_cb_tbl[i]);
+		atomic_set(&dpu_kms->irq_obj.enable_counts[i], 0);
+		atomic_set(&dpu_kms->irq_obj.irq_counts[i], 0);
+	}
+}
+
+int dpu_core_irq_postinstall(struct dpu_kms *dpu_kms)
+{
+	return 0;
+}
+
+void dpu_core_irq_uninstall(struct dpu_kms *dpu_kms)
+{
+	struct msm_drm_private *priv;
+	int i;
+
+	if (!dpu_kms) {
+		DPU_ERROR("invalid dpu_kms\n");
+		return;
+	} else if (!dpu_kms->dev) {
+		DPU_ERROR("invalid drm device\n");
+		return;
+	} else if (!dpu_kms->dev->dev_private) {
+		DPU_ERROR("invalid device private\n");
+		return;
+	}
+	priv = dpu_kms->dev->dev_private;
+
+	pm_runtime_get_sync(&dpu_kms->pdev->dev);
+	for (i = 0; i < dpu_kms->irq_obj.total_irqs; i++)
+		if (atomic_read(&dpu_kms->irq_obj.enable_counts[i]) ||
+				!list_empty(&dpu_kms->irq_obj.irq_cb_tbl[i]))
+			DPU_ERROR("irq_idx=%d still enabled/registered\n", i);
+
+	dpu_clear_all_irqs(dpu_kms);
+	dpu_disable_all_irqs(dpu_kms);
+	pm_runtime_put_sync(&dpu_kms->pdev->dev);
+
+	kfree(dpu_kms->irq_obj.irq_cb_tbl);
+	kfree(dpu_kms->irq_obj.enable_counts);
+	kfree(dpu_kms->irq_obj.irq_counts);
+	dpu_kms->irq_obj.irq_cb_tbl = NULL;
+	dpu_kms->irq_obj.enable_counts = NULL;
+	dpu_kms->irq_obj.irq_counts = NULL;
+	dpu_kms->irq_obj.total_irqs = 0;
+}
+
+irqreturn_t dpu_core_irq(struct dpu_kms *dpu_kms)
+{
+	/*
+	 * Read interrupt status from all sources. Interrupt status are
+	 * stored within hw_intr.
+	 * Function will also clear the interrupt status after reading.
+	 * Individual interrupt status bit will only get stored if it
+	 * is enabled.
+	 */
+	dpu_kms->hw_intr->ops.get_interrupt_statuses(dpu_kms->hw_intr);
+
+	/*
+	 * Dispatch to HW driver to handle interrupt lookup that is being
+	 * fired. When matching interrupt is located, HW driver will call to
+	 * dpu_core_irq_callback_handler with the irq_idx from the lookup table.
+	 * dpu_core_irq_callback_handler will perform the registered function
+	 * callback, and do the interrupt status clearing once the registered
+	 * callback is finished.
+	 */
+	dpu_kms->hw_intr->ops.dispatch_irqs(
+			dpu_kms->hw_intr,
+			dpu_core_irq_callback_handler,
+			dpu_kms);
+
+	return IRQ_HANDLED;
+}
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_core_irq.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_core_irq.h
new file mode 100644
index 000000000000..5e98bba46af5
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_core_irq.h
@@ -0,0 +1,153 @@
+/* Copyright (c) 2015-2018, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef __DPU_CORE_IRQ_H__
+#define __DPU_CORE_IRQ_H__
+
+#include "dpu_kms.h"
+#include "dpu_hw_interrupts.h"
+
+/**
+ * dpu_core_irq_preinstall - perform pre-installation of core IRQ handler
+ * @dpu_kms:		DPU handle
+ * @return:		none
+ */
+void dpu_core_irq_preinstall(struct dpu_kms *dpu_kms);
+
+/**
+ * dpu_core_irq_postinstall - perform post-installation of core IRQ handler
+ * @dpu_kms:		DPU handle
+ * @return:		0 if success; error code otherwise
+ */
+int dpu_core_irq_postinstall(struct dpu_kms *dpu_kms);
+
+/**
+ * dpu_core_irq_uninstall - uninstall core IRQ handler
+ * @dpu_kms:		DPU handle
+ * @return:		none
+ */
+void dpu_core_irq_uninstall(struct dpu_kms *dpu_kms);
+
+/**
+ * dpu_core_irq - core IRQ handler
+ * @dpu_kms:		DPU handle
+ * @return:		interrupt handling status
+ */
+irqreturn_t dpu_core_irq(struct dpu_kms *dpu_kms);
+
+/**
+ * dpu_core_irq_idx_lookup - IRQ helper function for lookup irq_idx from HW
+ *                      interrupt mapping table.
+ * @dpu_kms:		DPU handle
+ * @intr_type:		DPU HW interrupt type for lookup
+ * @instance_idx:	DPU HW block instance defined in dpu_hw_mdss.h
+ * @return:		irq_idx or -EINVAL when fail to lookup
+ */
+int dpu_core_irq_idx_lookup(
+		struct dpu_kms *dpu_kms,
+		enum dpu_intr_type intr_type,
+		uint32_t instance_idx);
+
+/**
+ * dpu_core_irq_enable - IRQ helper function for enabling one or more IRQs
+ * @dpu_kms:		DPU handle
+ * @irq_idxs:		Array of irq index
+ * @irq_count:		Number of irq_idx provided in the array
+ * @return:		0 for success enabling IRQ, otherwise failure
+ *
+ * This function increments count on each enable and decrements on each
+ * disable.  Interrupts is enabled if count is 0 before increment.
+ */
+int dpu_core_irq_enable(
+		struct dpu_kms *dpu_kms,
+		int *irq_idxs,
+		uint32_t irq_count);
+
+/**
+ * dpu_core_irq_disable - IRQ helper function for disabling one of more IRQs
+ * @dpu_kms:		DPU handle
+ * @irq_idxs:		Array of irq index
+ * @irq_count:		Number of irq_idx provided in the array
+ * @return:		0 for success disabling IRQ, otherwise failure
+ *
+ * This function increments count on each enable and decrements on each
+ * disable.  Interrupts is disabled if count is 0 after decrement.
+ */
+int dpu_core_irq_disable(
+		struct dpu_kms *dpu_kms,
+		int *irq_idxs,
+		uint32_t irq_count);
+
+/**
+ * dpu_core_irq_read - IRQ helper function for reading IRQ status
+ * @dpu_kms:		DPU handle
+ * @irq_idx:		irq index
+ * @clear:		True to clear the irq after read
+ * @return:		non-zero if irq detected; otherwise no irq detected
+ */
+u32 dpu_core_irq_read(
+		struct dpu_kms *dpu_kms,
+		int irq_idx,
+		bool clear);
+
+/**
+ * dpu_core_irq_register_callback - For registering callback function on IRQ
+ *                             interrupt
+ * @dpu_kms:		DPU handle
+ * @irq_idx:		irq index
+ * @irq_cb:		IRQ callback structure, containing callback function
+ *			and argument. Passing NULL for irq_cb will unregister
+ *			the callback for the given irq_idx
+ *			This must exist until un-registration.
+ * @return:		0 for success registering callback, otherwise failure
+ *
+ * This function supports registration of multiple callbacks for each interrupt.
+ */
+int dpu_core_irq_register_callback(
+		struct dpu_kms *dpu_kms,
+		int irq_idx,
+		struct dpu_irq_callback *irq_cb);
+
+/**
+ * dpu_core_irq_unregister_callback - For unregistering callback function on IRQ
+ *                             interrupt
+ * @dpu_kms:		DPU handle
+ * @irq_idx:		irq index
+ * @irq_cb:		IRQ callback structure, containing callback function
+ *			and argument. Passing NULL for irq_cb will unregister
+ *			the callback for the given irq_idx
+ *			This must match with registration.
+ * @return:		0 for success registering callback, otherwise failure
+ *
+ * This function supports registration of multiple callbacks for each interrupt.
+ */
+int dpu_core_irq_unregister_callback(
+		struct dpu_kms *dpu_kms,
+		int irq_idx,
+		struct dpu_irq_callback *irq_cb);
+
+/**
+ * dpu_debugfs_core_irq_init - register core irq debugfs
+ * @dpu_kms: pointer to kms
+ * @parent: debugfs directory root
+ * @Return: 0 on success
+ */
+int dpu_debugfs_core_irq_init(struct dpu_kms *dpu_kms,
+		struct dentry *parent);
+
+/**
+ * dpu_debugfs_core_irq_destroy - deregister core irq debugfs
+ * @dpu_kms: pointer to kms
+ */
+void dpu_debugfs_core_irq_destroy(struct dpu_kms *dpu_kms);
+
+#endif /* __DPU_CORE_IRQ_H__ */
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_core_perf.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_core_perf.c
new file mode 100644
index 000000000000..41c5191f9056
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_core_perf.c
@@ -0,0 +1,637 @@
+/* Copyright (c) 2016-2018, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#define pr_fmt(fmt)	"[drm:%s:%d] " fmt, __func__, __LINE__
+
+#include <linux/debugfs.h>
+#include <linux/errno.h>
+#include <linux/mutex.h>
+#include <linux/sort.h>
+#include <linux/clk.h>
+#include <linux/bitmap.h>
+
+#include "dpu_kms.h"
+#include "dpu_trace.h"
+#include "dpu_crtc.h"
+#include "dpu_core_perf.h"
+
+#define DPU_PERF_MODE_STRING_SIZE	128
+
+/**
+ * enum dpu_perf_mode - performance tuning mode
+ * @DPU_PERF_MODE_NORMAL: performance controlled by user mode client
+ * @DPU_PERF_MODE_MINIMUM: performance bounded by minimum setting
+ * @DPU_PERF_MODE_FIXED: performance bounded by fixed setting
+ */
+enum dpu_perf_mode {
+	DPU_PERF_MODE_NORMAL,
+	DPU_PERF_MODE_MINIMUM,
+	DPU_PERF_MODE_FIXED,
+	DPU_PERF_MODE_MAX
+};
+
+static struct dpu_kms *_dpu_crtc_get_kms(struct drm_crtc *crtc)
+{
+	struct msm_drm_private *priv;
+
+	if (!crtc->dev || !crtc->dev->dev_private) {
+		DPU_ERROR("invalid device\n");
+		return NULL;
+	}
+
+	priv = crtc->dev->dev_private;
+	if (!priv || !priv->kms) {
+		DPU_ERROR("invalid kms\n");
+		return NULL;
+	}
+
+	return to_dpu_kms(priv->kms);
+}
+
+static bool _dpu_core_perf_crtc_is_power_on(struct drm_crtc *crtc)
+{
+	return dpu_crtc_is_enabled(crtc);
+}
+
+static bool _dpu_core_video_mode_intf_connected(struct drm_crtc *crtc)
+{
+	struct drm_crtc *tmp_crtc;
+	bool intf_connected = false;
+
+	if (!crtc)
+		goto end;
+
+	drm_for_each_crtc(tmp_crtc, crtc->dev) {
+		if ((dpu_crtc_get_intf_mode(tmp_crtc) == INTF_MODE_VIDEO) &&
+				_dpu_core_perf_crtc_is_power_on(tmp_crtc)) {
+			DPU_DEBUG("video interface connected crtc:%d\n",
+				tmp_crtc->base.id);
+			intf_connected = true;
+			goto end;
+		}
+	}
+
+end:
+	return intf_connected;
+}
+
+static void _dpu_core_perf_calc_crtc(struct dpu_kms *kms,
+		struct drm_crtc *crtc,
+		struct drm_crtc_state *state,
+		struct dpu_core_perf_params *perf)
+{
+	struct dpu_crtc_state *dpu_cstate;
+	int i;
+
+	if (!kms || !kms->catalog || !crtc || !state || !perf) {
+		DPU_ERROR("invalid parameters\n");
+		return;
+	}
+
+	dpu_cstate = to_dpu_crtc_state(state);
+	memset(perf, 0, sizeof(struct dpu_core_perf_params));
+
+	if (!dpu_cstate->bw_control) {
+		for (i = 0; i < DPU_POWER_HANDLE_DBUS_ID_MAX; i++) {
+			perf->bw_ctl[i] = kms->catalog->perf.max_bw_high *
+					1000ULL;
+			perf->max_per_pipe_ib[i] = perf->bw_ctl[i];
+		}
+		perf->core_clk_rate = kms->perf.max_core_clk_rate;
+	} else if (kms->perf.perf_tune.mode == DPU_PERF_MODE_MINIMUM) {
+		for (i = 0; i < DPU_POWER_HANDLE_DBUS_ID_MAX; i++) {
+			perf->bw_ctl[i] = 0;
+			perf->max_per_pipe_ib[i] = 0;
+		}
+		perf->core_clk_rate = 0;
+	} else if (kms->perf.perf_tune.mode == DPU_PERF_MODE_FIXED) {
+		for (i = 0; i < DPU_POWER_HANDLE_DBUS_ID_MAX; i++) {
+			perf->bw_ctl[i] = kms->perf.fix_core_ab_vote;
+			perf->max_per_pipe_ib[i] = kms->perf.fix_core_ib_vote;
+		}
+		perf->core_clk_rate = kms->perf.fix_core_clk_rate;
+	}
+
+	DPU_DEBUG(
+		"crtc=%d clk_rate=%llu core_ib=%llu core_ab=%llu llcc_ib=%llu llcc_ab=%llu mem_ib=%llu mem_ab=%llu\n",
+			crtc->base.id, perf->core_clk_rate,
+			perf->max_per_pipe_ib[DPU_POWER_HANDLE_DBUS_ID_MNOC],
+			perf->bw_ctl[DPU_POWER_HANDLE_DBUS_ID_MNOC],
+			perf->max_per_pipe_ib[DPU_POWER_HANDLE_DBUS_ID_LLCC],
+			perf->bw_ctl[DPU_POWER_HANDLE_DBUS_ID_LLCC],
+			perf->max_per_pipe_ib[DPU_POWER_HANDLE_DBUS_ID_EBI],
+			perf->bw_ctl[DPU_POWER_HANDLE_DBUS_ID_EBI]);
+}
+
+int dpu_core_perf_crtc_check(struct drm_crtc *crtc,
+		struct drm_crtc_state *state)
+{
+	u32 bw, threshold;
+	u64 bw_sum_of_intfs = 0;
+	enum dpu_crtc_client_type curr_client_type;
+	bool is_video_mode;
+	struct dpu_crtc_state *dpu_cstate;
+	struct drm_crtc *tmp_crtc;
+	struct dpu_kms *kms;
+	int i;
+
+	if (!crtc || !state) {
+		DPU_ERROR("invalid crtc\n");
+		return -EINVAL;
+	}
+
+	kms = _dpu_crtc_get_kms(crtc);
+	if (!kms || !kms->catalog) {
+		DPU_ERROR("invalid parameters\n");
+		return 0;
+	}
+
+	/* we only need bandwidth check on real-time clients (interfaces) */
+	if (dpu_crtc_get_client_type(crtc) == NRT_CLIENT)
+		return 0;
+
+	dpu_cstate = to_dpu_crtc_state(state);
+
+	/* obtain new values */
+	_dpu_core_perf_calc_crtc(kms, crtc, state, &dpu_cstate->new_perf);
+
+	for (i = DPU_POWER_HANDLE_DBUS_ID_MNOC;
+			i < DPU_POWER_HANDLE_DBUS_ID_MAX; i++) {
+		bw_sum_of_intfs = dpu_cstate->new_perf.bw_ctl[i];
+		curr_client_type = dpu_crtc_get_client_type(crtc);
+
+		drm_for_each_crtc(tmp_crtc, crtc->dev) {
+			if (_dpu_core_perf_crtc_is_power_on(tmp_crtc) &&
+			    (dpu_crtc_get_client_type(tmp_crtc) ==
+					    curr_client_type) &&
+			    (tmp_crtc != crtc)) {
+				struct dpu_crtc_state *tmp_cstate =
+					to_dpu_crtc_state(tmp_crtc->state);
+
+				DPU_DEBUG("crtc:%d bw:%llu ctrl:%d\n",
+					tmp_crtc->base.id,
+					tmp_cstate->new_perf.bw_ctl[i],
+					tmp_cstate->bw_control);
+				/*
+				 * For bw check only use the bw if the
+				 * atomic property has been already set
+				 */
+				if (tmp_cstate->bw_control)
+					bw_sum_of_intfs +=
+						tmp_cstate->new_perf.bw_ctl[i];
+			}
+		}
+
+		/* convert bandwidth to kb */
+		bw = DIV_ROUND_UP_ULL(bw_sum_of_intfs, 1000);
+		DPU_DEBUG("calculated bandwidth=%uk\n", bw);
+
+		is_video_mode = dpu_crtc_get_intf_mode(crtc) == INTF_MODE_VIDEO;
+		threshold = (is_video_mode ||
+			_dpu_core_video_mode_intf_connected(crtc)) ?
+			kms->catalog->perf.max_bw_low :
+			kms->catalog->perf.max_bw_high;
+
+		DPU_DEBUG("final threshold bw limit = %d\n", threshold);
+
+		if (!dpu_cstate->bw_control) {
+			DPU_DEBUG("bypass bandwidth check\n");
+		} else if (!threshold) {
+			DPU_ERROR("no bandwidth limits specified\n");
+			return -E2BIG;
+		} else if (bw > threshold) {
+			DPU_ERROR("exceeds bandwidth: %ukb > %ukb\n", bw,
+					threshold);
+			return -E2BIG;
+		}
+	}
+
+	return 0;
+}
+
+static int _dpu_core_perf_crtc_update_bus(struct dpu_kms *kms,
+		struct drm_crtc *crtc, u32 bus_id)
+{
+	struct dpu_core_perf_params perf = { { 0 } };
+	enum dpu_crtc_client_type curr_client_type
+					= dpu_crtc_get_client_type(crtc);
+	struct drm_crtc *tmp_crtc;
+	struct dpu_crtc_state *dpu_cstate;
+	int ret = 0;
+
+	drm_for_each_crtc(tmp_crtc, crtc->dev) {
+		if (_dpu_core_perf_crtc_is_power_on(tmp_crtc) &&
+			curr_client_type ==
+				dpu_crtc_get_client_type(tmp_crtc)) {
+			dpu_cstate = to_dpu_crtc_state(tmp_crtc->state);
+
+			perf.max_per_pipe_ib[bus_id] =
+				max(perf.max_per_pipe_ib[bus_id],
+				dpu_cstate->new_perf.max_per_pipe_ib[bus_id]);
+
+			DPU_DEBUG("crtc=%d bus_id=%d bw=%llu\n",
+				tmp_crtc->base.id, bus_id,
+				dpu_cstate->new_perf.bw_ctl[bus_id]);
+		}
+	}
+	return ret;
+}
+
+/**
+ * @dpu_core_perf_crtc_release_bw() - request zero bandwidth
+ * @crtc - pointer to a crtc
+ *
+ * Function checks a state variable for the crtc, if all pending commit
+ * requests are done, meaning no more bandwidth is needed, release
+ * bandwidth request.
+ */
+void dpu_core_perf_crtc_release_bw(struct drm_crtc *crtc)
+{
+	struct drm_crtc *tmp_crtc;
+	struct dpu_crtc *dpu_crtc;
+	struct dpu_crtc_state *dpu_cstate;
+	struct dpu_kms *kms;
+	int i;
+
+	if (!crtc) {
+		DPU_ERROR("invalid crtc\n");
+		return;
+	}
+
+	kms = _dpu_crtc_get_kms(crtc);
+	if (!kms || !kms->catalog) {
+		DPU_ERROR("invalid kms\n");
+		return;
+	}
+
+	dpu_crtc = to_dpu_crtc(crtc);
+	dpu_cstate = to_dpu_crtc_state(crtc->state);
+
+	/* only do this for command mode rt client */
+	if (dpu_crtc_get_intf_mode(crtc) != INTF_MODE_CMD)
+		return;
+
+	/*
+	 * If video interface present, cmd panel bandwidth cannot be
+	 * released.
+	 */
+	if (dpu_crtc_get_intf_mode(crtc) == INTF_MODE_CMD)
+		drm_for_each_crtc(tmp_crtc, crtc->dev) {
+			if (_dpu_core_perf_crtc_is_power_on(tmp_crtc) &&
+				dpu_crtc_get_intf_mode(tmp_crtc) ==
+						INTF_MODE_VIDEO)
+				return;
+		}
+
+	/* Release the bandwidth */
+	if (kms->perf.enable_bw_release) {
+		trace_dpu_cmd_release_bw(crtc->base.id);
+		DPU_DEBUG("Release BW crtc=%d\n", crtc->base.id);
+		for (i = 0; i < DPU_POWER_HANDLE_DBUS_ID_MAX; i++) {
+			dpu_crtc->cur_perf.bw_ctl[i] = 0;
+			_dpu_core_perf_crtc_update_bus(kms, crtc, i);
+		}
+	}
+}
+
+static int _dpu_core_perf_set_core_clk_rate(struct dpu_kms *kms, u64 rate)
+{
+	struct dss_clk *core_clk = kms->perf.core_clk;
+
+	if (core_clk->max_rate && (rate > core_clk->max_rate))
+		rate = core_clk->max_rate;
+
+	core_clk->rate = rate;
+	return msm_dss_clk_set_rate(core_clk, 1);
+}
+
+static u64 _dpu_core_perf_get_core_clk_rate(struct dpu_kms *kms)
+{
+	u64 clk_rate = kms->perf.perf_tune.min_core_clk;
+	struct drm_crtc *crtc;
+	struct dpu_crtc_state *dpu_cstate;
+
+	drm_for_each_crtc(crtc, kms->dev) {
+		if (_dpu_core_perf_crtc_is_power_on(crtc)) {
+			dpu_cstate = to_dpu_crtc_state(crtc->state);
+			clk_rate = max(dpu_cstate->new_perf.core_clk_rate,
+							clk_rate);
+			clk_rate = clk_round_rate(kms->perf.core_clk->clk,
+					clk_rate);
+		}
+	}
+
+	if (kms->perf.perf_tune.mode == DPU_PERF_MODE_FIXED)
+		clk_rate = kms->perf.fix_core_clk_rate;
+
+	DPU_DEBUG("clk:%llu\n", clk_rate);
+
+	return clk_rate;
+}
+
+int dpu_core_perf_crtc_update(struct drm_crtc *crtc,
+		int params_changed, bool stop_req)
+{
+	struct dpu_core_perf_params *new, *old;
+	int update_bus = 0, update_clk = 0;
+	u64 clk_rate = 0;
+	struct dpu_crtc *dpu_crtc;
+	struct dpu_crtc_state *dpu_cstate;
+	int i;
+	struct msm_drm_private *priv;
+	struct dpu_kms *kms;
+	int ret;
+
+	if (!crtc) {
+		DPU_ERROR("invalid crtc\n");
+		return -EINVAL;
+	}
+
+	kms = _dpu_crtc_get_kms(crtc);
+	if (!kms || !kms->catalog) {
+		DPU_ERROR("invalid kms\n");
+		return -EINVAL;
+	}
+	priv = kms->dev->dev_private;
+
+	dpu_crtc = to_dpu_crtc(crtc);
+	dpu_cstate = to_dpu_crtc_state(crtc->state);
+
+	DPU_DEBUG("crtc:%d stop_req:%d core_clk:%llu\n",
+			crtc->base.id, stop_req, kms->perf.core_clk_rate);
+
+	old = &dpu_crtc->cur_perf;
+	new = &dpu_cstate->new_perf;
+
+	if (_dpu_core_perf_crtc_is_power_on(crtc) && !stop_req) {
+		for (i = 0; i < DPU_POWER_HANDLE_DBUS_ID_MAX; i++) {
+			/*
+			 * cases for bus bandwidth update.
+			 * 1. new bandwidth vote - "ab or ib vote" is higher
+			 *    than current vote for update request.
+			 * 2. new bandwidth vote - "ab or ib vote" is lower
+			 *    than current vote at end of commit or stop.
+			 */
+			if ((params_changed && ((new->bw_ctl[i] >
+						old->bw_ctl[i]) ||
+				  (new->max_per_pipe_ib[i] >
+						old->max_per_pipe_ib[i]))) ||
+			    (!params_changed && ((new->bw_ctl[i] <
+						old->bw_ctl[i]) ||
+				  (new->max_per_pipe_ib[i] <
+						old->max_per_pipe_ib[i])))) {
+				DPU_DEBUG(
+					"crtc=%d p=%d new_bw=%llu,old_bw=%llu\n",
+					crtc->base.id, params_changed,
+					new->bw_ctl[i], old->bw_ctl[i]);
+				old->bw_ctl[i] = new->bw_ctl[i];
+				old->max_per_pipe_ib[i] =
+						new->max_per_pipe_ib[i];
+				update_bus |= BIT(i);
+			}
+		}
+
+		if ((params_changed &&
+				(new->core_clk_rate > old->core_clk_rate)) ||
+				(!params_changed &&
+				(new->core_clk_rate < old->core_clk_rate))) {
+			old->core_clk_rate = new->core_clk_rate;
+			update_clk = 1;
+		}
+	} else {
+		DPU_DEBUG("crtc=%d disable\n", crtc->base.id);
+		memset(old, 0, sizeof(*old));
+		memset(new, 0, sizeof(*new));
+		update_bus = ~0;
+		update_clk = 1;
+	}
+	trace_dpu_perf_crtc_update(crtc->base.id,
+				new->bw_ctl[DPU_POWER_HANDLE_DBUS_ID_MNOC],
+				new->bw_ctl[DPU_POWER_HANDLE_DBUS_ID_LLCC],
+				new->bw_ctl[DPU_POWER_HANDLE_DBUS_ID_EBI],
+				new->core_clk_rate, stop_req,
+				update_bus, update_clk);
+
+	for (i = 0; i < DPU_POWER_HANDLE_DBUS_ID_MAX; i++) {
+		if (update_bus & BIT(i)) {
+			ret = _dpu_core_perf_crtc_update_bus(kms, crtc, i);
+			if (ret) {
+				DPU_ERROR("crtc-%d: failed to update bw vote for bus-%d\n",
+					  crtc->base.id, i);
+				return ret;
+			}
+		}
+	}
+
+	/*
+	 * Update the clock after bandwidth vote to ensure
+	 * bandwidth is available before clock rate is increased.
+	 */
+	if (update_clk) {
+		clk_rate = _dpu_core_perf_get_core_clk_rate(kms);
+
+		trace_dpu_core_perf_update_clk(kms->dev, stop_req, clk_rate);
+
+		ret = _dpu_core_perf_set_core_clk_rate(kms, clk_rate);
+		if (ret) {
+			DPU_ERROR("failed to set %s clock rate %llu\n",
+					kms->perf.core_clk->clk_name, clk_rate);
+			return ret;
+		}
+
+		kms->perf.core_clk_rate = clk_rate;
+		DPU_DEBUG("update clk rate = %lld HZ\n", clk_rate);
+	}
+	return 0;
+}
+
+#ifdef CONFIG_DEBUG_FS
+
+static ssize_t _dpu_core_perf_mode_write(struct file *file,
+		    const char __user *user_buf, size_t count, loff_t *ppos)
+{
+	struct dpu_core_perf *perf = file->private_data;
+	struct dpu_perf_cfg *cfg = &perf->catalog->perf;
+	u32 perf_mode = 0;
+	char buf[10];
+
+	if (!perf)
+		return -ENODEV;
+
+	if (count >= sizeof(buf))
+		return -EFAULT;
+
+	if (copy_from_user(buf, user_buf, count))
+		return -EFAULT;
+
+	buf[count] = 0;	/* end of string */
+
+	if (kstrtouint(buf, 0, &perf_mode))
+		return -EFAULT;
+
+	if (perf_mode >= DPU_PERF_MODE_MAX)
+		return -EFAULT;
+
+	if (perf_mode == DPU_PERF_MODE_FIXED) {
+		DRM_INFO("fix performance mode\n");
+	} else if (perf_mode == DPU_PERF_MODE_MINIMUM) {
+		/* run the driver with max clk and BW vote */
+		perf->perf_tune.min_core_clk = perf->max_core_clk_rate;
+		perf->perf_tune.min_bus_vote =
+				(u64) cfg->max_bw_high * 1000;
+		DRM_INFO("minimum performance mode\n");
+	} else if (perf_mode == DPU_PERF_MODE_NORMAL) {
+		/* reset the perf tune params to 0 */
+		perf->perf_tune.min_core_clk = 0;
+		perf->perf_tune.min_bus_vote = 0;
+		DRM_INFO("normal performance mode\n");
+	}
+	perf->perf_tune.mode = perf_mode;
+
+	return count;
+}
+
+static ssize_t _dpu_core_perf_mode_read(struct file *file,
+			char __user *buff, size_t count, loff_t *ppos)
+{
+	struct dpu_core_perf *perf = file->private_data;
+	int len = 0;
+	char buf[DPU_PERF_MODE_STRING_SIZE] = {'\0'};
+
+	if (!perf)
+		return -ENODEV;
+
+	if (*ppos)
+		return 0;	/* the end */
+
+	len = snprintf(buf, sizeof(buf),
+			"mode %d min_mdp_clk %llu min_bus_vote %llu\n",
+			perf->perf_tune.mode,
+			perf->perf_tune.min_core_clk,
+			perf->perf_tune.min_bus_vote);
+	if (len < 0 || len >= sizeof(buf))
+		return 0;
+
+	if ((count < sizeof(buf)) || copy_to_user(buff, buf, len))
+		return -EFAULT;
+
+	*ppos += len;   /* increase offset */
+
+	return len;
+}
+
+static const struct file_operations dpu_core_perf_mode_fops = {
+	.open = simple_open,
+	.read = _dpu_core_perf_mode_read,
+	.write = _dpu_core_perf_mode_write,
+};
+
+static void dpu_core_perf_debugfs_destroy(struct dpu_core_perf *perf)
+{
+	debugfs_remove_recursive(perf->debugfs_root);
+	perf->debugfs_root = NULL;
+}
+
+int dpu_core_perf_debugfs_init(struct dpu_core_perf *perf,
+		struct dentry *parent)
+{
+	struct dpu_mdss_cfg *catalog = perf->catalog;
+	struct msm_drm_private *priv;
+	struct dpu_kms *dpu_kms;
+
+	priv = perf->dev->dev_private;
+	if (!priv || !priv->kms) {
+		DPU_ERROR("invalid KMS reference\n");
+		return -EINVAL;
+	}
+
+	dpu_kms = to_dpu_kms(priv->kms);
+
+	perf->debugfs_root = debugfs_create_dir("core_perf", parent);
+	if (!perf->debugfs_root) {
+		DPU_ERROR("failed to create core perf debugfs\n");
+		return -EINVAL;
+	}
+
+	debugfs_create_u64("max_core_clk_rate", 0600, perf->debugfs_root,
+			&perf->max_core_clk_rate);
+	debugfs_create_u64("core_clk_rate", 0600, perf->debugfs_root,
+			&perf->core_clk_rate);
+	debugfs_create_u32("enable_bw_release", 0600, perf->debugfs_root,
+			(u32 *)&perf->enable_bw_release);
+	debugfs_create_u32("threshold_low", 0600, perf->debugfs_root,
+			(u32 *)&catalog->perf.max_bw_low);
+	debugfs_create_u32("threshold_high", 0600, perf->debugfs_root,
+			(u32 *)&catalog->perf.max_bw_high);
+	debugfs_create_u32("min_core_ib", 0600, perf->debugfs_root,
+			(u32 *)&catalog->perf.min_core_ib);
+	debugfs_create_u32("min_llcc_ib", 0600, perf->debugfs_root,
+			(u32 *)&catalog->perf.min_llcc_ib);
+	debugfs_create_u32("min_dram_ib", 0600, perf->debugfs_root,
+			(u32 *)&catalog->perf.min_dram_ib);
+	debugfs_create_file("perf_mode", 0600, perf->debugfs_root,
+			(u32 *)perf, &dpu_core_perf_mode_fops);
+	debugfs_create_u64("fix_core_clk_rate", 0600, perf->debugfs_root,
+			&perf->fix_core_clk_rate);
+	debugfs_create_u64("fix_core_ib_vote", 0600, perf->debugfs_root,
+			&perf->fix_core_ib_vote);
+	debugfs_create_u64("fix_core_ab_vote", 0600, perf->debugfs_root,
+			&perf->fix_core_ab_vote);
+
+	return 0;
+}
+#else
+static void dpu_core_perf_debugfs_destroy(struct dpu_core_perf *perf)
+{
+}
+
+int dpu_core_perf_debugfs_init(struct dpu_core_perf *perf,
+		struct dentry *parent)
+{
+	return 0;
+}
+#endif
+
+void dpu_core_perf_destroy(struct dpu_core_perf *perf)
+{
+	if (!perf) {
+		DPU_ERROR("invalid parameters\n");
+		return;
+	}
+
+	dpu_core_perf_debugfs_destroy(perf);
+	perf->max_core_clk_rate = 0;
+	perf->core_clk = NULL;
+	perf->phandle = NULL;
+	perf->catalog = NULL;
+	perf->dev = NULL;
+}
+
+int dpu_core_perf_init(struct dpu_core_perf *perf,
+		struct drm_device *dev,
+		struct dpu_mdss_cfg *catalog,
+		struct dpu_power_handle *phandle,
+		struct dss_clk *core_clk)
+{
+	perf->dev = dev;
+	perf->catalog = catalog;
+	perf->phandle = phandle;
+	perf->core_clk = core_clk;
+
+	perf->max_core_clk_rate = core_clk->max_rate;
+	if (!perf->max_core_clk_rate) {
+		DPU_DEBUG("optional max core clk rate, use default\n");
+		perf->max_core_clk_rate = DPU_PERF_DEFAULT_MAX_CORE_CLK_RATE;
+	}
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_core_perf.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_core_perf.h
new file mode 100644
index 000000000000..fbcbe0c7527a
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_core_perf.h
@@ -0,0 +1,133 @@
+/* Copyright (c) 2016-2018, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef _DPU_CORE_PERF_H_
+#define _DPU_CORE_PERF_H_
+
+#include <linux/types.h>
+#include <linux/dcache.h>
+#include <linux/mutex.h>
+#include <drm/drm_crtc.h>
+
+#include "dpu_hw_catalog.h"
+#include "dpu_power_handle.h"
+
+#define	DPU_PERF_DEFAULT_MAX_CORE_CLK_RATE	412500000
+
+/**
+ * struct dpu_core_perf_params - definition of performance parameters
+ * @max_per_pipe_ib: maximum instantaneous bandwidth request
+ * @bw_ctl: arbitrated bandwidth request
+ * @core_clk_rate: core clock rate request
+ */
+struct dpu_core_perf_params {
+	u64 max_per_pipe_ib[DPU_POWER_HANDLE_DBUS_ID_MAX];
+	u64 bw_ctl[DPU_POWER_HANDLE_DBUS_ID_MAX];
+	u64 core_clk_rate;
+};
+
+/**
+ * struct dpu_core_perf_tune - definition of performance tuning control
+ * @mode: performance mode
+ * @min_core_clk: minimum core clock
+ * @min_bus_vote: minimum bus vote
+ */
+struct dpu_core_perf_tune {
+	u32 mode;
+	u64 min_core_clk;
+	u64 min_bus_vote;
+};
+
+/**
+ * struct dpu_core_perf - definition of core performance context
+ * @dev: Pointer to drm device
+ * @debugfs_root: top level debug folder
+ * @catalog: Pointer to catalog configuration
+ * @phandle: Pointer to power handler
+ * @core_clk: Pointer to core clock structure
+ * @core_clk_rate: current core clock rate
+ * @max_core_clk_rate: maximum allowable core clock rate
+ * @perf_tune: debug control for performance tuning
+ * @enable_bw_release: debug control for bandwidth release
+ * @fix_core_clk_rate: fixed core clock request in Hz used in mode 2
+ * @fix_core_ib_vote: fixed core ib vote in bps used in mode 2
+ * @fix_core_ab_vote: fixed core ab vote in bps used in mode 2
+ */
+struct dpu_core_perf {
+	struct drm_device *dev;
+	struct dentry *debugfs_root;
+	struct dpu_mdss_cfg *catalog;
+	struct dpu_power_handle *phandle;
+	struct dss_clk *core_clk;
+	u64 core_clk_rate;
+	u64 max_core_clk_rate;
+	struct dpu_core_perf_tune perf_tune;
+	u32 enable_bw_release;
+	u64 fix_core_clk_rate;
+	u64 fix_core_ib_vote;
+	u64 fix_core_ab_vote;
+};
+
+/**
+ * dpu_core_perf_crtc_check - validate performance of the given crtc state
+ * @crtc: Pointer to crtc
+ * @state: Pointer to new crtc state
+ * return: zero if success, or error code otherwise
+ */
+int dpu_core_perf_crtc_check(struct drm_crtc *crtc,
+		struct drm_crtc_state *state);
+
+/**
+ * dpu_core_perf_crtc_update - update performance of the given crtc
+ * @crtc: Pointer to crtc
+ * @params_changed: true if crtc parameters are modified
+ * @stop_req: true if this is a stop request
+ * return: zero if success, or error code otherwise
+ */
+int dpu_core_perf_crtc_update(struct drm_crtc *crtc,
+		int params_changed, bool stop_req);
+
+/**
+ * dpu_core_perf_crtc_release_bw - release bandwidth of the given crtc
+ * @crtc: Pointer to crtc
+ */
+void dpu_core_perf_crtc_release_bw(struct drm_crtc *crtc);
+
+/**
+ * dpu_core_perf_destroy - destroy the given core performance context
+ * @perf: Pointer to core performance context
+ */
+void dpu_core_perf_destroy(struct dpu_core_perf *perf);
+
+/**
+ * dpu_core_perf_init - initialize the given core performance context
+ * @perf: Pointer to core performance context
+ * @dev: Pointer to drm device
+ * @catalog: Pointer to catalog
+ * @phandle: Pointer to power handle
+ * @core_clk: pointer to core clock
+ */
+int dpu_core_perf_init(struct dpu_core_perf *perf,
+		struct drm_device *dev,
+		struct dpu_mdss_cfg *catalog,
+		struct dpu_power_handle *phandle,
+		struct dss_clk *core_clk);
+
+/**
+ * dpu_core_perf_debugfs_init - initialize debugfs for core performance context
+ * @perf: Pointer to core performance context
+ * @debugfs_parent: Pointer to parent debugfs
+ */
+int dpu_core_perf_debugfs_init(struct dpu_core_perf *perf,
+		struct dentry *parent);
+
+#endif /* _DPU_CORE_PERF_H_ */
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c
new file mode 100644
index 000000000000..80cbf75bc2ff
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c
@@ -0,0 +1,2138 @@
+/*
+ * Copyright (c) 2014-2018 The Linux Foundation. All rights reserved.
+ * Copyright (C) 2013 Red Hat
+ * Author: Rob Clark <robdclark@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define pr_fmt(fmt)	"[drm:%s:%d] " fmt, __func__, __LINE__
+#include <linux/sort.h>
+#include <linux/debugfs.h>
+#include <linux/ktime.h>
+#include <drm/drm_mode.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_flip_work.h>
+#include <drm/drm_rect.h>
+
+#include "dpu_kms.h"
+#include "dpu_hw_lm.h"
+#include "dpu_hw_ctl.h"
+#include "dpu_crtc.h"
+#include "dpu_plane.h"
+#include "dpu_encoder.h"
+#include "dpu_vbif.h"
+#include "dpu_power_handle.h"
+#include "dpu_core_perf.h"
+#include "dpu_trace.h"
+
+#define DPU_DRM_BLEND_OP_NOT_DEFINED    0
+#define DPU_DRM_BLEND_OP_OPAQUE         1
+#define DPU_DRM_BLEND_OP_PREMULTIPLIED  2
+#define DPU_DRM_BLEND_OP_COVERAGE       3
+#define DPU_DRM_BLEND_OP_MAX            4
+
+/* layer mixer index on dpu_crtc */
+#define LEFT_MIXER 0
+#define RIGHT_MIXER 1
+
+#define MISR_BUFF_SIZE			256
+
+static inline struct dpu_kms *_dpu_crtc_get_kms(struct drm_crtc *crtc)
+{
+	struct msm_drm_private *priv;
+
+	if (!crtc || !crtc->dev || !crtc->dev->dev_private) {
+		DPU_ERROR("invalid crtc\n");
+		return NULL;
+	}
+	priv = crtc->dev->dev_private;
+	if (!priv || !priv->kms) {
+		DPU_ERROR("invalid kms\n");
+		return NULL;
+	}
+
+	return to_dpu_kms(priv->kms);
+}
+
+static inline int _dpu_crtc_power_enable(struct dpu_crtc *dpu_crtc, bool enable)
+{
+	struct drm_crtc *crtc;
+	struct msm_drm_private *priv;
+	struct dpu_kms *dpu_kms;
+
+	if (!dpu_crtc) {
+		DPU_ERROR("invalid dpu crtc\n");
+		return -EINVAL;
+	}
+
+	crtc = &dpu_crtc->base;
+	if (!crtc->dev || !crtc->dev->dev_private) {
+		DPU_ERROR("invalid drm device\n");
+		return -EINVAL;
+	}
+
+	priv = crtc->dev->dev_private;
+	if (!priv->kms) {
+		DPU_ERROR("invalid kms\n");
+		return -EINVAL;
+	}
+
+	dpu_kms = to_dpu_kms(priv->kms);
+
+	if (enable)
+		pm_runtime_get_sync(&dpu_kms->pdev->dev);
+	else
+		pm_runtime_put_sync(&dpu_kms->pdev->dev);
+
+	return 0;
+}
+
+/**
+ * _dpu_crtc_rp_to_crtc - get crtc from resource pool object
+ * @rp: Pointer to resource pool
+ * return: Pointer to drm crtc if success; null otherwise
+ */
+static struct drm_crtc *_dpu_crtc_rp_to_crtc(struct dpu_crtc_respool *rp)
+{
+	if (!rp)
+		return NULL;
+
+	return container_of(rp, struct dpu_crtc_state, rp)->base.crtc;
+}
+
+/**
+ * _dpu_crtc_rp_reclaim - reclaim unused, or all if forced, resources in pool
+ * @rp: Pointer to resource pool
+ * @force: True to reclaim all resources; otherwise, reclaim only unused ones
+ * return: None
+ */
+static void _dpu_crtc_rp_reclaim(struct dpu_crtc_respool *rp, bool force)
+{
+	struct dpu_crtc_res *res, *next;
+	struct drm_crtc *crtc;
+
+	crtc = _dpu_crtc_rp_to_crtc(rp);
+	if (!crtc) {
+		DPU_ERROR("invalid crtc\n");
+		return;
+	}
+
+	DPU_DEBUG("crtc%d.%u %s\n", crtc->base.id, rp->sequence_id,
+			force ? "destroy" : "free_unused");
+
+	list_for_each_entry_safe(res, next, &rp->res_list, list) {
+		if (!force && !(res->flags & DPU_CRTC_RES_FLAG_FREE))
+			continue;
+		DPU_DEBUG("crtc%d.%u reclaim res:0x%x/0x%llx/%pK/%d\n",
+				crtc->base.id, rp->sequence_id,
+				res->type, res->tag, res->val,
+				atomic_read(&res->refcount));
+		list_del(&res->list);
+		if (res->ops.put)
+			res->ops.put(res->val);
+		kfree(res);
+	}
+}
+
+/**
+ * _dpu_crtc_rp_free_unused - free unused resource in pool
+ * @rp: Pointer to resource pool
+ * return: none
+ */
+static void _dpu_crtc_rp_free_unused(struct dpu_crtc_respool *rp)
+{
+	mutex_lock(rp->rp_lock);
+	_dpu_crtc_rp_reclaim(rp, false);
+	mutex_unlock(rp->rp_lock);
+}
+
+/**
+ * _dpu_crtc_rp_destroy - destroy resource pool
+ * @rp: Pointer to resource pool
+ * return: None
+ */
+static void _dpu_crtc_rp_destroy(struct dpu_crtc_respool *rp)
+{
+	mutex_lock(rp->rp_lock);
+	list_del_init(&rp->rp_list);
+	_dpu_crtc_rp_reclaim(rp, true);
+	mutex_unlock(rp->rp_lock);
+}
+
+/**
+ * _dpu_crtc_hw_blk_get - get callback for hardware block
+ * @val: Resource handle
+ * @type: Resource type
+ * @tag: Search tag for given resource
+ * return: Resource handle
+ */
+static void *_dpu_crtc_hw_blk_get(void *val, u32 type, u64 tag)
+{
+	DPU_DEBUG("res:%d/0x%llx/%pK\n", type, tag, val);
+	return dpu_hw_blk_get(val, type, tag);
+}
+
+/**
+ * _dpu_crtc_hw_blk_put - put callback for hardware block
+ * @val: Resource handle
+ * return: None
+ */
+static void _dpu_crtc_hw_blk_put(void *val)
+{
+	DPU_DEBUG("res://%pK\n", val);
+	dpu_hw_blk_put(val);
+}
+
+/**
+ * _dpu_crtc_rp_duplicate - duplicate resource pool and reset reference count
+ * @rp: Pointer to original resource pool
+ * @dup_rp: Pointer to duplicated resource pool
+ * return: None
+ */
+static void _dpu_crtc_rp_duplicate(struct dpu_crtc_respool *rp,
+		struct dpu_crtc_respool *dup_rp)
+{
+	struct dpu_crtc_res *res, *dup_res;
+	struct drm_crtc *crtc;
+
+	if (!rp || !dup_rp || !rp->rp_head) {
+		DPU_ERROR("invalid resource pool\n");
+		return;
+	}
+
+	crtc = _dpu_crtc_rp_to_crtc(rp);
+	if (!crtc) {
+		DPU_ERROR("invalid crtc\n");
+		return;
+	}
+
+	DPU_DEBUG("crtc%d.%u duplicate\n", crtc->base.id, rp->sequence_id);
+
+	mutex_lock(rp->rp_lock);
+	dup_rp->sequence_id = rp->sequence_id + 1;
+	INIT_LIST_HEAD(&dup_rp->res_list);
+	dup_rp->ops = rp->ops;
+	list_for_each_entry(res, &rp->res_list, list) {
+		dup_res = kzalloc(sizeof(struct dpu_crtc_res), GFP_KERNEL);
+		if (!dup_res) {
+			mutex_unlock(rp->rp_lock);
+			return;
+		}
+		INIT_LIST_HEAD(&dup_res->list);
+		atomic_set(&dup_res->refcount, 0);
+		dup_res->type = res->type;
+		dup_res->tag = res->tag;
+		dup_res->val = res->val;
+		dup_res->ops = res->ops;
+		dup_res->flags = DPU_CRTC_RES_FLAG_FREE;
+		DPU_DEBUG("crtc%d.%u dup res:0x%x/0x%llx/%pK/%d\n",
+				crtc->base.id, dup_rp->sequence_id,
+				dup_res->type, dup_res->tag, dup_res->val,
+				atomic_read(&dup_res->refcount));
+		list_add_tail(&dup_res->list, &dup_rp->res_list);
+		if (dup_res->ops.get)
+			dup_res->ops.get(dup_res->val, 0, -1);
+	}
+
+	dup_rp->rp_lock = rp->rp_lock;
+	dup_rp->rp_head = rp->rp_head;
+	INIT_LIST_HEAD(&dup_rp->rp_list);
+	list_add_tail(&dup_rp->rp_list, rp->rp_head);
+	mutex_unlock(rp->rp_lock);
+}
+
+/**
+ * _dpu_crtc_rp_reset - reset resource pool after allocation
+ * @rp: Pointer to original resource pool
+ * @rp_lock: Pointer to serialization resource pool lock
+ * @rp_head: Pointer to crtc resource pool head
+ * return: None
+ */
+static void _dpu_crtc_rp_reset(struct dpu_crtc_respool *rp,
+		struct mutex *rp_lock, struct list_head *rp_head)
+{
+	if (!rp || !rp_lock || !rp_head) {
+		DPU_ERROR("invalid resource pool\n");
+		return;
+	}
+
+	mutex_lock(rp_lock);
+	rp->rp_lock = rp_lock;
+	rp->rp_head = rp_head;
+	INIT_LIST_HEAD(&rp->rp_list);
+	rp->sequence_id = 0;
+	INIT_LIST_HEAD(&rp->res_list);
+	rp->ops.get = _dpu_crtc_hw_blk_get;
+	rp->ops.put = _dpu_crtc_hw_blk_put;
+	list_add_tail(&rp->rp_list, rp->rp_head);
+	mutex_unlock(rp_lock);
+}
+
+static void dpu_crtc_destroy(struct drm_crtc *crtc)
+{
+	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
+
+	DPU_DEBUG("\n");
+
+	if (!crtc)
+		return;
+
+	dpu_crtc->phandle = NULL;
+
+	drm_crtc_cleanup(crtc);
+	mutex_destroy(&dpu_crtc->crtc_lock);
+	kfree(dpu_crtc);
+}
+
+static void _dpu_crtc_setup_blend_cfg(struct dpu_crtc_mixer *mixer,
+		struct dpu_plane_state *pstate)
+{
+	struct dpu_hw_mixer *lm = mixer->hw_lm;
+
+	/* default to opaque blending */
+	lm->ops.setup_blend_config(lm, pstate->stage, 0XFF, 0,
+				DPU_BLEND_FG_ALPHA_FG_CONST |
+				DPU_BLEND_BG_ALPHA_BG_CONST);
+}
+
+static void _dpu_crtc_program_lm_output_roi(struct drm_crtc *crtc)
+{
+	struct dpu_crtc *dpu_crtc;
+	struct dpu_crtc_state *crtc_state;
+	int lm_idx, lm_horiz_position;
+
+	dpu_crtc = to_dpu_crtc(crtc);
+	crtc_state = to_dpu_crtc_state(crtc->state);
+
+	lm_horiz_position = 0;
+	for (lm_idx = 0; lm_idx < dpu_crtc->num_mixers; lm_idx++) {
+		const struct drm_rect *lm_roi = &crtc_state->lm_bounds[lm_idx];
+		struct dpu_hw_mixer *hw_lm = dpu_crtc->mixers[lm_idx].hw_lm;
+		struct dpu_hw_mixer_cfg cfg;
+
+		if (!lm_roi || !drm_rect_visible(lm_roi))
+			continue;
+
+		cfg.out_width = drm_rect_width(lm_roi);
+		cfg.out_height = drm_rect_height(lm_roi);
+		cfg.right_mixer = lm_horiz_position++;
+		cfg.flags = 0;
+		hw_lm->ops.setup_mixer_out(hw_lm, &cfg);
+	}
+}
+
+static void _dpu_crtc_blend_setup_mixer(struct drm_crtc *crtc,
+	struct dpu_crtc *dpu_crtc, struct dpu_crtc_mixer *mixer)
+{
+	struct drm_plane *plane;
+	struct drm_framebuffer *fb;
+	struct drm_plane_state *state;
+	struct dpu_crtc_state *cstate;
+	struct dpu_plane_state *pstate = NULL;
+	struct dpu_format *format;
+	struct dpu_hw_ctl *ctl;
+	struct dpu_hw_mixer *lm;
+	struct dpu_hw_stage_cfg *stage_cfg;
+
+	u32 flush_mask;
+	uint32_t stage_idx, lm_idx;
+	int zpos_cnt[DPU_STAGE_MAX + 1] = { 0 };
+	bool bg_alpha_enable = false;
+
+	if (!dpu_crtc || !mixer) {
+		DPU_ERROR("invalid dpu_crtc or mixer\n");
+		return;
+	}
+
+	ctl = mixer->hw_ctl;
+	lm = mixer->hw_lm;
+	stage_cfg = &dpu_crtc->stage_cfg;
+	cstate = to_dpu_crtc_state(crtc->state);
+
+	drm_atomic_crtc_for_each_plane(plane, crtc) {
+		state = plane->state;
+		if (!state)
+			continue;
+
+		pstate = to_dpu_plane_state(state);
+		fb = state->fb;
+
+		dpu_plane_get_ctl_flush(plane, ctl, &flush_mask);
+
+		DPU_DEBUG("crtc %d stage:%d - plane %d sspp %d fb %d\n",
+				crtc->base.id,
+				pstate->stage,
+				plane->base.id,
+				dpu_plane_pipe(plane) - SSPP_VIG0,
+				state->fb ? state->fb->base.id : -1);
+
+		format = to_dpu_format(msm_framebuffer_format(pstate->base.fb));
+		if (!format) {
+			DPU_ERROR("invalid format\n");
+			return;
+		}
+
+		if (pstate->stage == DPU_STAGE_BASE && format->alpha_enable)
+			bg_alpha_enable = true;
+
+		stage_idx = zpos_cnt[pstate->stage]++;
+		stage_cfg->stage[pstate->stage][stage_idx] =
+					dpu_plane_pipe(plane);
+		stage_cfg->multirect_index[pstate->stage][stage_idx] =
+					pstate->multirect_index;
+
+		trace_dpu_crtc_setup_mixer(DRMID(crtc), DRMID(plane),
+					   state, pstate, stage_idx,
+					   dpu_plane_pipe(plane) - SSPP_VIG0,
+					   format->base.pixel_format,
+					   fb ? fb->modifier : 0);
+
+		/* blend config update */
+		for (lm_idx = 0; lm_idx < dpu_crtc->num_mixers; lm_idx++) {
+			_dpu_crtc_setup_blend_cfg(mixer + lm_idx, pstate);
+
+			mixer[lm_idx].flush_mask |= flush_mask;
+
+			if (bg_alpha_enable && !format->alpha_enable)
+				mixer[lm_idx].mixer_op_mode = 0;
+			else
+				mixer[lm_idx].mixer_op_mode |=
+						1 << pstate->stage;
+		}
+	}
+
+	 _dpu_crtc_program_lm_output_roi(crtc);
+}
+
+/**
+ * _dpu_crtc_blend_setup - configure crtc mixers
+ * @crtc: Pointer to drm crtc structure
+ */
+static void _dpu_crtc_blend_setup(struct drm_crtc *crtc)
+{
+	struct dpu_crtc *dpu_crtc;
+	struct dpu_crtc_state *dpu_crtc_state;
+	struct dpu_crtc_mixer *mixer;
+	struct dpu_hw_ctl *ctl;
+	struct dpu_hw_mixer *lm;
+
+	int i;
+
+	if (!crtc)
+		return;
+
+	dpu_crtc = to_dpu_crtc(crtc);
+	dpu_crtc_state = to_dpu_crtc_state(crtc->state);
+	mixer = dpu_crtc->mixers;
+
+	DPU_DEBUG("%s\n", dpu_crtc->name);
+
+	if (dpu_crtc->num_mixers > CRTC_DUAL_MIXERS) {
+		DPU_ERROR("invalid number mixers: %d\n", dpu_crtc->num_mixers);
+		return;
+	}
+
+	for (i = 0; i < dpu_crtc->num_mixers; i++) {
+		if (!mixer[i].hw_lm || !mixer[i].hw_ctl) {
+			DPU_ERROR("invalid lm or ctl assigned to mixer\n");
+			return;
+		}
+		mixer[i].mixer_op_mode = 0;
+		mixer[i].flush_mask = 0;
+		if (mixer[i].hw_ctl->ops.clear_all_blendstages)
+			mixer[i].hw_ctl->ops.clear_all_blendstages(
+					mixer[i].hw_ctl);
+	}
+
+	/* initialize stage cfg */
+	memset(&dpu_crtc->stage_cfg, 0, sizeof(struct dpu_hw_stage_cfg));
+
+	_dpu_crtc_blend_setup_mixer(crtc, dpu_crtc, mixer);
+
+	for (i = 0; i < dpu_crtc->num_mixers; i++) {
+		ctl = mixer[i].hw_ctl;
+		lm = mixer[i].hw_lm;
+
+		lm->ops.setup_alpha_out(lm, mixer[i].mixer_op_mode);
+
+		mixer[i].flush_mask |= ctl->ops.get_bitmask_mixer(ctl,
+			mixer[i].hw_lm->idx);
+
+		/* stage config flush mask */
+		ctl->ops.update_pending_flush(ctl, mixer[i].flush_mask);
+
+		DPU_DEBUG("lm %d, op_mode 0x%X, ctl %d, flush mask 0x%x\n",
+			mixer[i].hw_lm->idx - LM_0,
+			mixer[i].mixer_op_mode,
+			ctl->idx - CTL_0,
+			mixer[i].flush_mask);
+
+		ctl->ops.setup_blendstage(ctl, mixer[i].hw_lm->idx,
+			&dpu_crtc->stage_cfg);
+	}
+}
+
+/**
+ *  _dpu_crtc_complete_flip - signal pending page_flip events
+ * Any pending vblank events are added to the vblank_event_list
+ * so that the next vblank interrupt shall signal them.
+ * However PAGE_FLIP events are not handled through the vblank_event_list.
+ * This API signals any pending PAGE_FLIP events requested through
+ * DRM_IOCTL_MODE_PAGE_FLIP and are cached in the dpu_crtc->event.
+ * @crtc: Pointer to drm crtc structure
+ */
+static void _dpu_crtc_complete_flip(struct drm_crtc *crtc)
+{
+	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
+	struct drm_device *dev = crtc->dev;
+	unsigned long flags;
+
+	spin_lock_irqsave(&dev->event_lock, flags);
+	if (dpu_crtc->event) {
+		DRM_DEBUG_VBL("%s: send event: %pK\n", dpu_crtc->name,
+			      dpu_crtc->event);
+		trace_dpu_crtc_complete_flip(DRMID(crtc));
+		drm_crtc_send_vblank_event(crtc, dpu_crtc->event);
+		dpu_crtc->event = NULL;
+	}
+	spin_unlock_irqrestore(&dev->event_lock, flags);
+}
+
+enum dpu_intf_mode dpu_crtc_get_intf_mode(struct drm_crtc *crtc)
+{
+	struct drm_encoder *encoder;
+
+	if (!crtc || !crtc->dev) {
+		DPU_ERROR("invalid crtc\n");
+		return INTF_MODE_NONE;
+	}
+
+	drm_for_each_encoder(encoder, crtc->dev)
+		if (encoder->crtc == crtc)
+			return dpu_encoder_get_intf_mode(encoder);
+
+	return INTF_MODE_NONE;
+}
+
+static void dpu_crtc_vblank_cb(void *data)
+{
+	struct drm_crtc *crtc = (struct drm_crtc *)data;
+	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
+
+	/* keep statistics on vblank callback - with auto reset via debugfs */
+	if (ktime_compare(dpu_crtc->vblank_cb_time, ktime_set(0, 0)) == 0)
+		dpu_crtc->vblank_cb_time = ktime_get();
+	else
+		dpu_crtc->vblank_cb_count++;
+	_dpu_crtc_complete_flip(crtc);
+	drm_crtc_handle_vblank(crtc);
+	trace_dpu_crtc_vblank_cb(DRMID(crtc));
+}
+
+static void dpu_crtc_frame_event_work(struct kthread_work *work)
+{
+	struct msm_drm_private *priv;
+	struct dpu_crtc_frame_event *fevent;
+	struct drm_crtc *crtc;
+	struct dpu_crtc *dpu_crtc;
+	struct dpu_kms *dpu_kms;
+	unsigned long flags;
+	bool frame_done = false;
+
+	if (!work) {
+		DPU_ERROR("invalid work handle\n");
+		return;
+	}
+
+	fevent = container_of(work, struct dpu_crtc_frame_event, work);
+	if (!fevent->crtc || !fevent->crtc->state) {
+		DPU_ERROR("invalid crtc\n");
+		return;
+	}
+
+	crtc = fevent->crtc;
+	dpu_crtc = to_dpu_crtc(crtc);
+
+	dpu_kms = _dpu_crtc_get_kms(crtc);
+	if (!dpu_kms) {
+		DPU_ERROR("invalid kms handle\n");
+		return;
+	}
+	priv = dpu_kms->dev->dev_private;
+	DPU_ATRACE_BEGIN("crtc_frame_event");
+
+	DRM_DEBUG_KMS("crtc%d event:%u ts:%lld\n", crtc->base.id, fevent->event,
+			ktime_to_ns(fevent->ts));
+
+	if (fevent->event & (DPU_ENCODER_FRAME_EVENT_DONE
+				| DPU_ENCODER_FRAME_EVENT_ERROR
+				| DPU_ENCODER_FRAME_EVENT_PANEL_DEAD)) {
+
+		if (atomic_read(&dpu_crtc->frame_pending) < 1) {
+			/* this should not happen */
+			DRM_ERROR("crtc%d ev:%u ts:%lld frame_pending:%d\n",
+					crtc->base.id,
+					fevent->event,
+					ktime_to_ns(fevent->ts),
+					atomic_read(&dpu_crtc->frame_pending));
+		} else if (atomic_dec_return(&dpu_crtc->frame_pending) == 0) {
+			/* release bandwidth and other resources */
+			trace_dpu_crtc_frame_event_done(DRMID(crtc),
+							fevent->event);
+			dpu_core_perf_crtc_release_bw(crtc);
+		} else {
+			trace_dpu_crtc_frame_event_more_pending(DRMID(crtc),
+								fevent->event);
+		}
+
+		if (fevent->event & DPU_ENCODER_FRAME_EVENT_DONE)
+			dpu_core_perf_crtc_update(crtc, 0, false);
+
+		if (fevent->event & (DPU_ENCODER_FRAME_EVENT_DONE
+					| DPU_ENCODER_FRAME_EVENT_ERROR))
+			frame_done = true;
+	}
+
+	if (fevent->event & DPU_ENCODER_FRAME_EVENT_PANEL_DEAD)
+		DPU_ERROR("crtc%d ts:%lld received panel dead event\n",
+				crtc->base.id, ktime_to_ns(fevent->ts));
+
+	if (frame_done)
+		complete_all(&dpu_crtc->frame_done_comp);
+
+	spin_lock_irqsave(&dpu_crtc->spin_lock, flags);
+	list_add_tail(&fevent->list, &dpu_crtc->frame_event_list);
+	spin_unlock_irqrestore(&dpu_crtc->spin_lock, flags);
+	DPU_ATRACE_END("crtc_frame_event");
+}
+
+/*
+ * dpu_crtc_frame_event_cb - crtc frame event callback API. CRTC module
+ * registers this API to encoder for all frame event callbacks like
+ * frame_error, frame_done, idle_timeout, etc. Encoder may call different events
+ * from different context - IRQ, user thread, commit_thread, etc. Each event
+ * should be carefully reviewed and should be processed in proper task context
+ * to avoid schedulin delay or properly manage the irq context's bottom half
+ * processing.
+ */
+static void dpu_crtc_frame_event_cb(void *data, u32 event)
+{
+	struct drm_crtc *crtc = (struct drm_crtc *)data;
+	struct dpu_crtc *dpu_crtc;
+	struct msm_drm_private *priv;
+	struct dpu_crtc_frame_event *fevent;
+	unsigned long flags;
+	u32 crtc_id;
+
+	if (!crtc || !crtc->dev || !crtc->dev->dev_private) {
+		DPU_ERROR("invalid parameters\n");
+		return;
+	}
+
+	/* Nothing to do on idle event */
+	if (event & DPU_ENCODER_FRAME_EVENT_IDLE)
+		return;
+
+	dpu_crtc = to_dpu_crtc(crtc);
+	priv = crtc->dev->dev_private;
+	crtc_id = drm_crtc_index(crtc);
+
+	trace_dpu_crtc_frame_event_cb(DRMID(crtc), event);
+
+	spin_lock_irqsave(&dpu_crtc->spin_lock, flags);
+	fevent = list_first_entry_or_null(&dpu_crtc->frame_event_list,
+			struct dpu_crtc_frame_event, list);
+	if (fevent)
+		list_del_init(&fevent->list);
+	spin_unlock_irqrestore(&dpu_crtc->spin_lock, flags);
+
+	if (!fevent) {
+		DRM_ERROR("crtc%d event %d overflow\n", crtc->base.id, event);
+		return;
+	}
+
+	fevent->event = event;
+	fevent->crtc = crtc;
+	fevent->ts = ktime_get();
+	kthread_queue_work(&priv->event_thread[crtc_id].worker, &fevent->work);
+}
+
+void dpu_crtc_complete_commit(struct drm_crtc *crtc,
+		struct drm_crtc_state *old_state)
+{
+	if (!crtc || !crtc->state) {
+		DPU_ERROR("invalid crtc\n");
+		return;
+	}
+	trace_dpu_crtc_complete_commit(DRMID(crtc));
+}
+
+static void _dpu_crtc_setup_mixer_for_encoder(
+		struct drm_crtc *crtc,
+		struct drm_encoder *enc)
+{
+	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
+	struct dpu_kms *dpu_kms = _dpu_crtc_get_kms(crtc);
+	struct dpu_rm *rm = &dpu_kms->rm;
+	struct dpu_crtc_mixer *mixer;
+	struct dpu_hw_ctl *last_valid_ctl = NULL;
+	int i;
+	struct dpu_rm_hw_iter lm_iter, ctl_iter;
+
+	dpu_rm_init_hw_iter(&lm_iter, enc->base.id, DPU_HW_BLK_LM);
+	dpu_rm_init_hw_iter(&ctl_iter, enc->base.id, DPU_HW_BLK_CTL);
+
+	/* Set up all the mixers and ctls reserved by this encoder */
+	for (i = dpu_crtc->num_mixers; i < ARRAY_SIZE(dpu_crtc->mixers); i++) {
+		mixer = &dpu_crtc->mixers[i];
+
+		if (!dpu_rm_get_hw(rm, &lm_iter))
+			break;
+		mixer->hw_lm = (struct dpu_hw_mixer *)lm_iter.hw;
+
+		/* CTL may be <= LMs, if <, multiple LMs controlled by 1 CTL */
+		if (!dpu_rm_get_hw(rm, &ctl_iter)) {
+			DPU_DEBUG("no ctl assigned to lm %d, using previous\n",
+					mixer->hw_lm->idx - LM_0);
+			mixer->hw_ctl = last_valid_ctl;
+		} else {
+			mixer->hw_ctl = (struct dpu_hw_ctl *)ctl_iter.hw;
+			last_valid_ctl = mixer->hw_ctl;
+		}
+
+		/* Shouldn't happen, mixers are always >= ctls */
+		if (!mixer->hw_ctl) {
+			DPU_ERROR("no valid ctls found for lm %d\n",
+					mixer->hw_lm->idx - LM_0);
+			return;
+		}
+
+		mixer->encoder = enc;
+
+		dpu_crtc->num_mixers++;
+		DPU_DEBUG("setup mixer %d: lm %d\n",
+				i, mixer->hw_lm->idx - LM_0);
+		DPU_DEBUG("setup mixer %d: ctl %d\n",
+				i, mixer->hw_ctl->idx - CTL_0);
+	}
+}
+
+static void _dpu_crtc_setup_mixers(struct drm_crtc *crtc)
+{
+	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
+	struct drm_encoder *enc;
+
+	dpu_crtc->num_mixers = 0;
+	dpu_crtc->mixers_swapped = false;
+	memset(dpu_crtc->mixers, 0, sizeof(dpu_crtc->mixers));
+
+	mutex_lock(&dpu_crtc->crtc_lock);
+	/* Check for mixers on all encoders attached to this crtc */
+	list_for_each_entry(enc, &crtc->dev->mode_config.encoder_list, head) {
+		if (enc->crtc != crtc)
+			continue;
+
+		_dpu_crtc_setup_mixer_for_encoder(crtc, enc);
+	}
+
+	mutex_unlock(&dpu_crtc->crtc_lock);
+}
+
+static void _dpu_crtc_setup_lm_bounds(struct drm_crtc *crtc,
+		struct drm_crtc_state *state)
+{
+	struct dpu_crtc *dpu_crtc;
+	struct dpu_crtc_state *cstate;
+	struct drm_display_mode *adj_mode;
+	u32 crtc_split_width;
+	int i;
+
+	if (!crtc || !state) {
+		DPU_ERROR("invalid args\n");
+		return;
+	}
+
+	dpu_crtc = to_dpu_crtc(crtc);
+	cstate = to_dpu_crtc_state(state);
+
+	adj_mode = &state->adjusted_mode;
+	crtc_split_width = dpu_crtc_get_mixer_width(dpu_crtc, cstate, adj_mode);
+
+	for (i = 0; i < dpu_crtc->num_mixers; i++) {
+		struct drm_rect *r = &cstate->lm_bounds[i];
+		r->x1 = crtc_split_width * i;
+		r->y1 = 0;
+		r->x2 = r->x1 + crtc_split_width;
+		r->y2 = dpu_crtc_get_mixer_height(dpu_crtc, cstate, adj_mode);
+
+		trace_dpu_crtc_setup_lm_bounds(DRMID(crtc), i, r);
+	}
+
+	drm_mode_debug_printmodeline(adj_mode);
+}
+
+static void dpu_crtc_atomic_begin(struct drm_crtc *crtc,
+		struct drm_crtc_state *old_state)
+{
+	struct dpu_crtc *dpu_crtc;
+	struct drm_encoder *encoder;
+	struct drm_device *dev;
+	unsigned long flags;
+	struct dpu_crtc_smmu_state_data *smmu_state;
+
+	if (!crtc) {
+		DPU_ERROR("invalid crtc\n");
+		return;
+	}
+
+	if (!crtc->state->enable) {
+		DPU_DEBUG("crtc%d -> enable %d, skip atomic_begin\n",
+				crtc->base.id, crtc->state->enable);
+		return;
+	}
+
+	DPU_DEBUG("crtc%d\n", crtc->base.id);
+
+	dpu_crtc = to_dpu_crtc(crtc);
+	dev = crtc->dev;
+	smmu_state = &dpu_crtc->smmu_state;
+
+	if (!dpu_crtc->num_mixers) {
+		_dpu_crtc_setup_mixers(crtc);
+		_dpu_crtc_setup_lm_bounds(crtc, crtc->state);
+	}
+
+	if (dpu_crtc->event) {
+		WARN_ON(dpu_crtc->event);
+	} else {
+		spin_lock_irqsave(&dev->event_lock, flags);
+		dpu_crtc->event = crtc->state->event;
+		crtc->state->event = NULL;
+		spin_unlock_irqrestore(&dev->event_lock, flags);
+	}
+
+	list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
+		if (encoder->crtc != crtc)
+			continue;
+
+		/* encoder will trigger pending mask now */
+		dpu_encoder_trigger_kickoff_pending(encoder);
+	}
+
+	/*
+	 * If no mixers have been allocated in dpu_crtc_atomic_check(),
+	 * it means we are trying to flush a CRTC whose state is disabled:
+	 * nothing else needs to be done.
+	 */
+	if (unlikely(!dpu_crtc->num_mixers))
+		return;
+
+	_dpu_crtc_blend_setup(crtc);
+
+	/*
+	 * PP_DONE irq is only used by command mode for now.
+	 * It is better to request pending before FLUSH and START trigger
+	 * to make sure no pp_done irq missed.
+	 * This is safe because no pp_done will happen before SW trigger
+	 * in command mode.
+	 */
+}
+
+static void dpu_crtc_atomic_flush(struct drm_crtc *crtc,
+		struct drm_crtc_state *old_crtc_state)
+{
+	struct dpu_crtc *dpu_crtc;
+	struct drm_device *dev;
+	struct drm_plane *plane;
+	struct msm_drm_private *priv;
+	struct msm_drm_thread *event_thread;
+	unsigned long flags;
+	struct dpu_crtc_state *cstate;
+
+	if (!crtc || !crtc->dev || !crtc->dev->dev_private) {
+		DPU_ERROR("invalid crtc\n");
+		return;
+	}
+
+	if (!crtc->state->enable) {
+		DPU_DEBUG("crtc%d -> enable %d, skip atomic_flush\n",
+				crtc->base.id, crtc->state->enable);
+		return;
+	}
+
+	DPU_DEBUG("crtc%d\n", crtc->base.id);
+
+	dpu_crtc = to_dpu_crtc(crtc);
+	cstate = to_dpu_crtc_state(crtc->state);
+	dev = crtc->dev;
+	priv = dev->dev_private;
+
+	if (crtc->index >= ARRAY_SIZE(priv->event_thread)) {
+		DPU_ERROR("invalid crtc index[%d]\n", crtc->index);
+		return;
+	}
+
+	event_thread = &priv->event_thread[crtc->index];
+
+	if (dpu_crtc->event) {
+		DPU_DEBUG("already received dpu_crtc->event\n");
+	} else {
+		spin_lock_irqsave(&dev->event_lock, flags);
+		dpu_crtc->event = crtc->state->event;
+		crtc->state->event = NULL;
+		spin_unlock_irqrestore(&dev->event_lock, flags);
+	}
+
+	/*
+	 * If no mixers has been allocated in dpu_crtc_atomic_check(),
+	 * it means we are trying to flush a CRTC whose state is disabled:
+	 * nothing else needs to be done.
+	 */
+	if (unlikely(!dpu_crtc->num_mixers))
+		return;
+
+	/*
+	 * For planes without commit update, drm framework will not add
+	 * those planes to current state since hardware update is not
+	 * required. However, if those planes were power collapsed since
+	 * last commit cycle, driver has to restore the hardware state
+	 * of those planes explicitly here prior to plane flush.
+	 */
+	drm_atomic_crtc_for_each_plane(plane, crtc)
+		dpu_plane_restore(plane);
+
+	/* update performance setting before crtc kickoff */
+	dpu_core_perf_crtc_update(crtc, 1, false);
+
+	/*
+	 * Final plane updates: Give each plane a chance to complete all
+	 *                      required writes/flushing before crtc's "flush
+	 *                      everything" call below.
+	 */
+	drm_atomic_crtc_for_each_plane(plane, crtc) {
+		if (dpu_crtc->smmu_state.transition_error)
+			dpu_plane_set_error(plane, true);
+		dpu_plane_flush(plane);
+	}
+
+	/* Kickoff will be scheduled by outer layer */
+}
+
+/**
+ * dpu_crtc_destroy_state - state destroy hook
+ * @crtc: drm CRTC
+ * @state: CRTC state object to release
+ */
+static void dpu_crtc_destroy_state(struct drm_crtc *crtc,
+		struct drm_crtc_state *state)
+{
+	struct dpu_crtc *dpu_crtc;
+	struct dpu_crtc_state *cstate;
+
+	if (!crtc || !state) {
+		DPU_ERROR("invalid argument(s)\n");
+		return;
+	}
+
+	dpu_crtc = to_dpu_crtc(crtc);
+	cstate = to_dpu_crtc_state(state);
+
+	DPU_DEBUG("crtc%d\n", crtc->base.id);
+
+	_dpu_crtc_rp_destroy(&cstate->rp);
+
+	__drm_atomic_helper_crtc_destroy_state(state);
+
+	kfree(cstate);
+}
+
+static int _dpu_crtc_wait_for_frame_done(struct drm_crtc *crtc)
+{
+	struct dpu_crtc *dpu_crtc;
+	int ret, rc = 0;
+
+	if (!crtc) {
+		DPU_ERROR("invalid argument\n");
+		return -EINVAL;
+	}
+	dpu_crtc = to_dpu_crtc(crtc);
+
+	if (!atomic_read(&dpu_crtc->frame_pending)) {
+		DPU_DEBUG("no frames pending\n");
+		return 0;
+	}
+
+	DPU_ATRACE_BEGIN("frame done completion wait");
+	ret = wait_for_completion_timeout(&dpu_crtc->frame_done_comp,
+			msecs_to_jiffies(DPU_FRAME_DONE_TIMEOUT));
+	if (!ret) {
+		DRM_ERROR("frame done wait timed out, ret:%d\n", ret);
+		rc = -ETIMEDOUT;
+	}
+	DPU_ATRACE_END("frame done completion wait");
+
+	return rc;
+}
+
+void dpu_crtc_commit_kickoff(struct drm_crtc *crtc)
+{
+	struct drm_encoder *encoder;
+	struct drm_device *dev;
+	struct dpu_crtc *dpu_crtc;
+	struct msm_drm_private *priv;
+	struct dpu_kms *dpu_kms;
+	struct dpu_crtc_state *cstate;
+	int ret;
+
+	if (!crtc) {
+		DPU_ERROR("invalid argument\n");
+		return;
+	}
+	dev = crtc->dev;
+	dpu_crtc = to_dpu_crtc(crtc);
+	dpu_kms = _dpu_crtc_get_kms(crtc);
+
+	if (!dpu_kms || !dpu_kms->dev || !dpu_kms->dev->dev_private) {
+		DPU_ERROR("invalid argument\n");
+		return;
+	}
+
+	priv = dpu_kms->dev->dev_private;
+	cstate = to_dpu_crtc_state(crtc->state);
+
+	/*
+	 * If no mixers has been allocated in dpu_crtc_atomic_check(),
+	 * it means we are trying to start a CRTC whose state is disabled:
+	 * nothing else needs to be done.
+	 */
+	if (unlikely(!dpu_crtc->num_mixers))
+		return;
+
+	DPU_ATRACE_BEGIN("crtc_commit");
+
+	list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
+		struct dpu_encoder_kickoff_params params = { 0 };
+
+		if (encoder->crtc != crtc)
+			continue;
+
+		/*
+		 * Encoder will flush/start now, unless it has a tx pending.
+		 * If so, it may delay and flush at an irq event (e.g. ppdone)
+		 */
+		dpu_encoder_prepare_for_kickoff(encoder, &params);
+	}
+
+	/* wait for frame_event_done completion */
+	DPU_ATRACE_BEGIN("wait_for_frame_done_event");
+	ret = _dpu_crtc_wait_for_frame_done(crtc);
+	DPU_ATRACE_END("wait_for_frame_done_event");
+	if (ret) {
+		DPU_ERROR("crtc%d wait for frame done failed;frame_pending%d\n",
+				crtc->base.id,
+				atomic_read(&dpu_crtc->frame_pending));
+		goto end;
+	}
+
+	if (atomic_inc_return(&dpu_crtc->frame_pending) == 1) {
+		/* acquire bandwidth and other resources */
+		DPU_DEBUG("crtc%d first commit\n", crtc->base.id);
+	} else
+		DPU_DEBUG("crtc%d commit\n", crtc->base.id);
+
+	dpu_crtc->play_count++;
+
+	dpu_vbif_clear_errors(dpu_kms);
+
+	list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
+		if (encoder->crtc != crtc)
+			continue;
+
+		dpu_encoder_kickoff(encoder);
+	}
+
+end:
+	reinit_completion(&dpu_crtc->frame_done_comp);
+	DPU_ATRACE_END("crtc_commit");
+}
+
+/**
+ * _dpu_crtc_vblank_enable_no_lock - update power resource and vblank request
+ * @dpu_crtc: Pointer to dpu crtc structure
+ * @enable: Whether to enable/disable vblanks
+ *
+ * @Return: error code
+ */
+static int _dpu_crtc_vblank_enable_no_lock(
+		struct dpu_crtc *dpu_crtc, bool enable)
+{
+	struct drm_device *dev;
+	struct drm_crtc *crtc;
+	struct drm_encoder *enc;
+
+	if (!dpu_crtc) {
+		DPU_ERROR("invalid crtc\n");
+		return -EINVAL;
+	}
+
+	crtc = &dpu_crtc->base;
+	dev = crtc->dev;
+
+	if (enable) {
+		int ret;
+
+		/* drop lock since power crtc cb may try to re-acquire lock */
+		mutex_unlock(&dpu_crtc->crtc_lock);
+		ret = _dpu_crtc_power_enable(dpu_crtc, true);
+		mutex_lock(&dpu_crtc->crtc_lock);
+		if (ret)
+			return ret;
+
+		list_for_each_entry(enc, &dev->mode_config.encoder_list, head) {
+			if (enc->crtc != crtc)
+				continue;
+
+			trace_dpu_crtc_vblank_enable(DRMID(&dpu_crtc->base),
+						     DRMID(enc), enable,
+						     dpu_crtc);
+
+			dpu_encoder_register_vblank_callback(enc,
+					dpu_crtc_vblank_cb, (void *)crtc);
+		}
+	} else {
+		list_for_each_entry(enc, &dev->mode_config.encoder_list, head) {
+			if (enc->crtc != crtc)
+				continue;
+
+			trace_dpu_crtc_vblank_enable(DRMID(&dpu_crtc->base),
+						     DRMID(enc), enable,
+						     dpu_crtc);
+
+			dpu_encoder_register_vblank_callback(enc, NULL, NULL);
+		}
+
+		/* drop lock since power crtc cb may try to re-acquire lock */
+		mutex_unlock(&dpu_crtc->crtc_lock);
+		_dpu_crtc_power_enable(dpu_crtc, false);
+		mutex_lock(&dpu_crtc->crtc_lock);
+	}
+
+	return 0;
+}
+
+/**
+ * _dpu_crtc_set_suspend - notify crtc of suspend enable/disable
+ * @crtc: Pointer to drm crtc object
+ * @enable: true to enable suspend, false to indicate resume
+ */
+static void _dpu_crtc_set_suspend(struct drm_crtc *crtc, bool enable)
+{
+	struct dpu_crtc *dpu_crtc;
+	struct msm_drm_private *priv;
+	struct dpu_kms *dpu_kms;
+	int ret = 0;
+
+	if (!crtc || !crtc->dev || !crtc->dev->dev_private) {
+		DPU_ERROR("invalid crtc\n");
+		return;
+	}
+	dpu_crtc = to_dpu_crtc(crtc);
+	priv = crtc->dev->dev_private;
+
+	if (!priv->kms) {
+		DPU_ERROR("invalid crtc kms\n");
+		return;
+	}
+	dpu_kms = to_dpu_kms(priv->kms);
+
+	DRM_DEBUG_KMS("crtc%d suspend = %d\n", crtc->base.id, enable);
+
+	mutex_lock(&dpu_crtc->crtc_lock);
+
+	/*
+	 * If the vblank is enabled, release a power reference on suspend
+	 * and take it back during resume (if it is still enabled).
+	 */
+	trace_dpu_crtc_set_suspend(DRMID(&dpu_crtc->base), enable, dpu_crtc);
+	if (dpu_crtc->suspend == enable)
+		DPU_DEBUG("crtc%d suspend already set to %d, ignoring update\n",
+				crtc->base.id, enable);
+	else if (dpu_crtc->enabled && dpu_crtc->vblank_requested) {
+		ret = _dpu_crtc_vblank_enable_no_lock(dpu_crtc, !enable);
+		if (ret)
+			DPU_ERROR("%s vblank enable failed: %d\n",
+					dpu_crtc->name, ret);
+	}
+
+	dpu_crtc->suspend = enable;
+	mutex_unlock(&dpu_crtc->crtc_lock);
+}
+
+/**
+ * dpu_crtc_duplicate_state - state duplicate hook
+ * @crtc: Pointer to drm crtc structure
+ * @Returns: Pointer to new drm_crtc_state structure
+ */
+static struct drm_crtc_state *dpu_crtc_duplicate_state(struct drm_crtc *crtc)
+{
+	struct dpu_crtc *dpu_crtc;
+	struct dpu_crtc_state *cstate, *old_cstate;
+
+	if (!crtc || !crtc->state) {
+		DPU_ERROR("invalid argument(s)\n");
+		return NULL;
+	}
+
+	dpu_crtc = to_dpu_crtc(crtc);
+	old_cstate = to_dpu_crtc_state(crtc->state);
+	cstate = kmemdup(old_cstate, sizeof(*old_cstate), GFP_KERNEL);
+	if (!cstate) {
+		DPU_ERROR("failed to allocate state\n");
+		return NULL;
+	}
+
+	/* duplicate base helper */
+	__drm_atomic_helper_crtc_duplicate_state(crtc, &cstate->base);
+
+	_dpu_crtc_rp_duplicate(&old_cstate->rp, &cstate->rp);
+
+	return &cstate->base;
+}
+
+/**
+ * dpu_crtc_reset - reset hook for CRTCs
+ * Resets the atomic state for @crtc by freeing the state pointer (which might
+ * be NULL, e.g. at driver load time) and allocating a new empty state object.
+ * @crtc: Pointer to drm crtc structure
+ */
+static void dpu_crtc_reset(struct drm_crtc *crtc)
+{
+	struct dpu_crtc *dpu_crtc;
+	struct dpu_crtc_state *cstate;
+
+	if (!crtc) {
+		DPU_ERROR("invalid crtc\n");
+		return;
+	}
+
+	/* revert suspend actions, if necessary */
+	if (dpu_kms_is_suspend_state(crtc->dev))
+		_dpu_crtc_set_suspend(crtc, false);
+
+	/* remove previous state, if present */
+	if (crtc->state) {
+		dpu_crtc_destroy_state(crtc, crtc->state);
+		crtc->state = 0;
+	}
+
+	dpu_crtc = to_dpu_crtc(crtc);
+	cstate = kzalloc(sizeof(*cstate), GFP_KERNEL);
+	if (!cstate) {
+		DPU_ERROR("failed to allocate state\n");
+		return;
+	}
+
+	_dpu_crtc_rp_reset(&cstate->rp, &dpu_crtc->rp_lock,
+			&dpu_crtc->rp_head);
+
+	cstate->base.crtc = crtc;
+	crtc->state = &cstate->base;
+}
+
+static void dpu_crtc_handle_power_event(u32 event_type, void *arg)
+{
+	struct drm_crtc *crtc = arg;
+	struct dpu_crtc *dpu_crtc;
+	struct drm_encoder *encoder;
+	struct dpu_crtc_mixer *m;
+	u32 i, misr_status;
+
+	if (!crtc) {
+		DPU_ERROR("invalid crtc\n");
+		return;
+	}
+	dpu_crtc = to_dpu_crtc(crtc);
+
+	mutex_lock(&dpu_crtc->crtc_lock);
+
+	trace_dpu_crtc_handle_power_event(DRMID(crtc), event_type);
+
+	switch (event_type) {
+	case DPU_POWER_EVENT_POST_ENABLE:
+		/* restore encoder; crtc will be programmed during commit */
+		drm_for_each_encoder(encoder, crtc->dev) {
+			if (encoder->crtc != crtc)
+				continue;
+
+			dpu_encoder_virt_restore(encoder);
+		}
+
+		for (i = 0; i < dpu_crtc->num_mixers; ++i) {
+			m = &dpu_crtc->mixers[i];
+			if (!m->hw_lm || !m->hw_lm->ops.setup_misr ||
+					!dpu_crtc->misr_enable)
+				continue;
+
+			m->hw_lm->ops.setup_misr(m->hw_lm, true,
+					dpu_crtc->misr_frame_count);
+		}
+		break;
+	case DPU_POWER_EVENT_PRE_DISABLE:
+		for (i = 0; i < dpu_crtc->num_mixers; ++i) {
+			m = &dpu_crtc->mixers[i];
+			if (!m->hw_lm || !m->hw_lm->ops.collect_misr ||
+					!dpu_crtc->misr_enable)
+				continue;
+
+			misr_status = m->hw_lm->ops.collect_misr(m->hw_lm);
+			dpu_crtc->misr_data[i] = misr_status ? misr_status :
+							dpu_crtc->misr_data[i];
+		}
+		break;
+	case DPU_POWER_EVENT_POST_DISABLE:
+		/**
+		 * Nothing to do. All the planes on the CRTC will be
+		 * programmed for every frame
+		 */
+		break;
+	default:
+		DPU_DEBUG("event:%d not handled\n", event_type);
+		break;
+	}
+
+	mutex_unlock(&dpu_crtc->crtc_lock);
+}
+
+static void dpu_crtc_disable(struct drm_crtc *crtc)
+{
+	struct dpu_crtc *dpu_crtc;
+	struct dpu_crtc_state *cstate;
+	struct drm_display_mode *mode;
+	struct drm_encoder *encoder;
+	struct msm_drm_private *priv;
+	int ret;
+	unsigned long flags;
+
+	if (!crtc || !crtc->dev || !crtc->dev->dev_private || !crtc->state) {
+		DPU_ERROR("invalid crtc\n");
+		return;
+	}
+	dpu_crtc = to_dpu_crtc(crtc);
+	cstate = to_dpu_crtc_state(crtc->state);
+	mode = &cstate->base.adjusted_mode;
+	priv = crtc->dev->dev_private;
+
+	DRM_DEBUG_KMS("crtc%d\n", crtc->base.id);
+
+	if (dpu_kms_is_suspend_state(crtc->dev))
+		_dpu_crtc_set_suspend(crtc, true);
+
+	/* Disable/save vblank irq handling */
+	drm_crtc_vblank_off(crtc);
+
+	mutex_lock(&dpu_crtc->crtc_lock);
+
+	/* wait for frame_event_done completion */
+	if (_dpu_crtc_wait_for_frame_done(crtc))
+		DPU_ERROR("crtc%d wait for frame done failed;frame_pending%d\n",
+				crtc->base.id,
+				atomic_read(&dpu_crtc->frame_pending));
+
+	trace_dpu_crtc_disable(DRMID(crtc), false, dpu_crtc);
+	if (dpu_crtc->enabled && !dpu_crtc->suspend &&
+			dpu_crtc->vblank_requested) {
+		ret = _dpu_crtc_vblank_enable_no_lock(dpu_crtc, false);
+		if (ret)
+			DPU_ERROR("%s vblank enable failed: %d\n",
+					dpu_crtc->name, ret);
+	}
+	dpu_crtc->enabled = false;
+
+	if (atomic_read(&dpu_crtc->frame_pending)) {
+		trace_dpu_crtc_disable_frame_pending(DRMID(crtc),
+				     atomic_read(&dpu_crtc->frame_pending));
+		dpu_core_perf_crtc_release_bw(crtc);
+		atomic_set(&dpu_crtc->frame_pending, 0);
+	}
+
+	dpu_core_perf_crtc_update(crtc, 0, true);
+
+	drm_for_each_encoder(encoder, crtc->dev) {
+		if (encoder->crtc != crtc)
+			continue;
+		dpu_encoder_register_frame_event_callback(encoder, NULL, NULL);
+	}
+
+	if (dpu_crtc->power_event)
+		dpu_power_handle_unregister_event(dpu_crtc->phandle,
+				dpu_crtc->power_event);
+
+	memset(dpu_crtc->mixers, 0, sizeof(dpu_crtc->mixers));
+	dpu_crtc->num_mixers = 0;
+	dpu_crtc->mixers_swapped = false;
+
+	/* disable clk & bw control until clk & bw properties are set */
+	cstate->bw_control = false;
+	cstate->bw_split_vote = false;
+
+	mutex_unlock(&dpu_crtc->crtc_lock);
+
+	if (crtc->state->event && !crtc->state->active) {
+		spin_lock_irqsave(&crtc->dev->event_lock, flags);
+		drm_crtc_send_vblank_event(crtc, crtc->state->event);
+		crtc->state->event = NULL;
+		spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
+	}
+}
+
+static void dpu_crtc_enable(struct drm_crtc *crtc,
+		struct drm_crtc_state *old_crtc_state)
+{
+	struct dpu_crtc *dpu_crtc;
+	struct drm_encoder *encoder;
+	struct msm_drm_private *priv;
+	int ret;
+
+	if (!crtc || !crtc->dev || !crtc->dev->dev_private) {
+		DPU_ERROR("invalid crtc\n");
+		return;
+	}
+	priv = crtc->dev->dev_private;
+
+	DRM_DEBUG_KMS("crtc%d\n", crtc->base.id);
+	dpu_crtc = to_dpu_crtc(crtc);
+
+	drm_for_each_encoder(encoder, crtc->dev) {
+		if (encoder->crtc != crtc)
+			continue;
+		dpu_encoder_register_frame_event_callback(encoder,
+				dpu_crtc_frame_event_cb, (void *)crtc);
+	}
+
+	mutex_lock(&dpu_crtc->crtc_lock);
+	trace_dpu_crtc_enable(DRMID(crtc), true, dpu_crtc);
+	if (!dpu_crtc->enabled && !dpu_crtc->suspend &&
+			dpu_crtc->vblank_requested) {
+		ret = _dpu_crtc_vblank_enable_no_lock(dpu_crtc, true);
+		if (ret)
+			DPU_ERROR("%s vblank enable failed: %d\n",
+					dpu_crtc->name, ret);
+	}
+	dpu_crtc->enabled = true;
+
+	mutex_unlock(&dpu_crtc->crtc_lock);
+
+	/* Enable/restore vblank irq handling */
+	drm_crtc_vblank_on(crtc);
+
+	dpu_crtc->power_event = dpu_power_handle_register_event(
+		dpu_crtc->phandle,
+		DPU_POWER_EVENT_POST_ENABLE | DPU_POWER_EVENT_POST_DISABLE |
+		DPU_POWER_EVENT_PRE_DISABLE,
+		dpu_crtc_handle_power_event, crtc, dpu_crtc->name);
+
+}
+
+struct plane_state {
+	struct dpu_plane_state *dpu_pstate;
+	const struct drm_plane_state *drm_pstate;
+	int stage;
+	u32 pipe_id;
+};
+
+static int dpu_crtc_atomic_check(struct drm_crtc *crtc,
+		struct drm_crtc_state *state)
+{
+	struct dpu_crtc *dpu_crtc;
+	struct plane_state *pstates;
+	struct dpu_crtc_state *cstate;
+
+	const struct drm_plane_state *pstate;
+	struct drm_plane *plane;
+	struct drm_display_mode *mode;
+
+	int cnt = 0, rc = 0, mixer_width, i, z_pos;
+
+	struct dpu_multirect_plane_states multirect_plane[DPU_STAGE_MAX * 2];
+	int multirect_count = 0;
+	const struct drm_plane_state *pipe_staged[SSPP_MAX];
+	int left_zpos_cnt = 0, right_zpos_cnt = 0;
+	struct drm_rect crtc_rect = { 0 };
+
+	if (!crtc) {
+		DPU_ERROR("invalid crtc\n");
+		return -EINVAL;
+	}
+
+	pstates = kzalloc(sizeof(*pstates) * DPU_STAGE_MAX * 4, GFP_KERNEL);
+
+	dpu_crtc = to_dpu_crtc(crtc);
+	cstate = to_dpu_crtc_state(state);
+
+	if (!state->enable || !state->active) {
+		DPU_DEBUG("crtc%d -> enable %d, active %d, skip atomic_check\n",
+				crtc->base.id, state->enable, state->active);
+		goto end;
+	}
+
+	mode = &state->adjusted_mode;
+	DPU_DEBUG("%s: check", dpu_crtc->name);
+
+	/* force a full mode set if active state changed */
+	if (state->active_changed)
+		state->mode_changed = true;
+
+	memset(pipe_staged, 0, sizeof(pipe_staged));
+
+	mixer_width = dpu_crtc_get_mixer_width(dpu_crtc, cstate, mode);
+
+	_dpu_crtc_setup_lm_bounds(crtc, state);
+
+	crtc_rect.x2 = mode->hdisplay;
+	crtc_rect.y2 = mode->vdisplay;
+
+	 /* get plane state for all drm planes associated with crtc state */
+	drm_atomic_crtc_state_for_each_plane_state(plane, pstate, state) {
+		struct drm_rect dst, clip = crtc_rect;
+
+		if (IS_ERR_OR_NULL(pstate)) {
+			rc = PTR_ERR(pstate);
+			DPU_ERROR("%s: failed to get plane%d state, %d\n",
+					dpu_crtc->name, plane->base.id, rc);
+			goto end;
+		}
+		if (cnt >= DPU_STAGE_MAX * 4)
+			continue;
+
+		pstates[cnt].dpu_pstate = to_dpu_plane_state(pstate);
+		pstates[cnt].drm_pstate = pstate;
+		pstates[cnt].stage = pstate->normalized_zpos;
+		pstates[cnt].pipe_id = dpu_plane_pipe(plane);
+
+		if (pipe_staged[pstates[cnt].pipe_id]) {
+			multirect_plane[multirect_count].r0 =
+				pipe_staged[pstates[cnt].pipe_id];
+			multirect_plane[multirect_count].r1 = pstate;
+			multirect_count++;
+
+			pipe_staged[pstates[cnt].pipe_id] = NULL;
+		} else {
+			pipe_staged[pstates[cnt].pipe_id] = pstate;
+		}
+
+		cnt++;
+
+		dst = drm_plane_state_dest(pstate);
+		if (!drm_rect_intersect(&clip, &dst) ||
+		    !drm_rect_equals(&clip, &dst)) {
+			DPU_ERROR("invalid vertical/horizontal destination\n");
+			DPU_ERROR("display: " DRM_RECT_FMT " plane: "
+				  DRM_RECT_FMT "\n", DRM_RECT_ARG(&crtc_rect),
+				  DRM_RECT_ARG(&dst));
+			rc = -E2BIG;
+			goto end;
+		}
+	}
+
+	for (i = 1; i < SSPP_MAX; i++) {
+		if (pipe_staged[i]) {
+			dpu_plane_clear_multirect(pipe_staged[i]);
+
+			if (is_dpu_plane_virtual(pipe_staged[i]->plane)) {
+				DPU_ERROR(
+					"r1 only virt plane:%d not supported\n",
+					pipe_staged[i]->plane->base.id);
+				rc  = -EINVAL;
+				goto end;
+			}
+		}
+	}
+
+	z_pos = -1;
+	for (i = 0; i < cnt; i++) {
+		/* reset counts at every new blend stage */
+		if (pstates[i].stage != z_pos) {
+			left_zpos_cnt = 0;
+			right_zpos_cnt = 0;
+			z_pos = pstates[i].stage;
+		}
+
+		/* verify z_pos setting before using it */
+		if (z_pos >= DPU_STAGE_MAX - DPU_STAGE_0) {
+			DPU_ERROR("> %d plane stages assigned\n",
+					DPU_STAGE_MAX - DPU_STAGE_0);
+			rc = -EINVAL;
+			goto end;
+		} else if (pstates[i].drm_pstate->crtc_x < mixer_width) {
+			if (left_zpos_cnt == 2) {
+				DPU_ERROR("> 2 planes @ stage %d on left\n",
+					z_pos);
+				rc = -EINVAL;
+				goto end;
+			}
+			left_zpos_cnt++;
+
+		} else {
+			if (right_zpos_cnt == 2) {
+				DPU_ERROR("> 2 planes @ stage %d on right\n",
+					z_pos);
+				rc = -EINVAL;
+				goto end;
+			}
+			right_zpos_cnt++;
+		}
+
+		pstates[i].dpu_pstate->stage = z_pos + DPU_STAGE_0;
+		DPU_DEBUG("%s: zpos %d", dpu_crtc->name, z_pos);
+	}
+
+	for (i = 0; i < multirect_count; i++) {
+		if (dpu_plane_validate_multirect_v2(&multirect_plane[i])) {
+			DPU_ERROR(
+			"multirect validation failed for planes (%d - %d)\n",
+					multirect_plane[i].r0->plane->base.id,
+					multirect_plane[i].r1->plane->base.id);
+			rc = -EINVAL;
+			goto end;
+		}
+	}
+
+	rc = dpu_core_perf_crtc_check(crtc, state);
+	if (rc) {
+		DPU_ERROR("crtc%d failed performance check %d\n",
+				crtc->base.id, rc);
+		goto end;
+	}
+
+	/* validate source split:
+	 * use pstates sorted by stage to check planes on same stage
+	 * we assume that all pipes are in source split so its valid to compare
+	 * without taking into account left/right mixer placement
+	 */
+	for (i = 1; i < cnt; i++) {
+		struct plane_state *prv_pstate, *cur_pstate;
+		struct drm_rect left_rect, right_rect;
+		int32_t left_pid, right_pid;
+		int32_t stage;
+
+		prv_pstate = &pstates[i - 1];
+		cur_pstate = &pstates[i];
+		if (prv_pstate->stage != cur_pstate->stage)
+			continue;
+
+		stage = cur_pstate->stage;
+
+		left_pid = prv_pstate->dpu_pstate->base.plane->base.id;
+		left_rect = drm_plane_state_dest(prv_pstate->drm_pstate);
+
+		right_pid = cur_pstate->dpu_pstate->base.plane->base.id;
+		right_rect = drm_plane_state_dest(cur_pstate->drm_pstate);
+
+		if (right_rect.x1 < left_rect.x1) {
+			swap(left_pid, right_pid);
+			swap(left_rect, right_rect);
+		}
+
+		/**
+		 * - planes are enumerated in pipe-priority order such that
+		 *   planes with lower drm_id must be left-most in a shared
+		 *   blend-stage when using source split.
+		 * - planes in source split must be contiguous in width
+		 * - planes in source split must have same dest yoff and height
+		 */
+		if (right_pid < left_pid) {
+			DPU_ERROR(
+				"invalid src split cfg. priority mismatch. stage: %d left: %d right: %d\n",
+				stage, left_pid, right_pid);
+			rc = -EINVAL;
+			goto end;
+		} else if (right_rect.x1 != drm_rect_width(&left_rect)) {
+			DPU_ERROR("non-contiguous coordinates for src split. "
+				  "stage: %d left: " DRM_RECT_FMT " right: "
+				  DRM_RECT_FMT "\n", stage,
+				  DRM_RECT_ARG(&left_rect),
+				  DRM_RECT_ARG(&right_rect));
+			rc = -EINVAL;
+			goto end;
+		} else if (left_rect.y1 != right_rect.y1 ||
+			   drm_rect_height(&left_rect) != drm_rect_height(&right_rect)) {
+			DPU_ERROR("source split at stage: %d. invalid "
+				  "yoff/height: left: " DRM_RECT_FMT " right: "
+				  DRM_RECT_FMT "\n", stage,
+				  DRM_RECT_ARG(&left_rect),
+				  DRM_RECT_ARG(&right_rect));
+			rc = -EINVAL;
+			goto end;
+		}
+	}
+
+end:
+	_dpu_crtc_rp_free_unused(&cstate->rp);
+	kfree(pstates);
+	return rc;
+}
+
+int dpu_crtc_vblank(struct drm_crtc *crtc, bool en)
+{
+	struct dpu_crtc *dpu_crtc;
+	int ret;
+
+	if (!crtc) {
+		DPU_ERROR("invalid crtc\n");
+		return -EINVAL;
+	}
+	dpu_crtc = to_dpu_crtc(crtc);
+
+	mutex_lock(&dpu_crtc->crtc_lock);
+	trace_dpu_crtc_vblank(DRMID(&dpu_crtc->base), en, dpu_crtc);
+	if (dpu_crtc->enabled && !dpu_crtc->suspend) {
+		ret = _dpu_crtc_vblank_enable_no_lock(dpu_crtc, en);
+		if (ret)
+			DPU_ERROR("%s vblank enable failed: %d\n",
+					dpu_crtc->name, ret);
+	}
+	dpu_crtc->vblank_requested = en;
+	mutex_unlock(&dpu_crtc->crtc_lock);
+
+	return 0;
+}
+
+#ifdef CONFIG_DEBUG_FS
+static int _dpu_debugfs_status_show(struct seq_file *s, void *data)
+{
+	struct dpu_crtc *dpu_crtc;
+	struct dpu_plane_state *pstate = NULL;
+	struct dpu_crtc_mixer *m;
+
+	struct drm_crtc *crtc;
+	struct drm_plane *plane;
+	struct drm_display_mode *mode;
+	struct drm_framebuffer *fb;
+	struct drm_plane_state *state;
+	struct dpu_crtc_state *cstate;
+
+	int i, out_width;
+
+	if (!s || !s->private)
+		return -EINVAL;
+
+	dpu_crtc = s->private;
+	crtc = &dpu_crtc->base;
+	cstate = to_dpu_crtc_state(crtc->state);
+
+	mutex_lock(&dpu_crtc->crtc_lock);
+	mode = &crtc->state->adjusted_mode;
+	out_width = dpu_crtc_get_mixer_width(dpu_crtc, cstate, mode);
+
+	seq_printf(s, "crtc:%d width:%d height:%d\n", crtc->base.id,
+				mode->hdisplay, mode->vdisplay);
+
+	seq_puts(s, "\n");
+
+	for (i = 0; i < dpu_crtc->num_mixers; ++i) {
+		m = &dpu_crtc->mixers[i];
+		if (!m->hw_lm)
+			seq_printf(s, "\tmixer[%d] has no lm\n", i);
+		else if (!m->hw_ctl)
+			seq_printf(s, "\tmixer[%d] has no ctl\n", i);
+		else
+			seq_printf(s, "\tmixer:%d ctl:%d width:%d height:%d\n",
+				m->hw_lm->idx - LM_0, m->hw_ctl->idx - CTL_0,
+				out_width, mode->vdisplay);
+	}
+
+	seq_puts(s, "\n");
+
+	drm_atomic_crtc_for_each_plane(plane, crtc) {
+		pstate = to_dpu_plane_state(plane->state);
+		state = plane->state;
+
+		if (!pstate || !state)
+			continue;
+
+		seq_printf(s, "\tplane:%u stage:%d\n", plane->base.id,
+			pstate->stage);
+
+		if (plane->state->fb) {
+			fb = plane->state->fb;
+
+			seq_printf(s, "\tfb:%d image format:%4.4s wxh:%ux%u ",
+				fb->base.id, (char *) &fb->format->format,
+				fb->width, fb->height);
+			for (i = 0; i < ARRAY_SIZE(fb->format->cpp); ++i)
+				seq_printf(s, "cpp[%d]:%u ",
+						i, fb->format->cpp[i]);
+			seq_puts(s, "\n\t");
+
+			seq_printf(s, "modifier:%8llu ", fb->modifier);
+			seq_puts(s, "\n");
+
+			seq_puts(s, "\t");
+			for (i = 0; i < ARRAY_SIZE(fb->pitches); i++)
+				seq_printf(s, "pitches[%d]:%8u ", i,
+							fb->pitches[i]);
+			seq_puts(s, "\n");
+
+			seq_puts(s, "\t");
+			for (i = 0; i < ARRAY_SIZE(fb->offsets); i++)
+				seq_printf(s, "offsets[%d]:%8u ", i,
+							fb->offsets[i]);
+			seq_puts(s, "\n");
+		}
+
+		seq_printf(s, "\tsrc_x:%4d src_y:%4d src_w:%4d src_h:%4d\n",
+			state->src_x, state->src_y, state->src_w, state->src_h);
+
+		seq_printf(s, "\tdst x:%4d dst_y:%4d dst_w:%4d dst_h:%4d\n",
+			state->crtc_x, state->crtc_y, state->crtc_w,
+			state->crtc_h);
+		seq_printf(s, "\tmultirect: mode: %d index: %d\n",
+			pstate->multirect_mode, pstate->multirect_index);
+
+		seq_puts(s, "\n");
+	}
+	if (dpu_crtc->vblank_cb_count) {
+		ktime_t diff = ktime_sub(ktime_get(), dpu_crtc->vblank_cb_time);
+		s64 diff_ms = ktime_to_ms(diff);
+		s64 fps = diff_ms ? div_s64(
+				dpu_crtc->vblank_cb_count * 1000, diff_ms) : 0;
+
+		seq_printf(s,
+			"vblank fps:%lld count:%u total:%llums total_framecount:%llu\n",
+				fps, dpu_crtc->vblank_cb_count,
+				ktime_to_ms(diff), dpu_crtc->play_count);
+
+		/* reset time & count for next measurement */
+		dpu_crtc->vblank_cb_count = 0;
+		dpu_crtc->vblank_cb_time = ktime_set(0, 0);
+	}
+
+	seq_printf(s, "vblank_enable:%d\n", dpu_crtc->vblank_requested);
+
+	mutex_unlock(&dpu_crtc->crtc_lock);
+
+	return 0;
+}
+
+static int _dpu_debugfs_status_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, _dpu_debugfs_status_show, inode->i_private);
+}
+
+static ssize_t _dpu_crtc_misr_setup(struct file *file,
+		const char __user *user_buf, size_t count, loff_t *ppos)
+{
+	struct dpu_crtc *dpu_crtc;
+	struct dpu_crtc_mixer *m;
+	int i = 0, rc;
+	char buf[MISR_BUFF_SIZE + 1];
+	u32 frame_count, enable;
+	size_t buff_copy;
+
+	if (!file || !file->private_data)
+		return -EINVAL;
+
+	dpu_crtc = file->private_data;
+	buff_copy = min_t(size_t, count, MISR_BUFF_SIZE);
+	if (copy_from_user(buf, user_buf, buff_copy)) {
+		DPU_ERROR("buffer copy failed\n");
+		return -EINVAL;
+	}
+
+	buf[buff_copy] = 0; /* end of string */
+
+	if (sscanf(buf, "%u %u", &enable, &frame_count) != 2)
+		return -EINVAL;
+
+	rc = _dpu_crtc_power_enable(dpu_crtc, true);
+	if (rc)
+		return rc;
+
+	mutex_lock(&dpu_crtc->crtc_lock);
+	dpu_crtc->misr_enable = enable;
+	dpu_crtc->misr_frame_count = frame_count;
+	for (i = 0; i < dpu_crtc->num_mixers; ++i) {
+		dpu_crtc->misr_data[i] = 0;
+		m = &dpu_crtc->mixers[i];
+		if (!m->hw_lm || !m->hw_lm->ops.setup_misr)
+			continue;
+
+		m->hw_lm->ops.setup_misr(m->hw_lm, enable, frame_count);
+	}
+	mutex_unlock(&dpu_crtc->crtc_lock);
+	_dpu_crtc_power_enable(dpu_crtc, false);
+
+	return count;
+}
+
+static ssize_t _dpu_crtc_misr_read(struct file *file,
+		char __user *user_buff, size_t count, loff_t *ppos)
+{
+	struct dpu_crtc *dpu_crtc;
+	struct dpu_crtc_mixer *m;
+	int i = 0, rc;
+	u32 misr_status;
+	ssize_t len = 0;
+	char buf[MISR_BUFF_SIZE + 1] = {'\0'};
+
+	if (*ppos)
+		return 0;
+
+	if (!file || !file->private_data)
+		return -EINVAL;
+
+	dpu_crtc = file->private_data;
+	rc = _dpu_crtc_power_enable(dpu_crtc, true);
+	if (rc)
+		return rc;
+
+	mutex_lock(&dpu_crtc->crtc_lock);
+	if (!dpu_crtc->misr_enable) {
+		len += snprintf(buf + len, MISR_BUFF_SIZE - len,
+			"disabled\n");
+		goto buff_check;
+	}
+
+	for (i = 0; i < dpu_crtc->num_mixers; ++i) {
+		m = &dpu_crtc->mixers[i];
+		if (!m->hw_lm || !m->hw_lm->ops.collect_misr)
+			continue;
+
+		misr_status = m->hw_lm->ops.collect_misr(m->hw_lm);
+		dpu_crtc->misr_data[i] = misr_status ? misr_status :
+							dpu_crtc->misr_data[i];
+		len += snprintf(buf + len, MISR_BUFF_SIZE - len, "lm idx:%d\n",
+					m->hw_lm->idx - LM_0);
+		len += snprintf(buf + len, MISR_BUFF_SIZE - len, "0x%x\n",
+							dpu_crtc->misr_data[i]);
+	}
+
+buff_check:
+	if (count <= len) {
+		len = 0;
+		goto end;
+	}
+
+	if (copy_to_user(user_buff, buf, len)) {
+		len = -EFAULT;
+		goto end;
+	}
+
+	*ppos += len;   /* increase offset */
+
+end:
+	mutex_unlock(&dpu_crtc->crtc_lock);
+	_dpu_crtc_power_enable(dpu_crtc, false);
+	return len;
+}
+
+#define DEFINE_DPU_DEBUGFS_SEQ_FOPS(__prefix)                          \
+static int __prefix ## _open(struct inode *inode, struct file *file)	\
+{									\
+	return single_open(file, __prefix ## _show, inode->i_private);	\
+}									\
+static const struct file_operations __prefix ## _fops = {		\
+	.owner = THIS_MODULE,						\
+	.open = __prefix ## _open,					\
+	.release = single_release,					\
+	.read = seq_read,						\
+	.llseek = seq_lseek,						\
+}
+
+static int dpu_crtc_debugfs_state_show(struct seq_file *s, void *v)
+{
+	struct drm_crtc *crtc = (struct drm_crtc *) s->private;
+	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
+	struct dpu_crtc_res *res;
+	struct dpu_crtc_respool *rp;
+	int i;
+
+	seq_printf(s, "client type: %d\n", dpu_crtc_get_client_type(crtc));
+	seq_printf(s, "intf_mode: %d\n", dpu_crtc_get_intf_mode(crtc));
+	seq_printf(s, "core_clk_rate: %llu\n",
+			dpu_crtc->cur_perf.core_clk_rate);
+	for (i = DPU_POWER_HANDLE_DBUS_ID_MNOC;
+			i < DPU_POWER_HANDLE_DBUS_ID_MAX; i++) {
+		seq_printf(s, "bw_ctl[%s]: %llu\n",
+				dpu_power_handle_get_dbus_name(i),
+				dpu_crtc->cur_perf.bw_ctl[i]);
+		seq_printf(s, "max_per_pipe_ib[%s]: %llu\n",
+				dpu_power_handle_get_dbus_name(i),
+				dpu_crtc->cur_perf.max_per_pipe_ib[i]);
+	}
+
+	mutex_lock(&dpu_crtc->rp_lock);
+	list_for_each_entry(rp, &dpu_crtc->rp_head, rp_list) {
+		seq_printf(s, "rp.%d: ", rp->sequence_id);
+		list_for_each_entry(res, &rp->res_list, list)
+			seq_printf(s, "0x%x/0x%llx/%pK/%d ",
+					res->type, res->tag, res->val,
+					atomic_read(&res->refcount));
+		seq_puts(s, "\n");
+	}
+	mutex_unlock(&dpu_crtc->rp_lock);
+
+	return 0;
+}
+DEFINE_DPU_DEBUGFS_SEQ_FOPS(dpu_crtc_debugfs_state);
+
+static int _dpu_crtc_init_debugfs(struct drm_crtc *crtc)
+{
+	struct dpu_crtc *dpu_crtc;
+	struct dpu_kms *dpu_kms;
+
+	static const struct file_operations debugfs_status_fops = {
+		.open =		_dpu_debugfs_status_open,
+		.read =		seq_read,
+		.llseek =	seq_lseek,
+		.release =	single_release,
+	};
+	static const struct file_operations debugfs_misr_fops = {
+		.open =		simple_open,
+		.read =		_dpu_crtc_misr_read,
+		.write =	_dpu_crtc_misr_setup,
+	};
+
+	if (!crtc)
+		return -EINVAL;
+	dpu_crtc = to_dpu_crtc(crtc);
+
+	dpu_kms = _dpu_crtc_get_kms(crtc);
+	if (!dpu_kms)
+		return -EINVAL;
+
+	dpu_crtc->debugfs_root = debugfs_create_dir(dpu_crtc->name,
+			crtc->dev->primary->debugfs_root);
+	if (!dpu_crtc->debugfs_root)
+		return -ENOMEM;
+
+	/* don't error check these */
+	debugfs_create_file("status", 0400,
+			dpu_crtc->debugfs_root,
+			dpu_crtc, &debugfs_status_fops);
+	debugfs_create_file("state", 0600,
+			dpu_crtc->debugfs_root,
+			&dpu_crtc->base,
+			&dpu_crtc_debugfs_state_fops);
+	debugfs_create_file("misr_data", 0600, dpu_crtc->debugfs_root,
+					dpu_crtc, &debugfs_misr_fops);
+
+	return 0;
+}
+
+static void _dpu_crtc_destroy_debugfs(struct drm_crtc *crtc)
+{
+	struct dpu_crtc *dpu_crtc;
+
+	if (!crtc)
+		return;
+	dpu_crtc = to_dpu_crtc(crtc);
+	debugfs_remove_recursive(dpu_crtc->debugfs_root);
+}
+#else
+static int _dpu_crtc_init_debugfs(struct drm_crtc *crtc)
+{
+	return 0;
+}
+
+static void _dpu_crtc_destroy_debugfs(struct drm_crtc *crtc)
+{
+}
+#endif /* CONFIG_DEBUG_FS */
+
+static int dpu_crtc_late_register(struct drm_crtc *crtc)
+{
+	return _dpu_crtc_init_debugfs(crtc);
+}
+
+static void dpu_crtc_early_unregister(struct drm_crtc *crtc)
+{
+	_dpu_crtc_destroy_debugfs(crtc);
+}
+
+static const struct drm_crtc_funcs dpu_crtc_funcs = {
+	.set_config = drm_atomic_helper_set_config,
+	.destroy = dpu_crtc_destroy,
+	.page_flip = drm_atomic_helper_page_flip,
+	.reset = dpu_crtc_reset,
+	.atomic_duplicate_state = dpu_crtc_duplicate_state,
+	.atomic_destroy_state = dpu_crtc_destroy_state,
+	.late_register = dpu_crtc_late_register,
+	.early_unregister = dpu_crtc_early_unregister,
+};
+
+static const struct drm_crtc_helper_funcs dpu_crtc_helper_funcs = {
+	.disable = dpu_crtc_disable,
+	.atomic_enable = dpu_crtc_enable,
+	.atomic_check = dpu_crtc_atomic_check,
+	.atomic_begin = dpu_crtc_atomic_begin,
+	.atomic_flush = dpu_crtc_atomic_flush,
+};
+
+/* initialize crtc */
+struct drm_crtc *dpu_crtc_init(struct drm_device *dev, struct drm_plane *plane)
+{
+	struct drm_crtc *crtc = NULL;
+	struct dpu_crtc *dpu_crtc = NULL;
+	struct msm_drm_private *priv = NULL;
+	struct dpu_kms *kms = NULL;
+	int i;
+
+	priv = dev->dev_private;
+	kms = to_dpu_kms(priv->kms);
+
+	dpu_crtc = kzalloc(sizeof(*dpu_crtc), GFP_KERNEL);
+	if (!dpu_crtc)
+		return ERR_PTR(-ENOMEM);
+
+	crtc = &dpu_crtc->base;
+	crtc->dev = dev;
+
+	mutex_init(&dpu_crtc->crtc_lock);
+	spin_lock_init(&dpu_crtc->spin_lock);
+	atomic_set(&dpu_crtc->frame_pending, 0);
+
+	mutex_init(&dpu_crtc->rp_lock);
+	INIT_LIST_HEAD(&dpu_crtc->rp_head);
+
+	init_completion(&dpu_crtc->frame_done_comp);
+
+	INIT_LIST_HEAD(&dpu_crtc->frame_event_list);
+
+	for (i = 0; i < ARRAY_SIZE(dpu_crtc->frame_events); i++) {
+		INIT_LIST_HEAD(&dpu_crtc->frame_events[i].list);
+		list_add(&dpu_crtc->frame_events[i].list,
+				&dpu_crtc->frame_event_list);
+		kthread_init_work(&dpu_crtc->frame_events[i].work,
+				dpu_crtc_frame_event_work);
+	}
+
+	drm_crtc_init_with_planes(dev, crtc, plane, NULL, &dpu_crtc_funcs,
+				NULL);
+
+	drm_crtc_helper_add(crtc, &dpu_crtc_helper_funcs);
+	plane->crtc = crtc;
+
+	/* save user friendly CRTC name for later */
+	snprintf(dpu_crtc->name, DPU_CRTC_NAME_SIZE, "crtc%u", crtc->base.id);
+
+	/* initialize event handling */
+	spin_lock_init(&dpu_crtc->event_lock);
+
+	dpu_crtc->phandle = &kms->phandle;
+
+	DPU_DEBUG("%s: successfully initialized crtc\n", dpu_crtc->name);
+	return crtc;
+}
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.h
new file mode 100644
index 000000000000..e87109e608e9
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.h
@@ -0,0 +1,423 @@
+/*
+ * Copyright (c) 2015-2018 The Linux Foundation. All rights reserved.
+ * Copyright (C) 2013 Red Hat
+ * Author: Rob Clark <robdclark@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _DPU_CRTC_H_
+#define _DPU_CRTC_H_
+
+#include <linux/kthread.h>
+#include <drm/drm_crtc.h>
+#include "dpu_kms.h"
+#include "dpu_core_perf.h"
+#include "dpu_hw_blk.h"
+
+#define DPU_CRTC_NAME_SIZE	12
+
+/* define the maximum number of in-flight frame events */
+#define DPU_CRTC_FRAME_EVENT_SIZE	4
+
+/**
+ * enum dpu_crtc_client_type: crtc client type
+ * @RT_CLIENT:	RealTime client like video/cmd mode display
+ *              voting through apps rsc
+ * @NRT_CLIENT:	Non-RealTime client like WB display
+ *              voting through apps rsc
+ */
+enum dpu_crtc_client_type {
+	RT_CLIENT,
+	NRT_CLIENT,
+};
+
+/**
+ * enum dpu_crtc_smmu_state:	smmu state
+ * @ATTACHED:	 all the context banks are attached.
+ * @DETACHED:	 all the context banks are detached.
+ * @ATTACH_ALL_REQ:	 transient state of attaching context banks.
+ * @DETACH_ALL_REQ:	 transient state of detaching context banks.
+ */
+enum dpu_crtc_smmu_state {
+	ATTACHED = 0,
+	DETACHED,
+	ATTACH_ALL_REQ,
+	DETACH_ALL_REQ,
+};
+
+/**
+ * enum dpu_crtc_smmu_state_transition_type: state transition type
+ * @NONE: no pending state transitions
+ * @PRE_COMMIT: state transitions should be done before processing the commit
+ * @POST_COMMIT: state transitions to be done after processing the commit.
+ */
+enum dpu_crtc_smmu_state_transition_type {
+	NONE,
+	PRE_COMMIT,
+	POST_COMMIT
+};
+
+/**
+ * struct dpu_crtc_smmu_state_data: stores the smmu state and transition type
+ * @state: current state of smmu context banks
+ * @transition_type: transition request type
+ * @transition_error: whether there is error while transitioning the state
+ */
+struct dpu_crtc_smmu_state_data {
+	uint32_t state;
+	uint32_t transition_type;
+	uint32_t transition_error;
+};
+
+/**
+ * struct dpu_crtc_mixer: stores the map for each virtual pipeline in the CRTC
+ * @hw_lm:	LM HW Driver context
+ * @hw_ctl:	CTL Path HW driver context
+ * @encoder:	Encoder attached to this lm & ctl
+ * @mixer_op_mode:	mixer blending operation mode
+ * @flush_mask:	mixer flush mask for ctl, mixer and pipe
+ */
+struct dpu_crtc_mixer {
+	struct dpu_hw_mixer *hw_lm;
+	struct dpu_hw_ctl *hw_ctl;
+	struct drm_encoder *encoder;
+	u32 mixer_op_mode;
+	u32 flush_mask;
+};
+
+/**
+ * struct dpu_crtc_frame_event: stores crtc frame event for crtc processing
+ * @work:	base work structure
+ * @crtc:	Pointer to crtc handling this event
+ * @list:	event list
+ * @ts:		timestamp at queue entry
+ * @event:	event identifier
+ */
+struct dpu_crtc_frame_event {
+	struct kthread_work work;
+	struct drm_crtc *crtc;
+	struct list_head list;
+	ktime_t ts;
+	u32 event;
+};
+
+/*
+ * Maximum number of free event structures to cache
+ */
+#define DPU_CRTC_MAX_EVENT_COUNT	16
+
+/**
+ * struct dpu_crtc - virtualized CRTC data structure
+ * @base          : Base drm crtc structure
+ * @name          : ASCII description of this crtc
+ * @num_ctls      : Number of ctl paths in use
+ * @num_mixers    : Number of mixers in use
+ * @mixers_swapped: Whether the mixers have been swapped for left/right update
+ *                  especially in the case of DSC Merge.
+ * @mixers        : List of active mixers
+ * @event         : Pointer to last received drm vblank event. If there is a
+ *                  pending vblank event, this will be non-null.
+ * @vsync_count   : Running count of received vsync events
+ * @drm_requested_vblank : Whether vblanks have been enabled in the encoder
+ * @property_info : Opaque structure for generic property support
+ * @property_defaults : Array of default values for generic property support
+ * @stage_cfg     : H/w mixer stage configuration
+ * @debugfs_root  : Parent of debugfs node
+ * @vblank_cb_count : count of vblank callback since last reset
+ * @play_count    : frame count between crtc enable and disable
+ * @vblank_cb_time  : ktime at vblank count reset
+ * @vblank_requested : whether the user has requested vblank events
+ * @suspend         : whether or not a suspend operation is in progress
+ * @enabled       : whether the DPU CRTC is currently enabled. updated in the
+ *                  commit-thread, not state-swap time which is earlier, so
+ *                  safe to make decisions on during VBLANK on/off work
+ * @feature_list  : list of color processing features supported on a crtc
+ * @active_list   : list of color processing features are active
+ * @dirty_list    : list of color processing features are dirty
+ * @ad_dirty: list containing ad properties that are dirty
+ * @ad_active: list containing ad properties that are active
+ * @crtc_lock     : crtc lock around create, destroy and access.
+ * @frame_pending : Whether or not an update is pending
+ * @frame_events  : static allocation of in-flight frame events
+ * @frame_event_list : available frame event list
+ * @spin_lock     : spin lock for frame event, transaction status, etc...
+ * @frame_done_comp    : for frame_event_done synchronization
+ * @event_thread  : Pointer to event handler thread
+ * @event_worker  : Event worker queue
+ * @event_lock    : Spinlock around event handling code
+ * @misr_enable   : boolean entry indicates misr enable/disable status.
+ * @misr_frame_count  : misr frame count provided by client
+ * @misr_data     : store misr data before turning off the clocks.
+ * @phandle: Pointer to power handler
+ * @power_event   : registered power event handle
+ * @cur_perf      : current performance committed to clock/bandwidth driver
+ * @rp_lock       : serialization lock for resource pool
+ * @rp_head       : list of active resource pool
+ * @scl3_cfg_lut  : qseed3 lut config
+ */
+struct dpu_crtc {
+	struct drm_crtc base;
+	char name[DPU_CRTC_NAME_SIZE];
+
+	/* HW Resources reserved for the crtc */
+	u32 num_ctls;
+	u32 num_mixers;
+	bool mixers_swapped;
+	struct dpu_crtc_mixer mixers[CRTC_DUAL_MIXERS];
+	struct dpu_hw_scaler3_lut_cfg *scl3_lut_cfg;
+
+	struct drm_pending_vblank_event *event;
+	u32 vsync_count;
+
+	struct dpu_hw_stage_cfg stage_cfg;
+	struct dentry *debugfs_root;
+
+	u32 vblank_cb_count;
+	u64 play_count;
+	ktime_t vblank_cb_time;
+	bool vblank_requested;
+	bool suspend;
+	bool enabled;
+
+	struct list_head feature_list;
+	struct list_head active_list;
+	struct list_head dirty_list;
+	struct list_head ad_dirty;
+	struct list_head ad_active;
+
+	struct mutex crtc_lock;
+
+	atomic_t frame_pending;
+	struct dpu_crtc_frame_event frame_events[DPU_CRTC_FRAME_EVENT_SIZE];
+	struct list_head frame_event_list;
+	spinlock_t spin_lock;
+	struct completion frame_done_comp;
+
+	/* for handling internal event thread */
+	spinlock_t event_lock;
+	bool misr_enable;
+	u32 misr_frame_count;
+	u32 misr_data[CRTC_DUAL_MIXERS];
+
+	struct dpu_power_handle *phandle;
+	struct dpu_power_event *power_event;
+
+	struct dpu_core_perf_params cur_perf;
+
+	struct mutex rp_lock;
+	struct list_head rp_head;
+
+	struct dpu_crtc_smmu_state_data smmu_state;
+};
+
+#define to_dpu_crtc(x) container_of(x, struct dpu_crtc, base)
+
+/**
+ * struct dpu_crtc_res_ops - common operations for crtc resources
+ * @get: get given resource
+ * @put: put given resource
+ */
+struct dpu_crtc_res_ops {
+	void *(*get)(void *val, u32 type, u64 tag);
+	void (*put)(void *val);
+};
+
+#define DPU_CRTC_RES_FLAG_FREE		BIT(0)
+
+/**
+ * struct dpu_crtc_res - definition of crtc resources
+ * @list: list of crtc resource
+ * @type: crtc resource type
+ * @tag: unique identifier per type
+ * @refcount: reference/usage count
+ * @ops: callback operations
+ * @val: resource handle associated with type/tag
+ * @flags: customization flags
+ */
+struct dpu_crtc_res {
+	struct list_head list;
+	u32 type;
+	u64 tag;
+	atomic_t refcount;
+	struct dpu_crtc_res_ops ops;
+	void *val;
+	u32 flags;
+};
+
+/**
+ * dpu_crtc_respool - crtc resource pool
+ * @rp_lock: pointer to serialization lock
+ * @rp_head: pointer to head of active resource pools of this crtc
+ * @rp_list: list of crtc resource pool
+ * @sequence_id: sequence identifier, incremented per state duplication
+ * @res_list: list of resource managed by this resource pool
+ * @ops: resource operations for parent resource pool
+ */
+struct dpu_crtc_respool {
+	struct mutex *rp_lock;
+	struct list_head *rp_head;
+	struct list_head rp_list;
+	u32 sequence_id;
+	struct list_head res_list;
+	struct dpu_crtc_res_ops ops;
+};
+
+/**
+ * struct dpu_crtc_state - dpu container for atomic crtc state
+ * @base: Base drm crtc state structure
+ * @is_ppsplit    : Whether current topology requires PPSplit special handling
+ * @bw_control    : true if bw/clk controlled by core bw/clk properties
+ * @bw_split_vote : true if bw controlled by llcc/dram bw properties
+ * @lm_bounds     : LM boundaries based on current mode full resolution, no ROI.
+ *                  Origin top left of CRTC.
+ * @property_state: Local storage for msm_prop properties
+ * @property_values: Current crtc property values
+ * @input_fence_timeout_ns : Cached input fence timeout, in ns
+ * @new_perf: new performance state being requested
+ */
+struct dpu_crtc_state {
+	struct drm_crtc_state base;
+
+	bool bw_control;
+	bool bw_split_vote;
+
+	bool is_ppsplit;
+	struct drm_rect lm_bounds[CRTC_DUAL_MIXERS];
+
+	uint64_t input_fence_timeout_ns;
+
+	struct dpu_core_perf_params new_perf;
+	struct dpu_crtc_respool rp;
+};
+
+#define to_dpu_crtc_state(x) \
+	container_of(x, struct dpu_crtc_state, base)
+
+/**
+ * dpu_crtc_get_mixer_width - get the mixer width
+ * Mixer width will be same as panel width(/2 for split)
+ */
+static inline int dpu_crtc_get_mixer_width(struct dpu_crtc *dpu_crtc,
+	struct dpu_crtc_state *cstate, struct drm_display_mode *mode)
+{
+	u32 mixer_width;
+
+	if (!dpu_crtc || !cstate || !mode)
+		return 0;
+
+	mixer_width = (dpu_crtc->num_mixers == CRTC_DUAL_MIXERS ?
+			mode->hdisplay / CRTC_DUAL_MIXERS : mode->hdisplay);
+
+	return mixer_width;
+}
+
+/**
+ * dpu_crtc_get_mixer_height - get the mixer height
+ * Mixer height will be same as panel height
+ */
+static inline int dpu_crtc_get_mixer_height(struct dpu_crtc *dpu_crtc,
+		struct dpu_crtc_state *cstate, struct drm_display_mode *mode)
+{
+	if (!dpu_crtc || !cstate || !mode)
+		return 0;
+
+	return mode->vdisplay;
+}
+
+/**
+ * dpu_crtc_frame_pending - retun the number of pending frames
+ * @crtc: Pointer to drm crtc object
+ */
+static inline int dpu_crtc_frame_pending(struct drm_crtc *crtc)
+{
+	struct dpu_crtc *dpu_crtc;
+
+	if (!crtc)
+		return -EINVAL;
+
+	dpu_crtc = to_dpu_crtc(crtc);
+	return atomic_read(&dpu_crtc->frame_pending);
+}
+
+/**
+ * dpu_crtc_vblank - enable or disable vblanks for this crtc
+ * @crtc: Pointer to drm crtc object
+ * @en: true to enable vblanks, false to disable
+ */
+int dpu_crtc_vblank(struct drm_crtc *crtc, bool en);
+
+/**
+ * dpu_crtc_commit_kickoff - trigger kickoff of the commit for this crtc
+ * @crtc: Pointer to drm crtc object
+ */
+void dpu_crtc_commit_kickoff(struct drm_crtc *crtc);
+
+/**
+ * dpu_crtc_complete_commit - callback signalling completion of current commit
+ * @crtc: Pointer to drm crtc object
+ * @old_state: Pointer to drm crtc old state object
+ */
+void dpu_crtc_complete_commit(struct drm_crtc *crtc,
+		struct drm_crtc_state *old_state);
+
+/**
+ * dpu_crtc_init - create a new crtc object
+ * @dev: dpu device
+ * @plane: base plane
+ * @Return: new crtc object or error
+ */
+struct drm_crtc *dpu_crtc_init(struct drm_device *dev, struct drm_plane *plane);
+
+/**
+ * dpu_crtc_register_custom_event - api for enabling/disabling crtc event
+ * @kms: Pointer to dpu_kms
+ * @crtc_drm: Pointer to crtc object
+ * @event: Event that client is interested
+ * @en: Flag to enable/disable the event
+ */
+int dpu_crtc_register_custom_event(struct dpu_kms *kms,
+		struct drm_crtc *crtc_drm, u32 event, bool en);
+
+/**
+ * dpu_crtc_get_intf_mode - get interface mode of the given crtc
+ * @crtc: Pointert to crtc
+ */
+enum dpu_intf_mode dpu_crtc_get_intf_mode(struct drm_crtc *crtc);
+
+/**
+ * dpu_crtc_get_client_type - check the crtc type- rt, nrt etc.
+ * @crtc: Pointer to crtc
+ */
+static inline enum dpu_crtc_client_type dpu_crtc_get_client_type(
+						struct drm_crtc *crtc)
+{
+	struct dpu_crtc_state *cstate =
+			crtc ? to_dpu_crtc_state(crtc->state) : NULL;
+
+	if (!cstate)
+		return NRT_CLIENT;
+
+	return RT_CLIENT;
+}
+
+/**
+ * dpu_crtc_is_enabled - check if dpu crtc is enabled or not
+ * @crtc: Pointer to crtc
+ */
+static inline bool dpu_crtc_is_enabled(struct drm_crtc *crtc)
+{
+	return crtc ? crtc->enabled : false;
+}
+
+#endif /* _DPU_CRTC_H_ */
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_dbg.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_dbg.c
new file mode 100644
index 000000000000..ae2aee7ed9e1
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_dbg.c
@@ -0,0 +1,2393 @@
+/* Copyright (c) 2009-2018, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#define pr_fmt(fmt)	"[drm:%s:%d] " fmt, __func__, __LINE__
+
+#include <linux/delay.h>
+#include <linux/spinlock.h>
+#include <linux/ktime.h>
+#include <linux/debugfs.h>
+#include <linux/uaccess.h>
+#include <linux/dma-buf.h>
+#include <linux/slab.h>
+#include <linux/list_sort.h>
+#include <linux/pm_runtime.h>
+
+#include "dpu_dbg.h"
+#include "disp/dpu1/dpu_hw_catalog.h"
+
+
+#define DEFAULT_DBGBUS_DPU	DPU_DBG_DUMP_IN_MEM
+#define DEFAULT_DBGBUS_VBIFRT	DPU_DBG_DUMP_IN_MEM
+#define REG_BASE_NAME_LEN	80
+
+#define DBGBUS_FLAGS_DSPP	BIT(0)
+#define DBGBUS_DSPP_STATUS	0x34C
+
+#define DBGBUS_NAME_DPU		"dpu"
+#define DBGBUS_NAME_VBIF_RT	"vbif_rt"
+
+/* offsets from dpu top address for the debug buses */
+#define DBGBUS_SSPP0	0x188
+#define DBGBUS_AXI_INTF	0x194
+#define DBGBUS_SSPP1	0x298
+#define DBGBUS_DSPP	0x348
+#define DBGBUS_PERIPH	0x418
+
+#define TEST_MASK(id, tp)	((id << 4) | (tp << 1) | BIT(0))
+
+/* following offsets are with respect to MDP VBIF base for DBG BUS access */
+#define MMSS_VBIF_CLKON			0x4
+#define MMSS_VBIF_TEST_BUS_OUT_CTRL	0x210
+#define MMSS_VBIF_TEST_BUS_OUT		0x230
+
+/* Vbif error info */
+#define MMSS_VBIF_PND_ERR		0x190
+#define MMSS_VBIF_SRC_ERR		0x194
+#define MMSS_VBIF_XIN_HALT_CTRL1	0x204
+#define MMSS_VBIF_ERR_INFO		0X1a0
+#define MMSS_VBIF_ERR_INFO_1		0x1a4
+#define MMSS_VBIF_CLIENT_NUM		14
+
+/**
+ * struct dpu_dbg_reg_base - register region base.
+ *	may sub-ranges: sub-ranges are used for dumping
+ *	or may not have sub-ranges: dumping is base -> max_offset
+ * @reg_base_head: head of this node
+ * @name: register base name
+ * @base: base pointer
+ * @off: cached offset of region for manual register dumping
+ * @cnt: cached range of region for manual register dumping
+ * @max_offset: length of region
+ * @buf: buffer used for manual register dumping
+ * @buf_len:  buffer length used for manual register dumping
+ * @cb: callback for external dump function, null if not defined
+ * @cb_ptr: private pointer to callback function
+ */
+struct dpu_dbg_reg_base {
+	struct list_head reg_base_head;
+	char name[REG_BASE_NAME_LEN];
+	void __iomem *base;
+	size_t off;
+	size_t cnt;
+	size_t max_offset;
+	char *buf;
+	size_t buf_len;
+	void (*cb)(void *ptr);
+	void *cb_ptr;
+};
+
+struct dpu_debug_bus_entry {
+	u32 wr_addr;
+	u32 block_id;
+	u32 test_id;
+	void (*analyzer)(void __iomem *mem_base,
+				struct dpu_debug_bus_entry *entry, u32 val);
+};
+
+struct vbif_debug_bus_entry {
+	u32 disable_bus_addr;
+	u32 block_bus_addr;
+	u32 bit_offset;
+	u32 block_cnt;
+	u32 test_pnt_start;
+	u32 test_pnt_cnt;
+};
+
+struct dpu_dbg_debug_bus_common {
+	char *name;
+	u32 enable_mask;
+	bool include_in_deferred_work;
+	u32 flags;
+	u32 entries_size;
+	u32 *dumped_content;
+};
+
+struct dpu_dbg_dpu_debug_bus {
+	struct dpu_dbg_debug_bus_common cmn;
+	struct dpu_debug_bus_entry *entries;
+	u32 top_blk_off;
+};
+
+struct dpu_dbg_vbif_debug_bus {
+	struct dpu_dbg_debug_bus_common cmn;
+	struct vbif_debug_bus_entry *entries;
+};
+
+/**
+ * struct dpu_dbg_base - global dpu debug base structure
+ * @reg_base_list: list of register dumping regions
+ * @dev: device pointer
+ * @dump_work: work struct for deferring register dump work to separate thread
+ * @dbgbus_dpu: debug bus structure for the dpu
+ * @dbgbus_vbif_rt: debug bus structure for the realtime vbif
+ */
+static struct dpu_dbg_base {
+	struct list_head reg_base_list;
+	struct device *dev;
+
+	struct work_struct dump_work;
+
+	struct dpu_dbg_dpu_debug_bus dbgbus_dpu;
+	struct dpu_dbg_vbif_debug_bus dbgbus_vbif_rt;
+} dpu_dbg_base;
+
+static void _dpu_debug_bus_xbar_dump(void __iomem *mem_base,
+		struct dpu_debug_bus_entry *entry, u32 val)
+{
+	dev_err(dpu_dbg_base.dev, "xbar 0x%x %d %d 0x%x\n",
+			entry->wr_addr, entry->block_id, entry->test_id, val);
+}
+
+static void _dpu_debug_bus_lm_dump(void __iomem *mem_base,
+		struct dpu_debug_bus_entry *entry, u32 val)
+{
+	if (!(val & 0xFFF000))
+		return;
+
+	dev_err(dpu_dbg_base.dev, "lm 0x%x %d %d 0x%x\n",
+			entry->wr_addr, entry->block_id, entry->test_id, val);
+}
+
+static void _dpu_debug_bus_ppb0_dump(void __iomem *mem_base,
+		struct dpu_debug_bus_entry *entry, u32 val)
+{
+	if (!(val & BIT(15)))
+		return;
+
+	dev_err(dpu_dbg_base.dev, "ppb0 0x%x %d %d 0x%x\n",
+			entry->wr_addr, entry->block_id, entry->test_id, val);
+}
+
+static void _dpu_debug_bus_ppb1_dump(void __iomem *mem_base,
+		struct dpu_debug_bus_entry *entry, u32 val)
+{
+	if (!(val & BIT(15)))
+		return;
+
+	dev_err(dpu_dbg_base.dev, "ppb1 0x%x %d %d 0x%x\n",
+			entry->wr_addr, entry->block_id, entry->test_id, val);
+}
+
+static struct dpu_debug_bus_entry dbg_bus_dpu_8998[] = {
+
+	/* Unpack 0 sspp 0*/
+	{ DBGBUS_SSPP0, 50, 2 },
+	{ DBGBUS_SSPP0, 60, 2 },
+	{ DBGBUS_SSPP0, 70, 2 },
+	{ DBGBUS_SSPP0, 85, 2 },
+
+	/* Upack 0 sspp 1*/
+	{ DBGBUS_SSPP1, 50, 2 },
+	{ DBGBUS_SSPP1, 60, 2 },
+	{ DBGBUS_SSPP1, 70, 2 },
+	{ DBGBUS_SSPP1, 85, 2 },
+
+	/* scheduler */
+	{ DBGBUS_DSPP, 130, 0 },
+	{ DBGBUS_DSPP, 130, 1 },
+	{ DBGBUS_DSPP, 130, 2 },
+	{ DBGBUS_DSPP, 130, 3 },
+	{ DBGBUS_DSPP, 130, 4 },
+	{ DBGBUS_DSPP, 130, 5 },
+
+	/* qseed */
+	{ DBGBUS_SSPP0, 6, 0},
+	{ DBGBUS_SSPP0, 6, 1},
+	{ DBGBUS_SSPP0, 26, 0},
+	{ DBGBUS_SSPP0, 26, 1},
+	{ DBGBUS_SSPP1, 6, 0},
+	{ DBGBUS_SSPP1, 6, 1},
+	{ DBGBUS_SSPP1, 26, 0},
+	{ DBGBUS_SSPP1, 26, 1},
+
+	/* scale */
+	{ DBGBUS_SSPP0, 16, 0},
+	{ DBGBUS_SSPP0, 16, 1},
+	{ DBGBUS_SSPP0, 36, 0},
+	{ DBGBUS_SSPP0, 36, 1},
+	{ DBGBUS_SSPP1, 16, 0},
+	{ DBGBUS_SSPP1, 16, 1},
+	{ DBGBUS_SSPP1, 36, 0},
+	{ DBGBUS_SSPP1, 36, 1},
+
+	/* fetch sspp0 */
+
+	/* vig 0 */
+	{ DBGBUS_SSPP0, 0, 0 },
+	{ DBGBUS_SSPP0, 0, 1 },
+	{ DBGBUS_SSPP0, 0, 2 },
+	{ DBGBUS_SSPP0, 0, 3 },
+	{ DBGBUS_SSPP0, 0, 4 },
+	{ DBGBUS_SSPP0, 0, 5 },
+	{ DBGBUS_SSPP0, 0, 6 },
+	{ DBGBUS_SSPP0, 0, 7 },
+
+	{ DBGBUS_SSPP0, 1, 0 },
+	{ DBGBUS_SSPP0, 1, 1 },
+	{ DBGBUS_SSPP0, 1, 2 },
+	{ DBGBUS_SSPP0, 1, 3 },
+	{ DBGBUS_SSPP0, 1, 4 },
+	{ DBGBUS_SSPP0, 1, 5 },
+	{ DBGBUS_SSPP0, 1, 6 },
+	{ DBGBUS_SSPP0, 1, 7 },
+
+	{ DBGBUS_SSPP0, 2, 0 },
+	{ DBGBUS_SSPP0, 2, 1 },
+	{ DBGBUS_SSPP0, 2, 2 },
+	{ DBGBUS_SSPP0, 2, 3 },
+	{ DBGBUS_SSPP0, 2, 4 },
+	{ DBGBUS_SSPP0, 2, 5 },
+	{ DBGBUS_SSPP0, 2, 6 },
+	{ DBGBUS_SSPP0, 2, 7 },
+
+	{ DBGBUS_SSPP0, 4, 0 },
+	{ DBGBUS_SSPP0, 4, 1 },
+	{ DBGBUS_SSPP0, 4, 2 },
+	{ DBGBUS_SSPP0, 4, 3 },
+	{ DBGBUS_SSPP0, 4, 4 },
+	{ DBGBUS_SSPP0, 4, 5 },
+	{ DBGBUS_SSPP0, 4, 6 },
+	{ DBGBUS_SSPP0, 4, 7 },
+
+	{ DBGBUS_SSPP0, 5, 0 },
+	{ DBGBUS_SSPP0, 5, 1 },
+	{ DBGBUS_SSPP0, 5, 2 },
+	{ DBGBUS_SSPP0, 5, 3 },
+	{ DBGBUS_SSPP0, 5, 4 },
+	{ DBGBUS_SSPP0, 5, 5 },
+	{ DBGBUS_SSPP0, 5, 6 },
+	{ DBGBUS_SSPP0, 5, 7 },
+
+	/* vig 2 */
+	{ DBGBUS_SSPP0, 20, 0 },
+	{ DBGBUS_SSPP0, 20, 1 },
+	{ DBGBUS_SSPP0, 20, 2 },
+	{ DBGBUS_SSPP0, 20, 3 },
+	{ DBGBUS_SSPP0, 20, 4 },
+	{ DBGBUS_SSPP0, 20, 5 },
+	{ DBGBUS_SSPP0, 20, 6 },
+	{ DBGBUS_SSPP0, 20, 7 },
+
+	{ DBGBUS_SSPP0, 21, 0 },
+	{ DBGBUS_SSPP0, 21, 1 },
+	{ DBGBUS_SSPP0, 21, 2 },
+	{ DBGBUS_SSPP0, 21, 3 },
+	{ DBGBUS_SSPP0, 21, 4 },
+	{ DBGBUS_SSPP0, 21, 5 },
+	{ DBGBUS_SSPP0, 21, 6 },
+	{ DBGBUS_SSPP0, 21, 7 },
+
+	{ DBGBUS_SSPP0, 22, 0 },
+	{ DBGBUS_SSPP0, 22, 1 },
+	{ DBGBUS_SSPP0, 22, 2 },
+	{ DBGBUS_SSPP0, 22, 3 },
+	{ DBGBUS_SSPP0, 22, 4 },
+	{ DBGBUS_SSPP0, 22, 5 },
+	{ DBGBUS_SSPP0, 22, 6 },
+	{ DBGBUS_SSPP0, 22, 7 },
+
+	{ DBGBUS_SSPP0, 24, 0 },
+	{ DBGBUS_SSPP0, 24, 1 },
+	{ DBGBUS_SSPP0, 24, 2 },
+	{ DBGBUS_SSPP0, 24, 3 },
+	{ DBGBUS_SSPP0, 24, 4 },
+	{ DBGBUS_SSPP0, 24, 5 },
+	{ DBGBUS_SSPP0, 24, 6 },
+	{ DBGBUS_SSPP0, 24, 7 },
+
+	{ DBGBUS_SSPP0, 25, 0 },
+	{ DBGBUS_SSPP0, 25, 1 },
+	{ DBGBUS_SSPP0, 25, 2 },
+	{ DBGBUS_SSPP0, 25, 3 },
+	{ DBGBUS_SSPP0, 25, 4 },
+	{ DBGBUS_SSPP0, 25, 5 },
+	{ DBGBUS_SSPP0, 25, 6 },
+	{ DBGBUS_SSPP0, 25, 7 },
+
+	/* dma 2 */
+	{ DBGBUS_SSPP0, 30, 0 },
+	{ DBGBUS_SSPP0, 30, 1 },
+	{ DBGBUS_SSPP0, 30, 2 },
+	{ DBGBUS_SSPP0, 30, 3 },
+	{ DBGBUS_SSPP0, 30, 4 },
+	{ DBGBUS_SSPP0, 30, 5 },
+	{ DBGBUS_SSPP0, 30, 6 },
+	{ DBGBUS_SSPP0, 30, 7 },
+
+	{ DBGBUS_SSPP0, 31, 0 },
+	{ DBGBUS_SSPP0, 31, 1 },
+	{ DBGBUS_SSPP0, 31, 2 },
+	{ DBGBUS_SSPP0, 31, 3 },
+	{ DBGBUS_SSPP0, 31, 4 },
+	{ DBGBUS_SSPP0, 31, 5 },
+	{ DBGBUS_SSPP0, 31, 6 },
+	{ DBGBUS_SSPP0, 31, 7 },
+
+	{ DBGBUS_SSPP0, 32, 0 },
+	{ DBGBUS_SSPP0, 32, 1 },
+	{ DBGBUS_SSPP0, 32, 2 },
+	{ DBGBUS_SSPP0, 32, 3 },
+	{ DBGBUS_SSPP0, 32, 4 },
+	{ DBGBUS_SSPP0, 32, 5 },
+	{ DBGBUS_SSPP0, 32, 6 },
+	{ DBGBUS_SSPP0, 32, 7 },
+
+	{ DBGBUS_SSPP0, 33, 0 },
+	{ DBGBUS_SSPP0, 33, 1 },
+	{ DBGBUS_SSPP0, 33, 2 },
+	{ DBGBUS_SSPP0, 33, 3 },
+	{ DBGBUS_SSPP0, 33, 4 },
+	{ DBGBUS_SSPP0, 33, 5 },
+	{ DBGBUS_SSPP0, 33, 6 },
+	{ DBGBUS_SSPP0, 33, 7 },
+
+	{ DBGBUS_SSPP0, 34, 0 },
+	{ DBGBUS_SSPP0, 34, 1 },
+	{ DBGBUS_SSPP0, 34, 2 },
+	{ DBGBUS_SSPP0, 34, 3 },
+	{ DBGBUS_SSPP0, 34, 4 },
+	{ DBGBUS_SSPP0, 34, 5 },
+	{ DBGBUS_SSPP0, 34, 6 },
+	{ DBGBUS_SSPP0, 34, 7 },
+
+	{ DBGBUS_SSPP0, 35, 0 },
+	{ DBGBUS_SSPP0, 35, 1 },
+	{ DBGBUS_SSPP0, 35, 2 },
+	{ DBGBUS_SSPP0, 35, 3 },
+
+	/* dma 0 */
+	{ DBGBUS_SSPP0, 40, 0 },
+	{ DBGBUS_SSPP0, 40, 1 },
+	{ DBGBUS_SSPP0, 40, 2 },
+	{ DBGBUS_SSPP0, 40, 3 },
+	{ DBGBUS_SSPP0, 40, 4 },
+	{ DBGBUS_SSPP0, 40, 5 },
+	{ DBGBUS_SSPP0, 40, 6 },
+	{ DBGBUS_SSPP0, 40, 7 },
+
+	{ DBGBUS_SSPP0, 41, 0 },
+	{ DBGBUS_SSPP0, 41, 1 },
+	{ DBGBUS_SSPP0, 41, 2 },
+	{ DBGBUS_SSPP0, 41, 3 },
+	{ DBGBUS_SSPP0, 41, 4 },
+	{ DBGBUS_SSPP0, 41, 5 },
+	{ DBGBUS_SSPP0, 41, 6 },
+	{ DBGBUS_SSPP0, 41, 7 },
+
+	{ DBGBUS_SSPP0, 42, 0 },
+	{ DBGBUS_SSPP0, 42, 1 },
+	{ DBGBUS_SSPP0, 42, 2 },
+	{ DBGBUS_SSPP0, 42, 3 },
+	{ DBGBUS_SSPP0, 42, 4 },
+	{ DBGBUS_SSPP0, 42, 5 },
+	{ DBGBUS_SSPP0, 42, 6 },
+	{ DBGBUS_SSPP0, 42, 7 },
+
+	{ DBGBUS_SSPP0, 44, 0 },
+	{ DBGBUS_SSPP0, 44, 1 },
+	{ DBGBUS_SSPP0, 44, 2 },
+	{ DBGBUS_SSPP0, 44, 3 },
+	{ DBGBUS_SSPP0, 44, 4 },
+	{ DBGBUS_SSPP0, 44, 5 },
+	{ DBGBUS_SSPP0, 44, 6 },
+	{ DBGBUS_SSPP0, 44, 7 },
+
+	{ DBGBUS_SSPP0, 45, 0 },
+	{ DBGBUS_SSPP0, 45, 1 },
+	{ DBGBUS_SSPP0, 45, 2 },
+	{ DBGBUS_SSPP0, 45, 3 },
+	{ DBGBUS_SSPP0, 45, 4 },
+	{ DBGBUS_SSPP0, 45, 5 },
+	{ DBGBUS_SSPP0, 45, 6 },
+	{ DBGBUS_SSPP0, 45, 7 },
+
+	/* fetch sspp1 */
+	/* vig 1 */
+	{ DBGBUS_SSPP1, 0, 0 },
+	{ DBGBUS_SSPP1, 0, 1 },
+	{ DBGBUS_SSPP1, 0, 2 },
+	{ DBGBUS_SSPP1, 0, 3 },
+	{ DBGBUS_SSPP1, 0, 4 },
+	{ DBGBUS_SSPP1, 0, 5 },
+	{ DBGBUS_SSPP1, 0, 6 },
+	{ DBGBUS_SSPP1, 0, 7 },
+
+	{ DBGBUS_SSPP1, 1, 0 },
+	{ DBGBUS_SSPP1, 1, 1 },
+	{ DBGBUS_SSPP1, 1, 2 },
+	{ DBGBUS_SSPP1, 1, 3 },
+	{ DBGBUS_SSPP1, 1, 4 },
+	{ DBGBUS_SSPP1, 1, 5 },
+	{ DBGBUS_SSPP1, 1, 6 },
+	{ DBGBUS_SSPP1, 1, 7 },
+
+	{ DBGBUS_SSPP1, 2, 0 },
+	{ DBGBUS_SSPP1, 2, 1 },
+	{ DBGBUS_SSPP1, 2, 2 },
+	{ DBGBUS_SSPP1, 2, 3 },
+	{ DBGBUS_SSPP1, 2, 4 },
+	{ DBGBUS_SSPP1, 2, 5 },
+	{ DBGBUS_SSPP1, 2, 6 },
+	{ DBGBUS_SSPP1, 2, 7 },
+
+	{ DBGBUS_SSPP1, 4, 0 },
+	{ DBGBUS_SSPP1, 4, 1 },
+	{ DBGBUS_SSPP1, 4, 2 },
+	{ DBGBUS_SSPP1, 4, 3 },
+	{ DBGBUS_SSPP1, 4, 4 },
+	{ DBGBUS_SSPP1, 4, 5 },
+	{ DBGBUS_SSPP1, 4, 6 },
+	{ DBGBUS_SSPP1, 4, 7 },
+
+	{ DBGBUS_SSPP1, 5, 0 },
+	{ DBGBUS_SSPP1, 5, 1 },
+	{ DBGBUS_SSPP1, 5, 2 },
+	{ DBGBUS_SSPP1, 5, 3 },
+	{ DBGBUS_SSPP1, 5, 4 },
+	{ DBGBUS_SSPP1, 5, 5 },
+	{ DBGBUS_SSPP1, 5, 6 },
+	{ DBGBUS_SSPP1, 5, 7 },
+
+	/* vig 3 */
+	{ DBGBUS_SSPP1, 20, 0 },
+	{ DBGBUS_SSPP1, 20, 1 },
+	{ DBGBUS_SSPP1, 20, 2 },
+	{ DBGBUS_SSPP1, 20, 3 },
+	{ DBGBUS_SSPP1, 20, 4 },
+	{ DBGBUS_SSPP1, 20, 5 },
+	{ DBGBUS_SSPP1, 20, 6 },
+	{ DBGBUS_SSPP1, 20, 7 },
+
+	{ DBGBUS_SSPP1, 21, 0 },
+	{ DBGBUS_SSPP1, 21, 1 },
+	{ DBGBUS_SSPP1, 21, 2 },
+	{ DBGBUS_SSPP1, 21, 3 },
+	{ DBGBUS_SSPP1, 21, 4 },
+	{ DBGBUS_SSPP1, 21, 5 },
+	{ DBGBUS_SSPP1, 21, 6 },
+	{ DBGBUS_SSPP1, 21, 7 },
+
+	{ DBGBUS_SSPP1, 22, 0 },
+	{ DBGBUS_SSPP1, 22, 1 },
+	{ DBGBUS_SSPP1, 22, 2 },
+	{ DBGBUS_SSPP1, 22, 3 },
+	{ DBGBUS_SSPP1, 22, 4 },
+	{ DBGBUS_SSPP1, 22, 5 },
+	{ DBGBUS_SSPP1, 22, 6 },
+	{ DBGBUS_SSPP1, 22, 7 },
+
+	{ DBGBUS_SSPP1, 24, 0 },
+	{ DBGBUS_SSPP1, 24, 1 },
+	{ DBGBUS_SSPP1, 24, 2 },
+	{ DBGBUS_SSPP1, 24, 3 },
+	{ DBGBUS_SSPP1, 24, 4 },
+	{ DBGBUS_SSPP1, 24, 5 },
+	{ DBGBUS_SSPP1, 24, 6 },
+	{ DBGBUS_SSPP1, 24, 7 },
+
+	{ DBGBUS_SSPP1, 25, 0 },
+	{ DBGBUS_SSPP1, 25, 1 },
+	{ DBGBUS_SSPP1, 25, 2 },
+	{ DBGBUS_SSPP1, 25, 3 },
+	{ DBGBUS_SSPP1, 25, 4 },
+	{ DBGBUS_SSPP1, 25, 5 },
+	{ DBGBUS_SSPP1, 25, 6 },
+	{ DBGBUS_SSPP1, 25, 7 },
+
+	/* dma 3 */
+	{ DBGBUS_SSPP1, 30, 0 },
+	{ DBGBUS_SSPP1, 30, 1 },
+	{ DBGBUS_SSPP1, 30, 2 },
+	{ DBGBUS_SSPP1, 30, 3 },
+	{ DBGBUS_SSPP1, 30, 4 },
+	{ DBGBUS_SSPP1, 30, 5 },
+	{ DBGBUS_SSPP1, 30, 6 },
+	{ DBGBUS_SSPP1, 30, 7 },
+
+	{ DBGBUS_SSPP1, 31, 0 },
+	{ DBGBUS_SSPP1, 31, 1 },
+	{ DBGBUS_SSPP1, 31, 2 },
+	{ DBGBUS_SSPP1, 31, 3 },
+	{ DBGBUS_SSPP1, 31, 4 },
+	{ DBGBUS_SSPP1, 31, 5 },
+	{ DBGBUS_SSPP1, 31, 6 },
+	{ DBGBUS_SSPP1, 31, 7 },
+
+	{ DBGBUS_SSPP1, 32, 0 },
+	{ DBGBUS_SSPP1, 32, 1 },
+	{ DBGBUS_SSPP1, 32, 2 },
+	{ DBGBUS_SSPP1, 32, 3 },
+	{ DBGBUS_SSPP1, 32, 4 },
+	{ DBGBUS_SSPP1, 32, 5 },
+	{ DBGBUS_SSPP1, 32, 6 },
+	{ DBGBUS_SSPP1, 32, 7 },
+
+	{ DBGBUS_SSPP1, 33, 0 },
+	{ DBGBUS_SSPP1, 33, 1 },
+	{ DBGBUS_SSPP1, 33, 2 },
+	{ DBGBUS_SSPP1, 33, 3 },
+	{ DBGBUS_SSPP1, 33, 4 },
+	{ DBGBUS_SSPP1, 33, 5 },
+	{ DBGBUS_SSPP1, 33, 6 },
+	{ DBGBUS_SSPP1, 33, 7 },
+
+	{ DBGBUS_SSPP1, 34, 0 },
+	{ DBGBUS_SSPP1, 34, 1 },
+	{ DBGBUS_SSPP1, 34, 2 },
+	{ DBGBUS_SSPP1, 34, 3 },
+	{ DBGBUS_SSPP1, 34, 4 },
+	{ DBGBUS_SSPP1, 34, 5 },
+	{ DBGBUS_SSPP1, 34, 6 },
+	{ DBGBUS_SSPP1, 34, 7 },
+
+	{ DBGBUS_SSPP1, 35, 0 },
+	{ DBGBUS_SSPP1, 35, 1 },
+	{ DBGBUS_SSPP1, 35, 2 },
+
+	/* dma 1 */
+	{ DBGBUS_SSPP1, 40, 0 },
+	{ DBGBUS_SSPP1, 40, 1 },
+	{ DBGBUS_SSPP1, 40, 2 },
+	{ DBGBUS_SSPP1, 40, 3 },
+	{ DBGBUS_SSPP1, 40, 4 },
+	{ DBGBUS_SSPP1, 40, 5 },
+	{ DBGBUS_SSPP1, 40, 6 },
+	{ DBGBUS_SSPP1, 40, 7 },
+
+	{ DBGBUS_SSPP1, 41, 0 },
+	{ DBGBUS_SSPP1, 41, 1 },
+	{ DBGBUS_SSPP1, 41, 2 },
+	{ DBGBUS_SSPP1, 41, 3 },
+	{ DBGBUS_SSPP1, 41, 4 },
+	{ DBGBUS_SSPP1, 41, 5 },
+	{ DBGBUS_SSPP1, 41, 6 },
+	{ DBGBUS_SSPP1, 41, 7 },
+
+	{ DBGBUS_SSPP1, 42, 0 },
+	{ DBGBUS_SSPP1, 42, 1 },
+	{ DBGBUS_SSPP1, 42, 2 },
+	{ DBGBUS_SSPP1, 42, 3 },
+	{ DBGBUS_SSPP1, 42, 4 },
+	{ DBGBUS_SSPP1, 42, 5 },
+	{ DBGBUS_SSPP1, 42, 6 },
+	{ DBGBUS_SSPP1, 42, 7 },
+
+	{ DBGBUS_SSPP1, 44, 0 },
+	{ DBGBUS_SSPP1, 44, 1 },
+	{ DBGBUS_SSPP1, 44, 2 },
+	{ DBGBUS_SSPP1, 44, 3 },
+	{ DBGBUS_SSPP1, 44, 4 },
+	{ DBGBUS_SSPP1, 44, 5 },
+	{ DBGBUS_SSPP1, 44, 6 },
+	{ DBGBUS_SSPP1, 44, 7 },
+
+	{ DBGBUS_SSPP1, 45, 0 },
+	{ DBGBUS_SSPP1, 45, 1 },
+	{ DBGBUS_SSPP1, 45, 2 },
+	{ DBGBUS_SSPP1, 45, 3 },
+	{ DBGBUS_SSPP1, 45, 4 },
+	{ DBGBUS_SSPP1, 45, 5 },
+	{ DBGBUS_SSPP1, 45, 6 },
+	{ DBGBUS_SSPP1, 45, 7 },
+
+	/* cursor 1 */
+	{ DBGBUS_SSPP1, 80, 0 },
+	{ DBGBUS_SSPP1, 80, 1 },
+	{ DBGBUS_SSPP1, 80, 2 },
+	{ DBGBUS_SSPP1, 80, 3 },
+	{ DBGBUS_SSPP1, 80, 4 },
+	{ DBGBUS_SSPP1, 80, 5 },
+	{ DBGBUS_SSPP1, 80, 6 },
+	{ DBGBUS_SSPP1, 80, 7 },
+
+	{ DBGBUS_SSPP1, 81, 0 },
+	{ DBGBUS_SSPP1, 81, 1 },
+	{ DBGBUS_SSPP1, 81, 2 },
+	{ DBGBUS_SSPP1, 81, 3 },
+	{ DBGBUS_SSPP1, 81, 4 },
+	{ DBGBUS_SSPP1, 81, 5 },
+	{ DBGBUS_SSPP1, 81, 6 },
+	{ DBGBUS_SSPP1, 81, 7 },
+
+	{ DBGBUS_SSPP1, 82, 0 },
+	{ DBGBUS_SSPP1, 82, 1 },
+	{ DBGBUS_SSPP1, 82, 2 },
+	{ DBGBUS_SSPP1, 82, 3 },
+	{ DBGBUS_SSPP1, 82, 4 },
+	{ DBGBUS_SSPP1, 82, 5 },
+	{ DBGBUS_SSPP1, 82, 6 },
+	{ DBGBUS_SSPP1, 82, 7 },
+
+	{ DBGBUS_SSPP1, 83, 0 },
+	{ DBGBUS_SSPP1, 83, 1 },
+	{ DBGBUS_SSPP1, 83, 2 },
+	{ DBGBUS_SSPP1, 83, 3 },
+	{ DBGBUS_SSPP1, 83, 4 },
+	{ DBGBUS_SSPP1, 83, 5 },
+	{ DBGBUS_SSPP1, 83, 6 },
+	{ DBGBUS_SSPP1, 83, 7 },
+
+	{ DBGBUS_SSPP1, 84, 0 },
+	{ DBGBUS_SSPP1, 84, 1 },
+	{ DBGBUS_SSPP1, 84, 2 },
+	{ DBGBUS_SSPP1, 84, 3 },
+	{ DBGBUS_SSPP1, 84, 4 },
+	{ DBGBUS_SSPP1, 84, 5 },
+	{ DBGBUS_SSPP1, 84, 6 },
+	{ DBGBUS_SSPP1, 84, 7 },
+
+	/* dspp */
+	{ DBGBUS_DSPP, 13, 0 },
+	{ DBGBUS_DSPP, 19, 0 },
+	{ DBGBUS_DSPP, 14, 0 },
+	{ DBGBUS_DSPP, 14, 1 },
+	{ DBGBUS_DSPP, 14, 3 },
+	{ DBGBUS_DSPP, 20, 0 },
+	{ DBGBUS_DSPP, 20, 1 },
+	{ DBGBUS_DSPP, 20, 3 },
+
+	/* ppb_0 */
+	{ DBGBUS_DSPP, 31, 0, _dpu_debug_bus_ppb0_dump },
+	{ DBGBUS_DSPP, 33, 0, _dpu_debug_bus_ppb0_dump },
+	{ DBGBUS_DSPP, 35, 0, _dpu_debug_bus_ppb0_dump },
+	{ DBGBUS_DSPP, 42, 0, _dpu_debug_bus_ppb0_dump },
+
+	/* ppb_1 */
+	{ DBGBUS_DSPP, 32, 0, _dpu_debug_bus_ppb1_dump },
+	{ DBGBUS_DSPP, 34, 0, _dpu_debug_bus_ppb1_dump },
+	{ DBGBUS_DSPP, 36, 0, _dpu_debug_bus_ppb1_dump },
+	{ DBGBUS_DSPP, 43, 0, _dpu_debug_bus_ppb1_dump },
+
+	/* lm_lut */
+	{ DBGBUS_DSPP, 109, 0 },
+	{ DBGBUS_DSPP, 105, 0 },
+	{ DBGBUS_DSPP, 103, 0 },
+
+	/* tear-check */
+	{ DBGBUS_PERIPH, 63, 0 },
+	{ DBGBUS_PERIPH, 64, 0 },
+	{ DBGBUS_PERIPH, 65, 0 },
+	{ DBGBUS_PERIPH, 73, 0 },
+	{ DBGBUS_PERIPH, 74, 0 },
+
+	/* crossbar */
+	{ DBGBUS_DSPP, 0, 0, _dpu_debug_bus_xbar_dump },
+
+	/* rotator */
+	{ DBGBUS_DSPP, 9, 0},
+
+	/* blend */
+	/* LM0 */
+	{ DBGBUS_DSPP, 63, 0},
+	{ DBGBUS_DSPP, 63, 1},
+	{ DBGBUS_DSPP, 63, 2},
+	{ DBGBUS_DSPP, 63, 3},
+	{ DBGBUS_DSPP, 63, 4},
+	{ DBGBUS_DSPP, 63, 5},
+	{ DBGBUS_DSPP, 63, 6},
+	{ DBGBUS_DSPP, 63, 7, _dpu_debug_bus_lm_dump },
+
+	{ DBGBUS_DSPP, 64, 0},
+	{ DBGBUS_DSPP, 64, 1},
+	{ DBGBUS_DSPP, 64, 2},
+	{ DBGBUS_DSPP, 64, 3},
+	{ DBGBUS_DSPP, 64, 4},
+	{ DBGBUS_DSPP, 64, 5},
+	{ DBGBUS_DSPP, 64, 6},
+	{ DBGBUS_DSPP, 64, 7, _dpu_debug_bus_lm_dump },
+
+	{ DBGBUS_DSPP, 65, 0},
+	{ DBGBUS_DSPP, 65, 1},
+	{ DBGBUS_DSPP, 65, 2},
+	{ DBGBUS_DSPP, 65, 3},
+	{ DBGBUS_DSPP, 65, 4},
+	{ DBGBUS_DSPP, 65, 5},
+	{ DBGBUS_DSPP, 65, 6},
+	{ DBGBUS_DSPP, 65, 7, _dpu_debug_bus_lm_dump },
+
+	{ DBGBUS_DSPP, 66, 0},
+	{ DBGBUS_DSPP, 66, 1},
+	{ DBGBUS_DSPP, 66, 2},
+	{ DBGBUS_DSPP, 66, 3},
+	{ DBGBUS_DSPP, 66, 4},
+	{ DBGBUS_DSPP, 66, 5},
+	{ DBGBUS_DSPP, 66, 6},
+	{ DBGBUS_DSPP, 66, 7, _dpu_debug_bus_lm_dump },
+
+	{ DBGBUS_DSPP, 67, 0},
+	{ DBGBUS_DSPP, 67, 1},
+	{ DBGBUS_DSPP, 67, 2},
+	{ DBGBUS_DSPP, 67, 3},
+	{ DBGBUS_DSPP, 67, 4},
+	{ DBGBUS_DSPP, 67, 5},
+	{ DBGBUS_DSPP, 67, 6},
+	{ DBGBUS_DSPP, 67, 7, _dpu_debug_bus_lm_dump },
+
+	{ DBGBUS_DSPP, 68, 0},
+	{ DBGBUS_DSPP, 68, 1},
+	{ DBGBUS_DSPP, 68, 2},
+	{ DBGBUS_DSPP, 68, 3},
+	{ DBGBUS_DSPP, 68, 4},
+	{ DBGBUS_DSPP, 68, 5},
+	{ DBGBUS_DSPP, 68, 6},
+	{ DBGBUS_DSPP, 68, 7, _dpu_debug_bus_lm_dump },
+
+	{ DBGBUS_DSPP, 69, 0},
+	{ DBGBUS_DSPP, 69, 1},
+	{ DBGBUS_DSPP, 69, 2},
+	{ DBGBUS_DSPP, 69, 3},
+	{ DBGBUS_DSPP, 69, 4},
+	{ DBGBUS_DSPP, 69, 5},
+	{ DBGBUS_DSPP, 69, 6},
+	{ DBGBUS_DSPP, 69, 7, _dpu_debug_bus_lm_dump },
+
+	/* LM1 */
+	{ DBGBUS_DSPP, 70, 0},
+	{ DBGBUS_DSPP, 70, 1},
+	{ DBGBUS_DSPP, 70, 2},
+	{ DBGBUS_DSPP, 70, 3},
+	{ DBGBUS_DSPP, 70, 4},
+	{ DBGBUS_DSPP, 70, 5},
+	{ DBGBUS_DSPP, 70, 6},
+	{ DBGBUS_DSPP, 70, 7, _dpu_debug_bus_lm_dump },
+
+	{ DBGBUS_DSPP, 71, 0},
+	{ DBGBUS_DSPP, 71, 1},
+	{ DBGBUS_DSPP, 71, 2},
+	{ DBGBUS_DSPP, 71, 3},
+	{ DBGBUS_DSPP, 71, 4},
+	{ DBGBUS_DSPP, 71, 5},
+	{ DBGBUS_DSPP, 71, 6},
+	{ DBGBUS_DSPP, 71, 7, _dpu_debug_bus_lm_dump },
+
+	{ DBGBUS_DSPP, 72, 0},
+	{ DBGBUS_DSPP, 72, 1},
+	{ DBGBUS_DSPP, 72, 2},
+	{ DBGBUS_DSPP, 72, 3},
+	{ DBGBUS_DSPP, 72, 4},
+	{ DBGBUS_DSPP, 72, 5},
+	{ DBGBUS_DSPP, 72, 6},
+	{ DBGBUS_DSPP, 72, 7, _dpu_debug_bus_lm_dump },
+
+	{ DBGBUS_DSPP, 73, 0},
+	{ DBGBUS_DSPP, 73, 1},
+	{ DBGBUS_DSPP, 73, 2},
+	{ DBGBUS_DSPP, 73, 3},
+	{ DBGBUS_DSPP, 73, 4},
+	{ DBGBUS_DSPP, 73, 5},
+	{ DBGBUS_DSPP, 73, 6},
+	{ DBGBUS_DSPP, 73, 7, _dpu_debug_bus_lm_dump },
+
+	{ DBGBUS_DSPP, 74, 0},
+	{ DBGBUS_DSPP, 74, 1},
+	{ DBGBUS_DSPP, 74, 2},
+	{ DBGBUS_DSPP, 74, 3},
+	{ DBGBUS_DSPP, 74, 4},
+	{ DBGBUS_DSPP, 74, 5},
+	{ DBGBUS_DSPP, 74, 6},
+	{ DBGBUS_DSPP, 74, 7, _dpu_debug_bus_lm_dump },
+
+	{ DBGBUS_DSPP, 75, 0},
+	{ DBGBUS_DSPP, 75, 1},
+	{ DBGBUS_DSPP, 75, 2},
+	{ DBGBUS_DSPP, 75, 3},
+	{ DBGBUS_DSPP, 75, 4},
+	{ DBGBUS_DSPP, 75, 5},
+	{ DBGBUS_DSPP, 75, 6},
+	{ DBGBUS_DSPP, 75, 7, _dpu_debug_bus_lm_dump },
+
+	{ DBGBUS_DSPP, 76, 0},
+	{ DBGBUS_DSPP, 76, 1},
+	{ DBGBUS_DSPP, 76, 2},
+	{ DBGBUS_DSPP, 76, 3},
+	{ DBGBUS_DSPP, 76, 4},
+	{ DBGBUS_DSPP, 76, 5},
+	{ DBGBUS_DSPP, 76, 6},
+	{ DBGBUS_DSPP, 76, 7, _dpu_debug_bus_lm_dump },
+
+	/* LM2 */
+	{ DBGBUS_DSPP, 77, 0},
+	{ DBGBUS_DSPP, 77, 1},
+	{ DBGBUS_DSPP, 77, 2},
+	{ DBGBUS_DSPP, 77, 3},
+	{ DBGBUS_DSPP, 77, 4},
+	{ DBGBUS_DSPP, 77, 5},
+	{ DBGBUS_DSPP, 77, 6},
+	{ DBGBUS_DSPP, 77, 7, _dpu_debug_bus_lm_dump },
+
+	{ DBGBUS_DSPP, 78, 0},
+	{ DBGBUS_DSPP, 78, 1},
+	{ DBGBUS_DSPP, 78, 2},
+	{ DBGBUS_DSPP, 78, 3},
+	{ DBGBUS_DSPP, 78, 4},
+	{ DBGBUS_DSPP, 78, 5},
+	{ DBGBUS_DSPP, 78, 6},
+	{ DBGBUS_DSPP, 78, 7, _dpu_debug_bus_lm_dump },
+
+	{ DBGBUS_DSPP, 79, 0},
+	{ DBGBUS_DSPP, 79, 1},
+	{ DBGBUS_DSPP, 79, 2},
+	{ DBGBUS_DSPP, 79, 3},
+	{ DBGBUS_DSPP, 79, 4},
+	{ DBGBUS_DSPP, 79, 5},
+	{ DBGBUS_DSPP, 79, 6},
+	{ DBGBUS_DSPP, 79, 7, _dpu_debug_bus_lm_dump },
+
+	{ DBGBUS_DSPP, 80, 0},
+	{ DBGBUS_DSPP, 80, 1},
+	{ DBGBUS_DSPP, 80, 2},
+	{ DBGBUS_DSPP, 80, 3},
+	{ DBGBUS_DSPP, 80, 4},
+	{ DBGBUS_DSPP, 80, 5},
+	{ DBGBUS_DSPP, 80, 6},
+	{ DBGBUS_DSPP, 80, 7, _dpu_debug_bus_lm_dump },
+
+	{ DBGBUS_DSPP, 81, 0},
+	{ DBGBUS_DSPP, 81, 1},
+	{ DBGBUS_DSPP, 81, 2},
+	{ DBGBUS_DSPP, 81, 3},
+	{ DBGBUS_DSPP, 81, 4},
+	{ DBGBUS_DSPP, 81, 5},
+	{ DBGBUS_DSPP, 81, 6},
+	{ DBGBUS_DSPP, 81, 7, _dpu_debug_bus_lm_dump },
+
+	{ DBGBUS_DSPP, 82, 0},
+	{ DBGBUS_DSPP, 82, 1},
+	{ DBGBUS_DSPP, 82, 2},
+	{ DBGBUS_DSPP, 82, 3},
+	{ DBGBUS_DSPP, 82, 4},
+	{ DBGBUS_DSPP, 82, 5},
+	{ DBGBUS_DSPP, 82, 6},
+	{ DBGBUS_DSPP, 82, 7, _dpu_debug_bus_lm_dump },
+
+	{ DBGBUS_DSPP, 83, 0},
+	{ DBGBUS_DSPP, 83, 1},
+	{ DBGBUS_DSPP, 83, 2},
+	{ DBGBUS_DSPP, 83, 3},
+	{ DBGBUS_DSPP, 83, 4},
+	{ DBGBUS_DSPP, 83, 5},
+	{ DBGBUS_DSPP, 83, 6},
+	{ DBGBUS_DSPP, 83, 7, _dpu_debug_bus_lm_dump },
+
+	/* csc */
+	{ DBGBUS_SSPP0, 7, 0},
+	{ DBGBUS_SSPP0, 7, 1},
+	{ DBGBUS_SSPP0, 27, 0},
+	{ DBGBUS_SSPP0, 27, 1},
+	{ DBGBUS_SSPP1, 7, 0},
+	{ DBGBUS_SSPP1, 7, 1},
+	{ DBGBUS_SSPP1, 27, 0},
+	{ DBGBUS_SSPP1, 27, 1},
+
+	/* pcc */
+	{ DBGBUS_SSPP0, 3,  3},
+	{ DBGBUS_SSPP0, 23, 3},
+	{ DBGBUS_SSPP0, 33, 3},
+	{ DBGBUS_SSPP0, 43, 3},
+	{ DBGBUS_SSPP1, 3,  3},
+	{ DBGBUS_SSPP1, 23, 3},
+	{ DBGBUS_SSPP1, 33, 3},
+	{ DBGBUS_SSPP1, 43, 3},
+
+	/* spa */
+	{ DBGBUS_SSPP0, 8,  0},
+	{ DBGBUS_SSPP0, 28, 0},
+	{ DBGBUS_SSPP1, 8,  0},
+	{ DBGBUS_SSPP1, 28, 0},
+	{ DBGBUS_DSPP, 13, 0},
+	{ DBGBUS_DSPP, 19, 0},
+
+	/* igc */
+	{ DBGBUS_SSPP0, 9,  0},
+	{ DBGBUS_SSPP0, 9,  1},
+	{ DBGBUS_SSPP0, 9,  3},
+	{ DBGBUS_SSPP0, 29, 0},
+	{ DBGBUS_SSPP0, 29, 1},
+	{ DBGBUS_SSPP0, 29, 3},
+	{ DBGBUS_SSPP0, 17, 0},
+	{ DBGBUS_SSPP0, 17, 1},
+	{ DBGBUS_SSPP0, 17, 3},
+	{ DBGBUS_SSPP0, 37, 0},
+	{ DBGBUS_SSPP0, 37, 1},
+	{ DBGBUS_SSPP0, 37, 3},
+	{ DBGBUS_SSPP0, 46, 0},
+	{ DBGBUS_SSPP0, 46, 1},
+	{ DBGBUS_SSPP0, 46, 3},
+
+	{ DBGBUS_SSPP1, 9,  0},
+	{ DBGBUS_SSPP1, 9,  1},
+	{ DBGBUS_SSPP1, 9,  3},
+	{ DBGBUS_SSPP1, 29, 0},
+	{ DBGBUS_SSPP1, 29, 1},
+	{ DBGBUS_SSPP1, 29, 3},
+	{ DBGBUS_SSPP1, 17, 0},
+	{ DBGBUS_SSPP1, 17, 1},
+	{ DBGBUS_SSPP1, 17, 3},
+	{ DBGBUS_SSPP1, 37, 0},
+	{ DBGBUS_SSPP1, 37, 1},
+	{ DBGBUS_SSPP1, 37, 3},
+	{ DBGBUS_SSPP1, 46, 0},
+	{ DBGBUS_SSPP1, 46, 1},
+	{ DBGBUS_SSPP1, 46, 3},
+
+	{ DBGBUS_DSPP, 14, 0},
+	{ DBGBUS_DSPP, 14, 1},
+	{ DBGBUS_DSPP, 14, 3},
+	{ DBGBUS_DSPP, 20, 0},
+	{ DBGBUS_DSPP, 20, 1},
+	{ DBGBUS_DSPP, 20, 3},
+
+	{ DBGBUS_PERIPH, 60, 0},
+};
+
+static struct dpu_debug_bus_entry dbg_bus_dpu_sdm845[] = {
+
+	/* Unpack 0 sspp 0*/
+	{ DBGBUS_SSPP0, 50, 2 },
+	{ DBGBUS_SSPP0, 60, 2 },
+	{ DBGBUS_SSPP0, 70, 2 },
+
+	/* Upack 0 sspp 1*/
+	{ DBGBUS_SSPP1, 50, 2 },
+	{ DBGBUS_SSPP1, 60, 2 },
+	{ DBGBUS_SSPP1, 70, 2 },
+
+	/* scheduler */
+	{ DBGBUS_DSPP, 130, 0 },
+	{ DBGBUS_DSPP, 130, 1 },
+	{ DBGBUS_DSPP, 130, 2 },
+	{ DBGBUS_DSPP, 130, 3 },
+	{ DBGBUS_DSPP, 130, 4 },
+	{ DBGBUS_DSPP, 130, 5 },
+
+	/* qseed */
+	{ DBGBUS_SSPP0, 6, 0},
+	{ DBGBUS_SSPP0, 6, 1},
+	{ DBGBUS_SSPP0, 26, 0},
+	{ DBGBUS_SSPP0, 26, 1},
+	{ DBGBUS_SSPP1, 6, 0},
+	{ DBGBUS_SSPP1, 6, 1},
+	{ DBGBUS_SSPP1, 26, 0},
+	{ DBGBUS_SSPP1, 26, 1},
+
+	/* scale */
+	{ DBGBUS_SSPP0, 16, 0},
+	{ DBGBUS_SSPP0, 16, 1},
+	{ DBGBUS_SSPP0, 36, 0},
+	{ DBGBUS_SSPP0, 36, 1},
+	{ DBGBUS_SSPP1, 16, 0},
+	{ DBGBUS_SSPP1, 16, 1},
+	{ DBGBUS_SSPP1, 36, 0},
+	{ DBGBUS_SSPP1, 36, 1},
+
+	/* fetch sspp0 */
+
+	/* vig 0 */
+	{ DBGBUS_SSPP0, 0, 0 },
+	{ DBGBUS_SSPP0, 0, 1 },
+	{ DBGBUS_SSPP0, 0, 2 },
+	{ DBGBUS_SSPP0, 0, 3 },
+	{ DBGBUS_SSPP0, 0, 4 },
+	{ DBGBUS_SSPP0, 0, 5 },
+	{ DBGBUS_SSPP0, 0, 6 },
+	{ DBGBUS_SSPP0, 0, 7 },
+
+	{ DBGBUS_SSPP0, 1, 0 },
+	{ DBGBUS_SSPP0, 1, 1 },
+	{ DBGBUS_SSPP0, 1, 2 },
+	{ DBGBUS_SSPP0, 1, 3 },
+	{ DBGBUS_SSPP0, 1, 4 },
+	{ DBGBUS_SSPP0, 1, 5 },
+	{ DBGBUS_SSPP0, 1, 6 },
+	{ DBGBUS_SSPP0, 1, 7 },
+
+	{ DBGBUS_SSPP0, 2, 0 },
+	{ DBGBUS_SSPP0, 2, 1 },
+	{ DBGBUS_SSPP0, 2, 2 },
+	{ DBGBUS_SSPP0, 2, 3 },
+	{ DBGBUS_SSPP0, 2, 4 },
+	{ DBGBUS_SSPP0, 2, 5 },
+	{ DBGBUS_SSPP0, 2, 6 },
+	{ DBGBUS_SSPP0, 2, 7 },
+
+	{ DBGBUS_SSPP0, 4, 0 },
+	{ DBGBUS_SSPP0, 4, 1 },
+	{ DBGBUS_SSPP0, 4, 2 },
+	{ DBGBUS_SSPP0, 4, 3 },
+	{ DBGBUS_SSPP0, 4, 4 },
+	{ DBGBUS_SSPP0, 4, 5 },
+	{ DBGBUS_SSPP0, 4, 6 },
+	{ DBGBUS_SSPP0, 4, 7 },
+
+	{ DBGBUS_SSPP0, 5, 0 },
+	{ DBGBUS_SSPP0, 5, 1 },
+	{ DBGBUS_SSPP0, 5, 2 },
+	{ DBGBUS_SSPP0, 5, 3 },
+	{ DBGBUS_SSPP0, 5, 4 },
+	{ DBGBUS_SSPP0, 5, 5 },
+	{ DBGBUS_SSPP0, 5, 6 },
+	{ DBGBUS_SSPP0, 5, 7 },
+
+	/* vig 2 */
+	{ DBGBUS_SSPP0, 20, 0 },
+	{ DBGBUS_SSPP0, 20, 1 },
+	{ DBGBUS_SSPP0, 20, 2 },
+	{ DBGBUS_SSPP0, 20, 3 },
+	{ DBGBUS_SSPP0, 20, 4 },
+	{ DBGBUS_SSPP0, 20, 5 },
+	{ DBGBUS_SSPP0, 20, 6 },
+	{ DBGBUS_SSPP0, 20, 7 },
+
+	{ DBGBUS_SSPP0, 21, 0 },
+	{ DBGBUS_SSPP0, 21, 1 },
+	{ DBGBUS_SSPP0, 21, 2 },
+	{ DBGBUS_SSPP0, 21, 3 },
+	{ DBGBUS_SSPP0, 21, 4 },
+	{ DBGBUS_SSPP0, 21, 5 },
+	{ DBGBUS_SSPP0, 21, 6 },
+	{ DBGBUS_SSPP0, 21, 7 },
+
+	{ DBGBUS_SSPP0, 22, 0 },
+	{ DBGBUS_SSPP0, 22, 1 },
+	{ DBGBUS_SSPP0, 22, 2 },
+	{ DBGBUS_SSPP0, 22, 3 },
+	{ DBGBUS_SSPP0, 22, 4 },
+	{ DBGBUS_SSPP0, 22, 5 },
+	{ DBGBUS_SSPP0, 22, 6 },
+	{ DBGBUS_SSPP0, 22, 7 },
+
+	{ DBGBUS_SSPP0, 24, 0 },
+	{ DBGBUS_SSPP0, 24, 1 },
+	{ DBGBUS_SSPP0, 24, 2 },
+	{ DBGBUS_SSPP0, 24, 3 },
+	{ DBGBUS_SSPP0, 24, 4 },
+	{ DBGBUS_SSPP0, 24, 5 },
+	{ DBGBUS_SSPP0, 24, 6 },
+	{ DBGBUS_SSPP0, 24, 7 },
+
+	{ DBGBUS_SSPP0, 25, 0 },
+	{ DBGBUS_SSPP0, 25, 1 },
+	{ DBGBUS_SSPP0, 25, 2 },
+	{ DBGBUS_SSPP0, 25, 3 },
+	{ DBGBUS_SSPP0, 25, 4 },
+	{ DBGBUS_SSPP0, 25, 5 },
+	{ DBGBUS_SSPP0, 25, 6 },
+	{ DBGBUS_SSPP0, 25, 7 },
+
+	/* dma 2 */
+	{ DBGBUS_SSPP0, 30, 0 },
+	{ DBGBUS_SSPP0, 30, 1 },
+	{ DBGBUS_SSPP0, 30, 2 },
+	{ DBGBUS_SSPP0, 30, 3 },
+	{ DBGBUS_SSPP0, 30, 4 },
+	{ DBGBUS_SSPP0, 30, 5 },
+	{ DBGBUS_SSPP0, 30, 6 },
+	{ DBGBUS_SSPP0, 30, 7 },
+
+	{ DBGBUS_SSPP0, 31, 0 },
+	{ DBGBUS_SSPP0, 31, 1 },
+	{ DBGBUS_SSPP0, 31, 2 },
+	{ DBGBUS_SSPP0, 31, 3 },
+	{ DBGBUS_SSPP0, 31, 4 },
+	{ DBGBUS_SSPP0, 31, 5 },
+	{ DBGBUS_SSPP0, 31, 6 },
+	{ DBGBUS_SSPP0, 31, 7 },
+
+	{ DBGBUS_SSPP0, 32, 0 },
+	{ DBGBUS_SSPP0, 32, 1 },
+	{ DBGBUS_SSPP0, 32, 2 },
+	{ DBGBUS_SSPP0, 32, 3 },
+	{ DBGBUS_SSPP0, 32, 4 },
+	{ DBGBUS_SSPP0, 32, 5 },
+	{ DBGBUS_SSPP0, 32, 6 },
+	{ DBGBUS_SSPP0, 32, 7 },
+
+	{ DBGBUS_SSPP0, 33, 0 },
+	{ DBGBUS_SSPP0, 33, 1 },
+	{ DBGBUS_SSPP0, 33, 2 },
+	{ DBGBUS_SSPP0, 33, 3 },
+	{ DBGBUS_SSPP0, 33, 4 },
+	{ DBGBUS_SSPP0, 33, 5 },
+	{ DBGBUS_SSPP0, 33, 6 },
+	{ DBGBUS_SSPP0, 33, 7 },
+
+	{ DBGBUS_SSPP0, 34, 0 },
+	{ DBGBUS_SSPP0, 34, 1 },
+	{ DBGBUS_SSPP0, 34, 2 },
+	{ DBGBUS_SSPP0, 34, 3 },
+	{ DBGBUS_SSPP0, 34, 4 },
+	{ DBGBUS_SSPP0, 34, 5 },
+	{ DBGBUS_SSPP0, 34, 6 },
+	{ DBGBUS_SSPP0, 34, 7 },
+
+	{ DBGBUS_SSPP0, 35, 0 },
+	{ DBGBUS_SSPP0, 35, 1 },
+	{ DBGBUS_SSPP0, 35, 2 },
+	{ DBGBUS_SSPP0, 35, 3 },
+
+	/* dma 0 */
+	{ DBGBUS_SSPP0, 40, 0 },
+	{ DBGBUS_SSPP0, 40, 1 },
+	{ DBGBUS_SSPP0, 40, 2 },
+	{ DBGBUS_SSPP0, 40, 3 },
+	{ DBGBUS_SSPP0, 40, 4 },
+	{ DBGBUS_SSPP0, 40, 5 },
+	{ DBGBUS_SSPP0, 40, 6 },
+	{ DBGBUS_SSPP0, 40, 7 },
+
+	{ DBGBUS_SSPP0, 41, 0 },
+	{ DBGBUS_SSPP0, 41, 1 },
+	{ DBGBUS_SSPP0, 41, 2 },
+	{ DBGBUS_SSPP0, 41, 3 },
+	{ DBGBUS_SSPP0, 41, 4 },
+	{ DBGBUS_SSPP0, 41, 5 },
+	{ DBGBUS_SSPP0, 41, 6 },
+	{ DBGBUS_SSPP0, 41, 7 },
+
+	{ DBGBUS_SSPP0, 42, 0 },
+	{ DBGBUS_SSPP0, 42, 1 },
+	{ DBGBUS_SSPP0, 42, 2 },
+	{ DBGBUS_SSPP0, 42, 3 },
+	{ DBGBUS_SSPP0, 42, 4 },
+	{ DBGBUS_SSPP0, 42, 5 },
+	{ DBGBUS_SSPP0, 42, 6 },
+	{ DBGBUS_SSPP0, 42, 7 },
+
+	{ DBGBUS_SSPP0, 44, 0 },
+	{ DBGBUS_SSPP0, 44, 1 },
+	{ DBGBUS_SSPP0, 44, 2 },
+	{ DBGBUS_SSPP0, 44, 3 },
+	{ DBGBUS_SSPP0, 44, 4 },
+	{ DBGBUS_SSPP0, 44, 5 },
+	{ DBGBUS_SSPP0, 44, 6 },
+	{ DBGBUS_SSPP0, 44, 7 },
+
+	{ DBGBUS_SSPP0, 45, 0 },
+	{ DBGBUS_SSPP0, 45, 1 },
+	{ DBGBUS_SSPP0, 45, 2 },
+	{ DBGBUS_SSPP0, 45, 3 },
+	{ DBGBUS_SSPP0, 45, 4 },
+	{ DBGBUS_SSPP0, 45, 5 },
+	{ DBGBUS_SSPP0, 45, 6 },
+	{ DBGBUS_SSPP0, 45, 7 },
+
+	/* fetch sspp1 */
+	/* vig 1 */
+	{ DBGBUS_SSPP1, 0, 0 },
+	{ DBGBUS_SSPP1, 0, 1 },
+	{ DBGBUS_SSPP1, 0, 2 },
+	{ DBGBUS_SSPP1, 0, 3 },
+	{ DBGBUS_SSPP1, 0, 4 },
+	{ DBGBUS_SSPP1, 0, 5 },
+	{ DBGBUS_SSPP1, 0, 6 },
+	{ DBGBUS_SSPP1, 0, 7 },
+
+	{ DBGBUS_SSPP1, 1, 0 },
+	{ DBGBUS_SSPP1, 1, 1 },
+	{ DBGBUS_SSPP1, 1, 2 },
+	{ DBGBUS_SSPP1, 1, 3 },
+	{ DBGBUS_SSPP1, 1, 4 },
+	{ DBGBUS_SSPP1, 1, 5 },
+	{ DBGBUS_SSPP1, 1, 6 },
+	{ DBGBUS_SSPP1, 1, 7 },
+
+	{ DBGBUS_SSPP1, 2, 0 },
+	{ DBGBUS_SSPP1, 2, 1 },
+	{ DBGBUS_SSPP1, 2, 2 },
+	{ DBGBUS_SSPP1, 2, 3 },
+	{ DBGBUS_SSPP1, 2, 4 },
+	{ DBGBUS_SSPP1, 2, 5 },
+	{ DBGBUS_SSPP1, 2, 6 },
+	{ DBGBUS_SSPP1, 2, 7 },
+
+	{ DBGBUS_SSPP1, 4, 0 },
+	{ DBGBUS_SSPP1, 4, 1 },
+	{ DBGBUS_SSPP1, 4, 2 },
+	{ DBGBUS_SSPP1, 4, 3 },
+	{ DBGBUS_SSPP1, 4, 4 },
+	{ DBGBUS_SSPP1, 4, 5 },
+	{ DBGBUS_SSPP1, 4, 6 },
+	{ DBGBUS_SSPP1, 4, 7 },
+
+	{ DBGBUS_SSPP1, 5, 0 },
+	{ DBGBUS_SSPP1, 5, 1 },
+	{ DBGBUS_SSPP1, 5, 2 },
+	{ DBGBUS_SSPP1, 5, 3 },
+	{ DBGBUS_SSPP1, 5, 4 },
+	{ DBGBUS_SSPP1, 5, 5 },
+	{ DBGBUS_SSPP1, 5, 6 },
+	{ DBGBUS_SSPP1, 5, 7 },
+
+	/* vig 3 */
+	{ DBGBUS_SSPP1, 20, 0 },
+	{ DBGBUS_SSPP1, 20, 1 },
+	{ DBGBUS_SSPP1, 20, 2 },
+	{ DBGBUS_SSPP1, 20, 3 },
+	{ DBGBUS_SSPP1, 20, 4 },
+	{ DBGBUS_SSPP1, 20, 5 },
+	{ DBGBUS_SSPP1, 20, 6 },
+	{ DBGBUS_SSPP1, 20, 7 },
+
+	{ DBGBUS_SSPP1, 21, 0 },
+	{ DBGBUS_SSPP1, 21, 1 },
+	{ DBGBUS_SSPP1, 21, 2 },
+	{ DBGBUS_SSPP1, 21, 3 },
+	{ DBGBUS_SSPP1, 21, 4 },
+	{ DBGBUS_SSPP1, 21, 5 },
+	{ DBGBUS_SSPP1, 21, 6 },
+	{ DBGBUS_SSPP1, 21, 7 },
+
+	{ DBGBUS_SSPP1, 22, 0 },
+	{ DBGBUS_SSPP1, 22, 1 },
+	{ DBGBUS_SSPP1, 22, 2 },
+	{ DBGBUS_SSPP1, 22, 3 },
+	{ DBGBUS_SSPP1, 22, 4 },
+	{ DBGBUS_SSPP1, 22, 5 },
+	{ DBGBUS_SSPP1, 22, 6 },
+	{ DBGBUS_SSPP1, 22, 7 },
+
+	{ DBGBUS_SSPP1, 24, 0 },
+	{ DBGBUS_SSPP1, 24, 1 },
+	{ DBGBUS_SSPP1, 24, 2 },
+	{ DBGBUS_SSPP1, 24, 3 },
+	{ DBGBUS_SSPP1, 24, 4 },
+	{ DBGBUS_SSPP1, 24, 5 },
+	{ DBGBUS_SSPP1, 24, 6 },
+	{ DBGBUS_SSPP1, 24, 7 },
+
+	{ DBGBUS_SSPP1, 25, 0 },
+	{ DBGBUS_SSPP1, 25, 1 },
+	{ DBGBUS_SSPP1, 25, 2 },
+	{ DBGBUS_SSPP1, 25, 3 },
+	{ DBGBUS_SSPP1, 25, 4 },
+	{ DBGBUS_SSPP1, 25, 5 },
+	{ DBGBUS_SSPP1, 25, 6 },
+	{ DBGBUS_SSPP1, 25, 7 },
+
+	/* dma 3 */
+	{ DBGBUS_SSPP1, 30, 0 },
+	{ DBGBUS_SSPP1, 30, 1 },
+	{ DBGBUS_SSPP1, 30, 2 },
+	{ DBGBUS_SSPP1, 30, 3 },
+	{ DBGBUS_SSPP1, 30, 4 },
+	{ DBGBUS_SSPP1, 30, 5 },
+	{ DBGBUS_SSPP1, 30, 6 },
+	{ DBGBUS_SSPP1, 30, 7 },
+
+	{ DBGBUS_SSPP1, 31, 0 },
+	{ DBGBUS_SSPP1, 31, 1 },
+	{ DBGBUS_SSPP1, 31, 2 },
+	{ DBGBUS_SSPP1, 31, 3 },
+	{ DBGBUS_SSPP1, 31, 4 },
+	{ DBGBUS_SSPP1, 31, 5 },
+	{ DBGBUS_SSPP1, 31, 6 },
+	{ DBGBUS_SSPP1, 31, 7 },
+
+	{ DBGBUS_SSPP1, 32, 0 },
+	{ DBGBUS_SSPP1, 32, 1 },
+	{ DBGBUS_SSPP1, 32, 2 },
+	{ DBGBUS_SSPP1, 32, 3 },
+	{ DBGBUS_SSPP1, 32, 4 },
+	{ DBGBUS_SSPP1, 32, 5 },
+	{ DBGBUS_SSPP1, 32, 6 },
+	{ DBGBUS_SSPP1, 32, 7 },
+
+	{ DBGBUS_SSPP1, 33, 0 },
+	{ DBGBUS_SSPP1, 33, 1 },
+	{ DBGBUS_SSPP1, 33, 2 },
+	{ DBGBUS_SSPP1, 33, 3 },
+	{ DBGBUS_SSPP1, 33, 4 },
+	{ DBGBUS_SSPP1, 33, 5 },
+	{ DBGBUS_SSPP1, 33, 6 },
+	{ DBGBUS_SSPP1, 33, 7 },
+
+	{ DBGBUS_SSPP1, 34, 0 },
+	{ DBGBUS_SSPP1, 34, 1 },
+	{ DBGBUS_SSPP1, 34, 2 },
+	{ DBGBUS_SSPP1, 34, 3 },
+	{ DBGBUS_SSPP1, 34, 4 },
+	{ DBGBUS_SSPP1, 34, 5 },
+	{ DBGBUS_SSPP1, 34, 6 },
+	{ DBGBUS_SSPP1, 34, 7 },
+
+	{ DBGBUS_SSPP1, 35, 0 },
+	{ DBGBUS_SSPP1, 35, 1 },
+	{ DBGBUS_SSPP1, 35, 2 },
+
+	/* dma 1 */
+	{ DBGBUS_SSPP1, 40, 0 },
+	{ DBGBUS_SSPP1, 40, 1 },
+	{ DBGBUS_SSPP1, 40, 2 },
+	{ DBGBUS_SSPP1, 40, 3 },
+	{ DBGBUS_SSPP1, 40, 4 },
+	{ DBGBUS_SSPP1, 40, 5 },
+	{ DBGBUS_SSPP1, 40, 6 },
+	{ DBGBUS_SSPP1, 40, 7 },
+
+	{ DBGBUS_SSPP1, 41, 0 },
+	{ DBGBUS_SSPP1, 41, 1 },
+	{ DBGBUS_SSPP1, 41, 2 },
+	{ DBGBUS_SSPP1, 41, 3 },
+	{ DBGBUS_SSPP1, 41, 4 },
+	{ DBGBUS_SSPP1, 41, 5 },
+	{ DBGBUS_SSPP1, 41, 6 },
+	{ DBGBUS_SSPP1, 41, 7 },
+
+	{ DBGBUS_SSPP1, 42, 0 },
+	{ DBGBUS_SSPP1, 42, 1 },
+	{ DBGBUS_SSPP1, 42, 2 },
+	{ DBGBUS_SSPP1, 42, 3 },
+	{ DBGBUS_SSPP1, 42, 4 },
+	{ DBGBUS_SSPP1, 42, 5 },
+	{ DBGBUS_SSPP1, 42, 6 },
+	{ DBGBUS_SSPP1, 42, 7 },
+
+	{ DBGBUS_SSPP1, 44, 0 },
+	{ DBGBUS_SSPP1, 44, 1 },
+	{ DBGBUS_SSPP1, 44, 2 },
+	{ DBGBUS_SSPP1, 44, 3 },
+	{ DBGBUS_SSPP1, 44, 4 },
+	{ DBGBUS_SSPP1, 44, 5 },
+	{ DBGBUS_SSPP1, 44, 6 },
+	{ DBGBUS_SSPP1, 44, 7 },
+
+	{ DBGBUS_SSPP1, 45, 0 },
+	{ DBGBUS_SSPP1, 45, 1 },
+	{ DBGBUS_SSPP1, 45, 2 },
+	{ DBGBUS_SSPP1, 45, 3 },
+	{ DBGBUS_SSPP1, 45, 4 },
+	{ DBGBUS_SSPP1, 45, 5 },
+	{ DBGBUS_SSPP1, 45, 6 },
+	{ DBGBUS_SSPP1, 45, 7 },
+
+	/* dspp */
+	{ DBGBUS_DSPP, 13, 0 },
+	{ DBGBUS_DSPP, 19, 0 },
+	{ DBGBUS_DSPP, 14, 0 },
+	{ DBGBUS_DSPP, 14, 1 },
+	{ DBGBUS_DSPP, 14, 3 },
+	{ DBGBUS_DSPP, 20, 0 },
+	{ DBGBUS_DSPP, 20, 1 },
+	{ DBGBUS_DSPP, 20, 3 },
+
+	/* ppb_0 */
+	{ DBGBUS_DSPP, 31, 0, _dpu_debug_bus_ppb0_dump },
+	{ DBGBUS_DSPP, 33, 0, _dpu_debug_bus_ppb0_dump },
+	{ DBGBUS_DSPP, 35, 0, _dpu_debug_bus_ppb0_dump },
+	{ DBGBUS_DSPP, 42, 0, _dpu_debug_bus_ppb0_dump },
+
+	/* ppb_1 */
+	{ DBGBUS_DSPP, 32, 0, _dpu_debug_bus_ppb1_dump },
+	{ DBGBUS_DSPP, 34, 0, _dpu_debug_bus_ppb1_dump },
+	{ DBGBUS_DSPP, 36, 0, _dpu_debug_bus_ppb1_dump },
+	{ DBGBUS_DSPP, 43, 0, _dpu_debug_bus_ppb1_dump },
+
+	/* lm_lut */
+	{ DBGBUS_DSPP, 109, 0 },
+	{ DBGBUS_DSPP, 105, 0 },
+	{ DBGBUS_DSPP, 103, 0 },
+
+	/* crossbar */
+	{ DBGBUS_DSPP, 0, 0, _dpu_debug_bus_xbar_dump },
+
+	/* rotator */
+	{ DBGBUS_DSPP, 9, 0},
+
+	/* blend */
+	/* LM0 */
+	{ DBGBUS_DSPP, 63, 1},
+	{ DBGBUS_DSPP, 63, 2},
+	{ DBGBUS_DSPP, 63, 3},
+	{ DBGBUS_DSPP, 63, 4},
+	{ DBGBUS_DSPP, 63, 5},
+	{ DBGBUS_DSPP, 63, 6},
+	{ DBGBUS_DSPP, 63, 7, _dpu_debug_bus_lm_dump },
+
+	{ DBGBUS_DSPP, 64, 1},
+	{ DBGBUS_DSPP, 64, 2},
+	{ DBGBUS_DSPP, 64, 3},
+	{ DBGBUS_DSPP, 64, 4},
+	{ DBGBUS_DSPP, 64, 5},
+	{ DBGBUS_DSPP, 64, 6},
+	{ DBGBUS_DSPP, 64, 7, _dpu_debug_bus_lm_dump },
+
+	{ DBGBUS_DSPP, 65, 1},
+	{ DBGBUS_DSPP, 65, 2},
+	{ DBGBUS_DSPP, 65, 3},
+	{ DBGBUS_DSPP, 65, 4},
+	{ DBGBUS_DSPP, 65, 5},
+	{ DBGBUS_DSPP, 65, 6},
+	{ DBGBUS_DSPP, 65, 7, _dpu_debug_bus_lm_dump },
+
+	{ DBGBUS_DSPP, 66, 1},
+	{ DBGBUS_DSPP, 66, 2},
+	{ DBGBUS_DSPP, 66, 3},
+	{ DBGBUS_DSPP, 66, 4},
+	{ DBGBUS_DSPP, 66, 5},
+	{ DBGBUS_DSPP, 66, 6},
+	{ DBGBUS_DSPP, 66, 7, _dpu_debug_bus_lm_dump },
+
+	{ DBGBUS_DSPP, 67, 1},
+	{ DBGBUS_DSPP, 67, 2},
+	{ DBGBUS_DSPP, 67, 3},
+	{ DBGBUS_DSPP, 67, 4},
+	{ DBGBUS_DSPP, 67, 5},
+	{ DBGBUS_DSPP, 67, 6},
+	{ DBGBUS_DSPP, 67, 7, _dpu_debug_bus_lm_dump },
+
+	{ DBGBUS_DSPP, 68, 1},
+	{ DBGBUS_DSPP, 68, 2},
+	{ DBGBUS_DSPP, 68, 3},
+	{ DBGBUS_DSPP, 68, 4},
+	{ DBGBUS_DSPP, 68, 5},
+	{ DBGBUS_DSPP, 68, 6},
+	{ DBGBUS_DSPP, 68, 7, _dpu_debug_bus_lm_dump },
+
+	{ DBGBUS_DSPP, 69, 1},
+	{ DBGBUS_DSPP, 69, 2},
+	{ DBGBUS_DSPP, 69, 3},
+	{ DBGBUS_DSPP, 69, 4},
+	{ DBGBUS_DSPP, 69, 5},
+	{ DBGBUS_DSPP, 69, 6},
+	{ DBGBUS_DSPP, 69, 7, _dpu_debug_bus_lm_dump },
+
+	{ DBGBUS_DSPP, 84, 1},
+	{ DBGBUS_DSPP, 84, 2},
+	{ DBGBUS_DSPP, 84, 3},
+	{ DBGBUS_DSPP, 84, 4},
+	{ DBGBUS_DSPP, 84, 5},
+	{ DBGBUS_DSPP, 84, 6},
+	{ DBGBUS_DSPP, 84, 7, _dpu_debug_bus_lm_dump },
+
+
+	{ DBGBUS_DSPP, 85, 1},
+	{ DBGBUS_DSPP, 85, 2},
+	{ DBGBUS_DSPP, 85, 3},
+	{ DBGBUS_DSPP, 85, 4},
+	{ DBGBUS_DSPP, 85, 5},
+	{ DBGBUS_DSPP, 85, 6},
+	{ DBGBUS_DSPP, 85, 7, _dpu_debug_bus_lm_dump },
+
+
+	{ DBGBUS_DSPP, 86, 1},
+	{ DBGBUS_DSPP, 86, 2},
+	{ DBGBUS_DSPP, 86, 3},
+	{ DBGBUS_DSPP, 86, 4},
+	{ DBGBUS_DSPP, 86, 5},
+	{ DBGBUS_DSPP, 86, 6},
+	{ DBGBUS_DSPP, 86, 7, _dpu_debug_bus_lm_dump },
+
+
+	{ DBGBUS_DSPP, 87, 1},
+	{ DBGBUS_DSPP, 87, 2},
+	{ DBGBUS_DSPP, 87, 3},
+	{ DBGBUS_DSPP, 87, 4},
+	{ DBGBUS_DSPP, 87, 5},
+	{ DBGBUS_DSPP, 87, 6},
+	{ DBGBUS_DSPP, 87, 7, _dpu_debug_bus_lm_dump },
+
+	/* LM1 */
+	{ DBGBUS_DSPP, 70, 1},
+	{ DBGBUS_DSPP, 70, 2},
+	{ DBGBUS_DSPP, 70, 3},
+	{ DBGBUS_DSPP, 70, 4},
+	{ DBGBUS_DSPP, 70, 5},
+	{ DBGBUS_DSPP, 70, 6},
+	{ DBGBUS_DSPP, 70, 7, _dpu_debug_bus_lm_dump },
+
+	{ DBGBUS_DSPP, 71, 1},
+	{ DBGBUS_DSPP, 71, 2},
+	{ DBGBUS_DSPP, 71, 3},
+	{ DBGBUS_DSPP, 71, 4},
+	{ DBGBUS_DSPP, 71, 5},
+	{ DBGBUS_DSPP, 71, 6},
+	{ DBGBUS_DSPP, 71, 7, _dpu_debug_bus_lm_dump },
+
+	{ DBGBUS_DSPP, 72, 1},
+	{ DBGBUS_DSPP, 72, 2},
+	{ DBGBUS_DSPP, 72, 3},
+	{ DBGBUS_DSPP, 72, 4},
+	{ DBGBUS_DSPP, 72, 5},
+	{ DBGBUS_DSPP, 72, 6},
+	{ DBGBUS_DSPP, 72, 7, _dpu_debug_bus_lm_dump },
+
+	{ DBGBUS_DSPP, 73, 1},
+	{ DBGBUS_DSPP, 73, 2},
+	{ DBGBUS_DSPP, 73, 3},
+	{ DBGBUS_DSPP, 73, 4},
+	{ DBGBUS_DSPP, 73, 5},
+	{ DBGBUS_DSPP, 73, 6},
+	{ DBGBUS_DSPP, 73, 7, _dpu_debug_bus_lm_dump },
+
+	{ DBGBUS_DSPP, 74, 1},
+	{ DBGBUS_DSPP, 74, 2},
+	{ DBGBUS_DSPP, 74, 3},
+	{ DBGBUS_DSPP, 74, 4},
+	{ DBGBUS_DSPP, 74, 5},
+	{ DBGBUS_DSPP, 74, 6},
+	{ DBGBUS_DSPP, 74, 7, _dpu_debug_bus_lm_dump },
+
+	{ DBGBUS_DSPP, 75, 1},
+	{ DBGBUS_DSPP, 75, 2},
+	{ DBGBUS_DSPP, 75, 3},
+	{ DBGBUS_DSPP, 75, 4},
+	{ DBGBUS_DSPP, 75, 5},
+	{ DBGBUS_DSPP, 75, 6},
+	{ DBGBUS_DSPP, 75, 7, _dpu_debug_bus_lm_dump },
+
+	{ DBGBUS_DSPP, 76, 1},
+	{ DBGBUS_DSPP, 76, 2},
+	{ DBGBUS_DSPP, 76, 3},
+	{ DBGBUS_DSPP, 76, 4},
+	{ DBGBUS_DSPP, 76, 5},
+	{ DBGBUS_DSPP, 76, 6},
+	{ DBGBUS_DSPP, 76, 7, _dpu_debug_bus_lm_dump },
+
+	{ DBGBUS_DSPP, 88, 1},
+	{ DBGBUS_DSPP, 88, 2},
+	{ DBGBUS_DSPP, 88, 3},
+	{ DBGBUS_DSPP, 88, 4},
+	{ DBGBUS_DSPP, 88, 5},
+	{ DBGBUS_DSPP, 88, 6},
+	{ DBGBUS_DSPP, 88, 7, _dpu_debug_bus_lm_dump },
+
+	{ DBGBUS_DSPP, 89, 1},
+	{ DBGBUS_DSPP, 89, 2},
+	{ DBGBUS_DSPP, 89, 3},
+	{ DBGBUS_DSPP, 89, 4},
+	{ DBGBUS_DSPP, 89, 5},
+	{ DBGBUS_DSPP, 89, 6},
+	{ DBGBUS_DSPP, 89, 7, _dpu_debug_bus_lm_dump },
+
+	{ DBGBUS_DSPP, 90, 1},
+	{ DBGBUS_DSPP, 90, 2},
+	{ DBGBUS_DSPP, 90, 3},
+	{ DBGBUS_DSPP, 90, 4},
+	{ DBGBUS_DSPP, 90, 5},
+	{ DBGBUS_DSPP, 90, 6},
+	{ DBGBUS_DSPP, 90, 7, _dpu_debug_bus_lm_dump },
+
+	{ DBGBUS_DSPP, 91, 1},
+	{ DBGBUS_DSPP, 91, 2},
+	{ DBGBUS_DSPP, 91, 3},
+	{ DBGBUS_DSPP, 91, 4},
+	{ DBGBUS_DSPP, 91, 5},
+	{ DBGBUS_DSPP, 91, 6},
+	{ DBGBUS_DSPP, 91, 7, _dpu_debug_bus_lm_dump },
+
+	/* LM2 */
+	{ DBGBUS_DSPP, 77, 0},
+	{ DBGBUS_DSPP, 77, 1},
+	{ DBGBUS_DSPP, 77, 2},
+	{ DBGBUS_DSPP, 77, 3},
+	{ DBGBUS_DSPP, 77, 4},
+	{ DBGBUS_DSPP, 77, 5},
+	{ DBGBUS_DSPP, 77, 6},
+	{ DBGBUS_DSPP, 77, 7, _dpu_debug_bus_lm_dump },
+
+	{ DBGBUS_DSPP, 78, 0},
+	{ DBGBUS_DSPP, 78, 1},
+	{ DBGBUS_DSPP, 78, 2},
+	{ DBGBUS_DSPP, 78, 3},
+	{ DBGBUS_DSPP, 78, 4},
+	{ DBGBUS_DSPP, 78, 5},
+	{ DBGBUS_DSPP, 78, 6},
+	{ DBGBUS_DSPP, 78, 7, _dpu_debug_bus_lm_dump },
+
+	{ DBGBUS_DSPP, 79, 0},
+	{ DBGBUS_DSPP, 79, 1},
+	{ DBGBUS_DSPP, 79, 2},
+	{ DBGBUS_DSPP, 79, 3},
+	{ DBGBUS_DSPP, 79, 4},
+	{ DBGBUS_DSPP, 79, 5},
+	{ DBGBUS_DSPP, 79, 6},
+	{ DBGBUS_DSPP, 79, 7, _dpu_debug_bus_lm_dump },
+
+	{ DBGBUS_DSPP, 80, 0},
+	{ DBGBUS_DSPP, 80, 1},
+	{ DBGBUS_DSPP, 80, 2},
+	{ DBGBUS_DSPP, 80, 3},
+	{ DBGBUS_DSPP, 80, 4},
+	{ DBGBUS_DSPP, 80, 5},
+	{ DBGBUS_DSPP, 80, 6},
+	{ DBGBUS_DSPP, 80, 7, _dpu_debug_bus_lm_dump },
+
+	{ DBGBUS_DSPP, 81, 0},
+	{ DBGBUS_DSPP, 81, 1},
+	{ DBGBUS_DSPP, 81, 2},
+	{ DBGBUS_DSPP, 81, 3},
+	{ DBGBUS_DSPP, 81, 4},
+	{ DBGBUS_DSPP, 81, 5},
+	{ DBGBUS_DSPP, 81, 6},
+	{ DBGBUS_DSPP, 81, 7, _dpu_debug_bus_lm_dump },
+
+	{ DBGBUS_DSPP, 82, 0},
+	{ DBGBUS_DSPP, 82, 1},
+	{ DBGBUS_DSPP, 82, 2},
+	{ DBGBUS_DSPP, 82, 3},
+	{ DBGBUS_DSPP, 82, 4},
+	{ DBGBUS_DSPP, 82, 5},
+	{ DBGBUS_DSPP, 82, 6},
+	{ DBGBUS_DSPP, 82, 7, _dpu_debug_bus_lm_dump },
+
+	{ DBGBUS_DSPP, 83, 0},
+	{ DBGBUS_DSPP, 83, 1},
+	{ DBGBUS_DSPP, 83, 2},
+	{ DBGBUS_DSPP, 83, 3},
+	{ DBGBUS_DSPP, 83, 4},
+	{ DBGBUS_DSPP, 83, 5},
+	{ DBGBUS_DSPP, 83, 6},
+	{ DBGBUS_DSPP, 83, 7, _dpu_debug_bus_lm_dump },
+
+	{ DBGBUS_DSPP, 92, 1},
+	{ DBGBUS_DSPP, 92, 2},
+	{ DBGBUS_DSPP, 92, 3},
+	{ DBGBUS_DSPP, 92, 4},
+	{ DBGBUS_DSPP, 92, 5},
+	{ DBGBUS_DSPP, 92, 6},
+	{ DBGBUS_DSPP, 92, 7, _dpu_debug_bus_lm_dump },
+
+	{ DBGBUS_DSPP, 93, 1},
+	{ DBGBUS_DSPP, 93, 2},
+	{ DBGBUS_DSPP, 93, 3},
+	{ DBGBUS_DSPP, 93, 4},
+	{ DBGBUS_DSPP, 93, 5},
+	{ DBGBUS_DSPP, 93, 6},
+	{ DBGBUS_DSPP, 93, 7, _dpu_debug_bus_lm_dump },
+
+	{ DBGBUS_DSPP, 94, 1},
+	{ DBGBUS_DSPP, 94, 2},
+	{ DBGBUS_DSPP, 94, 3},
+	{ DBGBUS_DSPP, 94, 4},
+	{ DBGBUS_DSPP, 94, 5},
+	{ DBGBUS_DSPP, 94, 6},
+	{ DBGBUS_DSPP, 94, 7, _dpu_debug_bus_lm_dump },
+
+	{ DBGBUS_DSPP, 95, 1},
+	{ DBGBUS_DSPP, 95, 2},
+	{ DBGBUS_DSPP, 95, 3},
+	{ DBGBUS_DSPP, 95, 4},
+	{ DBGBUS_DSPP, 95, 5},
+	{ DBGBUS_DSPP, 95, 6},
+	{ DBGBUS_DSPP, 95, 7, _dpu_debug_bus_lm_dump },
+
+	/* LM5 */
+	{ DBGBUS_DSPP, 110, 1},
+	{ DBGBUS_DSPP, 110, 2},
+	{ DBGBUS_DSPP, 110, 3},
+	{ DBGBUS_DSPP, 110, 4},
+	{ DBGBUS_DSPP, 110, 5},
+	{ DBGBUS_DSPP, 110, 6},
+	{ DBGBUS_DSPP, 110, 7, _dpu_debug_bus_lm_dump },
+
+	{ DBGBUS_DSPP, 111, 1},
+	{ DBGBUS_DSPP, 111, 2},
+	{ DBGBUS_DSPP, 111, 3},
+	{ DBGBUS_DSPP, 111, 4},
+	{ DBGBUS_DSPP, 111, 5},
+	{ DBGBUS_DSPP, 111, 6},
+	{ DBGBUS_DSPP, 111, 7, _dpu_debug_bus_lm_dump },
+
+	{ DBGBUS_DSPP, 112, 1},
+	{ DBGBUS_DSPP, 112, 2},
+	{ DBGBUS_DSPP, 112, 3},
+	{ DBGBUS_DSPP, 112, 4},
+	{ DBGBUS_DSPP, 112, 5},
+	{ DBGBUS_DSPP, 112, 6},
+	{ DBGBUS_DSPP, 112, 7, _dpu_debug_bus_lm_dump },
+
+	{ DBGBUS_DSPP, 113, 1},
+	{ DBGBUS_DSPP, 113, 2},
+	{ DBGBUS_DSPP, 113, 3},
+	{ DBGBUS_DSPP, 113, 4},
+	{ DBGBUS_DSPP, 113, 5},
+	{ DBGBUS_DSPP, 113, 6},
+	{ DBGBUS_DSPP, 113, 7, _dpu_debug_bus_lm_dump },
+
+	{ DBGBUS_DSPP, 114, 1},
+	{ DBGBUS_DSPP, 114, 2},
+	{ DBGBUS_DSPP, 114, 3},
+	{ DBGBUS_DSPP, 114, 4},
+	{ DBGBUS_DSPP, 114, 5},
+	{ DBGBUS_DSPP, 114, 6},
+	{ DBGBUS_DSPP, 114, 7, _dpu_debug_bus_lm_dump },
+
+	{ DBGBUS_DSPP, 115, 1},
+	{ DBGBUS_DSPP, 115, 2},
+	{ DBGBUS_DSPP, 115, 3},
+	{ DBGBUS_DSPP, 115, 4},
+	{ DBGBUS_DSPP, 115, 5},
+	{ DBGBUS_DSPP, 115, 6},
+	{ DBGBUS_DSPP, 115, 7, _dpu_debug_bus_lm_dump },
+
+	{ DBGBUS_DSPP, 116, 1},
+	{ DBGBUS_DSPP, 116, 2},
+	{ DBGBUS_DSPP, 116, 3},
+	{ DBGBUS_DSPP, 116, 4},
+	{ DBGBUS_DSPP, 116, 5},
+	{ DBGBUS_DSPP, 116, 6},
+	{ DBGBUS_DSPP, 116, 7, _dpu_debug_bus_lm_dump },
+
+	{ DBGBUS_DSPP, 117, 1},
+	{ DBGBUS_DSPP, 117, 2},
+	{ DBGBUS_DSPP, 117, 3},
+	{ DBGBUS_DSPP, 117, 4},
+	{ DBGBUS_DSPP, 117, 5},
+	{ DBGBUS_DSPP, 117, 6},
+	{ DBGBUS_DSPP, 117, 7, _dpu_debug_bus_lm_dump },
+
+	{ DBGBUS_DSPP, 118, 1},
+	{ DBGBUS_DSPP, 118, 2},
+	{ DBGBUS_DSPP, 118, 3},
+	{ DBGBUS_DSPP, 118, 4},
+	{ DBGBUS_DSPP, 118, 5},
+	{ DBGBUS_DSPP, 118, 6},
+	{ DBGBUS_DSPP, 118, 7, _dpu_debug_bus_lm_dump },
+
+	{ DBGBUS_DSPP, 119, 1},
+	{ DBGBUS_DSPP, 119, 2},
+	{ DBGBUS_DSPP, 119, 3},
+	{ DBGBUS_DSPP, 119, 4},
+	{ DBGBUS_DSPP, 119, 5},
+	{ DBGBUS_DSPP, 119, 6},
+	{ DBGBUS_DSPP, 119, 7, _dpu_debug_bus_lm_dump },
+
+	{ DBGBUS_DSPP, 120, 1},
+	{ DBGBUS_DSPP, 120, 2},
+	{ DBGBUS_DSPP, 120, 3},
+	{ DBGBUS_DSPP, 120, 4},
+	{ DBGBUS_DSPP, 120, 5},
+	{ DBGBUS_DSPP, 120, 6},
+	{ DBGBUS_DSPP, 120, 7, _dpu_debug_bus_lm_dump },
+
+	/* csc */
+	{ DBGBUS_SSPP0, 7, 0},
+	{ DBGBUS_SSPP0, 7, 1},
+	{ DBGBUS_SSPP0, 27, 0},
+	{ DBGBUS_SSPP0, 27, 1},
+	{ DBGBUS_SSPP1, 7, 0},
+	{ DBGBUS_SSPP1, 7, 1},
+	{ DBGBUS_SSPP1, 27, 0},
+	{ DBGBUS_SSPP1, 27, 1},
+
+	/* pcc */
+	{ DBGBUS_SSPP0, 3,  3},
+	{ DBGBUS_SSPP0, 23, 3},
+	{ DBGBUS_SSPP0, 33, 3},
+	{ DBGBUS_SSPP0, 43, 3},
+	{ DBGBUS_SSPP1, 3,  3},
+	{ DBGBUS_SSPP1, 23, 3},
+	{ DBGBUS_SSPP1, 33, 3},
+	{ DBGBUS_SSPP1, 43, 3},
+
+	/* spa */
+	{ DBGBUS_SSPP0, 8,  0},
+	{ DBGBUS_SSPP0, 28, 0},
+	{ DBGBUS_SSPP1, 8,  0},
+	{ DBGBUS_SSPP1, 28, 0},
+	{ DBGBUS_DSPP, 13, 0},
+	{ DBGBUS_DSPP, 19, 0},
+
+	/* igc */
+	{ DBGBUS_SSPP0, 17, 0},
+	{ DBGBUS_SSPP0, 17, 1},
+	{ DBGBUS_SSPP0, 17, 3},
+	{ DBGBUS_SSPP0, 37, 0},
+	{ DBGBUS_SSPP0, 37, 1},
+	{ DBGBUS_SSPP0, 37, 3},
+	{ DBGBUS_SSPP0, 46, 0},
+	{ DBGBUS_SSPP0, 46, 1},
+	{ DBGBUS_SSPP0, 46, 3},
+
+	{ DBGBUS_SSPP1, 17, 0},
+	{ DBGBUS_SSPP1, 17, 1},
+	{ DBGBUS_SSPP1, 17, 3},
+	{ DBGBUS_SSPP1, 37, 0},
+	{ DBGBUS_SSPP1, 37, 1},
+	{ DBGBUS_SSPP1, 37, 3},
+	{ DBGBUS_SSPP1, 46, 0},
+	{ DBGBUS_SSPP1, 46, 1},
+	{ DBGBUS_SSPP1, 46, 3},
+
+	{ DBGBUS_DSPP, 14, 0},
+	{ DBGBUS_DSPP, 14, 1},
+	{ DBGBUS_DSPP, 14, 3},
+	{ DBGBUS_DSPP, 20, 0},
+	{ DBGBUS_DSPP, 20, 1},
+	{ DBGBUS_DSPP, 20, 3},
+
+	/* intf0-3 */
+	{ DBGBUS_PERIPH, 0, 0},
+	{ DBGBUS_PERIPH, 1, 0},
+	{ DBGBUS_PERIPH, 2, 0},
+	{ DBGBUS_PERIPH, 3, 0},
+
+	/* te counter wrapper */
+	{ DBGBUS_PERIPH, 60, 0},
+
+	/* dsc0 */
+	{ DBGBUS_PERIPH, 47, 0},
+	{ DBGBUS_PERIPH, 47, 1},
+	{ DBGBUS_PERIPH, 47, 2},
+	{ DBGBUS_PERIPH, 47, 3},
+	{ DBGBUS_PERIPH, 47, 4},
+	{ DBGBUS_PERIPH, 47, 5},
+	{ DBGBUS_PERIPH, 47, 6},
+	{ DBGBUS_PERIPH, 47, 7},
+
+	/* dsc1 */
+	{ DBGBUS_PERIPH, 48, 0},
+	{ DBGBUS_PERIPH, 48, 1},
+	{ DBGBUS_PERIPH, 48, 2},
+	{ DBGBUS_PERIPH, 48, 3},
+	{ DBGBUS_PERIPH, 48, 4},
+	{ DBGBUS_PERIPH, 48, 5},
+	{ DBGBUS_PERIPH, 48, 6},
+	{ DBGBUS_PERIPH, 48, 7},
+
+	/* dsc2 */
+	{ DBGBUS_PERIPH, 51, 0},
+	{ DBGBUS_PERIPH, 51, 1},
+	{ DBGBUS_PERIPH, 51, 2},
+	{ DBGBUS_PERIPH, 51, 3},
+	{ DBGBUS_PERIPH, 51, 4},
+	{ DBGBUS_PERIPH, 51, 5},
+	{ DBGBUS_PERIPH, 51, 6},
+	{ DBGBUS_PERIPH, 51, 7},
+
+	/* dsc3 */
+	{ DBGBUS_PERIPH, 52, 0},
+	{ DBGBUS_PERIPH, 52, 1},
+	{ DBGBUS_PERIPH, 52, 2},
+	{ DBGBUS_PERIPH, 52, 3},
+	{ DBGBUS_PERIPH, 52, 4},
+	{ DBGBUS_PERIPH, 52, 5},
+	{ DBGBUS_PERIPH, 52, 6},
+	{ DBGBUS_PERIPH, 52, 7},
+
+	/* tear-check */
+	{ DBGBUS_PERIPH, 63, 0 },
+	{ DBGBUS_PERIPH, 64, 0 },
+	{ DBGBUS_PERIPH, 65, 0 },
+	{ DBGBUS_PERIPH, 73, 0 },
+	{ DBGBUS_PERIPH, 74, 0 },
+
+	/* cdwn */
+	{ DBGBUS_PERIPH, 80, 0},
+	{ DBGBUS_PERIPH, 80, 1},
+	{ DBGBUS_PERIPH, 80, 2},
+
+	{ DBGBUS_PERIPH, 81, 0},
+	{ DBGBUS_PERIPH, 81, 1},
+	{ DBGBUS_PERIPH, 81, 2},
+
+	{ DBGBUS_PERIPH, 82, 0},
+	{ DBGBUS_PERIPH, 82, 1},
+	{ DBGBUS_PERIPH, 82, 2},
+	{ DBGBUS_PERIPH, 82, 3},
+	{ DBGBUS_PERIPH, 82, 4},
+	{ DBGBUS_PERIPH, 82, 5},
+	{ DBGBUS_PERIPH, 82, 6},
+	{ DBGBUS_PERIPH, 82, 7},
+
+	/* hdmi */
+	{ DBGBUS_PERIPH, 68, 0},
+	{ DBGBUS_PERIPH, 68, 1},
+	{ DBGBUS_PERIPH, 68, 2},
+	{ DBGBUS_PERIPH, 68, 3},
+	{ DBGBUS_PERIPH, 68, 4},
+	{ DBGBUS_PERIPH, 68, 5},
+
+	/* edp */
+	{ DBGBUS_PERIPH, 69, 0},
+	{ DBGBUS_PERIPH, 69, 1},
+	{ DBGBUS_PERIPH, 69, 2},
+	{ DBGBUS_PERIPH, 69, 3},
+	{ DBGBUS_PERIPH, 69, 4},
+	{ DBGBUS_PERIPH, 69, 5},
+
+	/* dsi0 */
+	{ DBGBUS_PERIPH, 70, 0},
+	{ DBGBUS_PERIPH, 70, 1},
+	{ DBGBUS_PERIPH, 70, 2},
+	{ DBGBUS_PERIPH, 70, 3},
+	{ DBGBUS_PERIPH, 70, 4},
+	{ DBGBUS_PERIPH, 70, 5},
+
+	/* dsi1 */
+	{ DBGBUS_PERIPH, 71, 0},
+	{ DBGBUS_PERIPH, 71, 1},
+	{ DBGBUS_PERIPH, 71, 2},
+	{ DBGBUS_PERIPH, 71, 3},
+	{ DBGBUS_PERIPH, 71, 4},
+	{ DBGBUS_PERIPH, 71, 5},
+};
+
+static struct vbif_debug_bus_entry vbif_dbg_bus_msm8998[] = {
+	{0x214, 0x21c, 16, 2, 0x0, 0xd},     /* arb clients */
+	{0x214, 0x21c, 16, 2, 0x80, 0xc0},   /* arb clients */
+	{0x214, 0x21c, 16, 2, 0x100, 0x140}, /* arb clients */
+	{0x214, 0x21c, 0, 16, 0x0, 0xf},     /* xin blocks - axi side */
+	{0x214, 0x21c, 0, 16, 0x80, 0xa4},   /* xin blocks - axi side */
+	{0x214, 0x21c, 0, 15, 0x100, 0x124}, /* xin blocks - axi side */
+	{0x21c, 0x214, 0, 14, 0, 0xc}, /* xin blocks - clock side */
+};
+
+/**
+ * _dpu_dbg_enable_power - use callback to turn power on for hw register access
+ * @enable: whether to turn power on or off
+ */
+static inline void _dpu_dbg_enable_power(int enable)
+{
+	if (enable)
+		pm_runtime_get_sync(dpu_dbg_base.dev);
+	else
+		pm_runtime_put_sync(dpu_dbg_base.dev);
+}
+
+static void _dpu_dbg_dump_dpu_dbg_bus(struct dpu_dbg_dpu_debug_bus *bus)
+{
+	bool in_log, in_mem;
+	u32 **dump_mem = NULL;
+	u32 *dump_addr = NULL;
+	u32 status = 0;
+	struct dpu_debug_bus_entry *head;
+	phys_addr_t phys = 0;
+	int list_size;
+	int i;
+	u32 offset;
+	void __iomem *mem_base = NULL;
+	struct dpu_dbg_reg_base *reg_base;
+
+	if (!bus || !bus->cmn.entries_size)
+		return;
+
+	list_for_each_entry(reg_base, &dpu_dbg_base.reg_base_list,
+			reg_base_head)
+		if (strlen(reg_base->name) &&
+			!strcmp(reg_base->name, bus->cmn.name))
+			mem_base = reg_base->base + bus->top_blk_off;
+
+	if (!mem_base) {
+		pr_err("unable to find mem_base for %s\n", bus->cmn.name);
+		return;
+	}
+
+	dump_mem = &bus->cmn.dumped_content;
+
+	/* will keep in memory 4 entries of 4 bytes each */
+	list_size = (bus->cmn.entries_size * 4 * 4);
+
+	in_log = (bus->cmn.enable_mask & DPU_DBG_DUMP_IN_LOG);
+	in_mem = (bus->cmn.enable_mask & DPU_DBG_DUMP_IN_MEM);
+
+	if (!in_log && !in_mem)
+		return;
+
+	dev_info(dpu_dbg_base.dev, "======== start %s dump =========\n",
+			bus->cmn.name);
+
+	if (in_mem) {
+		if (!(*dump_mem))
+			*dump_mem = dma_alloc_coherent(dpu_dbg_base.dev,
+				list_size, &phys, GFP_KERNEL);
+
+		if (*dump_mem) {
+			dump_addr = *dump_mem;
+			dev_info(dpu_dbg_base.dev,
+				"%s: start_addr:0x%pK len:0x%x\n",
+				__func__, dump_addr, list_size);
+		} else {
+			in_mem = false;
+			pr_err("dump_mem: allocation fails\n");
+		}
+	}
+
+	_dpu_dbg_enable_power(true);
+	for (i = 0; i < bus->cmn.entries_size; i++) {
+		head = bus->entries + i;
+		writel_relaxed(TEST_MASK(head->block_id, head->test_id),
+				mem_base + head->wr_addr);
+		wmb(); /* make sure test bits were written */
+
+		if (bus->cmn.flags & DBGBUS_FLAGS_DSPP) {
+			offset = DBGBUS_DSPP_STATUS;
+			/* keep DSPP test point enabled */
+			if (head->wr_addr != DBGBUS_DSPP)
+				writel_relaxed(0xF, mem_base + DBGBUS_DSPP);
+		} else {
+			offset = head->wr_addr + 0x4;
+		}
+
+		status = readl_relaxed(mem_base + offset);
+
+		if (in_log)
+			dev_info(dpu_dbg_base.dev,
+					"waddr=0x%x blk=%d tst=%d val=0x%x\n",
+					head->wr_addr, head->block_id,
+					head->test_id, status);
+
+		if (dump_addr && in_mem) {
+			dump_addr[i*4]     = head->wr_addr;
+			dump_addr[i*4 + 1] = head->block_id;
+			dump_addr[i*4 + 2] = head->test_id;
+			dump_addr[i*4 + 3] = status;
+		}
+
+		if (head->analyzer)
+			head->analyzer(mem_base, head, status);
+
+		/* Disable debug bus once we are done */
+		writel_relaxed(0, mem_base + head->wr_addr);
+		if (bus->cmn.flags & DBGBUS_FLAGS_DSPP &&
+						head->wr_addr != DBGBUS_DSPP)
+			writel_relaxed(0x0, mem_base + DBGBUS_DSPP);
+	}
+	_dpu_dbg_enable_power(false);
+
+	dev_info(dpu_dbg_base.dev, "======== end %s dump =========\n",
+			bus->cmn.name);
+}
+
+static void _dpu_dbg_dump_vbif_debug_bus_entry(
+		struct vbif_debug_bus_entry *head, void __iomem *mem_base,
+		u32 *dump_addr, bool in_log)
+{
+	int i, j;
+	u32 val;
+
+	if (!dump_addr && !in_log)
+		return;
+
+	for (i = 0; i < head->block_cnt; i++) {
+		writel_relaxed(1 << (i + head->bit_offset),
+				mem_base + head->block_bus_addr);
+		/* make sure that current bus blcok enable */
+		wmb();
+		for (j = head->test_pnt_start; j < head->test_pnt_cnt; j++) {
+			writel_relaxed(j, mem_base + head->block_bus_addr + 4);
+			/* make sure that test point is enabled */
+			wmb();
+			val = readl_relaxed(mem_base + MMSS_VBIF_TEST_BUS_OUT);
+			if (dump_addr) {
+				*dump_addr++ = head->block_bus_addr;
+				*dump_addr++ = i;
+				*dump_addr++ = j;
+				*dump_addr++ = val;
+			}
+			if (in_log)
+				dev_info(dpu_dbg_base.dev,
+					"testpoint:%x arb/xin id=%d index=%d val=0x%x\n",
+					head->block_bus_addr, i, j, val);
+		}
+	}
+}
+
+static void _dpu_dbg_dump_vbif_dbg_bus(struct dpu_dbg_vbif_debug_bus *bus)
+{
+	bool in_log, in_mem;
+	u32 **dump_mem = NULL;
+	u32 *dump_addr = NULL;
+	u32 value, d0, d1;
+	unsigned long reg, reg1, reg2;
+	struct vbif_debug_bus_entry *head;
+	phys_addr_t phys = 0;
+	int i, list_size = 0;
+	void __iomem *mem_base = NULL;
+	struct vbif_debug_bus_entry *dbg_bus;
+	u32 bus_size;
+	struct dpu_dbg_reg_base *reg_base;
+
+	if (!bus || !bus->cmn.entries_size)
+		return;
+
+	list_for_each_entry(reg_base, &dpu_dbg_base.reg_base_list,
+			reg_base_head)
+		if (strlen(reg_base->name) &&
+			!strcmp(reg_base->name, bus->cmn.name))
+			mem_base = reg_base->base;
+
+	if (!mem_base) {
+		pr_err("unable to find mem_base for %s\n", bus->cmn.name);
+		return;
+	}
+
+	dbg_bus = bus->entries;
+	bus_size = bus->cmn.entries_size;
+	list_size = bus->cmn.entries_size;
+	dump_mem = &bus->cmn.dumped_content;
+
+	dev_info(dpu_dbg_base.dev, "======== start %s dump =========\n",
+			bus->cmn.name);
+
+	if (!dump_mem || !dbg_bus || !bus_size || !list_size)
+		return;
+
+	/* allocate memory for each test point */
+	for (i = 0; i < bus_size; i++) {
+		head = dbg_bus + i;
+		list_size += (head->block_cnt * head->test_pnt_cnt);
+	}
+
+	/* 4 bytes * 4 entries for each test point*/
+	list_size *= 16;
+
+	in_log = (bus->cmn.enable_mask & DPU_DBG_DUMP_IN_LOG);
+	in_mem = (bus->cmn.enable_mask & DPU_DBG_DUMP_IN_MEM);
+
+	if (!in_log && !in_mem)
+		return;
+
+	if (in_mem) {
+		if (!(*dump_mem))
+			*dump_mem = dma_alloc_coherent(dpu_dbg_base.dev,
+				list_size, &phys, GFP_KERNEL);
+
+		if (*dump_mem) {
+			dump_addr = *dump_mem;
+			dev_info(dpu_dbg_base.dev,
+				"%s: start_addr:0x%pK len:0x%x\n",
+				__func__, dump_addr, list_size);
+		} else {
+			in_mem = false;
+			pr_err("dump_mem: allocation fails\n");
+		}
+	}
+
+	_dpu_dbg_enable_power(true);
+
+	value = readl_relaxed(mem_base + MMSS_VBIF_CLKON);
+	writel_relaxed(value | BIT(1), mem_base + MMSS_VBIF_CLKON);
+
+	/* make sure that vbif core is on */
+	wmb();
+
+	/**
+	 * Extract VBIF error info based on XIN halt and error status.
+	 * If the XIN client is not in HALT state, or an error is detected,
+	 * then retrieve the VBIF error info for it.
+	 */
+	reg = readl_relaxed(mem_base + MMSS_VBIF_XIN_HALT_CTRL1);
+	reg1 = readl_relaxed(mem_base + MMSS_VBIF_PND_ERR);
+	reg2 = readl_relaxed(mem_base + MMSS_VBIF_SRC_ERR);
+	dev_err(dpu_dbg_base.dev,
+			"XIN HALT:0x%lX, PND ERR:0x%lX, SRC ERR:0x%lX\n",
+			reg, reg1, reg2);
+	reg >>= 16;
+	reg &= ~(reg1 | reg2);
+	for (i = 0; i < MMSS_VBIF_CLIENT_NUM; i++) {
+		if (!test_bit(0, &reg)) {
+			writel_relaxed(i, mem_base + MMSS_VBIF_ERR_INFO);
+			/* make sure reg write goes through */
+			wmb();
+
+			d0 = readl_relaxed(mem_base + MMSS_VBIF_ERR_INFO);
+			d1 = readl_relaxed(mem_base + MMSS_VBIF_ERR_INFO_1);
+
+			dev_err(dpu_dbg_base.dev,
+					"Client:%d, errinfo=0x%X, errinfo1=0x%X\n",
+					i, d0, d1);
+		}
+		reg >>= 1;
+	}
+
+	for (i = 0; i < bus_size; i++) {
+		head = dbg_bus + i;
+
+		writel_relaxed(0, mem_base + head->disable_bus_addr);
+		writel_relaxed(BIT(0), mem_base + MMSS_VBIF_TEST_BUS_OUT_CTRL);
+		/* make sure that other bus is off */
+		wmb();
+
+		_dpu_dbg_dump_vbif_debug_bus_entry(head, mem_base, dump_addr,
+				in_log);
+		if (dump_addr)
+			dump_addr += (head->block_cnt * head->test_pnt_cnt * 4);
+	}
+
+	_dpu_dbg_enable_power(false);
+
+	dev_info(dpu_dbg_base.dev, "======== end %s dump =========\n",
+			bus->cmn.name);
+}
+
+/**
+ * _dpu_dump_array - dump array of register bases
+ * @name: string indicating origin of dump
+ * @dump_dbgbus_dpu: whether to dump the dpu debug bus
+ * @dump_dbgbus_vbif_rt: whether to dump the vbif rt debug bus
+ */
+static void _dpu_dump_array(const char *name, bool dump_dbgbus_dpu,
+			    bool dump_dbgbus_vbif_rt)
+{
+	if (dump_dbgbus_dpu)
+		_dpu_dbg_dump_dpu_dbg_bus(&dpu_dbg_base.dbgbus_dpu);
+
+	if (dump_dbgbus_vbif_rt)
+		_dpu_dbg_dump_vbif_dbg_bus(&dpu_dbg_base.dbgbus_vbif_rt);
+}
+
+/**
+ * _dpu_dump_work - deferred dump work function
+ * @work: work structure
+ */
+static void _dpu_dump_work(struct work_struct *work)
+{
+	_dpu_dump_array("dpudump_workitem",
+		dpu_dbg_base.dbgbus_dpu.cmn.include_in_deferred_work,
+		dpu_dbg_base.dbgbus_vbif_rt.cmn.include_in_deferred_work);
+}
+
+void dpu_dbg_dump(bool queue_work, const char *name, bool dump_dbgbus_dpu,
+		  bool dump_dbgbus_vbif_rt)
+{
+	if (queue_work && work_pending(&dpu_dbg_base.dump_work))
+		return;
+
+	if (!queue_work) {
+		_dpu_dump_array(name, dump_dbgbus_dpu, dump_dbgbus_vbif_rt);
+		return;
+	}
+
+	/* schedule work to dump later */
+	dpu_dbg_base.dbgbus_dpu.cmn.include_in_deferred_work = dump_dbgbus_dpu;
+	dpu_dbg_base.dbgbus_vbif_rt.cmn.include_in_deferred_work =
+			dump_dbgbus_vbif_rt;
+	schedule_work(&dpu_dbg_base.dump_work);
+}
+
+/*
+ * dpu_dbg_debugfs_open - debugfs open handler for debug dump
+ * @inode: debugfs inode
+ * @file: file handle
+ */
+static int dpu_dbg_debugfs_open(struct inode *inode, struct file *file)
+{
+	/* non-seekable */
+	file->f_mode &= ~(FMODE_LSEEK | FMODE_PREAD | FMODE_PWRITE);
+	file->private_data = inode->i_private;
+	return 0;
+}
+
+/**
+ * dpu_dbg_dump_write - debugfs write handler for debug dump
+ * @file: file handler
+ * @user_buf: user buffer content from debugfs
+ * @count: size of user buffer
+ * @ppos: position offset of user buffer
+ */
+static ssize_t dpu_dbg_dump_write(struct file *file,
+	const char __user *user_buf, size_t count, loff_t *ppos)
+{
+	_dpu_dump_array("dump_debugfs", true, true);
+	return count;
+}
+
+static const struct file_operations dpu_dbg_dump_fops = {
+	.open = dpu_dbg_debugfs_open,
+	.write = dpu_dbg_dump_write,
+};
+
+int dpu_dbg_debugfs_register(struct dentry *debugfs_root)
+{
+	static struct dpu_dbg_base *dbg = &dpu_dbg_base;
+	char debug_name[80] = "";
+
+	if (!debugfs_root)
+		return -EINVAL;
+
+	debugfs_create_file("dump", 0600, debugfs_root, NULL,
+			&dpu_dbg_dump_fops);
+
+	if (dbg->dbgbus_dpu.entries) {
+		dbg->dbgbus_dpu.cmn.name = DBGBUS_NAME_DPU;
+		snprintf(debug_name, sizeof(debug_name), "%s_dbgbus",
+				dbg->dbgbus_dpu.cmn.name);
+		dbg->dbgbus_dpu.cmn.enable_mask = DEFAULT_DBGBUS_DPU;
+		debugfs_create_u32(debug_name, 0600, debugfs_root,
+				&dbg->dbgbus_dpu.cmn.enable_mask);
+	}
+
+	if (dbg->dbgbus_vbif_rt.entries) {
+		dbg->dbgbus_vbif_rt.cmn.name = DBGBUS_NAME_VBIF_RT;
+		snprintf(debug_name, sizeof(debug_name), "%s_dbgbus",
+				dbg->dbgbus_vbif_rt.cmn.name);
+		dbg->dbgbus_vbif_rt.cmn.enable_mask = DEFAULT_DBGBUS_VBIFRT;
+		debugfs_create_u32(debug_name, 0600, debugfs_root,
+				&dbg->dbgbus_vbif_rt.cmn.enable_mask);
+	}
+
+	return 0;
+}
+
+static void _dpu_dbg_debugfs_destroy(void)
+{
+}
+
+void dpu_dbg_init_dbg_buses(u32 hwversion)
+{
+	static struct dpu_dbg_base *dbg = &dpu_dbg_base;
+
+	memset(&dbg->dbgbus_dpu, 0, sizeof(dbg->dbgbus_dpu));
+	memset(&dbg->dbgbus_vbif_rt, 0, sizeof(dbg->dbgbus_vbif_rt));
+
+	if (IS_MSM8998_TARGET(hwversion)) {
+		dbg->dbgbus_dpu.entries = dbg_bus_dpu_8998;
+		dbg->dbgbus_dpu.cmn.entries_size = ARRAY_SIZE(dbg_bus_dpu_8998);
+		dbg->dbgbus_dpu.cmn.flags = DBGBUS_FLAGS_DSPP;
+
+		dbg->dbgbus_vbif_rt.entries = vbif_dbg_bus_msm8998;
+		dbg->dbgbus_vbif_rt.cmn.entries_size =
+				ARRAY_SIZE(vbif_dbg_bus_msm8998);
+	} else if (IS_SDM845_TARGET(hwversion) || IS_SDM670_TARGET(hwversion)) {
+		dbg->dbgbus_dpu.entries = dbg_bus_dpu_sdm845;
+		dbg->dbgbus_dpu.cmn.entries_size =
+				ARRAY_SIZE(dbg_bus_dpu_sdm845);
+		dbg->dbgbus_dpu.cmn.flags = DBGBUS_FLAGS_DSPP;
+
+		/* vbif is unchanged vs 8998 */
+		dbg->dbgbus_vbif_rt.entries = vbif_dbg_bus_msm8998;
+		dbg->dbgbus_vbif_rt.cmn.entries_size =
+				ARRAY_SIZE(vbif_dbg_bus_msm8998);
+	} else {
+		pr_err("unsupported chipset id %X\n", hwversion);
+	}
+}
+
+int dpu_dbg_init(struct device *dev)
+{
+	if (!dev) {
+		pr_err("invalid params\n");
+		return -EINVAL;
+	}
+
+	INIT_LIST_HEAD(&dpu_dbg_base.reg_base_list);
+	dpu_dbg_base.dev = dev;
+
+	INIT_WORK(&dpu_dbg_base.dump_work, _dpu_dump_work);
+
+	return 0;
+}
+
+/**
+ * dpu_dbg_destroy - destroy dpu debug facilities
+ */
+void dpu_dbg_destroy(void)
+{
+	_dpu_dbg_debugfs_destroy();
+}
+
+void dpu_dbg_set_dpu_top_offset(u32 blk_off)
+{
+	dpu_dbg_base.dbgbus_dpu.top_blk_off = blk_off;
+}
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_dbg.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_dbg.h
new file mode 100644
index 000000000000..1e6fa945f98b
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_dbg.h
@@ -0,0 +1,103 @@
+/* Copyright (c) 2016-2018, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef DPU_DBG_H_
+#define DPU_DBG_H_
+
+#include <stdarg.h>
+#include <linux/debugfs.h>
+#include <linux/list.h>
+
+enum dpu_dbg_dump_flag {
+	DPU_DBG_DUMP_IN_LOG = BIT(0),
+	DPU_DBG_DUMP_IN_MEM = BIT(1),
+};
+
+#if defined(CONFIG_DEBUG_FS)
+
+/**
+ * dpu_dbg_init_dbg_buses - initialize debug bus dumping support for the chipset
+ * @hwversion:		Chipset revision
+ */
+void dpu_dbg_init_dbg_buses(u32 hwversion);
+
+/**
+ * dpu_dbg_init - initialize global dpu debug facilities: regdump
+ * @dev:		device handle
+ * Returns:		0 or -ERROR
+ */
+int dpu_dbg_init(struct device *dev);
+
+/**
+ * dpu_dbg_debugfs_register - register entries at the given debugfs dir
+ * @debugfs_root:	debugfs root in which to create dpu debug entries
+ * Returns:	0 or -ERROR
+ */
+int dpu_dbg_debugfs_register(struct dentry *debugfs_root);
+
+/**
+ * dpu_dbg_destroy - destroy the global dpu debug facilities
+ * Returns:	none
+ */
+void dpu_dbg_destroy(void);
+
+/**
+ * dpu_dbg_dump - trigger dumping of all dpu_dbg facilities
+ * @queue_work:	  whether to queue the dumping work to the work_struct
+ * @name:	  string indicating origin of dump
+ * @dump_dbgbus:  dump the dpu debug bus
+ * @dump_vbif_rt: dump the vbif rt bus
+ * Returns:	none
+ */
+void dpu_dbg_dump(bool queue_work, const char *name, bool dump_dbgbus_dpu,
+		  bool dump_dbgbus_vbif_rt);
+
+/**
+ * dpu_dbg_set_dpu_top_offset - set the target specific offset from mdss base
+ *	address of the top registers. Used for accessing debug bus controls.
+ * @blk_off: offset from mdss base of the top block
+ */
+void dpu_dbg_set_dpu_top_offset(u32 blk_off);
+
+#else
+
+static inline void dpu_dbg_init_dbg_buses(u32 hwversion)
+{
+}
+
+static inline int dpu_dbg_init(struct device *dev)
+{
+	return 0;
+}
+
+static inline int dpu_dbg_debugfs_register(struct dentry *debugfs_root)
+{
+	return 0;
+}
+
+static inline void dpu_dbg_destroy(void)
+{
+}
+
+static inline void dpu_dbg_dump(bool queue_work, const char *name,
+				bool dump_dbgbus_dpu, bool dump_dbgbus_vbif_rt)
+{
+}
+
+static inline void dpu_dbg_set_dpu_top_offset(u32 blk_off)
+{
+}
+
+#endif /* defined(CONFIG_DEBUG_FS) */
+
+
+#endif /* DPU_DBG_H_ */
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c
new file mode 100644
index 000000000000..1b4de3486ef9
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c
@@ -0,0 +1,2500 @@
+/*
+ * Copyright (c) 2014-2018, The Linux Foundation. All rights reserved.
+ * Copyright (C) 2013 Red Hat
+ * Author: Rob Clark <robdclark@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define pr_fmt(fmt)	"[drm:%s:%d] " fmt, __func__, __LINE__
+#include <linux/kthread.h>
+#include <linux/debugfs.h>
+#include <linux/seq_file.h>
+
+#include "msm_drv.h"
+#include "dpu_kms.h"
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
+#include "dpu_hwio.h"
+#include "dpu_hw_catalog.h"
+#include "dpu_hw_intf.h"
+#include "dpu_hw_ctl.h"
+#include "dpu_formats.h"
+#include "dpu_encoder_phys.h"
+#include "dpu_crtc.h"
+#include "dpu_trace.h"
+#include "dpu_core_irq.h"
+
+#define DPU_DEBUG_ENC(e, fmt, ...) DPU_DEBUG("enc%d " fmt,\
+		(e) ? (e)->base.base.id : -1, ##__VA_ARGS__)
+
+#define DPU_ERROR_ENC(e, fmt, ...) DPU_ERROR("enc%d " fmt,\
+		(e) ? (e)->base.base.id : -1, ##__VA_ARGS__)
+
+#define DPU_DEBUG_PHYS(p, fmt, ...) DPU_DEBUG("enc%d intf%d pp%d " fmt,\
+		(p) ? (p)->parent->base.id : -1, \
+		(p) ? (p)->intf_idx - INTF_0 : -1, \
+		(p) ? ((p)->hw_pp ? (p)->hw_pp->idx - PINGPONG_0 : -1) : -1, \
+		##__VA_ARGS__)
+
+#define DPU_ERROR_PHYS(p, fmt, ...) DPU_ERROR("enc%d intf%d pp%d " fmt,\
+		(p) ? (p)->parent->base.id : -1, \
+		(p) ? (p)->intf_idx - INTF_0 : -1, \
+		(p) ? ((p)->hw_pp ? (p)->hw_pp->idx - PINGPONG_0 : -1) : -1, \
+		##__VA_ARGS__)
+
+/*
+ * Two to anticipate panels that can do cmd/vid dynamic switching
+ * plan is to create all possible physical encoder types, and switch between
+ * them at runtime
+ */
+#define NUM_PHYS_ENCODER_TYPES 2
+
+#define MAX_PHYS_ENCODERS_PER_VIRTUAL \
+	(MAX_H_TILES_PER_DISPLAY * NUM_PHYS_ENCODER_TYPES)
+
+#define MAX_CHANNELS_PER_ENC 2
+
+#define MISR_BUFF_SIZE			256
+
+#define IDLE_SHORT_TIMEOUT	1
+
+#define MAX_VDISPLAY_SPLIT 1080
+
+/**
+ * enum dpu_enc_rc_events - events for resource control state machine
+ * @DPU_ENC_RC_EVENT_KICKOFF:
+ *	This event happens at NORMAL priority.
+ *	Event that signals the start of the transfer. When this event is
+ *	received, enable MDP/DSI core clocks. Regardless of the previous
+ *	state, the resource should be in ON state at the end of this event.
+ * @DPU_ENC_RC_EVENT_FRAME_DONE:
+ *	This event happens at INTERRUPT level.
+ *	Event signals the end of the data transfer after the PP FRAME_DONE
+ *	event. At the end of this event, a delayed work is scheduled to go to
+ *	IDLE_PC state after IDLE_TIMEOUT time.
+ * @DPU_ENC_RC_EVENT_PRE_STOP:
+ *	This event happens at NORMAL priority.
+ *	This event, when received during the ON state, leave the RC STATE
+ *	in the PRE_OFF state. It should be followed by the STOP event as
+ *	part of encoder disable.
+ *	If received during IDLE or OFF states, it will do nothing.
+ * @DPU_ENC_RC_EVENT_STOP:
+ *	This event happens at NORMAL priority.
+ *	When this event is received, disable all the MDP/DSI core clocks, and
+ *	disable IRQs. It should be called from the PRE_OFF or IDLE states.
+ *	IDLE is expected when IDLE_PC has run, and PRE_OFF did nothing.
+ *	PRE_OFF is expected when PRE_STOP was executed during the ON state.
+ *	Resource state should be in OFF at the end of the event.
+ * @DPU_ENC_RC_EVENT_ENTER_IDLE:
+ *	This event happens at NORMAL priority from a work item.
+ *	Event signals that there were no frame updates for IDLE_TIMEOUT time.
+ *	This would disable MDP/DSI core clocks and change the resource state
+ *	to IDLE.
+ */
+enum dpu_enc_rc_events {
+	DPU_ENC_RC_EVENT_KICKOFF = 1,
+	DPU_ENC_RC_EVENT_FRAME_DONE,
+	DPU_ENC_RC_EVENT_PRE_STOP,
+	DPU_ENC_RC_EVENT_STOP,
+	DPU_ENC_RC_EVENT_ENTER_IDLE
+};
+
+/*
+ * enum dpu_enc_rc_states - states that the resource control maintains
+ * @DPU_ENC_RC_STATE_OFF: Resource is in OFF state
+ * @DPU_ENC_RC_STATE_PRE_OFF: Resource is transitioning to OFF state
+ * @DPU_ENC_RC_STATE_ON: Resource is in ON state
+ * @DPU_ENC_RC_STATE_MODESET: Resource is in modeset state
+ * @DPU_ENC_RC_STATE_IDLE: Resource is in IDLE state
+ */
+enum dpu_enc_rc_states {
+	DPU_ENC_RC_STATE_OFF,
+	DPU_ENC_RC_STATE_PRE_OFF,
+	DPU_ENC_RC_STATE_ON,
+	DPU_ENC_RC_STATE_IDLE
+};
+
+/**
+ * struct dpu_encoder_virt - virtual encoder. Container of one or more physical
+ *	encoders. Virtual encoder manages one "logical" display. Physical
+ *	encoders manage one intf block, tied to a specific panel/sub-panel.
+ *	Virtual encoder defers as much as possible to the physical encoders.
+ *	Virtual encoder registers itself with the DRM Framework as the encoder.
+ * @base:		drm_encoder base class for registration with DRM
+ * @enc_spin_lock:	Virtual-Encoder-Wide Spin Lock for IRQ purposes
+ * @bus_scaling_client:	Client handle to the bus scaling interface
+ * @num_phys_encs:	Actual number of physical encoders contained.
+ * @phys_encs:		Container of physical encoders managed.
+ * @cur_master:		Pointer to the current master in this mode. Optimization
+ *			Only valid after enable. Cleared as disable.
+ * @hw_pp		Handle to the pingpong blocks used for the display. No.
+ *			pingpong blocks can be different than num_phys_encs.
+ * @intfs_swapped	Whether or not the phys_enc interfaces have been swapped
+ *			for partial update right-only cases, such as pingpong
+ *			split where virtual pingpong does not generate IRQs
+ * @crtc_vblank_cb:	Callback into the upper layer / CRTC for
+ *			notification of the VBLANK
+ * @crtc_vblank_cb_data:	Data from upper layer for VBLANK notification
+ * @crtc_kickoff_cb:		Callback into CRTC that will flush & start
+ *				all CTL paths
+ * @crtc_kickoff_cb_data:	Opaque user data given to crtc_kickoff_cb
+ * @debugfs_root:		Debug file system root file node
+ * @enc_lock:			Lock around physical encoder create/destroy and
+				access.
+ * @frame_busy_mask:		Bitmask tracking which phys_enc we are still
+ *				busy processing current command.
+ *				Bit0 = phys_encs[0] etc.
+ * @crtc_frame_event_cb:	callback handler for frame event
+ * @crtc_frame_event_cb_data:	callback handler private data
+ * @frame_done_timeout:		frame done timeout in Hz
+ * @frame_done_timer:		watchdog timer for frame done event
+ * @vsync_event_timer:		vsync timer
+ * @disp_info:			local copy of msm_display_info struct
+ * @misr_enable:		misr enable/disable status
+ * @misr_frame_count:		misr frame count before start capturing the data
+ * @idle_pc_supported:		indicate if idle power collaps is supported
+ * @rc_lock:			resource control mutex lock to protect
+ *				virt encoder over various state changes
+ * @rc_state:			resource controller state
+ * @delayed_off_work:		delayed worker to schedule disabling of
+ *				clks and resources after IDLE_TIMEOUT time.
+ * @vsync_event_work:		worker to handle vsync event for autorefresh
+ * @topology:                   topology of the display
+ * @mode_set_complete:          flag to indicate modeset completion
+ * @idle_timeout:		idle timeout duration in milliseconds
+ */
+struct dpu_encoder_virt {
+	struct drm_encoder base;
+	spinlock_t enc_spinlock;
+	uint32_t bus_scaling_client;
+
+	uint32_t display_num_of_h_tiles;
+
+	unsigned int num_phys_encs;
+	struct dpu_encoder_phys *phys_encs[MAX_PHYS_ENCODERS_PER_VIRTUAL];
+	struct dpu_encoder_phys *cur_master;
+	struct dpu_hw_pingpong *hw_pp[MAX_CHANNELS_PER_ENC];
+
+	bool intfs_swapped;
+
+	void (*crtc_vblank_cb)(void *);
+	void *crtc_vblank_cb_data;
+
+	struct dentry *debugfs_root;
+	struct mutex enc_lock;
+	DECLARE_BITMAP(frame_busy_mask, MAX_PHYS_ENCODERS_PER_VIRTUAL);
+	void (*crtc_frame_event_cb)(void *, u32 event);
+	void *crtc_frame_event_cb_data;
+
+	atomic_t frame_done_timeout;
+	struct timer_list frame_done_timer;
+	struct timer_list vsync_event_timer;
+
+	struct msm_display_info disp_info;
+	bool misr_enable;
+	u32 misr_frame_count;
+
+	bool idle_pc_supported;
+	struct mutex rc_lock;
+	enum dpu_enc_rc_states rc_state;
+	struct kthread_delayed_work delayed_off_work;
+	struct kthread_work vsync_event_work;
+	struct msm_display_topology topology;
+	bool mode_set_complete;
+
+	u32 idle_timeout;
+};
+
+#define to_dpu_encoder_virt(x) container_of(x, struct dpu_encoder_virt, base)
+static inline int _dpu_encoder_power_enable(struct dpu_encoder_virt *dpu_enc,
+								bool enable)
+{
+	struct drm_encoder *drm_enc;
+	struct msm_drm_private *priv;
+	struct dpu_kms *dpu_kms;
+
+	if (!dpu_enc) {
+		DPU_ERROR("invalid dpu enc\n");
+		return -EINVAL;
+	}
+
+	drm_enc = &dpu_enc->base;
+	if (!drm_enc->dev || !drm_enc->dev->dev_private) {
+		DPU_ERROR("drm device invalid\n");
+		return -EINVAL;
+	}
+
+	priv = drm_enc->dev->dev_private;
+	if (!priv->kms) {
+		DPU_ERROR("invalid kms\n");
+		return -EINVAL;
+	}
+
+	dpu_kms = to_dpu_kms(priv->kms);
+
+	if (enable)
+		pm_runtime_get_sync(&dpu_kms->pdev->dev);
+	else
+		pm_runtime_put_sync(&dpu_kms->pdev->dev);
+
+	return 0;
+}
+
+void dpu_encoder_helper_report_irq_timeout(struct dpu_encoder_phys *phys_enc,
+		enum dpu_intr_idx intr_idx)
+{
+	DRM_ERROR("irq timeout id=%u, intf=%d, pp=%d, intr=%d\n",
+		  DRMID(phys_enc->parent), phys_enc->intf_idx - INTF_0,
+		  phys_enc->hw_pp->idx - PINGPONG_0, intr_idx);
+
+	if (phys_enc->parent_ops->handle_frame_done)
+		phys_enc->parent_ops->handle_frame_done(
+				phys_enc->parent, phys_enc,
+				DPU_ENCODER_FRAME_EVENT_ERROR);
+}
+
+static int dpu_encoder_helper_wait_event_timeout(int32_t drm_id,
+		int32_t hw_id, struct dpu_encoder_wait_info *info);
+
+int dpu_encoder_helper_wait_for_irq(struct dpu_encoder_phys *phys_enc,
+		enum dpu_intr_idx intr_idx,
+		struct dpu_encoder_wait_info *wait_info)
+{
+	struct dpu_encoder_irq *irq;
+	u32 irq_status;
+	int ret;
+
+	if (!phys_enc || !wait_info || intr_idx >= INTR_IDX_MAX) {
+		DPU_ERROR("invalid params\n");
+		return -EINVAL;
+	}
+	irq = &phys_enc->irq[intr_idx];
+
+	/* note: do master / slave checking outside */
+
+	/* return EWOULDBLOCK since we know the wait isn't necessary */
+	if (phys_enc->enable_state == DPU_ENC_DISABLED) {
+		DRM_ERROR("encoder is disabled id=%u, intr=%d, hw=%d, irq=%d",
+			  DRMID(phys_enc->parent), intr_idx, irq->hw_idx,
+			  irq->irq_idx);
+		return -EWOULDBLOCK;
+	}
+
+	if (irq->irq_idx < 0) {
+		DRM_DEBUG_KMS("skip irq wait id=%u, intr=%d, hw=%d, irq=%s",
+			      DRMID(phys_enc->parent), intr_idx, irq->hw_idx,
+			      irq->name);
+		return 0;
+	}
+
+	DRM_DEBUG_KMS("id=%u, intr=%d, hw=%d, irq=%d, pp=%d, pending_cnt=%d",
+		      DRMID(phys_enc->parent), intr_idx, irq->hw_idx,
+		      irq->irq_idx, phys_enc->hw_pp->idx - PINGPONG_0,
+		      atomic_read(wait_info->atomic_cnt));
+
+	ret = dpu_encoder_helper_wait_event_timeout(
+			DRMID(phys_enc->parent),
+			irq->hw_idx,
+			wait_info);
+
+	if (ret <= 0) {
+		irq_status = dpu_core_irq_read(phys_enc->dpu_kms,
+				irq->irq_idx, true);
+		if (irq_status) {
+			unsigned long flags;
+
+			DRM_DEBUG_KMS("irq not triggered id=%u, intr=%d, "
+				      "hw=%d, irq=%d, pp=%d, atomic_cnt=%d",
+				      DRMID(phys_enc->parent), intr_idx,
+				      irq->hw_idx, irq->irq_idx,
+				      phys_enc->hw_pp->idx - PINGPONG_0,
+				      atomic_read(wait_info->atomic_cnt));
+			local_irq_save(flags);
+			irq->cb.func(phys_enc, irq->irq_idx);
+			local_irq_restore(flags);
+			ret = 0;
+		} else {
+			ret = -ETIMEDOUT;
+			DRM_DEBUG_KMS("irq timeout id=%u, intr=%d, "
+				      "hw=%d, irq=%d, pp=%d, atomic_cnt=%d",
+				      DRMID(phys_enc->parent), intr_idx,
+				      irq->hw_idx, irq->irq_idx,
+				      phys_enc->hw_pp->idx - PINGPONG_0,
+				      atomic_read(wait_info->atomic_cnt));
+		}
+	} else {
+		ret = 0;
+		trace_dpu_enc_irq_wait_success(DRMID(phys_enc->parent),
+			intr_idx, irq->hw_idx, irq->irq_idx,
+			phys_enc->hw_pp->idx - PINGPONG_0,
+			atomic_read(wait_info->atomic_cnt));
+	}
+
+	return ret;
+}
+
+int dpu_encoder_helper_register_irq(struct dpu_encoder_phys *phys_enc,
+		enum dpu_intr_idx intr_idx)
+{
+	struct dpu_encoder_irq *irq;
+	int ret = 0;
+
+	if (!phys_enc || intr_idx >= INTR_IDX_MAX) {
+		DPU_ERROR("invalid params\n");
+		return -EINVAL;
+	}
+	irq = &phys_enc->irq[intr_idx];
+
+	if (irq->irq_idx >= 0) {
+		DPU_DEBUG_PHYS(phys_enc,
+				"skipping already registered irq %s type %d\n",
+				irq->name, irq->intr_type);
+		return 0;
+	}
+
+	irq->irq_idx = dpu_core_irq_idx_lookup(phys_enc->dpu_kms,
+			irq->intr_type, irq->hw_idx);
+	if (irq->irq_idx < 0) {
+		DPU_ERROR_PHYS(phys_enc,
+			"failed to lookup IRQ index for %s type:%d\n",
+			irq->name, irq->intr_type);
+		return -EINVAL;
+	}
+
+	ret = dpu_core_irq_register_callback(phys_enc->dpu_kms, irq->irq_idx,
+			&irq->cb);
+	if (ret) {
+		DPU_ERROR_PHYS(phys_enc,
+			"failed to register IRQ callback for %s\n",
+			irq->name);
+		irq->irq_idx = -EINVAL;
+		return ret;
+	}
+
+	ret = dpu_core_irq_enable(phys_enc->dpu_kms, &irq->irq_idx, 1);
+	if (ret) {
+		DRM_ERROR("enable failed id=%u, intr=%d, hw=%d, irq=%d",
+			  DRMID(phys_enc->parent), intr_idx, irq->hw_idx,
+			  irq->irq_idx);
+		dpu_core_irq_unregister_callback(phys_enc->dpu_kms,
+				irq->irq_idx, &irq->cb);
+		irq->irq_idx = -EINVAL;
+		return ret;
+	}
+
+	trace_dpu_enc_irq_register_success(DRMID(phys_enc->parent), intr_idx,
+				irq->hw_idx, irq->irq_idx);
+
+	return ret;
+}
+
+int dpu_encoder_helper_unregister_irq(struct dpu_encoder_phys *phys_enc,
+		enum dpu_intr_idx intr_idx)
+{
+	struct dpu_encoder_irq *irq;
+	int ret;
+
+	if (!phys_enc) {
+		DPU_ERROR("invalid encoder\n");
+		return -EINVAL;
+	}
+	irq = &phys_enc->irq[intr_idx];
+
+	/* silently skip irqs that weren't registered */
+	if (irq->irq_idx < 0) {
+		DRM_ERROR("duplicate unregister id=%u, intr=%d, hw=%d, irq=%d",
+			  DRMID(phys_enc->parent), intr_idx, irq->hw_idx,
+			  irq->irq_idx);
+		return 0;
+	}
+
+	ret = dpu_core_irq_disable(phys_enc->dpu_kms, &irq->irq_idx, 1);
+	if (ret) {
+		DRM_ERROR("disable failed id=%u, intr=%d, hw=%d, irq=%d ret=%d",
+			  DRMID(phys_enc->parent), intr_idx, irq->hw_idx,
+			  irq->irq_idx, ret);
+	}
+
+	ret = dpu_core_irq_unregister_callback(phys_enc->dpu_kms, irq->irq_idx,
+			&irq->cb);
+	if (ret) {
+		DRM_ERROR("unreg cb fail id=%u, intr=%d, hw=%d, irq=%d ret=%d",
+			  DRMID(phys_enc->parent), intr_idx, irq->hw_idx,
+			  irq->irq_idx, ret);
+	}
+
+	trace_dpu_enc_irq_unregister_success(DRMID(phys_enc->parent), intr_idx,
+					     irq->hw_idx, irq->irq_idx);
+
+	irq->irq_idx = -EINVAL;
+
+	return 0;
+}
+
+void dpu_encoder_get_hw_resources(struct drm_encoder *drm_enc,
+		struct dpu_encoder_hw_resources *hw_res,
+		struct drm_connector_state *conn_state)
+{
+	struct dpu_encoder_virt *dpu_enc = NULL;
+	int i = 0;
+
+	if (!hw_res || !drm_enc || !conn_state) {
+		DPU_ERROR("invalid argument(s), drm_enc %d, res %d, state %d\n",
+				drm_enc != 0, hw_res != 0, conn_state != 0);
+		return;
+	}
+
+	dpu_enc = to_dpu_encoder_virt(drm_enc);
+	DPU_DEBUG_ENC(dpu_enc, "\n");
+
+	/* Query resources used by phys encs, expected to be without overlap */
+	memset(hw_res, 0, sizeof(*hw_res));
+	hw_res->display_num_of_h_tiles = dpu_enc->display_num_of_h_tiles;
+
+	for (i = 0; i < dpu_enc->num_phys_encs; i++) {
+		struct dpu_encoder_phys *phys = dpu_enc->phys_encs[i];
+
+		if (phys && phys->ops.get_hw_resources)
+			phys->ops.get_hw_resources(phys, hw_res, conn_state);
+	}
+}
+
+static void dpu_encoder_destroy(struct drm_encoder *drm_enc)
+{
+	struct dpu_encoder_virt *dpu_enc = NULL;
+	int i = 0;
+
+	if (!drm_enc) {
+		DPU_ERROR("invalid encoder\n");
+		return;
+	}
+
+	dpu_enc = to_dpu_encoder_virt(drm_enc);
+	DPU_DEBUG_ENC(dpu_enc, "\n");
+
+	mutex_lock(&dpu_enc->enc_lock);
+
+	for (i = 0; i < dpu_enc->num_phys_encs; i++) {
+		struct dpu_encoder_phys *phys = dpu_enc->phys_encs[i];
+
+		if (phys && phys->ops.destroy) {
+			phys->ops.destroy(phys);
+			--dpu_enc->num_phys_encs;
+			dpu_enc->phys_encs[i] = NULL;
+		}
+	}
+
+	if (dpu_enc->num_phys_encs)
+		DPU_ERROR_ENC(dpu_enc, "expected 0 num_phys_encs not %d\n",
+				dpu_enc->num_phys_encs);
+	dpu_enc->num_phys_encs = 0;
+	mutex_unlock(&dpu_enc->enc_lock);
+
+	drm_encoder_cleanup(drm_enc);
+	mutex_destroy(&dpu_enc->enc_lock);
+
+	kfree(dpu_enc);
+}
+
+void dpu_encoder_helper_split_config(
+		struct dpu_encoder_phys *phys_enc,
+		enum dpu_intf interface)
+{
+	struct dpu_encoder_virt *dpu_enc;
+	struct split_pipe_cfg cfg = { 0 };
+	struct dpu_hw_mdp *hw_mdptop;
+	struct msm_display_info *disp_info;
+
+	if (!phys_enc || !phys_enc->hw_mdptop || !phys_enc->parent) {
+		DPU_ERROR("invalid arg(s), encoder %d\n", phys_enc != 0);
+		return;
+	}
+
+	dpu_enc = to_dpu_encoder_virt(phys_enc->parent);
+	hw_mdptop = phys_enc->hw_mdptop;
+	disp_info = &dpu_enc->disp_info;
+
+	if (disp_info->intf_type != DRM_MODE_CONNECTOR_DSI)
+		return;
+
+	/**
+	 * disable split modes since encoder will be operating in as the only
+	 * encoder, either for the entire use case in the case of, for example,
+	 * single DSI, or for this frame in the case of left/right only partial
+	 * update.
+	 */
+	if (phys_enc->split_role == ENC_ROLE_SOLO) {
+		if (hw_mdptop->ops.setup_split_pipe)
+			hw_mdptop->ops.setup_split_pipe(hw_mdptop, &cfg);
+		return;
+	}
+
+	cfg.en = true;
+	cfg.mode = phys_enc->intf_mode;
+	cfg.intf = interface;
+
+	if (cfg.en && phys_enc->ops.needs_single_flush &&
+			phys_enc->ops.needs_single_flush(phys_enc))
+		cfg.split_flush_en = true;
+
+	if (phys_enc->split_role == ENC_ROLE_MASTER) {
+		DPU_DEBUG_ENC(dpu_enc, "enable %d\n", cfg.en);
+
+		if (hw_mdptop->ops.setup_split_pipe)
+			hw_mdptop->ops.setup_split_pipe(hw_mdptop, &cfg);
+	}
+}
+
+static void _dpu_encoder_adjust_mode(struct drm_connector *connector,
+		struct drm_display_mode *adj_mode)
+{
+	struct drm_display_mode *cur_mode;
+
+	if (!connector || !adj_mode)
+		return;
+
+	list_for_each_entry(cur_mode, &connector->modes, head) {
+		if (cur_mode->vdisplay == adj_mode->vdisplay &&
+			cur_mode->hdisplay == adj_mode->hdisplay &&
+			cur_mode->vrefresh == adj_mode->vrefresh) {
+			adj_mode->private = cur_mode->private;
+			adj_mode->private_flags |= cur_mode->private_flags;
+		}
+	}
+}
+
+static struct msm_display_topology dpu_encoder_get_topology(
+			struct dpu_encoder_virt *dpu_enc,
+			struct dpu_kms *dpu_kms,
+			struct drm_display_mode *mode)
+{
+	struct msm_display_topology topology;
+	int i, intf_count = 0;
+
+	for (i = 0; i < MAX_PHYS_ENCODERS_PER_VIRTUAL; i++)
+		if (dpu_enc->phys_encs[i])
+			intf_count++;
+
+	/* User split topology for width > 1080 */
+	topology.num_lm = (mode->vdisplay > MAX_VDISPLAY_SPLIT) ? 2 : 1;
+	topology.num_enc = 0;
+	topology.num_intf = intf_count;
+
+	return topology;
+}
+static int dpu_encoder_virt_atomic_check(
+		struct drm_encoder *drm_enc,
+		struct drm_crtc_state *crtc_state,
+		struct drm_connector_state *conn_state)
+{
+	struct dpu_encoder_virt *dpu_enc;
+	struct msm_drm_private *priv;
+	struct dpu_kms *dpu_kms;
+	const struct drm_display_mode *mode;
+	struct drm_display_mode *adj_mode;
+	struct msm_display_topology topology;
+	int i = 0;
+	int ret = 0;
+
+	if (!drm_enc || !crtc_state || !conn_state) {
+		DPU_ERROR("invalid arg(s), drm_enc %d, crtc/conn state %d/%d\n",
+				drm_enc != 0, crtc_state != 0, conn_state != 0);
+		return -EINVAL;
+	}
+
+	dpu_enc = to_dpu_encoder_virt(drm_enc);
+	DPU_DEBUG_ENC(dpu_enc, "\n");
+
+	priv = drm_enc->dev->dev_private;
+	dpu_kms = to_dpu_kms(priv->kms);
+	mode = &crtc_state->mode;
+	adj_mode = &crtc_state->adjusted_mode;
+	trace_dpu_enc_atomic_check(DRMID(drm_enc));
+
+	/*
+	 * display drivers may populate private fields of the drm display mode
+	 * structure while registering possible modes of a connector with DRM.
+	 * These private fields are not populated back while DRM invokes
+	 * the mode_set callbacks. This module retrieves and populates the
+	 * private fields of the given mode.
+	 */
+	_dpu_encoder_adjust_mode(conn_state->connector, adj_mode);
+
+	/* perform atomic check on the first physical encoder (master) */
+	for (i = 0; i < dpu_enc->num_phys_encs; i++) {
+		struct dpu_encoder_phys *phys = dpu_enc->phys_encs[i];
+
+		if (phys && phys->ops.atomic_check)
+			ret = phys->ops.atomic_check(phys, crtc_state,
+					conn_state);
+		else if (phys && phys->ops.mode_fixup)
+			if (!phys->ops.mode_fixup(phys, mode, adj_mode))
+				ret = -EINVAL;
+
+		if (ret) {
+			DPU_ERROR_ENC(dpu_enc,
+					"mode unsupported, phys idx %d\n", i);
+			break;
+		}
+	}
+
+	topology = dpu_encoder_get_topology(dpu_enc, dpu_kms, adj_mode);
+
+	/* Reserve dynamic resources now. Indicating AtomicTest phase */
+	if (!ret) {
+		/*
+		 * Avoid reserving resources when mode set is pending. Topology
+		 * info may not be available to complete reservation.
+		 */
+		if (drm_atomic_crtc_needs_modeset(crtc_state)
+				&& dpu_enc->mode_set_complete) {
+			ret = dpu_rm_reserve(&dpu_kms->rm, drm_enc, crtc_state,
+				conn_state, topology, true);
+			dpu_enc->mode_set_complete = false;
+		}
+	}
+
+	if (!ret)
+		drm_mode_set_crtcinfo(adj_mode, 0);
+
+	trace_dpu_enc_atomic_check_flags(DRMID(drm_enc), adj_mode->flags,
+			adj_mode->private_flags);
+
+	return ret;
+}
+
+static void _dpu_encoder_update_vsync_source(struct dpu_encoder_virt *dpu_enc,
+			struct msm_display_info *disp_info)
+{
+	struct dpu_vsync_source_cfg vsync_cfg = { 0 };
+	struct msm_drm_private *priv;
+	struct dpu_kms *dpu_kms;
+	struct dpu_hw_mdp *hw_mdptop;
+	struct drm_encoder *drm_enc;
+	int i;
+
+	if (!dpu_enc || !disp_info) {
+		DPU_ERROR("invalid param dpu_enc:%d or disp_info:%d\n",
+					dpu_enc != NULL, disp_info != NULL);
+		return;
+	} else if (dpu_enc->num_phys_encs > ARRAY_SIZE(dpu_enc->hw_pp)) {
+		DPU_ERROR("invalid num phys enc %d/%d\n",
+				dpu_enc->num_phys_encs,
+				(int) ARRAY_SIZE(dpu_enc->hw_pp));
+		return;
+	}
+
+	drm_enc = &dpu_enc->base;
+	/* this pointers are checked in virt_enable_helper */
+	priv = drm_enc->dev->dev_private;
+
+	dpu_kms = to_dpu_kms(priv->kms);
+	if (!dpu_kms) {
+		DPU_ERROR("invalid dpu_kms\n");
+		return;
+	}
+
+	hw_mdptop = dpu_kms->hw_mdp;
+	if (!hw_mdptop) {
+		DPU_ERROR("invalid mdptop\n");
+		return;
+	}
+
+	if (hw_mdptop->ops.setup_vsync_source &&
+			disp_info->capabilities & MSM_DISPLAY_CAP_CMD_MODE) {
+		for (i = 0; i < dpu_enc->num_phys_encs; i++)
+			vsync_cfg.ppnumber[i] = dpu_enc->hw_pp[i]->idx;
+
+		vsync_cfg.pp_count = dpu_enc->num_phys_encs;
+		if (disp_info->is_te_using_watchdog_timer)
+			vsync_cfg.vsync_source = DPU_VSYNC_SOURCE_WD_TIMER_0;
+		else
+			vsync_cfg.vsync_source = DPU_VSYNC0_SOURCE_GPIO;
+
+		hw_mdptop->ops.setup_vsync_source(hw_mdptop, &vsync_cfg);
+	}
+}
+
+static void _dpu_encoder_irq_control(struct drm_encoder *drm_enc, bool enable)
+{
+	struct dpu_encoder_virt *dpu_enc;
+	int i;
+
+	if (!drm_enc) {
+		DPU_ERROR("invalid encoder\n");
+		return;
+	}
+
+	dpu_enc = to_dpu_encoder_virt(drm_enc);
+
+	DPU_DEBUG_ENC(dpu_enc, "enable:%d\n", enable);
+	for (i = 0; i < dpu_enc->num_phys_encs; i++) {
+		struct dpu_encoder_phys *phys = dpu_enc->phys_encs[i];
+
+		if (phys && phys->ops.irq_control)
+			phys->ops.irq_control(phys, enable);
+	}
+
+}
+
+static void _dpu_encoder_resource_control_helper(struct drm_encoder *drm_enc,
+		bool enable)
+{
+	struct msm_drm_private *priv;
+	struct dpu_kms *dpu_kms;
+	struct dpu_encoder_virt *dpu_enc;
+
+	dpu_enc = to_dpu_encoder_virt(drm_enc);
+	priv = drm_enc->dev->dev_private;
+	dpu_kms = to_dpu_kms(priv->kms);
+
+	trace_dpu_enc_rc_helper(DRMID(drm_enc), enable);
+
+	if (!dpu_enc->cur_master) {
+		DPU_ERROR("encoder master not set\n");
+		return;
+	}
+
+	if (enable) {
+		/* enable DPU core clks */
+		pm_runtime_get_sync(&dpu_kms->pdev->dev);
+
+		/* enable all the irq */
+		_dpu_encoder_irq_control(drm_enc, true);
+
+	} else {
+		/* disable all the irq */
+		_dpu_encoder_irq_control(drm_enc, false);
+
+		/* disable DPU core clks */
+		pm_runtime_put_sync(&dpu_kms->pdev->dev);
+	}
+
+}
+
+static int dpu_encoder_resource_control(struct drm_encoder *drm_enc,
+		u32 sw_event)
+{
+	struct dpu_encoder_virt *dpu_enc;
+	struct msm_drm_private *priv;
+	struct msm_drm_thread *disp_thread;
+	bool is_vid_mode = false;
+
+	if (!drm_enc || !drm_enc->dev || !drm_enc->dev->dev_private ||
+			!drm_enc->crtc) {
+		DPU_ERROR("invalid parameters\n");
+		return -EINVAL;
+	}
+	dpu_enc = to_dpu_encoder_virt(drm_enc);
+	priv = drm_enc->dev->dev_private;
+	is_vid_mode = dpu_enc->disp_info.capabilities &
+						MSM_DISPLAY_CAP_VID_MODE;
+
+	if (drm_enc->crtc->index >= ARRAY_SIZE(priv->disp_thread)) {
+		DPU_ERROR("invalid crtc index\n");
+		return -EINVAL;
+	}
+	disp_thread = &priv->disp_thread[drm_enc->crtc->index];
+
+	/*
+	 * when idle_pc is not supported, process only KICKOFF, STOP and MODESET
+	 * events and return early for other events (ie wb display).
+	 */
+	if (!dpu_enc->idle_pc_supported &&
+			(sw_event != DPU_ENC_RC_EVENT_KICKOFF &&
+			sw_event != DPU_ENC_RC_EVENT_STOP &&
+			sw_event != DPU_ENC_RC_EVENT_PRE_STOP))
+		return 0;
+
+	trace_dpu_enc_rc(DRMID(drm_enc), sw_event, dpu_enc->idle_pc_supported,
+			 dpu_enc->rc_state, "begin");
+
+	switch (sw_event) {
+	case DPU_ENC_RC_EVENT_KICKOFF:
+		/* cancel delayed off work, if any */
+		if (kthread_cancel_delayed_work_sync(
+				&dpu_enc->delayed_off_work))
+			DPU_DEBUG_ENC(dpu_enc, "sw_event:%d, work cancelled\n",
+					sw_event);
+
+		mutex_lock(&dpu_enc->rc_lock);
+
+		/* return if the resource control is already in ON state */
+		if (dpu_enc->rc_state == DPU_ENC_RC_STATE_ON) {
+			DRM_DEBUG_KMS("id;%u, sw_event:%d, rc in ON state\n",
+				      DRMID(drm_enc), sw_event);
+			mutex_unlock(&dpu_enc->rc_lock);
+			return 0;
+		} else if (dpu_enc->rc_state != DPU_ENC_RC_STATE_OFF &&
+				dpu_enc->rc_state != DPU_ENC_RC_STATE_IDLE) {
+			DRM_DEBUG_KMS("id;%u, sw_event:%d, rc in state %d\n",
+				      DRMID(drm_enc), sw_event,
+				      dpu_enc->rc_state);
+			mutex_unlock(&dpu_enc->rc_lock);
+			return -EINVAL;
+		}
+
+		if (is_vid_mode && dpu_enc->rc_state == DPU_ENC_RC_STATE_IDLE)
+			_dpu_encoder_irq_control(drm_enc, true);
+		else
+			_dpu_encoder_resource_control_helper(drm_enc, true);
+
+		dpu_enc->rc_state = DPU_ENC_RC_STATE_ON;
+
+		trace_dpu_enc_rc(DRMID(drm_enc), sw_event,
+				 dpu_enc->idle_pc_supported, dpu_enc->rc_state,
+				 "kickoff");
+
+		mutex_unlock(&dpu_enc->rc_lock);
+		break;
+
+	case DPU_ENC_RC_EVENT_FRAME_DONE:
+		/*
+		 * mutex lock is not used as this event happens at interrupt
+		 * context. And locking is not required as, the other events
+		 * like KICKOFF and STOP does a wait-for-idle before executing
+		 * the resource_control
+		 */
+		if (dpu_enc->rc_state != DPU_ENC_RC_STATE_ON) {
+			DRM_DEBUG_KMS("id:%d, sw_event:%d,rc:%d-unexpected\n",
+				      DRMID(drm_enc), sw_event,
+				      dpu_enc->rc_state);
+			return -EINVAL;
+		}
+
+		/*
+		 * schedule off work item only when there are no
+		 * frames pending
+		 */
+		if (dpu_crtc_frame_pending(drm_enc->crtc) > 1) {
+			DRM_DEBUG_KMS("id:%d skip schedule work\n",
+				      DRMID(drm_enc));
+			return 0;
+		}
+
+		kthread_queue_delayed_work(
+			&disp_thread->worker,
+			&dpu_enc->delayed_off_work,
+			msecs_to_jiffies(dpu_enc->idle_timeout));
+
+		trace_dpu_enc_rc(DRMID(drm_enc), sw_event,
+				 dpu_enc->idle_pc_supported, dpu_enc->rc_state,
+				 "frame done");
+		break;
+
+	case DPU_ENC_RC_EVENT_PRE_STOP:
+		/* cancel delayed off work, if any */
+		if (kthread_cancel_delayed_work_sync(
+				&dpu_enc->delayed_off_work))
+			DPU_DEBUG_ENC(dpu_enc, "sw_event:%d, work cancelled\n",
+					sw_event);
+
+		mutex_lock(&dpu_enc->rc_lock);
+
+		if (is_vid_mode &&
+			  dpu_enc->rc_state == DPU_ENC_RC_STATE_IDLE) {
+			_dpu_encoder_irq_control(drm_enc, true);
+		}
+		/* skip if is already OFF or IDLE, resources are off already */
+		else if (dpu_enc->rc_state == DPU_ENC_RC_STATE_OFF ||
+				dpu_enc->rc_state == DPU_ENC_RC_STATE_IDLE) {
+			DRM_DEBUG_KMS("id:%u, sw_event:%d, rc in %d state\n",
+				      DRMID(drm_enc), sw_event,
+				      dpu_enc->rc_state);
+			mutex_unlock(&dpu_enc->rc_lock);
+			return 0;
+		}
+
+		dpu_enc->rc_state = DPU_ENC_RC_STATE_PRE_OFF;
+
+		trace_dpu_enc_rc(DRMID(drm_enc), sw_event,
+				 dpu_enc->idle_pc_supported, dpu_enc->rc_state,
+				 "pre stop");
+
+		mutex_unlock(&dpu_enc->rc_lock);
+		break;
+
+	case DPU_ENC_RC_EVENT_STOP:
+		mutex_lock(&dpu_enc->rc_lock);
+
+		/* return if the resource control is already in OFF state */
+		if (dpu_enc->rc_state == DPU_ENC_RC_STATE_OFF) {
+			DRM_DEBUG_KMS("id: %u, sw_event:%d, rc in OFF state\n",
+				      DRMID(drm_enc), sw_event);
+			mutex_unlock(&dpu_enc->rc_lock);
+			return 0;
+		} else if (dpu_enc->rc_state == DPU_ENC_RC_STATE_ON) {
+			DRM_ERROR("id: %u, sw_event:%d, rc in state %d\n",
+				  DRMID(drm_enc), sw_event, dpu_enc->rc_state);
+			mutex_unlock(&dpu_enc->rc_lock);
+			return -EINVAL;
+		}
+
+		/**
+		 * expect to arrive here only if in either idle state or pre-off
+		 * and in IDLE state the resources are already disabled
+		 */
+		if (dpu_enc->rc_state == DPU_ENC_RC_STATE_PRE_OFF)
+			_dpu_encoder_resource_control_helper(drm_enc, false);
+
+		dpu_enc->rc_state = DPU_ENC_RC_STATE_OFF;
+
+		trace_dpu_enc_rc(DRMID(drm_enc), sw_event,
+				 dpu_enc->idle_pc_supported, dpu_enc->rc_state,
+				 "stop");
+
+		mutex_unlock(&dpu_enc->rc_lock);
+		break;
+
+	case DPU_ENC_RC_EVENT_ENTER_IDLE:
+		mutex_lock(&dpu_enc->rc_lock);
+
+		if (dpu_enc->rc_state != DPU_ENC_RC_STATE_ON) {
+			DRM_ERROR("id: %u, sw_event:%d, rc:%d !ON state\n",
+				  DRMID(drm_enc), sw_event, dpu_enc->rc_state);
+			mutex_unlock(&dpu_enc->rc_lock);
+			return 0;
+		}
+
+		/*
+		 * if we are in ON but a frame was just kicked off,
+		 * ignore the IDLE event, it's probably a stale timer event
+		 */
+		if (dpu_enc->frame_busy_mask[0]) {
+			DRM_ERROR("id:%u, sw_event:%d, rc:%d frame pending\n",
+				  DRMID(drm_enc), sw_event, dpu_enc->rc_state);
+			mutex_unlock(&dpu_enc->rc_lock);
+			return 0;
+		}
+
+		if (is_vid_mode)
+			_dpu_encoder_irq_control(drm_enc, false);
+		else
+			_dpu_encoder_resource_control_helper(drm_enc, false);
+
+		dpu_enc->rc_state = DPU_ENC_RC_STATE_IDLE;
+
+		trace_dpu_enc_rc(DRMID(drm_enc), sw_event,
+				 dpu_enc->idle_pc_supported, dpu_enc->rc_state,
+				 "idle");
+
+		mutex_unlock(&dpu_enc->rc_lock);
+		break;
+
+	default:
+		DRM_ERROR("id:%u, unexpected sw_event: %d\n", DRMID(drm_enc),
+			  sw_event);
+		trace_dpu_enc_rc(DRMID(drm_enc), sw_event,
+				 dpu_enc->idle_pc_supported, dpu_enc->rc_state,
+				 "error");
+		break;
+	}
+
+	trace_dpu_enc_rc(DRMID(drm_enc), sw_event,
+			 dpu_enc->idle_pc_supported, dpu_enc->rc_state,
+			 "end");
+	return 0;
+}
+
+static void dpu_encoder_virt_mode_set(struct drm_encoder *drm_enc,
+				      struct drm_display_mode *mode,
+				      struct drm_display_mode *adj_mode)
+{
+	struct dpu_encoder_virt *dpu_enc;
+	struct msm_drm_private *priv;
+	struct dpu_kms *dpu_kms;
+	struct list_head *connector_list;
+	struct drm_connector *conn = NULL, *conn_iter;
+	struct dpu_rm_hw_iter pp_iter;
+	struct msm_display_topology topology;
+	enum dpu_rm_topology_name topology_name;
+	int i = 0, ret;
+
+	if (!drm_enc) {
+		DPU_ERROR("invalid encoder\n");
+		return;
+	}
+
+	dpu_enc = to_dpu_encoder_virt(drm_enc);
+	DPU_DEBUG_ENC(dpu_enc, "\n");
+
+	priv = drm_enc->dev->dev_private;
+	dpu_kms = to_dpu_kms(priv->kms);
+	connector_list = &dpu_kms->dev->mode_config.connector_list;
+
+	trace_dpu_enc_mode_set(DRMID(drm_enc));
+
+	list_for_each_entry(conn_iter, connector_list, head)
+		if (conn_iter->encoder == drm_enc)
+			conn = conn_iter;
+
+	if (!conn) {
+		DPU_ERROR_ENC(dpu_enc, "failed to find attached connector\n");
+		return;
+	} else if (!conn->state) {
+		DPU_ERROR_ENC(dpu_enc, "invalid connector state\n");
+		return;
+	}
+
+	topology = dpu_encoder_get_topology(dpu_enc, dpu_kms, adj_mode);
+
+	/* Reserve dynamic resources now. Indicating non-AtomicTest phase */
+	ret = dpu_rm_reserve(&dpu_kms->rm, drm_enc, drm_enc->crtc->state,
+			conn->state, topology, false);
+	if (ret) {
+		DPU_ERROR_ENC(dpu_enc,
+				"failed to reserve hw resources, %d\n", ret);
+		return;
+	}
+
+	dpu_rm_init_hw_iter(&pp_iter, drm_enc->base.id, DPU_HW_BLK_PINGPONG);
+	for (i = 0; i < MAX_CHANNELS_PER_ENC; i++) {
+		dpu_enc->hw_pp[i] = NULL;
+		if (!dpu_rm_get_hw(&dpu_kms->rm, &pp_iter))
+			break;
+		dpu_enc->hw_pp[i] = (struct dpu_hw_pingpong *) pp_iter.hw;
+	}
+
+	topology_name = dpu_rm_get_topology_name(topology);
+	for (i = 0; i < dpu_enc->num_phys_encs; i++) {
+		struct dpu_encoder_phys *phys = dpu_enc->phys_encs[i];
+
+		if (phys) {
+			if (!dpu_enc->hw_pp[i]) {
+				DPU_ERROR_ENC(dpu_enc,
+				    "invalid pingpong block for the encoder\n");
+				return;
+			}
+			phys->hw_pp = dpu_enc->hw_pp[i];
+			phys->connector = conn->state->connector;
+			phys->topology_name = topology_name;
+			if (phys->ops.mode_set)
+				phys->ops.mode_set(phys, mode, adj_mode);
+		}
+	}
+
+	dpu_enc->mode_set_complete = true;
+}
+
+static void _dpu_encoder_virt_enable_helper(struct drm_encoder *drm_enc)
+{
+	struct dpu_encoder_virt *dpu_enc = NULL;
+	struct msm_drm_private *priv;
+	struct dpu_kms *dpu_kms;
+
+	if (!drm_enc || !drm_enc->dev || !drm_enc->dev->dev_private) {
+		DPU_ERROR("invalid parameters\n");
+		return;
+	}
+
+	priv = drm_enc->dev->dev_private;
+	dpu_kms = to_dpu_kms(priv->kms);
+	if (!dpu_kms) {
+		DPU_ERROR("invalid dpu_kms\n");
+		return;
+	}
+
+	dpu_enc = to_dpu_encoder_virt(drm_enc);
+	if (!dpu_enc || !dpu_enc->cur_master) {
+		DPU_ERROR("invalid dpu encoder/master\n");
+		return;
+	}
+
+	if (dpu_enc->disp_info.intf_type == DRM_MODE_CONNECTOR_DisplayPort &&
+	    dpu_enc->cur_master->hw_mdptop &&
+	    dpu_enc->cur_master->hw_mdptop->ops.intf_audio_select)
+		dpu_enc->cur_master->hw_mdptop->ops.intf_audio_select(
+					dpu_enc->cur_master->hw_mdptop);
+
+	if (dpu_enc->cur_master->hw_mdptop &&
+			dpu_enc->cur_master->hw_mdptop->ops.reset_ubwc)
+		dpu_enc->cur_master->hw_mdptop->ops.reset_ubwc(
+				dpu_enc->cur_master->hw_mdptop,
+				dpu_kms->catalog);
+
+	_dpu_encoder_update_vsync_source(dpu_enc, &dpu_enc->disp_info);
+}
+
+void dpu_encoder_virt_restore(struct drm_encoder *drm_enc)
+{
+	struct dpu_encoder_virt *dpu_enc = NULL;
+	int i;
+
+	if (!drm_enc) {
+		DPU_ERROR("invalid encoder\n");
+		return;
+	}
+	dpu_enc = to_dpu_encoder_virt(drm_enc);
+
+	for (i = 0; i < dpu_enc->num_phys_encs; i++) {
+		struct dpu_encoder_phys *phys = dpu_enc->phys_encs[i];
+
+		if (phys && (phys != dpu_enc->cur_master) && phys->ops.restore)
+			phys->ops.restore(phys);
+	}
+
+	if (dpu_enc->cur_master && dpu_enc->cur_master->ops.restore)
+		dpu_enc->cur_master->ops.restore(dpu_enc->cur_master);
+
+	_dpu_encoder_virt_enable_helper(drm_enc);
+}
+
+static void dpu_encoder_virt_enable(struct drm_encoder *drm_enc)
+{
+	struct dpu_encoder_virt *dpu_enc = NULL;
+	int i, ret = 0;
+	struct drm_display_mode *cur_mode = NULL;
+
+	if (!drm_enc) {
+		DPU_ERROR("invalid encoder\n");
+		return;
+	}
+	dpu_enc = to_dpu_encoder_virt(drm_enc);
+	cur_mode = &dpu_enc->base.crtc->state->adjusted_mode;
+
+	trace_dpu_enc_enable(DRMID(drm_enc), cur_mode->hdisplay,
+			     cur_mode->vdisplay);
+
+	dpu_enc->cur_master = NULL;
+	for (i = 0; i < dpu_enc->num_phys_encs; i++) {
+		struct dpu_encoder_phys *phys = dpu_enc->phys_encs[i];
+
+		if (phys && phys->ops.is_master && phys->ops.is_master(phys)) {
+			DPU_DEBUG_ENC(dpu_enc, "master is now idx %d\n", i);
+			dpu_enc->cur_master = phys;
+			break;
+		}
+	}
+
+	if (!dpu_enc->cur_master) {
+		DPU_ERROR("virt encoder has no master! num_phys %d\n", i);
+		return;
+	}
+
+	ret = dpu_encoder_resource_control(drm_enc, DPU_ENC_RC_EVENT_KICKOFF);
+	if (ret) {
+		DPU_ERROR_ENC(dpu_enc, "dpu resource control failed: %d\n",
+				ret);
+		return;
+	}
+
+	for (i = 0; i < dpu_enc->num_phys_encs; i++) {
+		struct dpu_encoder_phys *phys = dpu_enc->phys_encs[i];
+
+		if (!phys)
+			continue;
+
+		if (phys != dpu_enc->cur_master) {
+			if (phys->ops.enable)
+				phys->ops.enable(phys);
+		}
+
+		if (dpu_enc->misr_enable && (dpu_enc->disp_info.capabilities &
+		     MSM_DISPLAY_CAP_VID_MODE) && phys->ops.setup_misr)
+			phys->ops.setup_misr(phys, true,
+						dpu_enc->misr_frame_count);
+	}
+
+	if (dpu_enc->cur_master->ops.enable)
+		dpu_enc->cur_master->ops.enable(dpu_enc->cur_master);
+
+	_dpu_encoder_virt_enable_helper(drm_enc);
+}
+
+static void dpu_encoder_virt_disable(struct drm_encoder *drm_enc)
+{
+	struct dpu_encoder_virt *dpu_enc = NULL;
+	struct msm_drm_private *priv;
+	struct dpu_kms *dpu_kms;
+	struct drm_display_mode *mode;
+	int i = 0;
+
+	if (!drm_enc) {
+		DPU_ERROR("invalid encoder\n");
+		return;
+	} else if (!drm_enc->dev) {
+		DPU_ERROR("invalid dev\n");
+		return;
+	} else if (!drm_enc->dev->dev_private) {
+		DPU_ERROR("invalid dev_private\n");
+		return;
+	}
+
+	mode = &drm_enc->crtc->state->adjusted_mode;
+
+	dpu_enc = to_dpu_encoder_virt(drm_enc);
+	DPU_DEBUG_ENC(dpu_enc, "\n");
+
+	priv = drm_enc->dev->dev_private;
+	dpu_kms = to_dpu_kms(priv->kms);
+
+	trace_dpu_enc_disable(DRMID(drm_enc));
+
+	/* wait for idle */
+	dpu_encoder_wait_for_event(drm_enc, MSM_ENC_TX_COMPLETE);
+
+	dpu_encoder_resource_control(drm_enc, DPU_ENC_RC_EVENT_PRE_STOP);
+
+	for (i = 0; i < dpu_enc->num_phys_encs; i++) {
+		struct dpu_encoder_phys *phys = dpu_enc->phys_encs[i];
+
+		if (phys && phys->ops.disable)
+			phys->ops.disable(phys);
+	}
+
+	/* after phys waits for frame-done, should be no more frames pending */
+	if (atomic_xchg(&dpu_enc->frame_done_timeout, 0)) {
+		DPU_ERROR("enc%d timeout pending\n", drm_enc->base.id);
+		del_timer_sync(&dpu_enc->frame_done_timer);
+	}
+
+	dpu_encoder_resource_control(drm_enc, DPU_ENC_RC_EVENT_STOP);
+
+	for (i = 0; i < dpu_enc->num_phys_encs; i++) {
+		if (dpu_enc->phys_encs[i])
+			dpu_enc->phys_encs[i]->connector = NULL;
+	}
+
+	dpu_enc->cur_master = NULL;
+
+	DPU_DEBUG_ENC(dpu_enc, "encoder disabled\n");
+
+	dpu_rm_release(&dpu_kms->rm, drm_enc);
+}
+
+static enum dpu_intf dpu_encoder_get_intf(struct dpu_mdss_cfg *catalog,
+		enum dpu_intf_type type, u32 controller_id)
+{
+	int i = 0;
+
+	for (i = 0; i < catalog->intf_count; i++) {
+		if (catalog->intf[i].type == type
+		    && catalog->intf[i].controller_id == controller_id) {
+			return catalog->intf[i].id;
+		}
+	}
+
+	return INTF_MAX;
+}
+
+static void dpu_encoder_vblank_callback(struct drm_encoder *drm_enc,
+		struct dpu_encoder_phys *phy_enc)
+{
+	struct dpu_encoder_virt *dpu_enc = NULL;
+	unsigned long lock_flags;
+
+	if (!drm_enc || !phy_enc)
+		return;
+
+	DPU_ATRACE_BEGIN("encoder_vblank_callback");
+	dpu_enc = to_dpu_encoder_virt(drm_enc);
+
+	spin_lock_irqsave(&dpu_enc->enc_spinlock, lock_flags);
+	if (dpu_enc->crtc_vblank_cb)
+		dpu_enc->crtc_vblank_cb(dpu_enc->crtc_vblank_cb_data);
+	spin_unlock_irqrestore(&dpu_enc->enc_spinlock, lock_flags);
+
+	atomic_inc(&phy_enc->vsync_cnt);
+	DPU_ATRACE_END("encoder_vblank_callback");
+}
+
+static void dpu_encoder_underrun_callback(struct drm_encoder *drm_enc,
+		struct dpu_encoder_phys *phy_enc)
+{
+	if (!phy_enc)
+		return;
+
+	DPU_ATRACE_BEGIN("encoder_underrun_callback");
+	atomic_inc(&phy_enc->underrun_cnt);
+	trace_dpu_enc_underrun_cb(DRMID(drm_enc),
+				  atomic_read(&phy_enc->underrun_cnt));
+	DPU_ATRACE_END("encoder_underrun_callback");
+}
+
+void dpu_encoder_register_vblank_callback(struct drm_encoder *drm_enc,
+		void (*vbl_cb)(void *), void *vbl_data)
+{
+	struct dpu_encoder_virt *dpu_enc = to_dpu_encoder_virt(drm_enc);
+	unsigned long lock_flags;
+	bool enable;
+	int i;
+
+	enable = vbl_cb ? true : false;
+
+	if (!drm_enc) {
+		DPU_ERROR("invalid encoder\n");
+		return;
+	}
+	trace_dpu_enc_vblank_cb(DRMID(drm_enc), enable);
+
+	spin_lock_irqsave(&dpu_enc->enc_spinlock, lock_flags);
+	dpu_enc->crtc_vblank_cb = vbl_cb;
+	dpu_enc->crtc_vblank_cb_data = vbl_data;
+	spin_unlock_irqrestore(&dpu_enc->enc_spinlock, lock_flags);
+
+	for (i = 0; i < dpu_enc->num_phys_encs; i++) {
+		struct dpu_encoder_phys *phys = dpu_enc->phys_encs[i];
+
+		if (phys && phys->ops.control_vblank_irq)
+			phys->ops.control_vblank_irq(phys, enable);
+	}
+}
+
+void dpu_encoder_register_frame_event_callback(struct drm_encoder *drm_enc,
+		void (*frame_event_cb)(void *, u32 event),
+		void *frame_event_cb_data)
+{
+	struct dpu_encoder_virt *dpu_enc = to_dpu_encoder_virt(drm_enc);
+	unsigned long lock_flags;
+	bool enable;
+
+	enable = frame_event_cb ? true : false;
+
+	if (!drm_enc) {
+		DPU_ERROR("invalid encoder\n");
+		return;
+	}
+	trace_dpu_enc_frame_event_cb(DRMID(drm_enc), enable);
+
+	spin_lock_irqsave(&dpu_enc->enc_spinlock, lock_flags);
+	dpu_enc->crtc_frame_event_cb = frame_event_cb;
+	dpu_enc->crtc_frame_event_cb_data = frame_event_cb_data;
+	spin_unlock_irqrestore(&dpu_enc->enc_spinlock, lock_flags);
+}
+
+static void dpu_encoder_frame_done_callback(
+		struct drm_encoder *drm_enc,
+		struct dpu_encoder_phys *ready_phys, u32 event)
+{
+	struct dpu_encoder_virt *dpu_enc = to_dpu_encoder_virt(drm_enc);
+	unsigned int i;
+
+	if (event & (DPU_ENCODER_FRAME_EVENT_DONE
+			| DPU_ENCODER_FRAME_EVENT_ERROR
+			| DPU_ENCODER_FRAME_EVENT_PANEL_DEAD)) {
+
+		if (!dpu_enc->frame_busy_mask[0]) {
+			/**
+			 * suppress frame_done without waiter,
+			 * likely autorefresh
+			 */
+			trace_dpu_enc_frame_done_cb_not_busy(DRMID(drm_enc),
+					event, ready_phys->intf_idx);
+			return;
+		}
+
+		/* One of the physical encoders has become idle */
+		for (i = 0; i < dpu_enc->num_phys_encs; i++) {
+			if (dpu_enc->phys_encs[i] == ready_phys) {
+				clear_bit(i, dpu_enc->frame_busy_mask);
+				trace_dpu_enc_frame_done_cb(DRMID(drm_enc), i,
+						dpu_enc->frame_busy_mask[0]);
+			}
+		}
+
+		if (!dpu_enc->frame_busy_mask[0]) {
+			atomic_set(&dpu_enc->frame_done_timeout, 0);
+			del_timer(&dpu_enc->frame_done_timer);
+
+			dpu_encoder_resource_control(drm_enc,
+					DPU_ENC_RC_EVENT_FRAME_DONE);
+
+			if (dpu_enc->crtc_frame_event_cb)
+				dpu_enc->crtc_frame_event_cb(
+					dpu_enc->crtc_frame_event_cb_data,
+					event);
+		}
+	} else {
+		if (dpu_enc->crtc_frame_event_cb)
+			dpu_enc->crtc_frame_event_cb(
+				dpu_enc->crtc_frame_event_cb_data, event);
+	}
+}
+
+static void dpu_encoder_off_work(struct kthread_work *work)
+{
+	struct dpu_encoder_virt *dpu_enc = container_of(work,
+			struct dpu_encoder_virt, delayed_off_work.work);
+
+	if (!dpu_enc) {
+		DPU_ERROR("invalid dpu encoder\n");
+		return;
+	}
+
+	dpu_encoder_resource_control(&dpu_enc->base,
+						DPU_ENC_RC_EVENT_ENTER_IDLE);
+
+	dpu_encoder_frame_done_callback(&dpu_enc->base, NULL,
+				DPU_ENCODER_FRAME_EVENT_IDLE);
+}
+
+/**
+ * _dpu_encoder_trigger_flush - trigger flush for a physical encoder
+ * drm_enc: Pointer to drm encoder structure
+ * phys: Pointer to physical encoder structure
+ * extra_flush_bits: Additional bit mask to include in flush trigger
+ */
+static inline void _dpu_encoder_trigger_flush(struct drm_encoder *drm_enc,
+		struct dpu_encoder_phys *phys, uint32_t extra_flush_bits)
+{
+	struct dpu_hw_ctl *ctl;
+	int pending_kickoff_cnt;
+	u32 ret = UINT_MAX;
+
+	if (!drm_enc || !phys) {
+		DPU_ERROR("invalid argument(s), drm_enc %d, phys_enc %d\n",
+				drm_enc != 0, phys != 0);
+		return;
+	}
+
+	if (!phys->hw_pp) {
+		DPU_ERROR("invalid pingpong hw\n");
+		return;
+	}
+
+	ctl = phys->hw_ctl;
+	if (!ctl || !ctl->ops.trigger_flush) {
+		DPU_ERROR("missing trigger cb\n");
+		return;
+	}
+
+	pending_kickoff_cnt = dpu_encoder_phys_inc_pending(phys);
+
+	if (extra_flush_bits && ctl->ops.update_pending_flush)
+		ctl->ops.update_pending_flush(ctl, extra_flush_bits);
+
+	ctl->ops.trigger_flush(ctl);
+
+	if (ctl->ops.get_pending_flush)
+		ret = ctl->ops.get_pending_flush(ctl);
+
+	trace_dpu_enc_trigger_flush(DRMID(drm_enc), phys->intf_idx,
+				    pending_kickoff_cnt, ctl->idx, ret);
+}
+
+/**
+ * _dpu_encoder_trigger_start - trigger start for a physical encoder
+ * phys: Pointer to physical encoder structure
+ */
+static inline void _dpu_encoder_trigger_start(struct dpu_encoder_phys *phys)
+{
+	if (!phys) {
+		DPU_ERROR("invalid argument(s)\n");
+		return;
+	}
+
+	if (!phys->hw_pp) {
+		DPU_ERROR("invalid pingpong hw\n");
+		return;
+	}
+
+	if (phys->ops.trigger_start && phys->enable_state != DPU_ENC_DISABLED)
+		phys->ops.trigger_start(phys);
+}
+
+void dpu_encoder_helper_trigger_start(struct dpu_encoder_phys *phys_enc)
+{
+	struct dpu_hw_ctl *ctl;
+
+	if (!phys_enc) {
+		DPU_ERROR("invalid encoder\n");
+		return;
+	}
+
+	ctl = phys_enc->hw_ctl;
+	if (ctl && ctl->ops.trigger_start) {
+		ctl->ops.trigger_start(ctl);
+		trace_dpu_enc_trigger_start(DRMID(phys_enc->parent), ctl->idx);
+	}
+}
+
+static int dpu_encoder_helper_wait_event_timeout(
+		int32_t drm_id,
+		int32_t hw_id,
+		struct dpu_encoder_wait_info *info)
+{
+	int rc = 0;
+	s64 expected_time = ktime_to_ms(ktime_get()) + info->timeout_ms;
+	s64 jiffies = msecs_to_jiffies(info->timeout_ms);
+	s64 time;
+
+	do {
+		rc = wait_event_timeout(*(info->wq),
+				atomic_read(info->atomic_cnt) == 0, jiffies);
+		time = ktime_to_ms(ktime_get());
+
+		trace_dpu_enc_wait_event_timeout(drm_id, hw_id, rc, time,
+						 expected_time,
+						 atomic_read(info->atomic_cnt));
+	/* If we timed out, counter is valid and time is less, wait again */
+	} while (atomic_read(info->atomic_cnt) && (rc == 0) &&
+			(time < expected_time));
+
+	return rc;
+}
+
+void dpu_encoder_helper_hw_reset(struct dpu_encoder_phys *phys_enc)
+{
+	struct dpu_encoder_virt *dpu_enc;
+	struct dpu_hw_ctl *ctl;
+	int rc;
+
+	if (!phys_enc) {
+		DPU_ERROR("invalid encoder\n");
+		return;
+	}
+	dpu_enc = to_dpu_encoder_virt(phys_enc->parent);
+	ctl = phys_enc->hw_ctl;
+
+	if (!ctl || !ctl->ops.reset)
+		return;
+
+	DRM_DEBUG_KMS("id:%u ctl %d reset\n", DRMID(phys_enc->parent),
+		      ctl->idx);
+
+	rc = ctl->ops.reset(ctl);
+	if (rc) {
+		DPU_ERROR_ENC(dpu_enc, "ctl %d reset failure\n",  ctl->idx);
+		dpu_dbg_dump(false, __func__, true, true);
+	}
+
+	phys_enc->enable_state = DPU_ENC_ENABLED;
+}
+
+/**
+ * _dpu_encoder_kickoff_phys - handle physical encoder kickoff
+ *	Iterate through the physical encoders and perform consolidated flush
+ *	and/or control start triggering as needed. This is done in the virtual
+ *	encoder rather than the individual physical ones in order to handle
+ *	use cases that require visibility into multiple physical encoders at
+ *	a time.
+ * dpu_enc: Pointer to virtual encoder structure
+ */
+static void _dpu_encoder_kickoff_phys(struct dpu_encoder_virt *dpu_enc)
+{
+	struct dpu_hw_ctl *ctl;
+	uint32_t i, pending_flush;
+	unsigned long lock_flags;
+
+	if (!dpu_enc) {
+		DPU_ERROR("invalid encoder\n");
+		return;
+	}
+
+	pending_flush = 0x0;
+
+	/* update pending counts and trigger kickoff ctl flush atomically */
+	spin_lock_irqsave(&dpu_enc->enc_spinlock, lock_flags);
+
+	/* don't perform flush/start operations for slave encoders */
+	for (i = 0; i < dpu_enc->num_phys_encs; i++) {
+		struct dpu_encoder_phys *phys = dpu_enc->phys_encs[i];
+
+		if (!phys || phys->enable_state == DPU_ENC_DISABLED)
+			continue;
+
+		ctl = phys->hw_ctl;
+		if (!ctl)
+			continue;
+
+		if (phys->split_role != ENC_ROLE_SLAVE)
+			set_bit(i, dpu_enc->frame_busy_mask);
+		if (!phys->ops.needs_single_flush ||
+				!phys->ops.needs_single_flush(phys))
+			_dpu_encoder_trigger_flush(&dpu_enc->base, phys, 0x0);
+		else if (ctl->ops.get_pending_flush)
+			pending_flush |= ctl->ops.get_pending_flush(ctl);
+	}
+
+	/* for split flush, combine pending flush masks and send to master */
+	if (pending_flush && dpu_enc->cur_master) {
+		_dpu_encoder_trigger_flush(
+				&dpu_enc->base,
+				dpu_enc->cur_master,
+				pending_flush);
+	}
+
+	_dpu_encoder_trigger_start(dpu_enc->cur_master);
+
+	spin_unlock_irqrestore(&dpu_enc->enc_spinlock, lock_flags);
+}
+
+void dpu_encoder_trigger_kickoff_pending(struct drm_encoder *drm_enc)
+{
+	struct dpu_encoder_virt *dpu_enc;
+	struct dpu_encoder_phys *phys;
+	unsigned int i;
+	struct dpu_hw_ctl *ctl;
+	struct msm_display_info *disp_info;
+
+	if (!drm_enc) {
+		DPU_ERROR("invalid encoder\n");
+		return;
+	}
+	dpu_enc = to_dpu_encoder_virt(drm_enc);
+	disp_info = &dpu_enc->disp_info;
+
+	for (i = 0; i < dpu_enc->num_phys_encs; i++) {
+		phys = dpu_enc->phys_encs[i];
+
+		if (phys && phys->hw_ctl) {
+			ctl = phys->hw_ctl;
+			if (ctl->ops.clear_pending_flush)
+				ctl->ops.clear_pending_flush(ctl);
+
+			/* update only for command mode primary ctl */
+			if ((phys == dpu_enc->cur_master) &&
+			   (disp_info->capabilities & MSM_DISPLAY_CAP_CMD_MODE)
+			    && ctl->ops.trigger_pending)
+				ctl->ops.trigger_pending(ctl);
+		}
+	}
+}
+
+static u32 _dpu_encoder_calculate_linetime(struct dpu_encoder_virt *dpu_enc,
+		struct drm_display_mode *mode)
+{
+	u64 pclk_rate;
+	u32 pclk_period;
+	u32 line_time;
+
+	/*
+	 * For linetime calculation, only operate on master encoder.
+	 */
+	if (!dpu_enc->cur_master)
+		return 0;
+
+	if (!dpu_enc->cur_master->ops.get_line_count) {
+		DPU_ERROR("get_line_count function not defined\n");
+		return 0;
+	}
+
+	pclk_rate = mode->clock; /* pixel clock in kHz */
+	if (pclk_rate == 0) {
+		DPU_ERROR("pclk is 0, cannot calculate line time\n");
+		return 0;
+	}
+
+	pclk_period = DIV_ROUND_UP_ULL(1000000000ull, pclk_rate);
+	if (pclk_period == 0) {
+		DPU_ERROR("pclk period is 0\n");
+		return 0;
+	}
+
+	/*
+	 * Line time calculation based on Pixel clock and HTOTAL.
+	 * Final unit is in ns.
+	 */
+	line_time = (pclk_period * mode->htotal) / 1000;
+	if (line_time == 0) {
+		DPU_ERROR("line time calculation is 0\n");
+		return 0;
+	}
+
+	DPU_DEBUG_ENC(dpu_enc,
+			"clk_rate=%lldkHz, clk_period=%d, linetime=%dns\n",
+			pclk_rate, pclk_period, line_time);
+
+	return line_time;
+}
+
+static int _dpu_encoder_wakeup_time(struct drm_encoder *drm_enc,
+		ktime_t *wakeup_time)
+{
+	struct drm_display_mode *mode;
+	struct dpu_encoder_virt *dpu_enc;
+	u32 cur_line;
+	u32 line_time;
+	u32 vtotal, time_to_vsync;
+	ktime_t cur_time;
+
+	dpu_enc = to_dpu_encoder_virt(drm_enc);
+
+	if (!drm_enc->crtc || !drm_enc->crtc->state) {
+		DPU_ERROR("crtc/crtc state object is NULL\n");
+		return -EINVAL;
+	}
+	mode = &drm_enc->crtc->state->adjusted_mode;
+
+	line_time = _dpu_encoder_calculate_linetime(dpu_enc, mode);
+	if (!line_time)
+		return -EINVAL;
+
+	cur_line = dpu_enc->cur_master->ops.get_line_count(dpu_enc->cur_master);
+
+	vtotal = mode->vtotal;
+	if (cur_line >= vtotal)
+		time_to_vsync = line_time * vtotal;
+	else
+		time_to_vsync = line_time * (vtotal - cur_line);
+
+	if (time_to_vsync == 0) {
+		DPU_ERROR("time to vsync should not be zero, vtotal=%d\n",
+				vtotal);
+		return -EINVAL;
+	}
+
+	cur_time = ktime_get();
+	*wakeup_time = ktime_add_ns(cur_time, time_to_vsync);
+
+	DPU_DEBUG_ENC(dpu_enc,
+			"cur_line=%u vtotal=%u time_to_vsync=%u, cur_time=%lld, wakeup_time=%lld\n",
+			cur_line, vtotal, time_to_vsync,
+			ktime_to_ms(cur_time),
+			ktime_to_ms(*wakeup_time));
+	return 0;
+}
+
+static void dpu_encoder_vsync_event_handler(struct timer_list *t)
+{
+	struct dpu_encoder_virt *dpu_enc = from_timer(dpu_enc, t,
+			vsync_event_timer);
+	struct drm_encoder *drm_enc = &dpu_enc->base;
+	struct msm_drm_private *priv;
+	struct msm_drm_thread *event_thread;
+
+	if (!drm_enc->dev || !drm_enc->dev->dev_private ||
+			!drm_enc->crtc) {
+		DPU_ERROR("invalid parameters\n");
+		return;
+	}
+
+	priv = drm_enc->dev->dev_private;
+
+	if (drm_enc->crtc->index >= ARRAY_SIZE(priv->event_thread)) {
+		DPU_ERROR("invalid crtc index\n");
+		return;
+	}
+	event_thread = &priv->event_thread[drm_enc->crtc->index];
+	if (!event_thread) {
+		DPU_ERROR("event_thread not found for crtc:%d\n",
+				drm_enc->crtc->index);
+		return;
+	}
+
+	del_timer(&dpu_enc->vsync_event_timer);
+}
+
+static void dpu_encoder_vsync_event_work_handler(struct kthread_work *work)
+{
+	struct dpu_encoder_virt *dpu_enc = container_of(work,
+			struct dpu_encoder_virt, vsync_event_work);
+	ktime_t wakeup_time;
+
+	if (!dpu_enc) {
+		DPU_ERROR("invalid dpu encoder\n");
+		return;
+	}
+
+	if (_dpu_encoder_wakeup_time(&dpu_enc->base, &wakeup_time))
+		return;
+
+	trace_dpu_enc_vsync_event_work(DRMID(&dpu_enc->base), wakeup_time);
+	mod_timer(&dpu_enc->vsync_event_timer,
+			nsecs_to_jiffies(ktime_to_ns(wakeup_time)));
+}
+
+void dpu_encoder_prepare_for_kickoff(struct drm_encoder *drm_enc,
+		struct dpu_encoder_kickoff_params *params)
+{
+	struct dpu_encoder_virt *dpu_enc;
+	struct dpu_encoder_phys *phys;
+	bool needs_hw_reset = false;
+	unsigned int i;
+
+	if (!drm_enc || !params) {
+		DPU_ERROR("invalid args\n");
+		return;
+	}
+	dpu_enc = to_dpu_encoder_virt(drm_enc);
+
+	trace_dpu_enc_prepare_kickoff(DRMID(drm_enc));
+
+	/* prepare for next kickoff, may include waiting on previous kickoff */
+	DPU_ATRACE_BEGIN("enc_prepare_for_kickoff");
+	for (i = 0; i < dpu_enc->num_phys_encs; i++) {
+		phys = dpu_enc->phys_encs[i];
+		if (phys) {
+			if (phys->ops.prepare_for_kickoff)
+				phys->ops.prepare_for_kickoff(phys, params);
+			if (phys->enable_state == DPU_ENC_ERR_NEEDS_HW_RESET)
+				needs_hw_reset = true;
+		}
+	}
+	DPU_ATRACE_END("enc_prepare_for_kickoff");
+
+	dpu_encoder_resource_control(drm_enc, DPU_ENC_RC_EVENT_KICKOFF);
+
+	/* if any phys needs reset, reset all phys, in-order */
+	if (needs_hw_reset) {
+		trace_dpu_enc_prepare_kickoff_reset(DRMID(drm_enc));
+		for (i = 0; i < dpu_enc->num_phys_encs; i++) {
+			phys = dpu_enc->phys_encs[i];
+			if (phys && phys->ops.hw_reset)
+				phys->ops.hw_reset(phys);
+		}
+	}
+}
+
+void dpu_encoder_kickoff(struct drm_encoder *drm_enc)
+{
+	struct dpu_encoder_virt *dpu_enc;
+	struct dpu_encoder_phys *phys;
+	ktime_t wakeup_time;
+	unsigned int i;
+
+	if (!drm_enc) {
+		DPU_ERROR("invalid encoder\n");
+		return;
+	}
+	DPU_ATRACE_BEGIN("encoder_kickoff");
+	dpu_enc = to_dpu_encoder_virt(drm_enc);
+
+	trace_dpu_enc_kickoff(DRMID(drm_enc));
+
+	atomic_set(&dpu_enc->frame_done_timeout,
+			DPU_FRAME_DONE_TIMEOUT * 1000 /
+			drm_enc->crtc->state->adjusted_mode.vrefresh);
+	mod_timer(&dpu_enc->frame_done_timer, jiffies +
+		((atomic_read(&dpu_enc->frame_done_timeout) * HZ) / 1000));
+
+	/* All phys encs are ready to go, trigger the kickoff */
+	_dpu_encoder_kickoff_phys(dpu_enc);
+
+	/* allow phys encs to handle any post-kickoff business */
+	for (i = 0; i < dpu_enc->num_phys_encs; i++) {
+		phys = dpu_enc->phys_encs[i];
+		if (phys && phys->ops.handle_post_kickoff)
+			phys->ops.handle_post_kickoff(phys);
+	}
+
+	if (dpu_enc->disp_info.intf_type == DRM_MODE_CONNECTOR_DSI &&
+			!_dpu_encoder_wakeup_time(drm_enc, &wakeup_time)) {
+		trace_dpu_enc_early_kickoff(DRMID(drm_enc),
+					    ktime_to_ms(wakeup_time));
+		mod_timer(&dpu_enc->vsync_event_timer,
+				nsecs_to_jiffies(ktime_to_ns(wakeup_time)));
+	}
+
+	DPU_ATRACE_END("encoder_kickoff");
+}
+
+void dpu_encoder_prepare_commit(struct drm_encoder *drm_enc)
+{
+	struct dpu_encoder_virt *dpu_enc;
+	struct dpu_encoder_phys *phys;
+	int i;
+
+	if (!drm_enc) {
+		DPU_ERROR("invalid encoder\n");
+		return;
+	}
+	dpu_enc = to_dpu_encoder_virt(drm_enc);
+
+	for (i = 0; i < dpu_enc->num_phys_encs; i++) {
+		phys = dpu_enc->phys_encs[i];
+		if (phys && phys->ops.prepare_commit)
+			phys->ops.prepare_commit(phys);
+	}
+}
+
+#ifdef CONFIG_DEBUG_FS
+static int _dpu_encoder_status_show(struct seq_file *s, void *data)
+{
+	struct dpu_encoder_virt *dpu_enc;
+	int i;
+
+	if (!s || !s->private)
+		return -EINVAL;
+
+	dpu_enc = s->private;
+
+	mutex_lock(&dpu_enc->enc_lock);
+	for (i = 0; i < dpu_enc->num_phys_encs; i++) {
+		struct dpu_encoder_phys *phys = dpu_enc->phys_encs[i];
+
+		if (!phys)
+			continue;
+
+		seq_printf(s, "intf:%d    vsync:%8d     underrun:%8d    ",
+				phys->intf_idx - INTF_0,
+				atomic_read(&phys->vsync_cnt),
+				atomic_read(&phys->underrun_cnt));
+
+		switch (phys->intf_mode) {
+		case INTF_MODE_VIDEO:
+			seq_puts(s, "mode: video\n");
+			break;
+		case INTF_MODE_CMD:
+			seq_puts(s, "mode: command\n");
+			break;
+		default:
+			seq_puts(s, "mode: ???\n");
+			break;
+		}
+	}
+	mutex_unlock(&dpu_enc->enc_lock);
+
+	return 0;
+}
+
+static int _dpu_encoder_debugfs_status_open(struct inode *inode,
+		struct file *file)
+{
+	return single_open(file, _dpu_encoder_status_show, inode->i_private);
+}
+
+static ssize_t _dpu_encoder_misr_setup(struct file *file,
+		const char __user *user_buf, size_t count, loff_t *ppos)
+{
+	struct dpu_encoder_virt *dpu_enc;
+	int i = 0, rc;
+	char buf[MISR_BUFF_SIZE + 1];
+	size_t buff_copy;
+	u32 frame_count, enable;
+
+	if (!file || !file->private_data)
+		return -EINVAL;
+
+	dpu_enc = file->private_data;
+
+	buff_copy = min_t(size_t, count, MISR_BUFF_SIZE);
+	if (copy_from_user(buf, user_buf, buff_copy))
+		return -EINVAL;
+
+	buf[buff_copy] = 0; /* end of string */
+
+	if (sscanf(buf, "%u %u", &enable, &frame_count) != 2)
+		return -EINVAL;
+
+	rc = _dpu_encoder_power_enable(dpu_enc, true);
+	if (rc)
+		return rc;
+
+	mutex_lock(&dpu_enc->enc_lock);
+	dpu_enc->misr_enable = enable;
+	dpu_enc->misr_frame_count = frame_count;
+	for (i = 0; i < dpu_enc->num_phys_encs; i++) {
+		struct dpu_encoder_phys *phys = dpu_enc->phys_encs[i];
+
+		if (!phys || !phys->ops.setup_misr)
+			continue;
+
+		phys->ops.setup_misr(phys, enable, frame_count);
+	}
+	mutex_unlock(&dpu_enc->enc_lock);
+	_dpu_encoder_power_enable(dpu_enc, false);
+
+	return count;
+}
+
+static ssize_t _dpu_encoder_misr_read(struct file *file,
+		char __user *user_buff, size_t count, loff_t *ppos)
+{
+	struct dpu_encoder_virt *dpu_enc;
+	int i = 0, len = 0;
+	char buf[MISR_BUFF_SIZE + 1] = {'\0'};
+	int rc;
+
+	if (*ppos)
+		return 0;
+
+	if (!file || !file->private_data)
+		return -EINVAL;
+
+	dpu_enc = file->private_data;
+
+	rc = _dpu_encoder_power_enable(dpu_enc, true);
+	if (rc)
+		return rc;
+
+	mutex_lock(&dpu_enc->enc_lock);
+	if (!dpu_enc->misr_enable) {
+		len += snprintf(buf + len, MISR_BUFF_SIZE - len,
+			"disabled\n");
+		goto buff_check;
+	} else if (dpu_enc->disp_info.capabilities &
+						~MSM_DISPLAY_CAP_VID_MODE) {
+		len += snprintf(buf + len, MISR_BUFF_SIZE - len,
+			"unsupported\n");
+		goto buff_check;
+	}
+
+	for (i = 0; i < dpu_enc->num_phys_encs; i++) {
+		struct dpu_encoder_phys *phys = dpu_enc->phys_encs[i];
+
+		if (!phys || !phys->ops.collect_misr)
+			continue;
+
+		len += snprintf(buf + len, MISR_BUFF_SIZE - len,
+			"Intf idx:%d\n", phys->intf_idx - INTF_0);
+		len += snprintf(buf + len, MISR_BUFF_SIZE - len, "0x%x\n",
+					phys->ops.collect_misr(phys));
+	}
+
+buff_check:
+	if (count <= len) {
+		len = 0;
+		goto end;
+	}
+
+	if (copy_to_user(user_buff, buf, len)) {
+		len = -EFAULT;
+		goto end;
+	}
+
+	*ppos += len;   /* increase offset */
+
+end:
+	mutex_unlock(&dpu_enc->enc_lock);
+	_dpu_encoder_power_enable(dpu_enc, false);
+	return len;
+}
+
+static int _dpu_encoder_init_debugfs(struct drm_encoder *drm_enc)
+{
+	struct dpu_encoder_virt *dpu_enc;
+	struct msm_drm_private *priv;
+	struct dpu_kms *dpu_kms;
+	int i;
+
+	static const struct file_operations debugfs_status_fops = {
+		.open =		_dpu_encoder_debugfs_status_open,
+		.read =		seq_read,
+		.llseek =	seq_lseek,
+		.release =	single_release,
+	};
+
+	static const struct file_operations debugfs_misr_fops = {
+		.open = simple_open,
+		.read = _dpu_encoder_misr_read,
+		.write = _dpu_encoder_misr_setup,
+	};
+
+	char name[DPU_NAME_SIZE];
+
+	if (!drm_enc || !drm_enc->dev || !drm_enc->dev->dev_private) {
+		DPU_ERROR("invalid encoder or kms\n");
+		return -EINVAL;
+	}
+
+	dpu_enc = to_dpu_encoder_virt(drm_enc);
+	priv = drm_enc->dev->dev_private;
+	dpu_kms = to_dpu_kms(priv->kms);
+
+	snprintf(name, DPU_NAME_SIZE, "encoder%u", drm_enc->base.id);
+
+	/* create overall sub-directory for the encoder */
+	dpu_enc->debugfs_root = debugfs_create_dir(name,
+			drm_enc->dev->primary->debugfs_root);
+	if (!dpu_enc->debugfs_root)
+		return -ENOMEM;
+
+	/* don't error check these */
+	debugfs_create_file("status", 0600,
+		dpu_enc->debugfs_root, dpu_enc, &debugfs_status_fops);
+
+	debugfs_create_file("misr_data", 0600,
+		dpu_enc->debugfs_root, dpu_enc, &debugfs_misr_fops);
+
+	for (i = 0; i < dpu_enc->num_phys_encs; i++)
+		if (dpu_enc->phys_encs[i] &&
+				dpu_enc->phys_encs[i]->ops.late_register)
+			dpu_enc->phys_encs[i]->ops.late_register(
+					dpu_enc->phys_encs[i],
+					dpu_enc->debugfs_root);
+
+	return 0;
+}
+
+static void _dpu_encoder_destroy_debugfs(struct drm_encoder *drm_enc)
+{
+	struct dpu_encoder_virt *dpu_enc;
+
+	if (!drm_enc)
+		return;
+
+	dpu_enc = to_dpu_encoder_virt(drm_enc);
+	debugfs_remove_recursive(dpu_enc->debugfs_root);
+}
+#else
+static int _dpu_encoder_init_debugfs(struct drm_encoder *drm_enc)
+{
+	return 0;
+}
+
+static void _dpu_encoder_destroy_debugfs(struct drm_encoder *drm_enc)
+{
+}
+#endif
+
+static int dpu_encoder_late_register(struct drm_encoder *encoder)
+{
+	return _dpu_encoder_init_debugfs(encoder);
+}
+
+static void dpu_encoder_early_unregister(struct drm_encoder *encoder)
+{
+	_dpu_encoder_destroy_debugfs(encoder);
+}
+
+static int dpu_encoder_virt_add_phys_encs(
+		u32 display_caps,
+		struct dpu_encoder_virt *dpu_enc,
+		struct dpu_enc_phys_init_params *params)
+{
+	struct dpu_encoder_phys *enc = NULL;
+
+	DPU_DEBUG_ENC(dpu_enc, "\n");
+
+	/*
+	 * We may create up to NUM_PHYS_ENCODER_TYPES physical encoder types
+	 * in this function, check up-front.
+	 */
+	if (dpu_enc->num_phys_encs + NUM_PHYS_ENCODER_TYPES >=
+			ARRAY_SIZE(dpu_enc->phys_encs)) {
+		DPU_ERROR_ENC(dpu_enc, "too many physical encoders %d\n",
+			  dpu_enc->num_phys_encs);
+		return -EINVAL;
+	}
+
+	if (display_caps & MSM_DISPLAY_CAP_VID_MODE) {
+		enc = dpu_encoder_phys_vid_init(params);
+
+		if (IS_ERR_OR_NULL(enc)) {
+			DPU_ERROR_ENC(dpu_enc, "failed to init vid enc: %ld\n",
+				PTR_ERR(enc));
+			return enc == 0 ? -EINVAL : PTR_ERR(enc);
+		}
+
+		dpu_enc->phys_encs[dpu_enc->num_phys_encs] = enc;
+		++dpu_enc->num_phys_encs;
+	}
+
+	if (display_caps & MSM_DISPLAY_CAP_CMD_MODE) {
+		enc = dpu_encoder_phys_cmd_init(params);
+
+		if (IS_ERR_OR_NULL(enc)) {
+			DPU_ERROR_ENC(dpu_enc, "failed to init cmd enc: %ld\n",
+				PTR_ERR(enc));
+			return enc == 0 ? -EINVAL : PTR_ERR(enc);
+		}
+
+		dpu_enc->phys_encs[dpu_enc->num_phys_encs] = enc;
+		++dpu_enc->num_phys_encs;
+	}
+
+	return 0;
+}
+
+static const struct dpu_encoder_virt_ops dpu_encoder_parent_ops = {
+	.handle_vblank_virt = dpu_encoder_vblank_callback,
+	.handle_underrun_virt = dpu_encoder_underrun_callback,
+	.handle_frame_done = dpu_encoder_frame_done_callback,
+};
+
+static int dpu_encoder_setup_display(struct dpu_encoder_virt *dpu_enc,
+				 struct dpu_kms *dpu_kms,
+				 struct msm_display_info *disp_info,
+				 int *drm_enc_mode)
+{
+	int ret = 0;
+	int i = 0;
+	enum dpu_intf_type intf_type;
+	struct dpu_enc_phys_init_params phys_params;
+
+	if (!dpu_enc || !dpu_kms) {
+		DPU_ERROR("invalid arg(s), enc %d kms %d\n",
+				dpu_enc != 0, dpu_kms != 0);
+		return -EINVAL;
+	}
+
+	memset(&phys_params, 0, sizeof(phys_params));
+	phys_params.dpu_kms = dpu_kms;
+	phys_params.parent = &dpu_enc->base;
+	phys_params.parent_ops = &dpu_encoder_parent_ops;
+	phys_params.enc_spinlock = &dpu_enc->enc_spinlock;
+
+	DPU_DEBUG("\n");
+
+	if (disp_info->intf_type == DRM_MODE_CONNECTOR_DSI) {
+		*drm_enc_mode = DRM_MODE_ENCODER_DSI;
+		intf_type = INTF_DSI;
+	} else if (disp_info->intf_type == DRM_MODE_CONNECTOR_HDMIA) {
+		*drm_enc_mode = DRM_MODE_ENCODER_TMDS;
+		intf_type = INTF_HDMI;
+	} else if (disp_info->intf_type == DRM_MODE_CONNECTOR_DisplayPort) {
+		*drm_enc_mode = DRM_MODE_ENCODER_TMDS;
+		intf_type = INTF_DP;
+	} else {
+		DPU_ERROR_ENC(dpu_enc, "unsupported display interface type\n");
+		return -EINVAL;
+	}
+
+	WARN_ON(disp_info->num_of_h_tiles < 1);
+
+	dpu_enc->display_num_of_h_tiles = disp_info->num_of_h_tiles;
+
+	DPU_DEBUG("dsi_info->num_of_h_tiles %d\n", disp_info->num_of_h_tiles);
+
+	if ((disp_info->capabilities & MSM_DISPLAY_CAP_CMD_MODE) ||
+	    (disp_info->capabilities & MSM_DISPLAY_CAP_VID_MODE))
+		dpu_enc->idle_pc_supported =
+				dpu_kms->catalog->caps->has_idle_pc;
+
+	mutex_lock(&dpu_enc->enc_lock);
+	for (i = 0; i < disp_info->num_of_h_tiles && !ret; i++) {
+		/*
+		 * Left-most tile is at index 0, content is controller id
+		 * h_tile_instance_ids[2] = {0, 1}; DSI0 = left, DSI1 = right
+		 * h_tile_instance_ids[2] = {1, 0}; DSI1 = left, DSI0 = right
+		 */
+		u32 controller_id = disp_info->h_tile_instance[i];
+
+		if (disp_info->num_of_h_tiles > 1) {
+			if (i == 0)
+				phys_params.split_role = ENC_ROLE_MASTER;
+			else
+				phys_params.split_role = ENC_ROLE_SLAVE;
+		} else {
+			phys_params.split_role = ENC_ROLE_SOLO;
+		}
+
+		DPU_DEBUG("h_tile_instance %d = %d, split_role %d\n",
+				i, controller_id, phys_params.split_role);
+
+		phys_params.intf_idx = dpu_encoder_get_intf(dpu_kms->catalog,
+													intf_type,
+													controller_id);
+		if (phys_params.intf_idx == INTF_MAX) {
+			DPU_ERROR_ENC(dpu_enc, "could not get intf: type %d, id %d\n",
+						  intf_type, controller_id);
+			ret = -EINVAL;
+		}
+
+		if (!ret) {
+			ret = dpu_encoder_virt_add_phys_encs(disp_info->capabilities,
+												 dpu_enc,
+												 &phys_params);
+			if (ret)
+				DPU_ERROR_ENC(dpu_enc, "failed to add phys encs\n");
+		}
+	}
+
+	for (i = 0; i < dpu_enc->num_phys_encs; i++) {
+		struct dpu_encoder_phys *phys = dpu_enc->phys_encs[i];
+
+		if (phys) {
+			atomic_set(&phys->vsync_cnt, 0);
+			atomic_set(&phys->underrun_cnt, 0);
+		}
+	}
+	mutex_unlock(&dpu_enc->enc_lock);
+
+	return ret;
+}
+
+static void dpu_encoder_frame_done_timeout(struct timer_list *t)
+{
+	struct dpu_encoder_virt *dpu_enc = from_timer(dpu_enc, t,
+			frame_done_timer);
+	struct drm_encoder *drm_enc = &dpu_enc->base;
+	struct msm_drm_private *priv;
+	u32 event;
+
+	if (!drm_enc->dev || !drm_enc->dev->dev_private) {
+		DPU_ERROR("invalid parameters\n");
+		return;
+	}
+	priv = drm_enc->dev->dev_private;
+
+	if (!dpu_enc->frame_busy_mask[0] || !dpu_enc->crtc_frame_event_cb) {
+		DRM_DEBUG_KMS("id:%u invalid timeout frame_busy_mask=%lu\n",
+			      DRMID(drm_enc), dpu_enc->frame_busy_mask[0]);
+		return;
+	} else if (!atomic_xchg(&dpu_enc->frame_done_timeout, 0)) {
+		DRM_DEBUG_KMS("id:%u invalid timeout\n", DRMID(drm_enc));
+		return;
+	}
+
+	DPU_ERROR_ENC(dpu_enc, "frame done timeout\n");
+
+	event = DPU_ENCODER_FRAME_EVENT_ERROR;
+	trace_dpu_enc_frame_done_timeout(DRMID(drm_enc), event);
+	dpu_enc->crtc_frame_event_cb(dpu_enc->crtc_frame_event_cb_data, event);
+}
+
+static const struct drm_encoder_helper_funcs dpu_encoder_helper_funcs = {
+	.mode_set = dpu_encoder_virt_mode_set,
+	.disable = dpu_encoder_virt_disable,
+	.enable = dpu_kms_encoder_enable,
+	.atomic_check = dpu_encoder_virt_atomic_check,
+
+	/* This is called by dpu_kms_encoder_enable */
+	.commit = dpu_encoder_virt_enable,
+};
+
+static const struct drm_encoder_funcs dpu_encoder_funcs = {
+		.destroy = dpu_encoder_destroy,
+		.late_register = dpu_encoder_late_register,
+		.early_unregister = dpu_encoder_early_unregister,
+};
+
+int dpu_encoder_setup(struct drm_device *dev, struct drm_encoder *enc,
+		struct msm_display_info *disp_info)
+{
+	struct msm_drm_private *priv = dev->dev_private;
+	struct dpu_kms *dpu_kms = to_dpu_kms(priv->kms);
+	struct drm_encoder *drm_enc = NULL;
+	struct dpu_encoder_virt *dpu_enc = NULL;
+	int drm_enc_mode = DRM_MODE_ENCODER_NONE;
+	int ret = 0;
+
+	dpu_enc = to_dpu_encoder_virt(enc);
+
+	mutex_init(&dpu_enc->enc_lock);
+	ret = dpu_encoder_setup_display(dpu_enc, dpu_kms, disp_info,
+			&drm_enc_mode);
+	if (ret)
+		goto fail;
+
+	dpu_enc->cur_master = NULL;
+	spin_lock_init(&dpu_enc->enc_spinlock);
+
+	atomic_set(&dpu_enc->frame_done_timeout, 0);
+	timer_setup(&dpu_enc->frame_done_timer,
+			dpu_encoder_frame_done_timeout, 0);
+
+	if (disp_info->intf_type == DRM_MODE_CONNECTOR_DSI)
+		timer_setup(&dpu_enc->vsync_event_timer,
+				dpu_encoder_vsync_event_handler,
+				0);
+
+
+	mutex_init(&dpu_enc->rc_lock);
+	kthread_init_delayed_work(&dpu_enc->delayed_off_work,
+			dpu_encoder_off_work);
+	dpu_enc->idle_timeout = IDLE_TIMEOUT;
+
+	kthread_init_work(&dpu_enc->vsync_event_work,
+			dpu_encoder_vsync_event_work_handler);
+
+	memcpy(&dpu_enc->disp_info, disp_info, sizeof(*disp_info));
+
+	DPU_DEBUG_ENC(dpu_enc, "created\n");
+
+	return ret;
+
+fail:
+	DPU_ERROR("failed to create encoder\n");
+	if (drm_enc)
+		dpu_encoder_destroy(drm_enc);
+
+	return ret;
+
+
+}
+
+struct drm_encoder *dpu_encoder_init(struct drm_device *dev,
+		int drm_enc_mode)
+{
+	struct dpu_encoder_virt *dpu_enc = NULL;
+	int rc = 0;
+
+	dpu_enc = devm_kzalloc(dev->dev, sizeof(*dpu_enc), GFP_KERNEL);
+	if (!dpu_enc)
+		return ERR_PTR(ENOMEM);
+
+	rc = drm_encoder_init(dev, &dpu_enc->base, &dpu_encoder_funcs,
+			drm_enc_mode, NULL);
+	if (rc) {
+		devm_kfree(dev->dev, dpu_enc);
+		return ERR_PTR(rc);
+	}
+
+	drm_encoder_helper_add(&dpu_enc->base, &dpu_encoder_helper_funcs);
+
+	return &dpu_enc->base;
+}
+
+int dpu_encoder_wait_for_event(struct drm_encoder *drm_enc,
+	enum msm_event_wait event)
+{
+	int (*fn_wait)(struct dpu_encoder_phys *phys_enc) = NULL;
+	struct dpu_encoder_virt *dpu_enc = NULL;
+	int i, ret = 0;
+
+	if (!drm_enc) {
+		DPU_ERROR("invalid encoder\n");
+		return -EINVAL;
+	}
+	dpu_enc = to_dpu_encoder_virt(drm_enc);
+	DPU_DEBUG_ENC(dpu_enc, "\n");
+
+	for (i = 0; i < dpu_enc->num_phys_encs; i++) {
+		struct dpu_encoder_phys *phys = dpu_enc->phys_encs[i];
+		if (!phys)
+			continue;
+
+		switch (event) {
+		case MSM_ENC_COMMIT_DONE:
+			fn_wait = phys->ops.wait_for_commit_done;
+			break;
+		case MSM_ENC_TX_COMPLETE:
+			fn_wait = phys->ops.wait_for_tx_complete;
+			break;
+		case MSM_ENC_VBLANK:
+			fn_wait = phys->ops.wait_for_vblank;
+			break;
+		default:
+			DPU_ERROR_ENC(dpu_enc, "unknown wait event %d\n",
+					event);
+			return -EINVAL;
+		};
+
+		if (fn_wait) {
+			DPU_ATRACE_BEGIN("wait_for_completion_event");
+			ret = fn_wait(phys);
+			DPU_ATRACE_END("wait_for_completion_event");
+			if (ret)
+				return ret;
+		}
+	}
+
+	return ret;
+}
+
+enum dpu_intf_mode dpu_encoder_get_intf_mode(struct drm_encoder *encoder)
+{
+	struct dpu_encoder_virt *dpu_enc = NULL;
+	int i;
+
+	if (!encoder) {
+		DPU_ERROR("invalid encoder\n");
+		return INTF_MODE_NONE;
+	}
+	dpu_enc = to_dpu_encoder_virt(encoder);
+
+	if (dpu_enc->cur_master)
+		return dpu_enc->cur_master->intf_mode;
+
+	for (i = 0; i < dpu_enc->num_phys_encs; i++) {
+		struct dpu_encoder_phys *phys = dpu_enc->phys_encs[i];
+
+		if (phys)
+			return phys->intf_mode;
+	}
+
+	return INTF_MODE_NONE;
+}
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.h
new file mode 100644
index 000000000000..60f809fc7c13
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.h
@@ -0,0 +1,177 @@
+/*
+ * Copyright (c) 2015-2018, The Linux Foundation. All rights reserved.
+ * Copyright (C) 2013 Red Hat
+ * Author: Rob Clark <robdclark@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __DPU_ENCODER_H__
+#define __DPU_ENCODER_H__
+
+#include <drm/drm_crtc.h>
+#include "dpu_hw_mdss.h"
+
+#define DPU_ENCODER_FRAME_EVENT_DONE			BIT(0)
+#define DPU_ENCODER_FRAME_EVENT_ERROR			BIT(1)
+#define DPU_ENCODER_FRAME_EVENT_PANEL_DEAD		BIT(2)
+#define DPU_ENCODER_FRAME_EVENT_IDLE			BIT(3)
+
+#define IDLE_TIMEOUT	(66 - 16/2)
+
+/**
+ * Encoder functions and data types
+ * @intfs:	Interfaces this encoder is using, INTF_MODE_NONE if unused
+ * @needs_cdm:	Encoder requests a CDM based on pixel format conversion needs
+ * @display_num_of_h_tiles: Number of horizontal tiles in case of split
+ *                          interface
+ * @topology:   Topology of the display
+ */
+struct dpu_encoder_hw_resources {
+	enum dpu_intf_mode intfs[INTF_MAX];
+	bool needs_cdm;
+	u32 display_num_of_h_tiles;
+};
+
+/**
+ * dpu_encoder_kickoff_params - info encoder requires at kickoff
+ * @affected_displays:  bitmask, bit set means the ROI of the commit lies within
+ *                      the bounds of the physical display at the bit index
+ */
+struct dpu_encoder_kickoff_params {
+	unsigned long affected_displays;
+};
+
+/**
+ * dpu_encoder_get_hw_resources - Populate table of required hardware resources
+ * @encoder:	encoder pointer
+ * @hw_res:	resource table to populate with encoder required resources
+ * @conn_state:	report hw reqs based on this proposed connector state
+ */
+void dpu_encoder_get_hw_resources(struct drm_encoder *encoder,
+		struct dpu_encoder_hw_resources *hw_res,
+		struct drm_connector_state *conn_state);
+
+/**
+ * dpu_encoder_register_vblank_callback - provide callback to encoder that
+ *	will be called on the next vblank.
+ * @encoder:	encoder pointer
+ * @cb:		callback pointer, provide NULL to deregister and disable IRQs
+ * @data:	user data provided to callback
+ */
+void dpu_encoder_register_vblank_callback(struct drm_encoder *encoder,
+		void (*cb)(void *), void *data);
+
+/**
+ * dpu_encoder_register_frame_event_callback - provide callback to encoder that
+ *	will be called after the request is complete, or other events.
+ * @encoder:	encoder pointer
+ * @cb:		callback pointer, provide NULL to deregister
+ * @data:	user data provided to callback
+ */
+void dpu_encoder_register_frame_event_callback(struct drm_encoder *encoder,
+		void (*cb)(void *, u32), void *data);
+
+/**
+ * dpu_encoder_prepare_for_kickoff - schedule double buffer flip of the ctl
+ *	path (i.e. ctl flush and start) at next appropriate time.
+ *	Immediately: if no previous commit is outstanding.
+ *	Delayed: Block until next trigger can be issued.
+ * @encoder:	encoder pointer
+ * @params:	kickoff time parameters
+ */
+void dpu_encoder_prepare_for_kickoff(struct drm_encoder *encoder,
+		struct dpu_encoder_kickoff_params *params);
+
+/**
+ * dpu_encoder_trigger_kickoff_pending - Clear the flush bits from previous
+ *        kickoff and trigger the ctl prepare progress for command mode display.
+ * @encoder:	encoder pointer
+ */
+void dpu_encoder_trigger_kickoff_pending(struct drm_encoder *encoder);
+
+/**
+ * dpu_encoder_kickoff - trigger a double buffer flip of the ctl path
+ *	(i.e. ctl flush and start) immediately.
+ * @encoder:	encoder pointer
+ */
+void dpu_encoder_kickoff(struct drm_encoder *encoder);
+
+/**
+ * dpu_encoder_wait_for_event - Waits for encoder events
+ * @encoder:	encoder pointer
+ * @event:      event to wait for
+ * MSM_ENC_COMMIT_DONE -  Wait for hardware to have flushed the current pending
+ *                        frames to hardware at a vblank or ctl_start
+ *                        Encoders will map this differently depending on the
+ *                        panel type.
+ *	                  vid mode -> vsync_irq
+ *                        cmd mode -> ctl_start
+ * MSM_ENC_TX_COMPLETE -  Wait for the hardware to transfer all the pixels to
+ *                        the panel. Encoders will map this differently
+ *                        depending on the panel type.
+ *                        vid mode -> vsync_irq
+ *                        cmd mode -> pp_done
+ * Returns: 0 on success, -EWOULDBLOCK if already signaled, error otherwise
+ */
+int dpu_encoder_wait_for_event(struct drm_encoder *drm_encoder,
+						enum msm_event_wait event);
+
+/*
+ * dpu_encoder_get_intf_mode - get interface mode of the given encoder
+ * @encoder: Pointer to drm encoder object
+ */
+enum dpu_intf_mode dpu_encoder_get_intf_mode(struct drm_encoder *encoder);
+
+/**
+ * dpu_encoder_virt_restore - restore the encoder configs
+ * @encoder:	encoder pointer
+ */
+void dpu_encoder_virt_restore(struct drm_encoder *encoder);
+
+/**
+ * dpu_encoder_init - initialize virtual encoder object
+ * @dev:        Pointer to drm device structure
+ * @disp_info:  Pointer to display information structure
+ * Returns:     Pointer to newly created drm encoder
+ */
+struct drm_encoder *dpu_encoder_init(
+		struct drm_device *dev,
+		int drm_enc_mode);
+
+/**
+ * dpu_encoder_setup - setup dpu_encoder for the display probed
+ * @dev:		Pointer to drm device structure
+ * @enc:		Pointer to the drm_encoder
+ * @disp_info:	Pointer to the display info
+ */
+int dpu_encoder_setup(struct drm_device *dev, struct drm_encoder *enc,
+		struct msm_display_info *disp_info);
+
+/**
+ * dpu_encoder_prepare_commit - prepare encoder at the very beginning of an
+ *	atomic commit, before any registers are written
+ * @drm_enc:    Pointer to previously created drm encoder structure
+ */
+void dpu_encoder_prepare_commit(struct drm_encoder *drm_enc);
+
+/**
+ * dpu_encoder_set_idle_timeout - set the idle timeout for video
+ *                    and command mode encoders.
+ * @drm_enc:    Pointer to previously created drm encoder structure
+ * @idle_timeout:    idle timeout duration in milliseconds
+ */
+void dpu_encoder_set_idle_timeout(struct drm_encoder *drm_enc,
+							u32 idle_timeout);
+
+#endif /* __DPU_ENCODER_H__ */
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys.h
new file mode 100644
index 000000000000..c7df8aad6613
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys.h
@@ -0,0 +1,430 @@
+/*
+ * Copyright (c) 2015-2018 The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#ifndef __DPU_ENCODER_PHYS_H__
+#define __DPU_ENCODER_PHYS_H__
+
+#include <linux/jiffies.h>
+
+#include "dpu_kms.h"
+#include "dpu_hw_intf.h"
+#include "dpu_hw_pingpong.h"
+#include "dpu_hw_ctl.h"
+#include "dpu_hw_top.h"
+#include "dpu_hw_cdm.h"
+#include "dpu_encoder.h"
+
+#define DPU_ENCODER_NAME_MAX	16
+
+/* wait for at most 2 vsync for lowest refresh rate (24hz) */
+#define KICKOFF_TIMEOUT_MS		84
+#define KICKOFF_TIMEOUT_JIFFIES		msecs_to_jiffies(KICKOFF_TIMEOUT_MS)
+
+/**
+ * enum dpu_enc_split_role - Role this physical encoder will play in a
+ *	split-panel configuration, where one panel is master, and others slaves.
+ *	Masters have extra responsibilities, like managing the VBLANK IRQ.
+ * @ENC_ROLE_SOLO:	This is the one and only panel. This encoder is master.
+ * @ENC_ROLE_MASTER:	This encoder is the master of a split panel config.
+ * @ENC_ROLE_SLAVE:	This encoder is not the master of a split panel config.
+ */
+enum dpu_enc_split_role {
+	ENC_ROLE_SOLO,
+	ENC_ROLE_MASTER,
+	ENC_ROLE_SLAVE,
+};
+
+/**
+ * enum dpu_enc_enable_state - current enabled state of the physical encoder
+ * @DPU_ENC_DISABLING:	Encoder transitioning to disable state
+ *			Events bounding transition are encoder type specific
+ * @DPU_ENC_DISABLED:	Encoder is disabled
+ * @DPU_ENC_ENABLING:	Encoder transitioning to enabled
+ *			Events bounding transition are encoder type specific
+ * @DPU_ENC_ENABLED:	Encoder is enabled
+ * @DPU_ENC_ERR_NEEDS_HW_RESET:	Encoder is enabled, but requires a hw_reset
+ *				to recover from a previous error
+ */
+enum dpu_enc_enable_state {
+	DPU_ENC_DISABLING,
+	DPU_ENC_DISABLED,
+	DPU_ENC_ENABLING,
+	DPU_ENC_ENABLED,
+	DPU_ENC_ERR_NEEDS_HW_RESET
+};
+
+struct dpu_encoder_phys;
+
+/**
+ * struct dpu_encoder_virt_ops - Interface the containing virtual encoder
+ *	provides for the physical encoders to use to callback.
+ * @handle_vblank_virt:	Notify virtual encoder of vblank IRQ reception
+ *			Note: This is called from IRQ handler context.
+ * @handle_underrun_virt: Notify virtual encoder of underrun IRQ reception
+ *			Note: This is called from IRQ handler context.
+ * @handle_frame_done:	Notify virtual encoder that this phys encoder
+ *			completes last request frame.
+ */
+struct dpu_encoder_virt_ops {
+	void (*handle_vblank_virt)(struct drm_encoder *,
+			struct dpu_encoder_phys *phys);
+	void (*handle_underrun_virt)(struct drm_encoder *,
+			struct dpu_encoder_phys *phys);
+	void (*handle_frame_done)(struct drm_encoder *,
+			struct dpu_encoder_phys *phys, u32 event);
+};
+
+/**
+ * struct dpu_encoder_phys_ops - Interface the physical encoders provide to
+ *	the containing virtual encoder.
+ * @late_register:		DRM Call. Add Userspace interfaces, debugfs.
+ * @prepare_commit:		MSM Atomic Call, start of atomic commit sequence
+ * @is_master:			Whether this phys_enc is the current master
+ *				encoder. Can be switched at enable time. Based
+ *				on split_role and current mode (CMD/VID).
+ * @mode_fixup:			DRM Call. Fixup a DRM mode.
+ * @mode_set:			DRM Call. Set a DRM mode.
+ *				This likely caches the mode, for use at enable.
+ * @enable:			DRM Call. Enable a DRM mode.
+ * @disable:			DRM Call. Disable mode.
+ * @atomic_check:		DRM Call. Atomic check new DRM state.
+ * @destroy:			DRM Call. Destroy and release resources.
+ * @get_hw_resources:		Populate the structure with the hardware
+ *				resources that this phys_enc is using.
+ *				Expect no overlap between phys_encs.
+ * @control_vblank_irq		Register/Deregister for VBLANK IRQ
+ * @wait_for_commit_done:	Wait for hardware to have flushed the
+ *				current pending frames to hardware
+ * @wait_for_tx_complete:	Wait for hardware to transfer the pixels
+ *				to the panel
+ * @wait_for_vblank:		Wait for VBLANK, for sub-driver internal use
+ * @prepare_for_kickoff:	Do any work necessary prior to a kickoff
+ *				For CMD encoder, may wait for previous tx done
+ * @handle_post_kickoff:	Do any work necessary post-kickoff work
+ * @trigger_start:		Process start event on physical encoder
+ * @needs_single_flush:		Whether encoder slaves need to be flushed
+ * @setup_misr:		Sets up MISR, enable and disables based on sysfs
+ * @collect_misr:		Collects MISR data on frame update
+ * @hw_reset:			Issue HW recovery such as CTL reset and clear
+ *				DPU_ENC_ERR_NEEDS_HW_RESET state
+ * @irq_control:		Handler to enable/disable all the encoder IRQs
+ * @prepare_idle_pc:		phys encoder can update the vsync_enable status
+ *                              on idle power collapse prepare
+ * @restore:			Restore all the encoder configs.
+ * @get_line_count:		Obtain current vertical line count
+ */
+
+struct dpu_encoder_phys_ops {
+	int (*late_register)(struct dpu_encoder_phys *encoder,
+			struct dentry *debugfs_root);
+	void (*prepare_commit)(struct dpu_encoder_phys *encoder);
+	bool (*is_master)(struct dpu_encoder_phys *encoder);
+	bool (*mode_fixup)(struct dpu_encoder_phys *encoder,
+			const struct drm_display_mode *mode,
+			struct drm_display_mode *adjusted_mode);
+	void (*mode_set)(struct dpu_encoder_phys *encoder,
+			struct drm_display_mode *mode,
+			struct drm_display_mode *adjusted_mode);
+	void (*enable)(struct dpu_encoder_phys *encoder);
+	void (*disable)(struct dpu_encoder_phys *encoder);
+	int (*atomic_check)(struct dpu_encoder_phys *encoder,
+			    struct drm_crtc_state *crtc_state,
+			    struct drm_connector_state *conn_state);
+	void (*destroy)(struct dpu_encoder_phys *encoder);
+	void (*get_hw_resources)(struct dpu_encoder_phys *encoder,
+			struct dpu_encoder_hw_resources *hw_res,
+			struct drm_connector_state *conn_state);
+	int (*control_vblank_irq)(struct dpu_encoder_phys *enc, bool enable);
+	int (*wait_for_commit_done)(struct dpu_encoder_phys *phys_enc);
+	int (*wait_for_tx_complete)(struct dpu_encoder_phys *phys_enc);
+	int (*wait_for_vblank)(struct dpu_encoder_phys *phys_enc);
+	void (*prepare_for_kickoff)(struct dpu_encoder_phys *phys_enc,
+			struct dpu_encoder_kickoff_params *params);
+	void (*handle_post_kickoff)(struct dpu_encoder_phys *phys_enc);
+	void (*trigger_start)(struct dpu_encoder_phys *phys_enc);
+	bool (*needs_single_flush)(struct dpu_encoder_phys *phys_enc);
+
+	void (*setup_misr)(struct dpu_encoder_phys *phys_encs,
+				bool enable, u32 frame_count);
+	u32 (*collect_misr)(struct dpu_encoder_phys *phys_enc);
+	void (*hw_reset)(struct dpu_encoder_phys *phys_enc);
+	void (*irq_control)(struct dpu_encoder_phys *phys, bool enable);
+	void (*prepare_idle_pc)(struct dpu_encoder_phys *phys_enc);
+	void (*restore)(struct dpu_encoder_phys *phys);
+	int (*get_line_count)(struct dpu_encoder_phys *phys);
+};
+
+/**
+ * enum dpu_intr_idx - dpu encoder interrupt index
+ * @INTR_IDX_VSYNC:    Vsync interrupt for video mode panel
+ * @INTR_IDX_PINGPONG: Pingpong done unterrupt for cmd mode panel
+ * @INTR_IDX_UNDERRUN: Underrun unterrupt for video and cmd mode panel
+ * @INTR_IDX_RDPTR:    Readpointer done unterrupt for cmd mode panel
+ */
+enum dpu_intr_idx {
+	INTR_IDX_VSYNC,
+	INTR_IDX_PINGPONG,
+	INTR_IDX_UNDERRUN,
+	INTR_IDX_CTL_START,
+	INTR_IDX_RDPTR,
+	INTR_IDX_MAX,
+};
+
+/**
+ * dpu_encoder_irq - tracking structure for interrupts
+ * @name:		string name of interrupt
+ * @intr_type:		Encoder interrupt type
+ * @intr_idx:		Encoder interrupt enumeration
+ * @hw_idx:		HW Block ID
+ * @irq_idx:		IRQ interface lookup index from DPU IRQ framework
+ *			will be -EINVAL if IRQ is not registered
+ * @irq_cb:		interrupt callback
+ */
+struct dpu_encoder_irq {
+	const char *name;
+	enum dpu_intr_type intr_type;
+	enum dpu_intr_idx intr_idx;
+	int hw_idx;
+	int irq_idx;
+	struct dpu_irq_callback cb;
+};
+
+/**
+ * struct dpu_encoder_phys - physical encoder that drives a single INTF block
+ *	tied to a specific panel / sub-panel. Abstract type, sub-classed by
+ *	phys_vid or phys_cmd for video mode or command mode encs respectively.
+ * @parent:		Pointer to the containing virtual encoder
+ * @connector:		If a mode is set, cached pointer to the active connector
+ * @ops:		Operations exposed to the virtual encoder
+ * @parent_ops:		Callbacks exposed by the parent to the phys_enc
+ * @hw_mdptop:		Hardware interface to the top registers
+ * @hw_ctl:		Hardware interface to the ctl registers
+ * @hw_cdm:		Hardware interface to the cdm registers
+ * @cdm_cfg:		Chroma-down hardware configuration
+ * @hw_pp:		Hardware interface to the ping pong registers
+ * @dpu_kms:		Pointer to the dpu_kms top level
+ * @cached_mode:	DRM mode cached at mode_set time, acted on in enable
+ * @enabled:		Whether the encoder has enabled and running a mode
+ * @split_role:		Role to play in a split-panel configuration
+ * @intf_mode:		Interface mode
+ * @intf_idx:		Interface index on dpu hardware
+ * @topology_name:	topology selected for the display
+ * @enc_spinlock:	Virtual-Encoder-Wide Spin Lock for IRQ purposes
+ * @enable_state:	Enable state tracking
+ * @vblank_refcount:	Reference count of vblank request
+ * @vsync_cnt:		Vsync count for the physical encoder
+ * @underrun_cnt:	Underrun count for the physical encoder
+ * @pending_kickoff_cnt:	Atomic counter tracking the number of kickoffs
+ *				vs. the number of done/vblank irqs. Should hover
+ *				between 0-2 Incremented when a new kickoff is
+ *				scheduled. Decremented in irq handler
+ * @pending_ctlstart_cnt:	Atomic counter tracking the number of ctl start
+ *                              pending.
+ * @pending_kickoff_wq:		Wait queue for blocking until kickoff completes
+ * @irq:			IRQ tracking structures
+ */
+struct dpu_encoder_phys {
+	struct drm_encoder *parent;
+	struct drm_connector *connector;
+	struct dpu_encoder_phys_ops ops;
+	const struct dpu_encoder_virt_ops *parent_ops;
+	struct dpu_hw_mdp *hw_mdptop;
+	struct dpu_hw_ctl *hw_ctl;
+	struct dpu_hw_cdm *hw_cdm;
+	struct dpu_hw_cdm_cfg cdm_cfg;
+	struct dpu_hw_pingpong *hw_pp;
+	struct dpu_kms *dpu_kms;
+	struct drm_display_mode cached_mode;
+	enum dpu_enc_split_role split_role;
+	enum dpu_intf_mode intf_mode;
+	enum dpu_intf intf_idx;
+	enum dpu_rm_topology_name topology_name;
+	spinlock_t *enc_spinlock;
+	enum dpu_enc_enable_state enable_state;
+	atomic_t vblank_refcount;
+	atomic_t vsync_cnt;
+	atomic_t underrun_cnt;
+	atomic_t pending_ctlstart_cnt;
+	atomic_t pending_kickoff_cnt;
+	wait_queue_head_t pending_kickoff_wq;
+	struct dpu_encoder_irq irq[INTR_IDX_MAX];
+};
+
+static inline int dpu_encoder_phys_inc_pending(struct dpu_encoder_phys *phys)
+{
+	atomic_inc_return(&phys->pending_ctlstart_cnt);
+	return atomic_inc_return(&phys->pending_kickoff_cnt);
+}
+
+/**
+ * struct dpu_encoder_phys_vid - sub-class of dpu_encoder_phys to handle video
+ *	mode specific operations
+ * @base:	Baseclass physical encoder structure
+ * @hw_intf:	Hardware interface to the intf registers
+ * @timing_params: Current timing parameter
+ */
+struct dpu_encoder_phys_vid {
+	struct dpu_encoder_phys base;
+	struct dpu_hw_intf *hw_intf;
+	struct intf_timing_params timing_params;
+};
+
+/**
+ * struct dpu_encoder_phys_cmd - sub-class of dpu_encoder_phys to handle command
+ *	mode specific operations
+ * @base:	Baseclass physical encoder structure
+ * @intf_idx:	Intf Block index used by this phys encoder
+ * @stream_sel:	Stream selection for multi-stream interfaces
+ * @serialize_wait4pp:	serialize wait4pp feature waits for pp_done interrupt
+ *			after ctl_start instead of before next frame kickoff
+ * @pp_timeout_report_cnt: number of pingpong done irq timeout errors
+ * @pending_vblank_cnt: Atomic counter tracking pending wait for VBLANK
+ * @pending_vblank_wq: Wait queue for blocking until VBLANK received
+ */
+struct dpu_encoder_phys_cmd {
+	struct dpu_encoder_phys base;
+	int stream_sel;
+	bool serialize_wait4pp;
+	int pp_timeout_report_cnt;
+	atomic_t pending_vblank_cnt;
+	wait_queue_head_t pending_vblank_wq;
+};
+
+/**
+ * struct dpu_enc_phys_init_params - initialization parameters for phys encs
+ * @dpu_kms:		Pointer to the dpu_kms top level
+ * @parent:		Pointer to the containing virtual encoder
+ * @parent_ops:		Callbacks exposed by the parent to the phys_enc
+ * @split_role:		Role to play in a split-panel configuration
+ * @intf_idx:		Interface index this phys_enc will control
+ * @enc_spinlock:	Virtual-Encoder-Wide Spin Lock for IRQ purposes
+ */
+struct dpu_enc_phys_init_params {
+	struct dpu_kms *dpu_kms;
+	struct drm_encoder *parent;
+	const struct dpu_encoder_virt_ops *parent_ops;
+	enum dpu_enc_split_role split_role;
+	enum dpu_intf intf_idx;
+	spinlock_t *enc_spinlock;
+};
+
+/**
+ * dpu_encoder_wait_info - container for passing arguments to irq wait functions
+ * @wq: wait queue structure
+ * @atomic_cnt: wait until atomic_cnt equals zero
+ * @timeout_ms: timeout value in milliseconds
+ */
+struct dpu_encoder_wait_info {
+	wait_queue_head_t *wq;
+	atomic_t *atomic_cnt;
+	s64 timeout_ms;
+};
+
+/**
+ * dpu_encoder_phys_vid_init - Construct a new video mode physical encoder
+ * @p:	Pointer to init params structure
+ * Return: Error code or newly allocated encoder
+ */
+struct dpu_encoder_phys *dpu_encoder_phys_vid_init(
+		struct dpu_enc_phys_init_params *p);
+
+/**
+ * dpu_encoder_phys_cmd_init - Construct a new command mode physical encoder
+ * @p:	Pointer to init params structure
+ * Return: Error code or newly allocated encoder
+ */
+struct dpu_encoder_phys *dpu_encoder_phys_cmd_init(
+		struct dpu_enc_phys_init_params *p);
+
+/**
+ * dpu_encoder_helper_trigger_start - control start helper function
+ *	This helper function may be optionally specified by physical
+ *	encoders if they require ctl_start triggering.
+ * @phys_enc: Pointer to physical encoder structure
+ */
+void dpu_encoder_helper_trigger_start(struct dpu_encoder_phys *phys_enc);
+
+/**
+ * dpu_encoder_helper_hw_reset - issue ctl hw reset
+ *	This helper function may be optionally specified by physical
+ *	encoders if they require ctl hw reset. If state is currently
+ *	DPU_ENC_ERR_NEEDS_HW_RESET, it is set back to DPU_ENC_ENABLED.
+ * @phys_enc: Pointer to physical encoder structure
+ */
+void dpu_encoder_helper_hw_reset(struct dpu_encoder_phys *phys_enc);
+
+static inline enum dpu_3d_blend_mode dpu_encoder_helper_get_3d_blend_mode(
+		struct dpu_encoder_phys *phys_enc)
+{
+	if (!phys_enc || phys_enc->enable_state == DPU_ENC_DISABLING)
+		return BLEND_3D_NONE;
+
+	if (phys_enc->split_role == ENC_ROLE_SOLO &&
+	    phys_enc->topology_name == DPU_RM_TOPOLOGY_DUALPIPE_3DMERGE)
+		return BLEND_3D_H_ROW_INT;
+
+	return BLEND_3D_NONE;
+}
+
+/**
+ * dpu_encoder_helper_split_config - split display configuration helper function
+ *	This helper function may be used by physical encoders to configure
+ *	the split display related registers.
+ * @phys_enc: Pointer to physical encoder structure
+ * @interface: enum dpu_intf setting
+ */
+void dpu_encoder_helper_split_config(
+		struct dpu_encoder_phys *phys_enc,
+		enum dpu_intf interface);
+
+/**
+ * dpu_encoder_helper_report_irq_timeout - utility to report error that irq has
+ *	timed out, including reporting frame error event to crtc and debug dump
+ * @phys_enc: Pointer to physical encoder structure
+ * @intr_idx: Failing interrupt index
+ */
+void dpu_encoder_helper_report_irq_timeout(struct dpu_encoder_phys *phys_enc,
+		enum dpu_intr_idx intr_idx);
+
+/**
+ * dpu_encoder_helper_wait_for_irq - utility to wait on an irq.
+ *	note: will call dpu_encoder_helper_wait_for_irq on timeout
+ * @phys_enc: Pointer to physical encoder structure
+ * @intr_idx: encoder interrupt index
+ * @wait_info: wait info struct
+ * @Return: 0 or -ERROR
+ */
+int dpu_encoder_helper_wait_for_irq(struct dpu_encoder_phys *phys_enc,
+		enum dpu_intr_idx intr_idx,
+		struct dpu_encoder_wait_info *wait_info);
+
+/**
+ * dpu_encoder_helper_register_irq - register and enable an irq
+ * @phys_enc: Pointer to physical encoder structure
+ * @intr_idx: encoder interrupt index
+ * @Return: 0 or -ERROR
+ */
+int dpu_encoder_helper_register_irq(struct dpu_encoder_phys *phys_enc,
+		enum dpu_intr_idx intr_idx);
+
+/**
+ * dpu_encoder_helper_unregister_irq - unregister and disable an irq
+ * @phys_enc: Pointer to physical encoder structure
+ * @intr_idx: encoder interrupt index
+ * @Return: 0 or -ERROR
+ */
+int dpu_encoder_helper_unregister_irq(struct dpu_encoder_phys *phys_enc,
+		enum dpu_intr_idx intr_idx);
+
+#endif /* __dpu_encoder_phys_H__ */
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_cmd.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_cmd.c
new file mode 100644
index 000000000000..3084675ed425
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_cmd.c
@@ -0,0 +1,905 @@
+/*
+ * Copyright (c) 2015-2018 The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#define pr_fmt(fmt)	"[drm:%s:%d] " fmt, __func__, __LINE__
+#include "dpu_encoder_phys.h"
+#include "dpu_hw_interrupts.h"
+#include "dpu_core_irq.h"
+#include "dpu_formats.h"
+#include "dpu_trace.h"
+
+#define DPU_DEBUG_CMDENC(e, fmt, ...) DPU_DEBUG("enc%d intf%d " fmt, \
+		(e) && (e)->base.parent ? \
+		(e)->base.parent->base.id : -1, \
+		(e) ? (e)->base.intf_idx - INTF_0 : -1, ##__VA_ARGS__)
+
+#define DPU_ERROR_CMDENC(e, fmt, ...) DPU_ERROR("enc%d intf%d " fmt, \
+		(e) && (e)->base.parent ? \
+		(e)->base.parent->base.id : -1, \
+		(e) ? (e)->base.intf_idx - INTF_0 : -1, ##__VA_ARGS__)
+
+#define to_dpu_encoder_phys_cmd(x) \
+	container_of(x, struct dpu_encoder_phys_cmd, base)
+
+#define PP_TIMEOUT_MAX_TRIALS	10
+
+/*
+ * Tearcheck sync start and continue thresholds are empirically found
+ * based on common panels In the future, may want to allow panels to override
+ * these default values
+ */
+#define DEFAULT_TEARCHECK_SYNC_THRESH_START	4
+#define DEFAULT_TEARCHECK_SYNC_THRESH_CONTINUE	4
+
+#define DPU_ENC_WR_PTR_START_TIMEOUT_US 20000
+
+static inline int _dpu_encoder_phys_cmd_get_idle_timeout(
+		struct dpu_encoder_phys_cmd *cmd_enc)
+{
+	return KICKOFF_TIMEOUT_MS;
+}
+
+static inline bool dpu_encoder_phys_cmd_is_master(
+		struct dpu_encoder_phys *phys_enc)
+{
+	return (phys_enc->split_role != ENC_ROLE_SLAVE) ? true : false;
+}
+
+static bool dpu_encoder_phys_cmd_mode_fixup(
+		struct dpu_encoder_phys *phys_enc,
+		const struct drm_display_mode *mode,
+		struct drm_display_mode *adj_mode)
+{
+	if (phys_enc)
+		DPU_DEBUG_CMDENC(to_dpu_encoder_phys_cmd(phys_enc), "\n");
+	return true;
+}
+
+static void _dpu_encoder_phys_cmd_update_intf_cfg(
+		struct dpu_encoder_phys *phys_enc)
+{
+	struct dpu_encoder_phys_cmd *cmd_enc =
+			to_dpu_encoder_phys_cmd(phys_enc);
+	struct dpu_hw_ctl *ctl;
+	struct dpu_hw_intf_cfg intf_cfg = { 0 };
+
+	if (!phys_enc)
+		return;
+
+	ctl = phys_enc->hw_ctl;
+	if (!ctl || !ctl->ops.setup_intf_cfg)
+		return;
+
+	intf_cfg.intf = phys_enc->intf_idx;
+	intf_cfg.intf_mode_sel = DPU_CTL_MODE_SEL_CMD;
+	intf_cfg.stream_sel = cmd_enc->stream_sel;
+	intf_cfg.mode_3d = dpu_encoder_helper_get_3d_blend_mode(phys_enc);
+	ctl->ops.setup_intf_cfg(ctl, &intf_cfg);
+}
+
+static void dpu_encoder_phys_cmd_pp_tx_done_irq(void *arg, int irq_idx)
+{
+	struct dpu_encoder_phys *phys_enc = arg;
+	unsigned long lock_flags;
+	int new_cnt;
+	u32 event = DPU_ENCODER_FRAME_EVENT_DONE;
+
+	if (!phys_enc || !phys_enc->hw_pp)
+		return;
+
+	DPU_ATRACE_BEGIN("pp_done_irq");
+	/* notify all synchronous clients first, then asynchronous clients */
+	if (phys_enc->parent_ops->handle_frame_done)
+		phys_enc->parent_ops->handle_frame_done(phys_enc->parent,
+				phys_enc, event);
+
+	spin_lock_irqsave(phys_enc->enc_spinlock, lock_flags);
+	new_cnt = atomic_add_unless(&phys_enc->pending_kickoff_cnt, -1, 0);
+	spin_unlock_irqrestore(phys_enc->enc_spinlock, lock_flags);
+
+	trace_dpu_enc_phys_cmd_pp_tx_done(DRMID(phys_enc->parent),
+					  phys_enc->hw_pp->idx - PINGPONG_0,
+					  new_cnt, event);
+
+	/* Signal any waiting atomic commit thread */
+	wake_up_all(&phys_enc->pending_kickoff_wq);
+	DPU_ATRACE_END("pp_done_irq");
+}
+
+static void dpu_encoder_phys_cmd_pp_rd_ptr_irq(void *arg, int irq_idx)
+{
+	struct dpu_encoder_phys *phys_enc = arg;
+	struct dpu_encoder_phys_cmd *cmd_enc;
+
+	if (!phys_enc || !phys_enc->hw_pp)
+		return;
+
+	DPU_ATRACE_BEGIN("rd_ptr_irq");
+	cmd_enc = to_dpu_encoder_phys_cmd(phys_enc);
+
+	if (phys_enc->parent_ops->handle_vblank_virt)
+		phys_enc->parent_ops->handle_vblank_virt(phys_enc->parent,
+			phys_enc);
+
+	atomic_add_unless(&cmd_enc->pending_vblank_cnt, -1, 0);
+	wake_up_all(&cmd_enc->pending_vblank_wq);
+	DPU_ATRACE_END("rd_ptr_irq");
+}
+
+static void dpu_encoder_phys_cmd_ctl_start_irq(void *arg, int irq_idx)
+{
+	struct dpu_encoder_phys *phys_enc = arg;
+	struct dpu_encoder_phys_cmd *cmd_enc;
+
+	if (!phys_enc || !phys_enc->hw_ctl)
+		return;
+
+	DPU_ATRACE_BEGIN("ctl_start_irq");
+	cmd_enc = to_dpu_encoder_phys_cmd(phys_enc);
+
+	atomic_add_unless(&phys_enc->pending_ctlstart_cnt, -1, 0);
+
+	/* Signal any waiting ctl start interrupt */
+	wake_up_all(&phys_enc->pending_kickoff_wq);
+	DPU_ATRACE_END("ctl_start_irq");
+}
+
+static void dpu_encoder_phys_cmd_underrun_irq(void *arg, int irq_idx)
+{
+	struct dpu_encoder_phys *phys_enc = arg;
+
+	if (!phys_enc)
+		return;
+
+	if (phys_enc->parent_ops->handle_underrun_virt)
+		phys_enc->parent_ops->handle_underrun_virt(phys_enc->parent,
+			phys_enc);
+}
+
+static void _dpu_encoder_phys_cmd_setup_irq_hw_idx(
+		struct dpu_encoder_phys *phys_enc)
+{
+	struct dpu_encoder_irq *irq;
+
+	irq = &phys_enc->irq[INTR_IDX_CTL_START];
+	irq->hw_idx = phys_enc->hw_ctl->idx;
+	irq->irq_idx = -EINVAL;
+
+	irq = &phys_enc->irq[INTR_IDX_PINGPONG];
+	irq->hw_idx = phys_enc->hw_pp->idx;
+	irq->irq_idx = -EINVAL;
+
+	irq = &phys_enc->irq[INTR_IDX_RDPTR];
+	irq->hw_idx = phys_enc->hw_pp->idx;
+	irq->irq_idx = -EINVAL;
+
+	irq = &phys_enc->irq[INTR_IDX_UNDERRUN];
+	irq->hw_idx = phys_enc->intf_idx;
+	irq->irq_idx = -EINVAL;
+}
+
+static void dpu_encoder_phys_cmd_mode_set(
+		struct dpu_encoder_phys *phys_enc,
+		struct drm_display_mode *mode,
+		struct drm_display_mode *adj_mode)
+{
+	struct dpu_encoder_phys_cmd *cmd_enc =
+		to_dpu_encoder_phys_cmd(phys_enc);
+	struct dpu_rm *rm = &phys_enc->dpu_kms->rm;
+	struct dpu_rm_hw_iter iter;
+	int i, instance;
+
+	if (!phys_enc || !mode || !adj_mode) {
+		DPU_ERROR("invalid args\n");
+		return;
+	}
+	phys_enc->cached_mode = *adj_mode;
+	DPU_DEBUG_CMDENC(cmd_enc, "caching mode:\n");
+	drm_mode_debug_printmodeline(adj_mode);
+
+	instance = phys_enc->split_role == ENC_ROLE_SLAVE ? 1 : 0;
+
+	/* Retrieve previously allocated HW Resources. Shouldn't fail */
+	dpu_rm_init_hw_iter(&iter, phys_enc->parent->base.id, DPU_HW_BLK_CTL);
+	for (i = 0; i <= instance; i++) {
+		if (dpu_rm_get_hw(rm, &iter))
+			phys_enc->hw_ctl = (struct dpu_hw_ctl *)iter.hw;
+	}
+
+	if (IS_ERR_OR_NULL(phys_enc->hw_ctl)) {
+		DPU_ERROR_CMDENC(cmd_enc, "failed to init ctl: %ld\n",
+				PTR_ERR(phys_enc->hw_ctl));
+		phys_enc->hw_ctl = NULL;
+		return;
+	}
+
+	_dpu_encoder_phys_cmd_setup_irq_hw_idx(phys_enc);
+}
+
+static int _dpu_encoder_phys_cmd_handle_ppdone_timeout(
+		struct dpu_encoder_phys *phys_enc)
+{
+	struct dpu_encoder_phys_cmd *cmd_enc =
+			to_dpu_encoder_phys_cmd(phys_enc);
+	u32 frame_event = DPU_ENCODER_FRAME_EVENT_ERROR;
+	bool do_log = false;
+
+	if (!phys_enc || !phys_enc->hw_pp || !phys_enc->hw_ctl)
+		return -EINVAL;
+
+	cmd_enc->pp_timeout_report_cnt++;
+	if (cmd_enc->pp_timeout_report_cnt == PP_TIMEOUT_MAX_TRIALS) {
+		frame_event |= DPU_ENCODER_FRAME_EVENT_PANEL_DEAD;
+		do_log = true;
+	} else if (cmd_enc->pp_timeout_report_cnt == 1) {
+		do_log = true;
+	}
+
+	trace_dpu_enc_phys_cmd_pdone_timeout(DRMID(phys_enc->parent),
+		     phys_enc->hw_pp->idx - PINGPONG_0,
+		     cmd_enc->pp_timeout_report_cnt,
+		     atomic_read(&phys_enc->pending_kickoff_cnt),
+		     frame_event);
+
+	/* to avoid flooding, only log first time, and "dead" time */
+	if (do_log) {
+		DRM_ERROR("id:%d pp:%d kickoff timeout %d cnt %d koff_cnt %d\n",
+			  DRMID(phys_enc->parent),
+			  phys_enc->hw_pp->idx - PINGPONG_0,
+			  phys_enc->hw_ctl->idx - CTL_0,
+			  cmd_enc->pp_timeout_report_cnt,
+			  atomic_read(&phys_enc->pending_kickoff_cnt));
+
+		dpu_encoder_helper_unregister_irq(phys_enc, INTR_IDX_RDPTR);
+		dpu_dbg_dump(false, __func__, true, true);
+	}
+
+	atomic_add_unless(&phys_enc->pending_kickoff_cnt, -1, 0);
+
+	/* request a ctl reset before the next kickoff */
+	phys_enc->enable_state = DPU_ENC_ERR_NEEDS_HW_RESET;
+
+	if (phys_enc->parent_ops->handle_frame_done)
+		phys_enc->parent_ops->handle_frame_done(
+				phys_enc->parent, phys_enc, frame_event);
+
+	return -ETIMEDOUT;
+}
+
+static int _dpu_encoder_phys_cmd_wait_for_idle(
+		struct dpu_encoder_phys *phys_enc)
+{
+	struct dpu_encoder_phys_cmd *cmd_enc =
+			to_dpu_encoder_phys_cmd(phys_enc);
+	struct dpu_encoder_wait_info wait_info;
+	int ret;
+
+	if (!phys_enc) {
+		DPU_ERROR("invalid encoder\n");
+		return -EINVAL;
+	}
+
+	wait_info.wq = &phys_enc->pending_kickoff_wq;
+	wait_info.atomic_cnt = &phys_enc->pending_kickoff_cnt;
+	wait_info.timeout_ms = KICKOFF_TIMEOUT_MS;
+
+	ret = dpu_encoder_helper_wait_for_irq(phys_enc, INTR_IDX_PINGPONG,
+			&wait_info);
+	if (ret == -ETIMEDOUT)
+		_dpu_encoder_phys_cmd_handle_ppdone_timeout(phys_enc);
+	else if (!ret)
+		cmd_enc->pp_timeout_report_cnt = 0;
+
+	return ret;
+}
+
+static int dpu_encoder_phys_cmd_control_vblank_irq(
+		struct dpu_encoder_phys *phys_enc,
+		bool enable)
+{
+	int ret = 0;
+	int refcount;
+
+	if (!phys_enc || !phys_enc->hw_pp) {
+		DPU_ERROR("invalid encoder\n");
+		return -EINVAL;
+	}
+
+	refcount = atomic_read(&phys_enc->vblank_refcount);
+
+	/* Slave encoders don't report vblank */
+	if (!dpu_encoder_phys_cmd_is_master(phys_enc))
+		goto end;
+
+	/* protect against negative */
+	if (!enable && refcount == 0) {
+		ret = -EINVAL;
+		goto end;
+	}
+
+	DRM_DEBUG_KMS("id:%u pp:%d enable=%s/%d\n", DRMID(phys_enc->parent),
+		      phys_enc->hw_pp->idx - PINGPONG_0,
+		      enable ? "true" : "false", refcount);
+
+	if (enable && atomic_inc_return(&phys_enc->vblank_refcount) == 1)
+		ret = dpu_encoder_helper_register_irq(phys_enc, INTR_IDX_RDPTR);
+	else if (!enable && atomic_dec_return(&phys_enc->vblank_refcount) == 0)
+		ret = dpu_encoder_helper_unregister_irq(phys_enc,
+				INTR_IDX_RDPTR);
+
+end:
+	if (ret) {
+		DRM_ERROR("vblank irq err id:%u pp:%d ret:%d, enable %s/%d\n",
+			  DRMID(phys_enc->parent),
+			  phys_enc->hw_pp->idx - PINGPONG_0, ret,
+			  enable ? "true" : "false", refcount);
+	}
+
+	return ret;
+}
+
+static void dpu_encoder_phys_cmd_irq_control(struct dpu_encoder_phys *phys_enc,
+		bool enable)
+{
+	struct dpu_encoder_phys_cmd *cmd_enc;
+
+	if (!phys_enc)
+		return;
+
+	cmd_enc = to_dpu_encoder_phys_cmd(phys_enc);
+
+	trace_dpu_enc_phys_cmd_irq_ctrl(DRMID(phys_enc->parent),
+			phys_enc->hw_pp->idx - PINGPONG_0,
+			enable, atomic_read(&phys_enc->vblank_refcount));
+
+	if (enable) {
+		dpu_encoder_helper_register_irq(phys_enc, INTR_IDX_PINGPONG);
+		dpu_encoder_helper_register_irq(phys_enc, INTR_IDX_UNDERRUN);
+		dpu_encoder_phys_cmd_control_vblank_irq(phys_enc, true);
+
+		if (dpu_encoder_phys_cmd_is_master(phys_enc))
+			dpu_encoder_helper_register_irq(phys_enc,
+					INTR_IDX_CTL_START);
+	} else {
+		if (dpu_encoder_phys_cmd_is_master(phys_enc))
+			dpu_encoder_helper_unregister_irq(phys_enc,
+					INTR_IDX_CTL_START);
+
+		dpu_encoder_helper_unregister_irq(phys_enc, INTR_IDX_UNDERRUN);
+		dpu_encoder_phys_cmd_control_vblank_irq(phys_enc, false);
+		dpu_encoder_helper_unregister_irq(phys_enc, INTR_IDX_PINGPONG);
+	}
+}
+
+static void dpu_encoder_phys_cmd_tearcheck_config(
+		struct dpu_encoder_phys *phys_enc)
+{
+	struct dpu_encoder_phys_cmd *cmd_enc =
+		to_dpu_encoder_phys_cmd(phys_enc);
+	struct dpu_hw_tear_check tc_cfg = { 0 };
+	struct drm_display_mode *mode;
+	bool tc_enable = true;
+	u32 vsync_hz;
+	struct msm_drm_private *priv;
+	struct dpu_kms *dpu_kms;
+
+	if (!phys_enc || !phys_enc->hw_pp) {
+		DPU_ERROR("invalid encoder\n");
+		return;
+	}
+	mode = &phys_enc->cached_mode;
+
+	DPU_DEBUG_CMDENC(cmd_enc, "pp %d\n", phys_enc->hw_pp->idx - PINGPONG_0);
+
+	if (!phys_enc->hw_pp->ops.setup_tearcheck ||
+		!phys_enc->hw_pp->ops.enable_tearcheck) {
+		DPU_DEBUG_CMDENC(cmd_enc, "tearcheck not supported\n");
+		return;
+	}
+
+	dpu_kms = phys_enc->dpu_kms;
+	if (!dpu_kms || !dpu_kms->dev || !dpu_kms->dev->dev_private) {
+		DPU_ERROR("invalid device\n");
+		return;
+	}
+	priv = dpu_kms->dev->dev_private;
+
+	/*
+	 * TE default: dsi byte clock calculated base on 70 fps;
+	 * around 14 ms to complete a kickoff cycle if te disabled;
+	 * vclk_line base on 60 fps; write is faster than read;
+	 * init == start == rdptr;
+	 *
+	 * vsync_count is ratio of MDP VSYNC clock frequency to LCD panel
+	 * frequency divided by the no. of rows (lines) in the LCDpanel.
+	 */
+	vsync_hz = dpu_kms_get_clk_rate(dpu_kms, "vsync");
+	if (vsync_hz <= 0) {
+		DPU_DEBUG_CMDENC(cmd_enc, "invalid - vsync_hz %u\n",
+				 vsync_hz);
+		return;
+	}
+
+	tc_cfg.vsync_count = vsync_hz / (mode->vtotal * mode->vrefresh);
+
+	/* enable external TE after kickoff to avoid premature autorefresh */
+	tc_cfg.hw_vsync_mode = 0;
+
+	/*
+	 * By setting sync_cfg_height to near max register value, we essentially
+	 * disable dpu hw generated TE signal, since hw TE will arrive first.
+	 * Only caveat is if due to error, we hit wrap-around.
+	 */
+	tc_cfg.sync_cfg_height = 0xFFF0;
+	tc_cfg.vsync_init_val = mode->vdisplay;
+	tc_cfg.sync_threshold_start = DEFAULT_TEARCHECK_SYNC_THRESH_START;
+	tc_cfg.sync_threshold_continue = DEFAULT_TEARCHECK_SYNC_THRESH_CONTINUE;
+	tc_cfg.start_pos = mode->vdisplay;
+	tc_cfg.rd_ptr_irq = mode->vdisplay + 1;
+
+	DPU_DEBUG_CMDENC(cmd_enc,
+		"tc %d vsync_clk_speed_hz %u vtotal %u vrefresh %u\n",
+		phys_enc->hw_pp->idx - PINGPONG_0, vsync_hz,
+		mode->vtotal, mode->vrefresh);
+	DPU_DEBUG_CMDENC(cmd_enc,
+		"tc %d enable %u start_pos %u rd_ptr_irq %u\n",
+		phys_enc->hw_pp->idx - PINGPONG_0, tc_enable, tc_cfg.start_pos,
+		tc_cfg.rd_ptr_irq);
+	DPU_DEBUG_CMDENC(cmd_enc,
+		"tc %d hw_vsync_mode %u vsync_count %u vsync_init_val %u\n",
+		phys_enc->hw_pp->idx - PINGPONG_0, tc_cfg.hw_vsync_mode,
+		tc_cfg.vsync_count, tc_cfg.vsync_init_val);
+	DPU_DEBUG_CMDENC(cmd_enc,
+		"tc %d cfgheight %u thresh_start %u thresh_cont %u\n",
+		phys_enc->hw_pp->idx - PINGPONG_0, tc_cfg.sync_cfg_height,
+		tc_cfg.sync_threshold_start, tc_cfg.sync_threshold_continue);
+
+	phys_enc->hw_pp->ops.setup_tearcheck(phys_enc->hw_pp, &tc_cfg);
+	phys_enc->hw_pp->ops.enable_tearcheck(phys_enc->hw_pp, tc_enable);
+}
+
+static void _dpu_encoder_phys_cmd_pingpong_config(
+		struct dpu_encoder_phys *phys_enc)
+{
+	struct dpu_encoder_phys_cmd *cmd_enc =
+		to_dpu_encoder_phys_cmd(phys_enc);
+
+	if (!phys_enc || !phys_enc->hw_ctl || !phys_enc->hw_pp
+			|| !phys_enc->hw_ctl->ops.setup_intf_cfg) {
+		DPU_ERROR("invalid arg(s), enc %d\n", phys_enc != 0);
+		return;
+	}
+
+	DPU_DEBUG_CMDENC(cmd_enc, "pp %d, enabling mode:\n",
+			phys_enc->hw_pp->idx - PINGPONG_0);
+	drm_mode_debug_printmodeline(&phys_enc->cached_mode);
+
+	_dpu_encoder_phys_cmd_update_intf_cfg(phys_enc);
+	dpu_encoder_phys_cmd_tearcheck_config(phys_enc);
+}
+
+static bool dpu_encoder_phys_cmd_needs_single_flush(
+		struct dpu_encoder_phys *phys_enc)
+{
+	/**
+	 * we do separate flush for each CTL and let
+	 * CTL_START synchronize them
+	 */
+	return false;
+}
+
+static void dpu_encoder_phys_cmd_enable_helper(
+		struct dpu_encoder_phys *phys_enc)
+{
+	struct dpu_hw_ctl *ctl;
+	u32 flush_mask = 0;
+
+	if (!phys_enc || !phys_enc->hw_ctl || !phys_enc->hw_pp) {
+		DPU_ERROR("invalid arg(s), encoder %d\n", phys_enc != 0);
+		return;
+	}
+
+	dpu_encoder_helper_split_config(phys_enc, phys_enc->intf_idx);
+
+	_dpu_encoder_phys_cmd_pingpong_config(phys_enc);
+
+	if (!dpu_encoder_phys_cmd_is_master(phys_enc))
+		goto skip_flush;
+
+	ctl = phys_enc->hw_ctl;
+	ctl->ops.get_bitmask_intf(ctl, &flush_mask, phys_enc->intf_idx);
+	ctl->ops.update_pending_flush(ctl, flush_mask);
+
+skip_flush:
+	return;
+}
+
+static void dpu_encoder_phys_cmd_enable(struct dpu_encoder_phys *phys_enc)
+{
+	struct dpu_encoder_phys_cmd *cmd_enc =
+		to_dpu_encoder_phys_cmd(phys_enc);
+
+	if (!phys_enc || !phys_enc->hw_pp) {
+		DPU_ERROR("invalid phys encoder\n");
+		return;
+	}
+
+	DPU_DEBUG_CMDENC(cmd_enc, "pp %d\n", phys_enc->hw_pp->idx - PINGPONG_0);
+
+	if (phys_enc->enable_state == DPU_ENC_ENABLED) {
+		DPU_ERROR("already enabled\n");
+		return;
+	}
+
+	dpu_encoder_phys_cmd_enable_helper(phys_enc);
+	phys_enc->enable_state = DPU_ENC_ENABLED;
+}
+
+static void _dpu_encoder_phys_cmd_connect_te(
+		struct dpu_encoder_phys *phys_enc, bool enable)
+{
+	if (!phys_enc || !phys_enc->hw_pp ||
+			!phys_enc->hw_pp->ops.connect_external_te)
+		return;
+
+	trace_dpu_enc_phys_cmd_connect_te(DRMID(phys_enc->parent), enable);
+	phys_enc->hw_pp->ops.connect_external_te(phys_enc->hw_pp, enable);
+}
+
+static void dpu_encoder_phys_cmd_prepare_idle_pc(
+		struct dpu_encoder_phys *phys_enc)
+{
+	_dpu_encoder_phys_cmd_connect_te(phys_enc, false);
+}
+
+static int dpu_encoder_phys_cmd_get_line_count(
+		struct dpu_encoder_phys *phys_enc)
+{
+	struct dpu_hw_pingpong *hw_pp;
+
+	if (!phys_enc || !phys_enc->hw_pp)
+		return -EINVAL;
+
+	if (!dpu_encoder_phys_cmd_is_master(phys_enc))
+		return -EINVAL;
+
+	hw_pp = phys_enc->hw_pp;
+	if (!hw_pp->ops.get_line_count)
+		return -EINVAL;
+
+	return hw_pp->ops.get_line_count(hw_pp);
+}
+
+static void dpu_encoder_phys_cmd_disable(struct dpu_encoder_phys *phys_enc)
+{
+	struct dpu_encoder_phys_cmd *cmd_enc =
+		to_dpu_encoder_phys_cmd(phys_enc);
+
+	if (!phys_enc || !phys_enc->hw_pp) {
+		DPU_ERROR("invalid encoder\n");
+		return;
+	}
+	DRM_DEBUG_KMS("id:%u pp:%d state:%d\n", DRMID(phys_enc->parent),
+		      phys_enc->hw_pp->idx - PINGPONG_0,
+		      phys_enc->enable_state);
+
+	if (phys_enc->enable_state == DPU_ENC_DISABLED) {
+		DPU_ERROR_CMDENC(cmd_enc, "already disabled\n");
+		return;
+	}
+
+	if (phys_enc->hw_pp->ops.enable_tearcheck)
+		phys_enc->hw_pp->ops.enable_tearcheck(phys_enc->hw_pp, false);
+	phys_enc->enable_state = DPU_ENC_DISABLED;
+}
+
+static void dpu_encoder_phys_cmd_destroy(struct dpu_encoder_phys *phys_enc)
+{
+	struct dpu_encoder_phys_cmd *cmd_enc =
+		to_dpu_encoder_phys_cmd(phys_enc);
+
+	if (!phys_enc) {
+		DPU_ERROR("invalid encoder\n");
+		return;
+	}
+	kfree(cmd_enc);
+}
+
+static void dpu_encoder_phys_cmd_get_hw_resources(
+		struct dpu_encoder_phys *phys_enc,
+		struct dpu_encoder_hw_resources *hw_res,
+		struct drm_connector_state *conn_state)
+{
+	struct dpu_encoder_phys_cmd *cmd_enc =
+		to_dpu_encoder_phys_cmd(phys_enc);
+
+	if (!phys_enc) {
+		DPU_ERROR("invalid encoder\n");
+		return;
+	}
+
+	if ((phys_enc->intf_idx - INTF_0) >= INTF_MAX) {
+		DPU_ERROR("invalid intf idx:%d\n", phys_enc->intf_idx);
+		return;
+	}
+
+	DPU_DEBUG_CMDENC(cmd_enc, "\n");
+	hw_res->intfs[phys_enc->intf_idx - INTF_0] = INTF_MODE_CMD;
+}
+
+static void dpu_encoder_phys_cmd_prepare_for_kickoff(
+		struct dpu_encoder_phys *phys_enc,
+		struct dpu_encoder_kickoff_params *params)
+{
+	struct dpu_encoder_phys_cmd *cmd_enc =
+			to_dpu_encoder_phys_cmd(phys_enc);
+	int ret;
+
+	if (!phys_enc || !phys_enc->hw_pp) {
+		DPU_ERROR("invalid encoder\n");
+		return;
+	}
+	DRM_DEBUG_KMS("id:%u pp:%d pending_cnt:%d\n", DRMID(phys_enc->parent),
+		      phys_enc->hw_pp->idx - PINGPONG_0,
+		      atomic_read(&phys_enc->pending_kickoff_cnt));
+
+	/*
+	 * Mark kickoff request as outstanding. If there are more than one,
+	 * outstanding, then we have to wait for the previous one to complete
+	 */
+	ret = _dpu_encoder_phys_cmd_wait_for_idle(phys_enc);
+	if (ret) {
+		/* force pending_kickoff_cnt 0 to discard failed kickoff */
+		atomic_set(&phys_enc->pending_kickoff_cnt, 0);
+		DRM_ERROR("failed wait_for_idle: id:%u ret:%d pp:%d\n",
+			  DRMID(phys_enc->parent), ret,
+			  phys_enc->hw_pp->idx - PINGPONG_0);
+	}
+
+	DPU_DEBUG_CMDENC(cmd_enc, "pp:%d pending_cnt %d\n",
+			phys_enc->hw_pp->idx - PINGPONG_0,
+			atomic_read(&phys_enc->pending_kickoff_cnt));
+}
+
+static int _dpu_encoder_phys_cmd_wait_for_ctl_start(
+		struct dpu_encoder_phys *phys_enc)
+{
+	struct dpu_encoder_phys_cmd *cmd_enc =
+			to_dpu_encoder_phys_cmd(phys_enc);
+	struct dpu_encoder_wait_info wait_info;
+	int ret;
+
+	if (!phys_enc || !phys_enc->hw_ctl) {
+		DPU_ERROR("invalid argument(s)\n");
+		return -EINVAL;
+	}
+
+	wait_info.wq = &phys_enc->pending_kickoff_wq;
+	wait_info.atomic_cnt = &phys_enc->pending_ctlstart_cnt;
+	wait_info.timeout_ms = KICKOFF_TIMEOUT_MS;
+
+	ret = dpu_encoder_helper_wait_for_irq(phys_enc, INTR_IDX_CTL_START,
+			&wait_info);
+	if (ret == -ETIMEDOUT) {
+		DPU_ERROR_CMDENC(cmd_enc, "ctl start interrupt wait failed\n");
+		ret = -EINVAL;
+	} else if (!ret)
+		ret = 0;
+
+	return ret;
+}
+
+static int dpu_encoder_phys_cmd_wait_for_tx_complete(
+		struct dpu_encoder_phys *phys_enc)
+{
+	int rc;
+	struct dpu_encoder_phys_cmd *cmd_enc;
+
+	if (!phys_enc)
+		return -EINVAL;
+
+	cmd_enc = to_dpu_encoder_phys_cmd(phys_enc);
+
+	rc = _dpu_encoder_phys_cmd_wait_for_idle(phys_enc);
+	if (rc) {
+		DRM_ERROR("failed wait_for_idle: id:%u ret:%d intf:%d\n",
+			  DRMID(phys_enc->parent), rc,
+			  phys_enc->intf_idx - INTF_0);
+	}
+
+	return rc;
+}
+
+static int dpu_encoder_phys_cmd_wait_for_commit_done(
+		struct dpu_encoder_phys *phys_enc)
+{
+	int rc = 0;
+	struct dpu_encoder_phys_cmd *cmd_enc;
+
+	if (!phys_enc)
+		return -EINVAL;
+
+	cmd_enc = to_dpu_encoder_phys_cmd(phys_enc);
+
+	/* only required for master controller */
+	if (dpu_encoder_phys_cmd_is_master(phys_enc))
+		rc = _dpu_encoder_phys_cmd_wait_for_ctl_start(phys_enc);
+
+	/* required for both controllers */
+	if (!rc && cmd_enc->serialize_wait4pp)
+		dpu_encoder_phys_cmd_prepare_for_kickoff(phys_enc, NULL);
+
+	return rc;
+}
+
+static int dpu_encoder_phys_cmd_wait_for_vblank(
+		struct dpu_encoder_phys *phys_enc)
+{
+	int rc = 0;
+	struct dpu_encoder_phys_cmd *cmd_enc;
+	struct dpu_encoder_wait_info wait_info;
+
+	if (!phys_enc)
+		return -EINVAL;
+
+	cmd_enc = to_dpu_encoder_phys_cmd(phys_enc);
+
+	/* only required for master controller */
+	if (!dpu_encoder_phys_cmd_is_master(phys_enc))
+		return rc;
+
+	wait_info.wq = &cmd_enc->pending_vblank_wq;
+	wait_info.atomic_cnt = &cmd_enc->pending_vblank_cnt;
+	wait_info.timeout_ms = _dpu_encoder_phys_cmd_get_idle_timeout(cmd_enc);
+
+	atomic_inc(&cmd_enc->pending_vblank_cnt);
+
+	rc = dpu_encoder_helper_wait_for_irq(phys_enc, INTR_IDX_RDPTR,
+			&wait_info);
+
+	return rc;
+}
+
+static void dpu_encoder_phys_cmd_handle_post_kickoff(
+		struct dpu_encoder_phys *phys_enc)
+{
+	if (!phys_enc)
+		return;
+
+	/**
+	 * re-enable external TE, either for the first time after enabling
+	 * or if disabled for Autorefresh
+	 */
+	_dpu_encoder_phys_cmd_connect_te(phys_enc, true);
+}
+
+static void dpu_encoder_phys_cmd_trigger_start(
+		struct dpu_encoder_phys *phys_enc)
+{
+	if (!phys_enc)
+		return;
+
+	dpu_encoder_helper_trigger_start(phys_enc);
+}
+
+static void dpu_encoder_phys_cmd_init_ops(
+		struct dpu_encoder_phys_ops *ops)
+{
+	ops->is_master = dpu_encoder_phys_cmd_is_master;
+	ops->mode_set = dpu_encoder_phys_cmd_mode_set;
+	ops->mode_fixup = dpu_encoder_phys_cmd_mode_fixup;
+	ops->enable = dpu_encoder_phys_cmd_enable;
+	ops->disable = dpu_encoder_phys_cmd_disable;
+	ops->destroy = dpu_encoder_phys_cmd_destroy;
+	ops->get_hw_resources = dpu_encoder_phys_cmd_get_hw_resources;
+	ops->control_vblank_irq = dpu_encoder_phys_cmd_control_vblank_irq;
+	ops->wait_for_commit_done = dpu_encoder_phys_cmd_wait_for_commit_done;
+	ops->prepare_for_kickoff = dpu_encoder_phys_cmd_prepare_for_kickoff;
+	ops->wait_for_tx_complete = dpu_encoder_phys_cmd_wait_for_tx_complete;
+	ops->wait_for_vblank = dpu_encoder_phys_cmd_wait_for_vblank;
+	ops->trigger_start = dpu_encoder_phys_cmd_trigger_start;
+	ops->needs_single_flush = dpu_encoder_phys_cmd_needs_single_flush;
+	ops->hw_reset = dpu_encoder_helper_hw_reset;
+	ops->irq_control = dpu_encoder_phys_cmd_irq_control;
+	ops->restore = dpu_encoder_phys_cmd_enable_helper;
+	ops->prepare_idle_pc = dpu_encoder_phys_cmd_prepare_idle_pc;
+	ops->handle_post_kickoff = dpu_encoder_phys_cmd_handle_post_kickoff;
+	ops->get_line_count = dpu_encoder_phys_cmd_get_line_count;
+}
+
+struct dpu_encoder_phys *dpu_encoder_phys_cmd_init(
+		struct dpu_enc_phys_init_params *p)
+{
+	struct dpu_encoder_phys *phys_enc = NULL;
+	struct dpu_encoder_phys_cmd *cmd_enc = NULL;
+	struct dpu_hw_mdp *hw_mdp;
+	struct dpu_encoder_irq *irq;
+	int i, ret = 0;
+
+	DPU_DEBUG("intf %d\n", p->intf_idx - INTF_0);
+
+	cmd_enc = kzalloc(sizeof(*cmd_enc), GFP_KERNEL);
+	if (!cmd_enc) {
+		ret = -ENOMEM;
+		DPU_ERROR("failed to allocate\n");
+		goto fail;
+	}
+	phys_enc = &cmd_enc->base;
+
+	hw_mdp = dpu_rm_get_mdp(&p->dpu_kms->rm);
+	if (IS_ERR_OR_NULL(hw_mdp)) {
+		ret = PTR_ERR(hw_mdp);
+		DPU_ERROR("failed to get mdptop\n");
+		goto fail_mdp_init;
+	}
+	phys_enc->hw_mdptop = hw_mdp;
+	phys_enc->intf_idx = p->intf_idx;
+
+	dpu_encoder_phys_cmd_init_ops(&phys_enc->ops);
+	phys_enc->parent = p->parent;
+	phys_enc->parent_ops = p->parent_ops;
+	phys_enc->dpu_kms = p->dpu_kms;
+	phys_enc->split_role = p->split_role;
+	phys_enc->intf_mode = INTF_MODE_CMD;
+	phys_enc->enc_spinlock = p->enc_spinlock;
+	cmd_enc->stream_sel = 0;
+	phys_enc->enable_state = DPU_ENC_DISABLED;
+	for (i = 0; i < INTR_IDX_MAX; i++) {
+		irq = &phys_enc->irq[i];
+		INIT_LIST_HEAD(&irq->cb.list);
+		irq->irq_idx = -EINVAL;
+		irq->hw_idx = -EINVAL;
+		irq->cb.arg = phys_enc;
+	}
+
+	irq = &phys_enc->irq[INTR_IDX_CTL_START];
+	irq->name = "ctl_start";
+	irq->intr_type = DPU_IRQ_TYPE_CTL_START;
+	irq->intr_idx = INTR_IDX_CTL_START;
+	irq->cb.func = dpu_encoder_phys_cmd_ctl_start_irq;
+
+	irq = &phys_enc->irq[INTR_IDX_PINGPONG];
+	irq->name = "pp_done";
+	irq->intr_type = DPU_IRQ_TYPE_PING_PONG_COMP;
+	irq->intr_idx = INTR_IDX_PINGPONG;
+	irq->cb.func = dpu_encoder_phys_cmd_pp_tx_done_irq;
+
+	irq = &phys_enc->irq[INTR_IDX_RDPTR];
+	irq->name = "pp_rd_ptr";
+	irq->intr_type = DPU_IRQ_TYPE_PING_PONG_RD_PTR;
+	irq->intr_idx = INTR_IDX_RDPTR;
+	irq->cb.func = dpu_encoder_phys_cmd_pp_rd_ptr_irq;
+
+	irq = &phys_enc->irq[INTR_IDX_UNDERRUN];
+	irq->name = "underrun";
+	irq->intr_type = DPU_IRQ_TYPE_INTF_UNDER_RUN;
+	irq->intr_idx = INTR_IDX_UNDERRUN;
+	irq->cb.func = dpu_encoder_phys_cmd_underrun_irq;
+
+	atomic_set(&phys_enc->vblank_refcount, 0);
+	atomic_set(&phys_enc->pending_kickoff_cnt, 0);
+	atomic_set(&phys_enc->pending_ctlstart_cnt, 0);
+	atomic_set(&cmd_enc->pending_vblank_cnt, 0);
+	init_waitqueue_head(&phys_enc->pending_kickoff_wq);
+	init_waitqueue_head(&cmd_enc->pending_vblank_wq);
+
+	DPU_DEBUG_CMDENC(cmd_enc, "created\n");
+
+	return phys_enc;
+
+fail_mdp_init:
+	kfree(cmd_enc);
+fail:
+	return ERR_PTR(ret);
+}
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_vid.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_vid.c
new file mode 100644
index 000000000000..14fc7c2a6bb7
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_vid.c
@@ -0,0 +1,922 @@
+/* Copyright (c) 2015-2018, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#define pr_fmt(fmt)	"[drm:%s:%d] " fmt, __func__, __LINE__
+#include "dpu_encoder_phys.h"
+#include "dpu_hw_interrupts.h"
+#include "dpu_core_irq.h"
+#include "dpu_formats.h"
+#include "dpu_trace.h"
+
+#define DPU_DEBUG_VIDENC(e, fmt, ...) DPU_DEBUG("enc%d intf%d " fmt, \
+		(e) && (e)->base.parent ? \
+		(e)->base.parent->base.id : -1, \
+		(e) && (e)->hw_intf ? \
+		(e)->hw_intf->idx - INTF_0 : -1, ##__VA_ARGS__)
+
+#define DPU_ERROR_VIDENC(e, fmt, ...) DPU_ERROR("enc%d intf%d " fmt, \
+		(e) && (e)->base.parent ? \
+		(e)->base.parent->base.id : -1, \
+		(e) && (e)->hw_intf ? \
+		(e)->hw_intf->idx - INTF_0 : -1, ##__VA_ARGS__)
+
+#define to_dpu_encoder_phys_vid(x) \
+	container_of(x, struct dpu_encoder_phys_vid, base)
+
+static bool dpu_encoder_phys_vid_is_master(
+		struct dpu_encoder_phys *phys_enc)
+{
+	bool ret = false;
+
+	if (phys_enc->split_role != ENC_ROLE_SLAVE)
+		ret = true;
+
+	return ret;
+}
+
+static void drm_mode_to_intf_timing_params(
+		const struct dpu_encoder_phys_vid *vid_enc,
+		const struct drm_display_mode *mode,
+		struct intf_timing_params *timing)
+{
+	memset(timing, 0, sizeof(*timing));
+
+	if ((mode->htotal < mode->hsync_end)
+			|| (mode->hsync_start < mode->hdisplay)
+			|| (mode->vtotal < mode->vsync_end)
+			|| (mode->vsync_start < mode->vdisplay)
+			|| (mode->hsync_end < mode->hsync_start)
+			|| (mode->vsync_end < mode->vsync_start)) {
+		DPU_ERROR(
+		    "invalid params - hstart:%d,hend:%d,htot:%d,hdisplay:%d\n",
+				mode->hsync_start, mode->hsync_end,
+				mode->htotal, mode->hdisplay);
+		DPU_ERROR("vstart:%d,vend:%d,vtot:%d,vdisplay:%d\n",
+				mode->vsync_start, mode->vsync_end,
+				mode->vtotal, mode->vdisplay);
+		return;
+	}
+
+	/*
+	 * https://www.kernel.org/doc/htmldocs/drm/ch02s05.html
+	 *  Active Region      Front Porch   Sync   Back Porch
+	 * <-----------------><------------><-----><----------->
+	 * <- [hv]display --->
+	 * <--------- [hv]sync_start ------>
+	 * <----------------- [hv]sync_end ------->
+	 * <---------------------------- [hv]total ------------->
+	 */
+	timing->width = mode->hdisplay;	/* active width */
+	timing->height = mode->vdisplay;	/* active height */
+	timing->xres = timing->width;
+	timing->yres = timing->height;
+	timing->h_back_porch = mode->htotal - mode->hsync_end;
+	timing->h_front_porch = mode->hsync_start - mode->hdisplay;
+	timing->v_back_porch = mode->vtotal - mode->vsync_end;
+	timing->v_front_porch = mode->vsync_start - mode->vdisplay;
+	timing->hsync_pulse_width = mode->hsync_end - mode->hsync_start;
+	timing->vsync_pulse_width = mode->vsync_end - mode->vsync_start;
+	timing->hsync_polarity = (mode->flags & DRM_MODE_FLAG_NHSYNC) ? 1 : 0;
+	timing->vsync_polarity = (mode->flags & DRM_MODE_FLAG_NVSYNC) ? 1 : 0;
+	timing->border_clr = 0;
+	timing->underflow_clr = 0xff;
+	timing->hsync_skew = mode->hskew;
+
+	/* DSI controller cannot handle active-low sync signals. */
+	if (vid_enc->hw_intf->cap->type == INTF_DSI) {
+		timing->hsync_polarity = 0;
+		timing->vsync_polarity = 0;
+	}
+
+	/*
+	 * For edp only:
+	 * DISPLAY_V_START = (VBP * HCYCLE) + HBP
+	 * DISPLAY_V_END = (VBP + VACTIVE) * HCYCLE - 1 - HFP
+	 */
+	/*
+	 * if (vid_enc->hw->cap->type == INTF_EDP) {
+	 * display_v_start += mode->htotal - mode->hsync_start;
+	 * display_v_end -= mode->hsync_start - mode->hdisplay;
+	 * }
+	 */
+}
+
+static inline u32 get_horizontal_total(const struct intf_timing_params *timing)
+{
+	u32 active = timing->xres;
+	u32 inactive =
+	    timing->h_back_porch + timing->h_front_porch +
+	    timing->hsync_pulse_width;
+	return active + inactive;
+}
+
+static inline u32 get_vertical_total(const struct intf_timing_params *timing)
+{
+	u32 active = timing->yres;
+	u32 inactive =
+	    timing->v_back_porch + timing->v_front_porch +
+	    timing->vsync_pulse_width;
+	return active + inactive;
+}
+
+/*
+ * programmable_fetch_get_num_lines:
+ *	Number of fetch lines in vertical front porch
+ * @timing: Pointer to the intf timing information for the requested mode
+ *
+ * Returns the number of fetch lines in vertical front porch at which mdp
+ * can start fetching the next frame.
+ *
+ * Number of needed prefetch lines is anything that cannot be absorbed in the
+ * start of frame time (back porch + vsync pulse width).
+ *
+ * Some panels have very large VFP, however we only need a total number of
+ * lines based on the chip worst case latencies.
+ */
+static u32 programmable_fetch_get_num_lines(
+		struct dpu_encoder_phys_vid *vid_enc,
+		const struct intf_timing_params *timing)
+{
+	u32 worst_case_needed_lines =
+	    vid_enc->hw_intf->cap->prog_fetch_lines_worst_case;
+	u32 start_of_frame_lines =
+	    timing->v_back_porch + timing->vsync_pulse_width;
+	u32 needed_vfp_lines = worst_case_needed_lines - start_of_frame_lines;
+	u32 actual_vfp_lines = 0;
+
+	/* Fetch must be outside active lines, otherwise undefined. */
+	if (start_of_frame_lines >= worst_case_needed_lines) {
+		DPU_DEBUG_VIDENC(vid_enc,
+				"prog fetch is not needed, large vbp+vsw\n");
+		actual_vfp_lines = 0;
+	} else if (timing->v_front_porch < needed_vfp_lines) {
+		/* Warn fetch needed, but not enough porch in panel config */
+		pr_warn_once
+			("low vbp+vfp may lead to perf issues in some cases\n");
+		DPU_DEBUG_VIDENC(vid_enc,
+				"less vfp than fetch req, using entire vfp\n");
+		actual_vfp_lines = timing->v_front_porch;
+	} else {
+		DPU_DEBUG_VIDENC(vid_enc, "room in vfp for needed prefetch\n");
+		actual_vfp_lines = needed_vfp_lines;
+	}
+
+	DPU_DEBUG_VIDENC(vid_enc,
+		"v_front_porch %u v_back_porch %u vsync_pulse_width %u\n",
+		timing->v_front_porch, timing->v_back_porch,
+		timing->vsync_pulse_width);
+	DPU_DEBUG_VIDENC(vid_enc,
+		"wc_lines %u needed_vfp_lines %u actual_vfp_lines %u\n",
+		worst_case_needed_lines, needed_vfp_lines, actual_vfp_lines);
+
+	return actual_vfp_lines;
+}
+
+/*
+ * programmable_fetch_config: Programs HW to prefetch lines by offsetting
+ *	the start of fetch into the vertical front porch for cases where the
+ *	vsync pulse width and vertical back porch time is insufficient
+ *
+ *	Gets # of lines to pre-fetch, then calculate VSYNC counter value.
+ *	HW layer requires VSYNC counter of first pixel of tgt VFP line.
+ *
+ * @timing: Pointer to the intf timing information for the requested mode
+ */
+static void programmable_fetch_config(struct dpu_encoder_phys *phys_enc,
+				      const struct intf_timing_params *timing)
+{
+	struct dpu_encoder_phys_vid *vid_enc =
+		to_dpu_encoder_phys_vid(phys_enc);
+	struct intf_prog_fetch f = { 0 };
+	u32 vfp_fetch_lines = 0;
+	u32 horiz_total = 0;
+	u32 vert_total = 0;
+	u32 vfp_fetch_start_vsync_counter = 0;
+	unsigned long lock_flags;
+
+	if (WARN_ON_ONCE(!vid_enc->hw_intf->ops.setup_prg_fetch))
+		return;
+
+	vfp_fetch_lines = programmable_fetch_get_num_lines(vid_enc, timing);
+	if (vfp_fetch_lines) {
+		vert_total = get_vertical_total(timing);
+		horiz_total = get_horizontal_total(timing);
+		vfp_fetch_start_vsync_counter =
+		    (vert_total - vfp_fetch_lines) * horiz_total + 1;
+		f.enable = 1;
+		f.fetch_start = vfp_fetch_start_vsync_counter;
+	}
+
+	DPU_DEBUG_VIDENC(vid_enc,
+		"vfp_fetch_lines %u vfp_fetch_start_vsync_counter %u\n",
+		vfp_fetch_lines, vfp_fetch_start_vsync_counter);
+
+	spin_lock_irqsave(phys_enc->enc_spinlock, lock_flags);
+	vid_enc->hw_intf->ops.setup_prg_fetch(vid_enc->hw_intf, &f);
+	spin_unlock_irqrestore(phys_enc->enc_spinlock, lock_flags);
+}
+
+static bool dpu_encoder_phys_vid_mode_fixup(
+		struct dpu_encoder_phys *phys_enc,
+		const struct drm_display_mode *mode,
+		struct drm_display_mode *adj_mode)
+{
+	if (phys_enc)
+		DPU_DEBUG_VIDENC(to_dpu_encoder_phys_vid(phys_enc), "\n");
+
+	/*
+	 * Modifying mode has consequences when the mode comes back to us
+	 */
+	return true;
+}
+
+static void dpu_encoder_phys_vid_setup_timing_engine(
+		struct dpu_encoder_phys *phys_enc)
+{
+	struct dpu_encoder_phys_vid *vid_enc;
+	struct drm_display_mode mode;
+	struct intf_timing_params timing_params = { 0 };
+	const struct dpu_format *fmt = NULL;
+	u32 fmt_fourcc = DRM_FORMAT_RGB888;
+	unsigned long lock_flags;
+	struct dpu_hw_intf_cfg intf_cfg = { 0 };
+
+	if (!phys_enc || !phys_enc->hw_ctl->ops.setup_intf_cfg) {
+		DPU_ERROR("invalid encoder %d\n", phys_enc != 0);
+		return;
+	}
+
+	mode = phys_enc->cached_mode;
+	vid_enc = to_dpu_encoder_phys_vid(phys_enc);
+	if (!vid_enc->hw_intf->ops.setup_timing_gen) {
+		DPU_ERROR("timing engine setup is not supported\n");
+		return;
+	}
+
+	DPU_DEBUG_VIDENC(vid_enc, "enabling mode:\n");
+	drm_mode_debug_printmodeline(&mode);
+
+	if (phys_enc->split_role != ENC_ROLE_SOLO) {
+		mode.hdisplay >>= 1;
+		mode.htotal >>= 1;
+		mode.hsync_start >>= 1;
+		mode.hsync_end >>= 1;
+
+		DPU_DEBUG_VIDENC(vid_enc,
+			"split_role %d, halve horizontal %d %d %d %d\n",
+			phys_enc->split_role,
+			mode.hdisplay, mode.htotal,
+			mode.hsync_start, mode.hsync_end);
+	}
+
+	drm_mode_to_intf_timing_params(vid_enc, &mode, &timing_params);
+
+	fmt = dpu_get_dpu_format(fmt_fourcc);
+	DPU_DEBUG_VIDENC(vid_enc, "fmt_fourcc 0x%X\n", fmt_fourcc);
+
+	intf_cfg.intf = vid_enc->hw_intf->idx;
+	intf_cfg.intf_mode_sel = DPU_CTL_MODE_SEL_VID;
+	intf_cfg.stream_sel = 0; /* Don't care value for video mode */
+	intf_cfg.mode_3d = dpu_encoder_helper_get_3d_blend_mode(phys_enc);
+
+	spin_lock_irqsave(phys_enc->enc_spinlock, lock_flags);
+	vid_enc->hw_intf->ops.setup_timing_gen(vid_enc->hw_intf,
+			&timing_params, fmt);
+	phys_enc->hw_ctl->ops.setup_intf_cfg(phys_enc->hw_ctl, &intf_cfg);
+	spin_unlock_irqrestore(phys_enc->enc_spinlock, lock_flags);
+
+	programmable_fetch_config(phys_enc, &timing_params);
+
+	vid_enc->timing_params = timing_params;
+}
+
+static void dpu_encoder_phys_vid_vblank_irq(void *arg, int irq_idx)
+{
+	struct dpu_encoder_phys *phys_enc = arg;
+	struct dpu_hw_ctl *hw_ctl;
+	unsigned long lock_flags;
+	u32 flush_register = 0;
+	int new_cnt = -1, old_cnt = -1;
+
+	if (!phys_enc)
+		return;
+
+	hw_ctl = phys_enc->hw_ctl;
+	if (!hw_ctl)
+		return;
+
+	DPU_ATRACE_BEGIN("vblank_irq");
+
+	if (phys_enc->parent_ops->handle_vblank_virt)
+		phys_enc->parent_ops->handle_vblank_virt(phys_enc->parent,
+				phys_enc);
+
+	old_cnt  = atomic_read(&phys_enc->pending_kickoff_cnt);
+
+	/*
+	 * only decrement the pending flush count if we've actually flushed
+	 * hardware. due to sw irq latency, vblank may have already happened
+	 * so we need to double-check with hw that it accepted the flush bits
+	 */
+	spin_lock_irqsave(phys_enc->enc_spinlock, lock_flags);
+	if (hw_ctl && hw_ctl->ops.get_flush_register)
+		flush_register = hw_ctl->ops.get_flush_register(hw_ctl);
+
+	if (flush_register == 0)
+		new_cnt = atomic_add_unless(&phys_enc->pending_kickoff_cnt,
+				-1, 0);
+	spin_unlock_irqrestore(phys_enc->enc_spinlock, lock_flags);
+
+	/* Signal any waiting atomic commit thread */
+	wake_up_all(&phys_enc->pending_kickoff_wq);
+	DPU_ATRACE_END("vblank_irq");
+}
+
+static void dpu_encoder_phys_vid_underrun_irq(void *arg, int irq_idx)
+{
+	struct dpu_encoder_phys *phys_enc = arg;
+
+	if (!phys_enc)
+		return;
+
+	if (phys_enc->parent_ops->handle_underrun_virt)
+		phys_enc->parent_ops->handle_underrun_virt(phys_enc->parent,
+			phys_enc);
+}
+
+static bool _dpu_encoder_phys_is_dual_ctl(struct dpu_encoder_phys *phys_enc)
+{
+	if (!phys_enc)
+		return false;
+
+	if (phys_enc->topology_name == DPU_RM_TOPOLOGY_DUALPIPE)
+		return true;
+
+	return false;
+}
+
+static bool dpu_encoder_phys_vid_needs_single_flush(
+		struct dpu_encoder_phys *phys_enc)
+{
+	return (phys_enc && _dpu_encoder_phys_is_dual_ctl(phys_enc));
+}
+
+static void _dpu_encoder_phys_vid_setup_irq_hw_idx(
+		struct dpu_encoder_phys *phys_enc)
+{
+	struct dpu_encoder_irq *irq;
+
+	/*
+	 * Initialize irq->hw_idx only when irq is not registered.
+	 * Prevent invalidating irq->irq_idx as modeset may be
+	 * called many times during dfps.
+	 */
+
+	irq = &phys_enc->irq[INTR_IDX_VSYNC];
+	if (irq->irq_idx < 0)
+		irq->hw_idx = phys_enc->intf_idx;
+
+	irq = &phys_enc->irq[INTR_IDX_UNDERRUN];
+	if (irq->irq_idx < 0)
+		irq->hw_idx = phys_enc->intf_idx;
+}
+
+static void dpu_encoder_phys_vid_mode_set(
+		struct dpu_encoder_phys *phys_enc,
+		struct drm_display_mode *mode,
+		struct drm_display_mode *adj_mode)
+{
+	struct dpu_rm *rm;
+	struct dpu_rm_hw_iter iter;
+	int i, instance;
+	struct dpu_encoder_phys_vid *vid_enc;
+
+	if (!phys_enc || !phys_enc->dpu_kms) {
+		DPU_ERROR("invalid encoder/kms\n");
+		return;
+	}
+
+	rm = &phys_enc->dpu_kms->rm;
+	vid_enc = to_dpu_encoder_phys_vid(phys_enc);
+
+	if (adj_mode) {
+		phys_enc->cached_mode = *adj_mode;
+		drm_mode_debug_printmodeline(adj_mode);
+		DPU_DEBUG_VIDENC(vid_enc, "caching mode:\n");
+	}
+
+	instance = phys_enc->split_role == ENC_ROLE_SLAVE ? 1 : 0;
+
+	/* Retrieve previously allocated HW Resources. Shouldn't fail */
+	dpu_rm_init_hw_iter(&iter, phys_enc->parent->base.id, DPU_HW_BLK_CTL);
+	for (i = 0; i <= instance; i++) {
+		if (dpu_rm_get_hw(rm, &iter))
+			phys_enc->hw_ctl = (struct dpu_hw_ctl *)iter.hw;
+	}
+	if (IS_ERR_OR_NULL(phys_enc->hw_ctl)) {
+		DPU_ERROR_VIDENC(vid_enc, "failed to init ctl, %ld\n",
+				PTR_ERR(phys_enc->hw_ctl));
+		phys_enc->hw_ctl = NULL;
+		return;
+	}
+
+	_dpu_encoder_phys_vid_setup_irq_hw_idx(phys_enc);
+}
+
+static int dpu_encoder_phys_vid_control_vblank_irq(
+		struct dpu_encoder_phys *phys_enc,
+		bool enable)
+{
+	int ret = 0;
+	struct dpu_encoder_phys_vid *vid_enc;
+	int refcount;
+
+	if (!phys_enc) {
+		DPU_ERROR("invalid encoder\n");
+		return -EINVAL;
+	}
+
+	refcount = atomic_read(&phys_enc->vblank_refcount);
+	vid_enc = to_dpu_encoder_phys_vid(phys_enc);
+
+	/* Slave encoders don't report vblank */
+	if (!dpu_encoder_phys_vid_is_master(phys_enc))
+		goto end;
+
+	/* protect against negative */
+	if (!enable && refcount == 0) {
+		ret = -EINVAL;
+		goto end;
+	}
+
+	DRM_DEBUG_KMS("id:%u enable=%d/%d\n", DRMID(phys_enc->parent), enable,
+		      atomic_read(&phys_enc->vblank_refcount));
+
+	if (enable && atomic_inc_return(&phys_enc->vblank_refcount) == 1)
+		ret = dpu_encoder_helper_register_irq(phys_enc, INTR_IDX_VSYNC);
+	else if (!enable && atomic_dec_return(&phys_enc->vblank_refcount) == 0)
+		ret = dpu_encoder_helper_unregister_irq(phys_enc,
+				INTR_IDX_VSYNC);
+
+end:
+	if (ret) {
+		DRM_ERROR("failed: id:%u intf:%d ret:%d enable:%d refcnt:%d\n",
+			  DRMID(phys_enc->parent),
+			  vid_enc->hw_intf->idx - INTF_0, ret, enable,
+			  refcount);
+	}
+	return ret;
+}
+
+static void dpu_encoder_phys_vid_enable(struct dpu_encoder_phys *phys_enc)
+{
+	struct msm_drm_private *priv;
+	struct dpu_encoder_phys_vid *vid_enc;
+	struct dpu_hw_intf *intf;
+	struct dpu_hw_ctl *ctl;
+	u32 flush_mask = 0;
+
+	if (!phys_enc || !phys_enc->parent || !phys_enc->parent->dev ||
+			!phys_enc->parent->dev->dev_private) {
+		DPU_ERROR("invalid encoder/device\n");
+		return;
+	}
+	priv = phys_enc->parent->dev->dev_private;
+
+	vid_enc = to_dpu_encoder_phys_vid(phys_enc);
+	intf = vid_enc->hw_intf;
+	ctl = phys_enc->hw_ctl;
+	if (!vid_enc->hw_intf || !phys_enc->hw_ctl) {
+		DPU_ERROR("invalid hw_intf %d hw_ctl %d\n",
+				vid_enc->hw_intf != 0, phys_enc->hw_ctl != 0);
+		return;
+	}
+
+	DPU_DEBUG_VIDENC(vid_enc, "\n");
+
+	if (WARN_ON(!vid_enc->hw_intf->ops.enable_timing))
+		return;
+
+	dpu_encoder_helper_split_config(phys_enc, vid_enc->hw_intf->idx);
+
+	dpu_encoder_phys_vid_setup_timing_engine(phys_enc);
+
+	/*
+	 * For single flush cases (dual-ctl or pp-split), skip setting the
+	 * flush bit for the slave intf, since both intfs use same ctl
+	 * and HW will only flush the master.
+	 */
+	if (dpu_encoder_phys_vid_needs_single_flush(phys_enc) &&
+		!dpu_encoder_phys_vid_is_master(phys_enc))
+		goto skip_flush;
+
+	ctl->ops.get_bitmask_intf(ctl, &flush_mask, intf->idx);
+	ctl->ops.update_pending_flush(ctl, flush_mask);
+
+skip_flush:
+	DPU_DEBUG_VIDENC(vid_enc, "update pending flush ctl %d flush_mask %x\n",
+		ctl->idx - CTL_0, flush_mask);
+
+	/* ctl_flush & timing engine enable will be triggered by framework */
+	if (phys_enc->enable_state == DPU_ENC_DISABLED)
+		phys_enc->enable_state = DPU_ENC_ENABLING;
+}
+
+static void dpu_encoder_phys_vid_destroy(struct dpu_encoder_phys *phys_enc)
+{
+	struct dpu_encoder_phys_vid *vid_enc;
+
+	if (!phys_enc) {
+		DPU_ERROR("invalid encoder\n");
+		return;
+	}
+
+	vid_enc = to_dpu_encoder_phys_vid(phys_enc);
+	DPU_DEBUG_VIDENC(vid_enc, "\n");
+	kfree(vid_enc);
+}
+
+static void dpu_encoder_phys_vid_get_hw_resources(
+		struct dpu_encoder_phys *phys_enc,
+		struct dpu_encoder_hw_resources *hw_res,
+		struct drm_connector_state *conn_state)
+{
+	struct dpu_encoder_phys_vid *vid_enc;
+
+	if (!phys_enc || !hw_res) {
+		DPU_ERROR("invalid arg(s), enc %d hw_res %d conn_state %d\n",
+				phys_enc != 0, hw_res != 0, conn_state != 0);
+		return;
+	}
+
+	vid_enc = to_dpu_encoder_phys_vid(phys_enc);
+	if (!vid_enc->hw_intf) {
+		DPU_ERROR("invalid arg(s), hw_intf\n");
+		return;
+	}
+
+	DPU_DEBUG_VIDENC(vid_enc, "\n");
+	hw_res->intfs[vid_enc->hw_intf->idx - INTF_0] = INTF_MODE_VIDEO;
+}
+
+static int _dpu_encoder_phys_vid_wait_for_vblank(
+		struct dpu_encoder_phys *phys_enc, bool notify)
+{
+	struct dpu_encoder_wait_info wait_info;
+	int ret;
+
+	if (!phys_enc) {
+		pr_err("invalid encoder\n");
+		return -EINVAL;
+	}
+
+	wait_info.wq = &phys_enc->pending_kickoff_wq;
+	wait_info.atomic_cnt = &phys_enc->pending_kickoff_cnt;
+	wait_info.timeout_ms = KICKOFF_TIMEOUT_MS;
+
+	if (!dpu_encoder_phys_vid_is_master(phys_enc)) {
+		if (notify && phys_enc->parent_ops->handle_frame_done)
+			phys_enc->parent_ops->handle_frame_done(
+					phys_enc->parent, phys_enc,
+					DPU_ENCODER_FRAME_EVENT_DONE);
+		return 0;
+	}
+
+	/* Wait for kickoff to complete */
+	ret = dpu_encoder_helper_wait_for_irq(phys_enc, INTR_IDX_VSYNC,
+			&wait_info);
+
+	if (ret == -ETIMEDOUT) {
+		dpu_encoder_helper_report_irq_timeout(phys_enc, INTR_IDX_VSYNC);
+	} else if (!ret && notify && phys_enc->parent_ops->handle_frame_done)
+		phys_enc->parent_ops->handle_frame_done(
+				phys_enc->parent, phys_enc,
+				DPU_ENCODER_FRAME_EVENT_DONE);
+
+	return ret;
+}
+
+static int dpu_encoder_phys_vid_wait_for_vblank(
+		struct dpu_encoder_phys *phys_enc)
+{
+	return _dpu_encoder_phys_vid_wait_for_vblank(phys_enc, true);
+}
+
+static void dpu_encoder_phys_vid_prepare_for_kickoff(
+		struct dpu_encoder_phys *phys_enc,
+		struct dpu_encoder_kickoff_params *params)
+{
+	struct dpu_encoder_phys_vid *vid_enc;
+	struct dpu_hw_ctl *ctl;
+	int rc;
+
+	if (!phys_enc || !params) {
+		DPU_ERROR("invalid encoder/parameters\n");
+		return;
+	}
+	vid_enc = to_dpu_encoder_phys_vid(phys_enc);
+
+	ctl = phys_enc->hw_ctl;
+	if (!ctl || !ctl->ops.wait_reset_status)
+		return;
+
+	/*
+	 * hw supports hardware initiated ctl reset, so before we kickoff a new
+	 * frame, need to check and wait for hw initiated ctl reset completion
+	 */
+	rc = ctl->ops.wait_reset_status(ctl);
+	if (rc) {
+		DPU_ERROR_VIDENC(vid_enc, "ctl %d reset failure: %d\n",
+				ctl->idx, rc);
+		dpu_encoder_helper_unregister_irq(phys_enc, INTR_IDX_VSYNC);
+		dpu_dbg_dump(false, __func__, true, true);
+	}
+}
+
+static void dpu_encoder_phys_vid_disable(struct dpu_encoder_phys *phys_enc)
+{
+	struct msm_drm_private *priv;
+	struct dpu_encoder_phys_vid *vid_enc;
+	unsigned long lock_flags;
+	int ret;
+
+	if (!phys_enc || !phys_enc->parent || !phys_enc->parent->dev ||
+			!phys_enc->parent->dev->dev_private) {
+		DPU_ERROR("invalid encoder/device\n");
+		return;
+	}
+	priv = phys_enc->parent->dev->dev_private;
+
+	vid_enc = to_dpu_encoder_phys_vid(phys_enc);
+	if (!vid_enc->hw_intf || !phys_enc->hw_ctl) {
+		DPU_ERROR("invalid hw_intf %d hw_ctl %d\n",
+				vid_enc->hw_intf != 0, phys_enc->hw_ctl != 0);
+		return;
+	}
+
+	DPU_DEBUG_VIDENC(vid_enc, "\n");
+
+	if (WARN_ON(!vid_enc->hw_intf->ops.enable_timing))
+		return;
+
+	if (phys_enc->enable_state == DPU_ENC_DISABLED) {
+		DPU_ERROR("already disabled\n");
+		return;
+	}
+
+	spin_lock_irqsave(phys_enc->enc_spinlock, lock_flags);
+	vid_enc->hw_intf->ops.enable_timing(vid_enc->hw_intf, 0);
+	if (dpu_encoder_phys_vid_is_master(phys_enc))
+		dpu_encoder_phys_inc_pending(phys_enc);
+	spin_unlock_irqrestore(phys_enc->enc_spinlock, lock_flags);
+
+	/*
+	 * Wait for a vsync so we know the ENABLE=0 latched before
+	 * the (connector) source of the vsync's gets disabled,
+	 * otherwise we end up in a funny state if we re-enable
+	 * before the disable latches, which results that some of
+	 * the settings changes for the new modeset (like new
+	 * scanout buffer) don't latch properly..
+	 */
+	if (dpu_encoder_phys_vid_is_master(phys_enc)) {
+		ret = _dpu_encoder_phys_vid_wait_for_vblank(phys_enc, false);
+		if (ret) {
+			atomic_set(&phys_enc->pending_kickoff_cnt, 0);
+			DRM_ERROR("wait disable failed: id:%u intf:%d ret:%d\n",
+				  DRMID(phys_enc->parent),
+				  vid_enc->hw_intf->idx - INTF_0, ret);
+		}
+	}
+
+	phys_enc->enable_state = DPU_ENC_DISABLED;
+}
+
+static void dpu_encoder_phys_vid_handle_post_kickoff(
+		struct dpu_encoder_phys *phys_enc)
+{
+	unsigned long lock_flags;
+	struct dpu_encoder_phys_vid *vid_enc;
+
+	if (!phys_enc) {
+		DPU_ERROR("invalid encoder\n");
+		return;
+	}
+
+	vid_enc = to_dpu_encoder_phys_vid(phys_enc);
+	DPU_DEBUG_VIDENC(vid_enc, "enable_state %d\n", phys_enc->enable_state);
+
+	/*
+	 * Video mode must flush CTL before enabling timing engine
+	 * Video encoders need to turn on their interfaces now
+	 */
+	if (phys_enc->enable_state == DPU_ENC_ENABLING) {
+		trace_dpu_enc_phys_vid_post_kickoff(DRMID(phys_enc->parent),
+				    vid_enc->hw_intf->idx - INTF_0);
+		spin_lock_irqsave(phys_enc->enc_spinlock, lock_flags);
+		vid_enc->hw_intf->ops.enable_timing(vid_enc->hw_intf, 1);
+		spin_unlock_irqrestore(phys_enc->enc_spinlock, lock_flags);
+		phys_enc->enable_state = DPU_ENC_ENABLED;
+	}
+}
+
+static void dpu_encoder_phys_vid_irq_control(struct dpu_encoder_phys *phys_enc,
+		bool enable)
+{
+	struct dpu_encoder_phys_vid *vid_enc;
+	int ret;
+
+	if (!phys_enc)
+		return;
+
+	vid_enc = to_dpu_encoder_phys_vid(phys_enc);
+
+	trace_dpu_enc_phys_vid_irq_ctrl(DRMID(phys_enc->parent),
+			    vid_enc->hw_intf->idx - INTF_0,
+			    enable,
+			    atomic_read(&phys_enc->vblank_refcount));
+
+	if (enable) {
+		ret = dpu_encoder_phys_vid_control_vblank_irq(phys_enc, true);
+		if (ret)
+			return;
+
+		dpu_encoder_helper_register_irq(phys_enc, INTR_IDX_UNDERRUN);
+	} else {
+		dpu_encoder_phys_vid_control_vblank_irq(phys_enc, false);
+		dpu_encoder_helper_unregister_irq(phys_enc, INTR_IDX_UNDERRUN);
+	}
+}
+
+static void dpu_encoder_phys_vid_setup_misr(struct dpu_encoder_phys *phys_enc,
+						bool enable, u32 frame_count)
+{
+	struct dpu_encoder_phys_vid *vid_enc;
+
+	if (!phys_enc)
+		return;
+	vid_enc = to_dpu_encoder_phys_vid(phys_enc);
+
+	if (vid_enc->hw_intf && vid_enc->hw_intf->ops.setup_misr)
+		vid_enc->hw_intf->ops.setup_misr(vid_enc->hw_intf,
+							enable, frame_count);
+}
+
+static u32 dpu_encoder_phys_vid_collect_misr(struct dpu_encoder_phys *phys_enc)
+{
+	struct dpu_encoder_phys_vid *vid_enc;
+
+	if (!phys_enc)
+		return 0;
+	vid_enc = to_dpu_encoder_phys_vid(phys_enc);
+
+	return vid_enc->hw_intf && vid_enc->hw_intf->ops.collect_misr ?
+		vid_enc->hw_intf->ops.collect_misr(vid_enc->hw_intf) : 0;
+}
+
+static int dpu_encoder_phys_vid_get_line_count(
+		struct dpu_encoder_phys *phys_enc)
+{
+	struct dpu_encoder_phys_vid *vid_enc;
+
+	if (!phys_enc)
+		return -EINVAL;
+
+	if (!dpu_encoder_phys_vid_is_master(phys_enc))
+		return -EINVAL;
+
+	vid_enc = to_dpu_encoder_phys_vid(phys_enc);
+	if (!vid_enc->hw_intf || !vid_enc->hw_intf->ops.get_line_count)
+		return -EINVAL;
+
+	return vid_enc->hw_intf->ops.get_line_count(vid_enc->hw_intf);
+}
+
+static void dpu_encoder_phys_vid_init_ops(struct dpu_encoder_phys_ops *ops)
+{
+	ops->is_master = dpu_encoder_phys_vid_is_master;
+	ops->mode_set = dpu_encoder_phys_vid_mode_set;
+	ops->mode_fixup = dpu_encoder_phys_vid_mode_fixup;
+	ops->enable = dpu_encoder_phys_vid_enable;
+	ops->disable = dpu_encoder_phys_vid_disable;
+	ops->destroy = dpu_encoder_phys_vid_destroy;
+	ops->get_hw_resources = dpu_encoder_phys_vid_get_hw_resources;
+	ops->control_vblank_irq = dpu_encoder_phys_vid_control_vblank_irq;
+	ops->wait_for_commit_done = dpu_encoder_phys_vid_wait_for_vblank;
+	ops->wait_for_vblank = dpu_encoder_phys_vid_wait_for_vblank;
+	ops->wait_for_tx_complete = dpu_encoder_phys_vid_wait_for_vblank;
+	ops->irq_control = dpu_encoder_phys_vid_irq_control;
+	ops->prepare_for_kickoff = dpu_encoder_phys_vid_prepare_for_kickoff;
+	ops->handle_post_kickoff = dpu_encoder_phys_vid_handle_post_kickoff;
+	ops->needs_single_flush = dpu_encoder_phys_vid_needs_single_flush;
+	ops->setup_misr = dpu_encoder_phys_vid_setup_misr;
+	ops->collect_misr = dpu_encoder_phys_vid_collect_misr;
+	ops->hw_reset = dpu_encoder_helper_hw_reset;
+	ops->get_line_count = dpu_encoder_phys_vid_get_line_count;
+}
+
+struct dpu_encoder_phys *dpu_encoder_phys_vid_init(
+		struct dpu_enc_phys_init_params *p)
+{
+	struct dpu_encoder_phys *phys_enc = NULL;
+	struct dpu_encoder_phys_vid *vid_enc = NULL;
+	struct dpu_rm_hw_iter iter;
+	struct dpu_hw_mdp *hw_mdp;
+	struct dpu_encoder_irq *irq;
+	int i, ret = 0;
+
+	if (!p) {
+		ret = -EINVAL;
+		goto fail;
+	}
+
+	vid_enc = kzalloc(sizeof(*vid_enc), GFP_KERNEL);
+	if (!vid_enc) {
+		ret = -ENOMEM;
+		goto fail;
+	}
+
+	phys_enc = &vid_enc->base;
+
+	hw_mdp = dpu_rm_get_mdp(&p->dpu_kms->rm);
+	if (IS_ERR_OR_NULL(hw_mdp)) {
+		ret = PTR_ERR(hw_mdp);
+		DPU_ERROR("failed to get mdptop\n");
+		goto fail;
+	}
+	phys_enc->hw_mdptop = hw_mdp;
+	phys_enc->intf_idx = p->intf_idx;
+
+	/**
+	 * hw_intf resource permanently assigned to this encoder
+	 * Other resources allocated at atomic commit time by use case
+	 */
+	dpu_rm_init_hw_iter(&iter, 0, DPU_HW_BLK_INTF);
+	while (dpu_rm_get_hw(&p->dpu_kms->rm, &iter)) {
+		struct dpu_hw_intf *hw_intf = (struct dpu_hw_intf *)iter.hw;
+
+		if (hw_intf->idx == p->intf_idx) {
+			vid_enc->hw_intf = hw_intf;
+			break;
+		}
+	}
+
+	if (!vid_enc->hw_intf) {
+		ret = -EINVAL;
+		DPU_ERROR("failed to get hw_intf\n");
+		goto fail;
+	}
+
+	DPU_DEBUG_VIDENC(vid_enc, "\n");
+
+	dpu_encoder_phys_vid_init_ops(&phys_enc->ops);
+	phys_enc->parent = p->parent;
+	phys_enc->parent_ops = p->parent_ops;
+	phys_enc->dpu_kms = p->dpu_kms;
+	phys_enc->split_role = p->split_role;
+	phys_enc->intf_mode = INTF_MODE_VIDEO;
+	phys_enc->enc_spinlock = p->enc_spinlock;
+	for (i = 0; i < INTR_IDX_MAX; i++) {
+		irq = &phys_enc->irq[i];
+		INIT_LIST_HEAD(&irq->cb.list);
+		irq->irq_idx = -EINVAL;
+		irq->hw_idx = -EINVAL;
+		irq->cb.arg = phys_enc;
+	}
+
+	irq = &phys_enc->irq[INTR_IDX_VSYNC];
+	irq->name = "vsync_irq";
+	irq->intr_type = DPU_IRQ_TYPE_INTF_VSYNC;
+	irq->intr_idx = INTR_IDX_VSYNC;
+	irq->cb.func = dpu_encoder_phys_vid_vblank_irq;
+
+	irq = &phys_enc->irq[INTR_IDX_UNDERRUN];
+	irq->name = "underrun";
+	irq->intr_type = DPU_IRQ_TYPE_INTF_UNDER_RUN;
+	irq->intr_idx = INTR_IDX_UNDERRUN;
+	irq->cb.func = dpu_encoder_phys_vid_underrun_irq;
+
+	atomic_set(&phys_enc->vblank_refcount, 0);
+	atomic_set(&phys_enc->pending_kickoff_cnt, 0);
+	init_waitqueue_head(&phys_enc->pending_kickoff_wq);
+	phys_enc->enable_state = DPU_ENC_DISABLED;
+
+	DPU_DEBUG_VIDENC(vid_enc, "created intf idx:%d\n", p->intf_idx);
+
+	return phys_enc;
+
+fail:
+	DPU_ERROR("failed to create encoder\n");
+	if (vid_enc)
+		dpu_encoder_phys_vid_destroy(phys_enc);
+
+	return ERR_PTR(ret);
+}
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_formats.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_formats.c
new file mode 100644
index 000000000000..bfcd165e96df
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_formats.c
@@ -0,0 +1,1173 @@
+/* Copyright (c) 2015-2018, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#define pr_fmt(fmt)	"[drm:%s:%d] " fmt, __func__, __LINE__
+
+#include <uapi/drm/drm_fourcc.h>
+
+#include "msm_media_info.h"
+#include "dpu_kms.h"
+#include "dpu_formats.h"
+
+#define DPU_UBWC_META_MACRO_W_H		16
+#define DPU_UBWC_META_BLOCK_SIZE	256
+#define DPU_UBWC_PLANE_SIZE_ALIGNMENT	4096
+
+#define DPU_TILE_HEIGHT_DEFAULT	1
+#define DPU_TILE_HEIGHT_TILED	4
+#define DPU_TILE_HEIGHT_UBWC	4
+#define DPU_TILE_HEIGHT_NV12	8
+
+#define DPU_MAX_IMG_WIDTH		0x3FFF
+#define DPU_MAX_IMG_HEIGHT		0x3FFF
+
+/**
+ * DPU supported format packing, bpp, and other format
+ * information.
+ * DPU currently only supports interleaved RGB formats
+ * UBWC support for a pixel format is indicated by the flag,
+ * there is additional meta data plane for such formats
+ */
+
+#define INTERLEAVED_RGB_FMT(fmt, a, r, g, b, e0, e1, e2, e3, uc, alpha,   \
+bp, flg, fm, np)                                                          \
+{                                                                         \
+	.base.pixel_format = DRM_FORMAT_ ## fmt,                          \
+	.fetch_planes = DPU_PLANE_INTERLEAVED,                            \
+	.alpha_enable = alpha,                                            \
+	.element = { (e0), (e1), (e2), (e3) },                            \
+	.bits = { g, b, r, a },                                           \
+	.chroma_sample = DPU_CHROMA_RGB,                                  \
+	.unpack_align_msb = 0,                                            \
+	.unpack_tight = 1,                                                \
+	.unpack_count = uc,                                               \
+	.bpp = bp,                                                        \
+	.fetch_mode = fm,                                                 \
+	.flag = {(flg)},                                                  \
+	.num_planes = np,                                                 \
+	.tile_height = DPU_TILE_HEIGHT_DEFAULT                            \
+}
+
+#define INTERLEAVED_RGB_FMT_TILED(fmt, a, r, g, b, e0, e1, e2, e3, uc,    \
+alpha, bp, flg, fm, np, th)                                               \
+{                                                                         \
+	.base.pixel_format = DRM_FORMAT_ ## fmt,                          \
+	.fetch_planes = DPU_PLANE_INTERLEAVED,                            \
+	.alpha_enable = alpha,                                            \
+	.element = { (e0), (e1), (e2), (e3) },                            \
+	.bits = { g, b, r, a },                                           \
+	.chroma_sample = DPU_CHROMA_RGB,                                  \
+	.unpack_align_msb = 0,                                            \
+	.unpack_tight = 1,                                                \
+	.unpack_count = uc,                                               \
+	.bpp = bp,                                                        \
+	.fetch_mode = fm,                                                 \
+	.flag = {(flg)},                                                  \
+	.num_planes = np,                                                 \
+	.tile_height = th                                                 \
+}
+
+
+#define INTERLEAVED_YUV_FMT(fmt, a, r, g, b, e0, e1, e2, e3,              \
+alpha, chroma, count, bp, flg, fm, np)                                    \
+{                                                                         \
+	.base.pixel_format = DRM_FORMAT_ ## fmt,                          \
+	.fetch_planes = DPU_PLANE_INTERLEAVED,                            \
+	.alpha_enable = alpha,                                            \
+	.element = { (e0), (e1), (e2), (e3)},                             \
+	.bits = { g, b, r, a },                                           \
+	.chroma_sample = chroma,                                          \
+	.unpack_align_msb = 0,                                            \
+	.unpack_tight = 1,                                                \
+	.unpack_count = count,                                            \
+	.bpp = bp,                                                        \
+	.fetch_mode = fm,                                                 \
+	.flag = {(flg)},                                                  \
+	.num_planes = np,                                                 \
+	.tile_height = DPU_TILE_HEIGHT_DEFAULT                            \
+}
+
+#define PSEUDO_YUV_FMT(fmt, a, r, g, b, e0, e1, chroma, flg, fm, np)      \
+{                                                                         \
+	.base.pixel_format = DRM_FORMAT_ ## fmt,                          \
+	.fetch_planes = DPU_PLANE_PSEUDO_PLANAR,                          \
+	.alpha_enable = false,                                            \
+	.element = { (e0), (e1), 0, 0 },                                  \
+	.bits = { g, b, r, a },                                           \
+	.chroma_sample = chroma,                                          \
+	.unpack_align_msb = 0,                                            \
+	.unpack_tight = 1,                                                \
+	.unpack_count = 2,                                                \
+	.bpp = 2,                                                         \
+	.fetch_mode = fm,                                                 \
+	.flag = {(flg)},                                                  \
+	.num_planes = np,                                                 \
+	.tile_height = DPU_TILE_HEIGHT_DEFAULT                            \
+}
+
+#define PSEUDO_YUV_FMT_TILED(fmt, a, r, g, b, e0, e1, chroma,             \
+flg, fm, np, th)                                                          \
+{                                                                         \
+	.base.pixel_format = DRM_FORMAT_ ## fmt,                          \
+	.fetch_planes = DPU_PLANE_PSEUDO_PLANAR,                          \
+	.alpha_enable = false,                                            \
+	.element = { (e0), (e1), 0, 0 },                                  \
+	.bits = { g, b, r, a },                                           \
+	.chroma_sample = chroma,                                          \
+	.unpack_align_msb = 0,                                            \
+	.unpack_tight = 1,                                                \
+	.unpack_count = 2,                                                \
+	.bpp = 2,                                                         \
+	.fetch_mode = fm,                                                 \
+	.flag = {(flg)},                                                  \
+	.num_planes = np,                                                 \
+	.tile_height = th                                                 \
+}
+
+#define PSEUDO_YUV_FMT_LOOSE(fmt, a, r, g, b, e0, e1, chroma, flg, fm, np)\
+{                                                                         \
+	.base.pixel_format = DRM_FORMAT_ ## fmt,                          \
+	.fetch_planes = DPU_PLANE_PSEUDO_PLANAR,                          \
+	.alpha_enable = false,                                            \
+	.element = { (e0), (e1), 0, 0 },                                  \
+	.bits = { g, b, r, a },                                           \
+	.chroma_sample = chroma,                                          \
+	.unpack_align_msb = 1,                                            \
+	.unpack_tight = 0,                                                \
+	.unpack_count = 2,                                                \
+	.bpp = 2,                                                         \
+	.fetch_mode = fm,                                                 \
+	.flag = {(flg)},                                                  \
+	.num_planes = np,                                                 \
+	.tile_height = DPU_TILE_HEIGHT_DEFAULT                            \
+}
+
+#define PSEUDO_YUV_FMT_LOOSE_TILED(fmt, a, r, g, b, e0, e1, chroma,       \
+flg, fm, np, th)                                                          \
+{                                                                         \
+	.base.pixel_format = DRM_FORMAT_ ## fmt,                          \
+	.fetch_planes = DPU_PLANE_PSEUDO_PLANAR,                          \
+	.alpha_enable = false,                                            \
+	.element = { (e0), (e1), 0, 0 },                                  \
+	.bits = { g, b, r, a },                                           \
+	.chroma_sample = chroma,                                          \
+	.unpack_align_msb = 1,                                            \
+	.unpack_tight = 0,                                                \
+	.unpack_count = 2,                                                \
+	.bpp = 2,                                                         \
+	.fetch_mode = fm,                                                 \
+	.flag = {(flg)},                                                  \
+	.num_planes = np,                                                 \
+	.tile_height = th                                                 \
+}
+
+
+#define PLANAR_YUV_FMT(fmt, a, r, g, b, e0, e1, e2, alpha, chroma, bp,    \
+flg, fm, np)                                                      \
+{                                                                         \
+	.base.pixel_format = DRM_FORMAT_ ## fmt,                          \
+	.fetch_planes = DPU_PLANE_PLANAR,                                 \
+	.alpha_enable = alpha,                                            \
+	.element = { (e0), (e1), (e2), 0 },                               \
+	.bits = { g, b, r, a },                                           \
+	.chroma_sample = chroma,                                          \
+	.unpack_align_msb = 0,                                            \
+	.unpack_tight = 1,                                                \
+	.unpack_count = 1,                                                \
+	.bpp = bp,                                                        \
+	.fetch_mode = fm,                                                 \
+	.flag = {(flg)},                                                  \
+	.num_planes = np,                                                 \
+	.tile_height = DPU_TILE_HEIGHT_DEFAULT                            \
+}
+
+/*
+ * struct dpu_media_color_map - maps drm format to media format
+ * @format: DRM base pixel format
+ * @color: Media API color related to DRM format
+ */
+struct dpu_media_color_map {
+	uint32_t format;
+	uint32_t color;
+};
+
+static const struct dpu_format dpu_format_map[] = {
+	INTERLEAVED_RGB_FMT(ARGB8888,
+		COLOR_8BIT, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
+		C1_B_Cb, C0_G_Y, C2_R_Cr, C3_ALPHA, 4,
+		true, 4, 0,
+		DPU_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(ABGR8888,
+		COLOR_8BIT, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
+		C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4,
+		true, 4, 0,
+		DPU_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(XBGR8888,
+		COLOR_8BIT, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
+		C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4,
+		true, 4, 0,
+		DPU_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(RGBA8888,
+		COLOR_8BIT, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
+		C3_ALPHA, C1_B_Cb, C0_G_Y, C2_R_Cr, 4,
+		true, 4, 0,
+		DPU_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(BGRA8888,
+		COLOR_8BIT, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
+		C3_ALPHA, C2_R_Cr, C0_G_Y, C1_B_Cb, 4,
+		true, 4, 0,
+		DPU_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(BGRX8888,
+		COLOR_8BIT, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
+		C3_ALPHA, C2_R_Cr, C0_G_Y, C1_B_Cb, 4,
+		false, 4, 0,
+		DPU_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(XRGB8888,
+		COLOR_8BIT, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
+		C1_B_Cb, C0_G_Y, C2_R_Cr, C3_ALPHA, 4,
+		false, 4, 0,
+		DPU_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(RGBX8888,
+		COLOR_8BIT, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
+		C3_ALPHA, C1_B_Cb, C0_G_Y, C2_R_Cr, 4,
+		false, 4, 0,
+		DPU_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(RGB888,
+		0, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
+		C1_B_Cb, C0_G_Y, C2_R_Cr, 0, 3,
+		false, 3, 0,
+		DPU_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(BGR888,
+		0, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
+		C2_R_Cr, C0_G_Y, C1_B_Cb, 0, 3,
+		false, 3, 0,
+		DPU_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(RGB565,
+		0, COLOR_5BIT, COLOR_6BIT, COLOR_5BIT,
+		C1_B_Cb, C0_G_Y, C2_R_Cr, 0, 3,
+		false, 2, 0,
+		DPU_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(BGR565,
+		0, COLOR_5BIT, COLOR_6BIT, COLOR_5BIT,
+		C2_R_Cr, C0_G_Y, C1_B_Cb, 0, 3,
+		false, 2, 0,
+		DPU_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(ARGB1555,
+		COLOR_ALPHA_1BIT, COLOR_5BIT, COLOR_5BIT, COLOR_5BIT,
+		C1_B_Cb, C0_G_Y, C2_R_Cr, C3_ALPHA, 4,
+		true, 2, 0,
+		DPU_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(ABGR1555,
+		COLOR_ALPHA_1BIT, COLOR_5BIT, COLOR_5BIT, COLOR_5BIT,
+		C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4,
+		true, 2, 0,
+		DPU_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(RGBA5551,
+		COLOR_ALPHA_1BIT, COLOR_5BIT, COLOR_5BIT, COLOR_5BIT,
+		C3_ALPHA, C1_B_Cb, C0_G_Y, C2_R_Cr, 4,
+		true, 2, 0,
+		DPU_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(BGRA5551,
+		COLOR_ALPHA_1BIT, COLOR_5BIT, COLOR_5BIT, COLOR_5BIT,
+		C3_ALPHA, C2_R_Cr, C0_G_Y, C1_B_Cb, 4,
+		true, 2, 0,
+		DPU_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(XRGB1555,
+		COLOR_ALPHA_1BIT, COLOR_5BIT, COLOR_5BIT, COLOR_5BIT,
+		C1_B_Cb, C0_G_Y, C2_R_Cr, C3_ALPHA, 4,
+		false, 2, 0,
+		DPU_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(XBGR1555,
+		COLOR_ALPHA_1BIT, COLOR_5BIT, COLOR_5BIT, COLOR_5BIT,
+		C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4,
+		false, 2, 0,
+		DPU_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(RGBX5551,
+		COLOR_ALPHA_1BIT, COLOR_5BIT, COLOR_5BIT, COLOR_5BIT,
+		C3_ALPHA, C1_B_Cb, C0_G_Y, C2_R_Cr, 4,
+		false, 2, 0,
+		DPU_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(BGRX5551,
+		COLOR_ALPHA_1BIT, COLOR_5BIT, COLOR_5BIT, COLOR_5BIT,
+		C3_ALPHA, C2_R_Cr, C0_G_Y, C1_B_Cb, 4,
+		false, 2, 0,
+		DPU_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(ARGB4444,
+		COLOR_ALPHA_4BIT, COLOR_4BIT, COLOR_4BIT, COLOR_4BIT,
+		C1_B_Cb, C0_G_Y, C2_R_Cr, C3_ALPHA, 4,
+		true, 2, 0,
+		DPU_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(ABGR4444,
+		COLOR_ALPHA_4BIT, COLOR_4BIT, COLOR_4BIT, COLOR_4BIT,
+		C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4,
+		true, 2, 0,
+		DPU_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(RGBA4444,
+		COLOR_ALPHA_4BIT, COLOR_4BIT, COLOR_4BIT, COLOR_4BIT,
+		C3_ALPHA, C1_B_Cb, C0_G_Y, C2_R_Cr, 4,
+		true, 2, 0,
+		DPU_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(BGRA4444,
+		COLOR_ALPHA_4BIT, COLOR_4BIT, COLOR_4BIT, COLOR_4BIT,
+		C3_ALPHA, C2_R_Cr, C0_G_Y, C1_B_Cb, 4,
+		true, 2, 0,
+		DPU_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(XRGB4444,
+		COLOR_ALPHA_4BIT, COLOR_4BIT, COLOR_4BIT, COLOR_4BIT,
+		C1_B_Cb, C0_G_Y, C2_R_Cr, C3_ALPHA, 4,
+		false, 2, 0,
+		DPU_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(XBGR4444,
+		COLOR_ALPHA_4BIT, COLOR_4BIT, COLOR_4BIT, COLOR_4BIT,
+		C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4,
+		false, 2, 0,
+		DPU_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(RGBX4444,
+		COLOR_ALPHA_4BIT, COLOR_4BIT, COLOR_4BIT, COLOR_4BIT,
+		C3_ALPHA, C1_B_Cb, C0_G_Y, C2_R_Cr, 4,
+		false, 2, 0,
+		DPU_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(BGRX4444,
+		COLOR_ALPHA_4BIT, COLOR_4BIT, COLOR_4BIT, COLOR_4BIT,
+		C3_ALPHA, C2_R_Cr, C0_G_Y, C1_B_Cb, 4,
+		false, 2, 0,
+		DPU_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(BGRA1010102,
+		COLOR_8BIT, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
+		C3_ALPHA, C2_R_Cr, C0_G_Y, C1_B_Cb, 4,
+		true, 4, DPU_FORMAT_FLAG_DX,
+		DPU_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(RGBA1010102,
+		COLOR_8BIT, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
+		C3_ALPHA, C1_B_Cb, C0_G_Y, C2_R_Cr, 4,
+		true, 4, DPU_FORMAT_FLAG_DX,
+		DPU_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(ABGR2101010,
+		COLOR_8BIT, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
+		C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4,
+		true, 4, DPU_FORMAT_FLAG_DX,
+		DPU_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(ARGB2101010,
+		COLOR_8BIT, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
+		C1_B_Cb, C0_G_Y, C2_R_Cr, C3_ALPHA, 4,
+		true, 4, DPU_FORMAT_FLAG_DX,
+		DPU_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(XRGB2101010,
+		COLOR_8BIT, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
+		C1_B_Cb, C0_G_Y, C2_R_Cr, C3_ALPHA, 4,
+		false, 4, DPU_FORMAT_FLAG_DX,
+		DPU_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(BGRX1010102,
+		COLOR_8BIT, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
+		C3_ALPHA, C2_R_Cr, C0_G_Y, C1_B_Cb, 4,
+		false, 4, DPU_FORMAT_FLAG_DX,
+		DPU_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(XBGR2101010,
+		COLOR_8BIT, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
+		C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4,
+		false, 4, DPU_FORMAT_FLAG_DX,
+		DPU_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(RGBX1010102,
+		COLOR_8BIT, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
+		C3_ALPHA, C1_B_Cb, C0_G_Y, C2_R_Cr, 4,
+		false, 4, DPU_FORMAT_FLAG_DX,
+		DPU_FETCH_LINEAR, 1),
+
+	PSEUDO_YUV_FMT(NV12,
+		0, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
+		C1_B_Cb, C2_R_Cr,
+		DPU_CHROMA_420, DPU_FORMAT_FLAG_YUV,
+		DPU_FETCH_LINEAR, 2),
+
+	PSEUDO_YUV_FMT(NV21,
+		0, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
+		C2_R_Cr, C1_B_Cb,
+		DPU_CHROMA_420, DPU_FORMAT_FLAG_YUV,
+		DPU_FETCH_LINEAR, 2),
+
+	PSEUDO_YUV_FMT(NV16,
+		0, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
+		C1_B_Cb, C2_R_Cr,
+		DPU_CHROMA_H2V1, DPU_FORMAT_FLAG_YUV,
+		DPU_FETCH_LINEAR, 2),
+
+	PSEUDO_YUV_FMT(NV61,
+		0, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
+		C2_R_Cr, C1_B_Cb,
+		DPU_CHROMA_H2V1, DPU_FORMAT_FLAG_YUV,
+		DPU_FETCH_LINEAR, 2),
+
+	INTERLEAVED_YUV_FMT(VYUY,
+		0, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
+		C2_R_Cr, C0_G_Y, C1_B_Cb, C0_G_Y,
+		false, DPU_CHROMA_H2V1, 4, 2, DPU_FORMAT_FLAG_YUV,
+		DPU_FETCH_LINEAR, 2),
+
+	INTERLEAVED_YUV_FMT(UYVY,
+		0, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
+		C1_B_Cb, C0_G_Y, C2_R_Cr, C0_G_Y,
+		false, DPU_CHROMA_H2V1, 4, 2, DPU_FORMAT_FLAG_YUV,
+		DPU_FETCH_LINEAR, 2),
+
+	INTERLEAVED_YUV_FMT(YUYV,
+		0, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
+		C0_G_Y, C1_B_Cb, C0_G_Y, C2_R_Cr,
+		false, DPU_CHROMA_H2V1, 4, 2, DPU_FORMAT_FLAG_YUV,
+		DPU_FETCH_LINEAR, 2),
+
+	INTERLEAVED_YUV_FMT(YVYU,
+		0, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
+		C0_G_Y, C2_R_Cr, C0_G_Y, C1_B_Cb,
+		false, DPU_CHROMA_H2V1, 4, 2, DPU_FORMAT_FLAG_YUV,
+		DPU_FETCH_LINEAR, 2),
+
+	PLANAR_YUV_FMT(YUV420,
+		0, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
+		C2_R_Cr, C1_B_Cb, C0_G_Y,
+		false, DPU_CHROMA_420, 1, DPU_FORMAT_FLAG_YUV,
+		DPU_FETCH_LINEAR, 3),
+
+	PLANAR_YUV_FMT(YVU420,
+		0, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
+		C1_B_Cb, C2_R_Cr, C0_G_Y,
+		false, DPU_CHROMA_420, 1, DPU_FORMAT_FLAG_YUV,
+		DPU_FETCH_LINEAR, 3),
+};
+
+/*
+ * A5x tile formats tables:
+ * These tables hold the A5x tile formats supported.
+ */
+static const struct dpu_format dpu_format_map_tile[] = {
+	INTERLEAVED_RGB_FMT_TILED(BGR565,
+		0, COLOR_5BIT, COLOR_6BIT, COLOR_5BIT,
+		C2_R_Cr, C0_G_Y, C1_B_Cb, 0, 3,
+		false, 2, 0,
+		DPU_FETCH_UBWC, 1, DPU_TILE_HEIGHT_TILED),
+
+	INTERLEAVED_RGB_FMT_TILED(ARGB8888,
+		COLOR_8BIT, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
+		C3_ALPHA, C2_R_Cr, C0_G_Y, C1_B_Cb, 4,
+		true, 4, 0,
+		DPU_FETCH_UBWC, 1, DPU_TILE_HEIGHT_TILED),
+
+	INTERLEAVED_RGB_FMT_TILED(ABGR8888,
+		COLOR_8BIT, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
+		C3_ALPHA, C1_B_Cb, C0_G_Y, C2_R_Cr, 4,
+		true, 4, 0,
+		DPU_FETCH_UBWC, 1, DPU_TILE_HEIGHT_TILED),
+
+	INTERLEAVED_RGB_FMT_TILED(XBGR8888,
+		COLOR_8BIT, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
+		C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4,
+		false, 4, 0,
+		DPU_FETCH_UBWC, 1, DPU_TILE_HEIGHT_TILED),
+
+	INTERLEAVED_RGB_FMT_TILED(RGBA8888,
+		COLOR_8BIT, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
+		C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4,
+		true, 4, 0,
+		DPU_FETCH_UBWC, 1, DPU_TILE_HEIGHT_TILED),
+
+	INTERLEAVED_RGB_FMT_TILED(BGRA8888,
+		COLOR_8BIT, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
+		C1_B_Cb, C0_G_Y, C2_R_Cr, C3_ALPHA, 4,
+		true, 4, 0,
+		DPU_FETCH_UBWC, 1, DPU_TILE_HEIGHT_TILED),
+
+	INTERLEAVED_RGB_FMT_TILED(BGRX8888,
+		COLOR_8BIT, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
+		C1_B_Cb, C0_G_Y, C2_R_Cr, C3_ALPHA, 4,
+		false, 4, 0,
+		DPU_FETCH_UBWC, 1, DPU_TILE_HEIGHT_TILED),
+
+	INTERLEAVED_RGB_FMT_TILED(XRGB8888,
+		COLOR_8BIT, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
+		C3_ALPHA, C2_R_Cr, C0_G_Y, C1_B_Cb, 4,
+		false, 4, 0,
+		DPU_FETCH_UBWC, 1, DPU_TILE_HEIGHT_TILED),
+
+	INTERLEAVED_RGB_FMT_TILED(RGBX8888,
+		COLOR_8BIT, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
+		C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4,
+		false, 4, 0,
+		DPU_FETCH_UBWC, 1, DPU_TILE_HEIGHT_TILED),
+
+	INTERLEAVED_RGB_FMT_TILED(ABGR2101010,
+		COLOR_8BIT, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
+		C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4,
+		true, 4, DPU_FORMAT_FLAG_DX,
+		DPU_FETCH_UBWC, 1, DPU_TILE_HEIGHT_TILED),
+
+	INTERLEAVED_RGB_FMT_TILED(XBGR2101010,
+		COLOR_8BIT, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
+		C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4,
+		true, 4, DPU_FORMAT_FLAG_DX,
+		DPU_FETCH_UBWC, 1, DPU_TILE_HEIGHT_TILED),
+
+	PSEUDO_YUV_FMT_TILED(NV12,
+		0, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
+		C1_B_Cb, C2_R_Cr,
+		DPU_CHROMA_420, DPU_FORMAT_FLAG_YUV,
+		DPU_FETCH_UBWC, 2, DPU_TILE_HEIGHT_NV12),
+
+	PSEUDO_YUV_FMT_TILED(NV21,
+		0, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
+		C2_R_Cr, C1_B_Cb,
+		DPU_CHROMA_420, DPU_FORMAT_FLAG_YUV,
+		DPU_FETCH_UBWC, 2, DPU_TILE_HEIGHT_NV12),
+};
+
+/*
+ * UBWC formats table:
+ * This table holds the UBWC formats supported.
+ * If a compression ratio needs to be used for this or any other format,
+ * the data will be passed by user-space.
+ */
+static const struct dpu_format dpu_format_map_ubwc[] = {
+	INTERLEAVED_RGB_FMT_TILED(BGR565,
+		0, COLOR_5BIT, COLOR_6BIT, COLOR_5BIT,
+		C2_R_Cr, C0_G_Y, C1_B_Cb, 0, 3,
+		false, 2, DPU_FORMAT_FLAG_COMPRESSED,
+		DPU_FETCH_UBWC, 2, DPU_TILE_HEIGHT_UBWC),
+
+	INTERLEAVED_RGB_FMT_TILED(ABGR8888,
+		COLOR_8BIT, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
+		C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4,
+		true, 4, DPU_FORMAT_FLAG_COMPRESSED,
+		DPU_FETCH_UBWC, 2, DPU_TILE_HEIGHT_UBWC),
+
+	INTERLEAVED_RGB_FMT_TILED(XBGR8888,
+		COLOR_8BIT, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
+		C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4,
+		false, 4, DPU_FORMAT_FLAG_COMPRESSED,
+		DPU_FETCH_UBWC, 2, DPU_TILE_HEIGHT_UBWC),
+
+	INTERLEAVED_RGB_FMT_TILED(ABGR2101010,
+		COLOR_8BIT, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
+		C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4,
+		true, 4, DPU_FORMAT_FLAG_DX | DPU_FORMAT_FLAG_COMPRESSED,
+		DPU_FETCH_UBWC, 2, DPU_TILE_HEIGHT_UBWC),
+
+	INTERLEAVED_RGB_FMT_TILED(XBGR2101010,
+		COLOR_8BIT, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
+		C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4,
+		true, 4, DPU_FORMAT_FLAG_DX | DPU_FORMAT_FLAG_COMPRESSED,
+		DPU_FETCH_UBWC, 2, DPU_TILE_HEIGHT_UBWC),
+
+	PSEUDO_YUV_FMT_TILED(NV12,
+		0, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
+		C1_B_Cb, C2_R_Cr,
+		DPU_CHROMA_420, DPU_FORMAT_FLAG_YUV |
+				DPU_FORMAT_FLAG_COMPRESSED,
+		DPU_FETCH_UBWC, 4, DPU_TILE_HEIGHT_NV12),
+};
+
+static const struct dpu_format dpu_format_map_p010[] = {
+	PSEUDO_YUV_FMT_LOOSE(NV12,
+		0, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
+		C1_B_Cb, C2_R_Cr,
+		DPU_CHROMA_420, (DPU_FORMAT_FLAG_YUV | DPU_FORMAT_FLAG_DX),
+		DPU_FETCH_LINEAR, 2),
+};
+
+static const struct dpu_format dpu_format_map_p010_ubwc[] = {
+	PSEUDO_YUV_FMT_LOOSE_TILED(NV12,
+		0, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
+		C1_B_Cb, C2_R_Cr,
+		DPU_CHROMA_420, (DPU_FORMAT_FLAG_YUV | DPU_FORMAT_FLAG_DX |
+				DPU_FORMAT_FLAG_COMPRESSED),
+		DPU_FETCH_UBWC, 4, DPU_TILE_HEIGHT_NV12),
+};
+
+static const struct dpu_format dpu_format_map_tp10_ubwc[] = {
+	PSEUDO_YUV_FMT_TILED(NV12,
+		0, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
+		C1_B_Cb, C2_R_Cr,
+		DPU_CHROMA_420, (DPU_FORMAT_FLAG_YUV | DPU_FORMAT_FLAG_DX |
+				DPU_FORMAT_FLAG_COMPRESSED),
+		DPU_FETCH_UBWC, 4, DPU_TILE_HEIGHT_NV12),
+};
+
+/* _dpu_get_v_h_subsample_rate - Get subsample rates for all formats we support
+ *   Note: Not using the drm_format_*_subsampling since we have formats
+ */
+static void _dpu_get_v_h_subsample_rate(
+	enum dpu_chroma_samp_type chroma_sample,
+	uint32_t *v_sample,
+	uint32_t *h_sample)
+{
+	if (!v_sample || !h_sample)
+		return;
+
+	switch (chroma_sample) {
+	case DPU_CHROMA_H2V1:
+		*v_sample = 1;
+		*h_sample = 2;
+		break;
+	case DPU_CHROMA_H1V2:
+		*v_sample = 2;
+		*h_sample = 1;
+		break;
+	case DPU_CHROMA_420:
+		*v_sample = 2;
+		*h_sample = 2;
+		break;
+	default:
+		*v_sample = 1;
+		*h_sample = 1;
+		break;
+	}
+}
+
+static int _dpu_format_get_media_color_ubwc(const struct dpu_format *fmt)
+{
+	static const struct dpu_media_color_map dpu_media_ubwc_map[] = {
+		{DRM_FORMAT_ABGR8888, COLOR_FMT_RGBA8888_UBWC},
+		{DRM_FORMAT_XBGR8888, COLOR_FMT_RGBA8888_UBWC},
+		{DRM_FORMAT_ABGR2101010, COLOR_FMT_RGBA1010102_UBWC},
+		{DRM_FORMAT_XBGR2101010, COLOR_FMT_RGBA1010102_UBWC},
+		{DRM_FORMAT_BGR565, COLOR_FMT_RGB565_UBWC},
+	};
+	int color_fmt = -1;
+	int i;
+
+	if (fmt->base.pixel_format == DRM_FORMAT_NV12) {
+		if (DPU_FORMAT_IS_DX(fmt)) {
+			if (fmt->unpack_tight)
+				color_fmt = COLOR_FMT_NV12_BPP10_UBWC;
+			else
+				color_fmt = COLOR_FMT_P010_UBWC;
+		} else
+			color_fmt = COLOR_FMT_NV12_UBWC;
+		return color_fmt;
+	}
+
+	for (i = 0; i < ARRAY_SIZE(dpu_media_ubwc_map); ++i)
+		if (fmt->base.pixel_format == dpu_media_ubwc_map[i].format) {
+			color_fmt = dpu_media_ubwc_map[i].color;
+			break;
+		}
+	return color_fmt;
+}
+
+static int _dpu_format_get_plane_sizes_ubwc(
+		const struct dpu_format *fmt,
+		const uint32_t width,
+		const uint32_t height,
+		struct dpu_hw_fmt_layout *layout)
+{
+	int i;
+	int color;
+	bool meta = DPU_FORMAT_IS_UBWC(fmt);
+
+	memset(layout, 0, sizeof(struct dpu_hw_fmt_layout));
+	layout->format = fmt;
+	layout->width = width;
+	layout->height = height;
+	layout->num_planes = fmt->num_planes;
+
+	color = _dpu_format_get_media_color_ubwc(fmt);
+	if (color < 0) {
+		DRM_ERROR("UBWC format not supported for fmt: %4.4s\n",
+			(char *)&fmt->base.pixel_format);
+		return -EINVAL;
+	}
+
+	if (DPU_FORMAT_IS_YUV(layout->format)) {
+		uint32_t y_sclines, uv_sclines;
+		uint32_t y_meta_scanlines = 0;
+		uint32_t uv_meta_scanlines = 0;
+
+		layout->num_planes = 2;
+		layout->plane_pitch[0] = VENUS_Y_STRIDE(color, width);
+		y_sclines = VENUS_Y_SCANLINES(color, height);
+		layout->plane_size[0] = MSM_MEDIA_ALIGN(layout->plane_pitch[0] *
+			y_sclines, DPU_UBWC_PLANE_SIZE_ALIGNMENT);
+
+		layout->plane_pitch[1] = VENUS_UV_STRIDE(color, width);
+		uv_sclines = VENUS_UV_SCANLINES(color, height);
+		layout->plane_size[1] = MSM_MEDIA_ALIGN(layout->plane_pitch[1] *
+			uv_sclines, DPU_UBWC_PLANE_SIZE_ALIGNMENT);
+
+		if (!meta)
+			goto done;
+
+		layout->num_planes += 2;
+		layout->plane_pitch[2] = VENUS_Y_META_STRIDE(color, width);
+		y_meta_scanlines = VENUS_Y_META_SCANLINES(color, height);
+		layout->plane_size[2] = MSM_MEDIA_ALIGN(layout->plane_pitch[2] *
+			y_meta_scanlines, DPU_UBWC_PLANE_SIZE_ALIGNMENT);
+
+		layout->plane_pitch[3] = VENUS_UV_META_STRIDE(color, width);
+		uv_meta_scanlines = VENUS_UV_META_SCANLINES(color, height);
+		layout->plane_size[3] = MSM_MEDIA_ALIGN(layout->plane_pitch[3] *
+			uv_meta_scanlines, DPU_UBWC_PLANE_SIZE_ALIGNMENT);
+
+	} else {
+		uint32_t rgb_scanlines, rgb_meta_scanlines;
+
+		layout->num_planes = 1;
+
+		layout->plane_pitch[0] = VENUS_RGB_STRIDE(color, width);
+		rgb_scanlines = VENUS_RGB_SCANLINES(color, height);
+		layout->plane_size[0] = MSM_MEDIA_ALIGN(layout->plane_pitch[0] *
+			rgb_scanlines, DPU_UBWC_PLANE_SIZE_ALIGNMENT);
+
+		if (!meta)
+			goto done;
+		layout->num_planes += 2;
+		layout->plane_pitch[2] = VENUS_RGB_META_STRIDE(color, width);
+		rgb_meta_scanlines = VENUS_RGB_META_SCANLINES(color, height);
+		layout->plane_size[2] = MSM_MEDIA_ALIGN(layout->plane_pitch[2] *
+			rgb_meta_scanlines, DPU_UBWC_PLANE_SIZE_ALIGNMENT);
+	}
+
+done:
+	for (i = 0; i < DPU_MAX_PLANES; i++)
+		layout->total_size += layout->plane_size[i];
+
+	return 0;
+}
+
+static int _dpu_format_get_plane_sizes_linear(
+		const struct dpu_format *fmt,
+		const uint32_t width,
+		const uint32_t height,
+		struct dpu_hw_fmt_layout *layout,
+		const uint32_t *pitches)
+{
+	int i;
+
+	memset(layout, 0, sizeof(struct dpu_hw_fmt_layout));
+	layout->format = fmt;
+	layout->width = width;
+	layout->height = height;
+	layout->num_planes = fmt->num_planes;
+
+	/* Due to memset above, only need to set planes of interest */
+	if (fmt->fetch_planes == DPU_PLANE_INTERLEAVED) {
+		layout->num_planes = 1;
+		layout->plane_size[0] = width * height * layout->format->bpp;
+		layout->plane_pitch[0] = width * layout->format->bpp;
+	} else {
+		uint32_t v_subsample, h_subsample;
+		uint32_t chroma_samp;
+		uint32_t bpp = 1;
+
+		chroma_samp = fmt->chroma_sample;
+		_dpu_get_v_h_subsample_rate(chroma_samp, &v_subsample,
+				&h_subsample);
+
+		if (width % h_subsample || height % v_subsample) {
+			DRM_ERROR("mismatch in subsample vs dimensions\n");
+			return -EINVAL;
+		}
+
+		if ((fmt->base.pixel_format == DRM_FORMAT_NV12) &&
+			(DPU_FORMAT_IS_DX(fmt)))
+			bpp = 2;
+		layout->plane_pitch[0] = width * bpp;
+		layout->plane_pitch[1] = layout->plane_pitch[0] / h_subsample;
+		layout->plane_size[0] = layout->plane_pitch[0] * height;
+		layout->plane_size[1] = layout->plane_pitch[1] *
+				(height / v_subsample);
+
+		if (fmt->fetch_planes == DPU_PLANE_PSEUDO_PLANAR) {
+			layout->num_planes = 2;
+			layout->plane_size[1] *= 2;
+			layout->plane_pitch[1] *= 2;
+		} else {
+			/* planar */
+			layout->num_planes = 3;
+			layout->plane_size[2] = layout->plane_size[1];
+			layout->plane_pitch[2] = layout->plane_pitch[1];
+		}
+	}
+
+	/*
+	 * linear format: allow user allocated pitches if they are greater than
+	 * the requirement.
+	 * ubwc format: pitch values are computed uniformly across
+	 * all the components based on ubwc specifications.
+	 */
+	for (i = 0; i < layout->num_planes && i < DPU_MAX_PLANES; ++i) {
+		if (pitches && layout->plane_pitch[i] < pitches[i])
+			layout->plane_pitch[i] = pitches[i];
+	}
+
+	for (i = 0; i < DPU_MAX_PLANES; i++)
+		layout->total_size += layout->plane_size[i];
+
+	return 0;
+}
+
+static int dpu_format_get_plane_sizes(
+		const struct dpu_format *fmt,
+		const uint32_t w,
+		const uint32_t h,
+		struct dpu_hw_fmt_layout *layout,
+		const uint32_t *pitches)
+{
+	if (!layout || !fmt) {
+		DRM_ERROR("invalid pointer\n");
+		return -EINVAL;
+	}
+
+	if ((w > DPU_MAX_IMG_WIDTH) || (h > DPU_MAX_IMG_HEIGHT)) {
+		DRM_ERROR("image dimensions outside max range\n");
+		return -ERANGE;
+	}
+
+	if (DPU_FORMAT_IS_UBWC(fmt) || DPU_FORMAT_IS_TILE(fmt))
+		return _dpu_format_get_plane_sizes_ubwc(fmt, w, h, layout);
+
+	return _dpu_format_get_plane_sizes_linear(fmt, w, h, layout, pitches);
+}
+
+static int _dpu_format_populate_addrs_ubwc(
+		struct msm_gem_address_space *aspace,
+		struct drm_framebuffer *fb,
+		struct dpu_hw_fmt_layout *layout)
+{
+	uint32_t base_addr = 0;
+	bool meta;
+
+	if (!fb || !layout) {
+		DRM_ERROR("invalid pointers\n");
+		return -EINVAL;
+	}
+
+	if (aspace)
+		base_addr = msm_framebuffer_iova(fb, aspace, 0);
+	if (!base_addr) {
+		DRM_ERROR("failed to retrieve base addr\n");
+		return -EFAULT;
+	}
+
+	meta = DPU_FORMAT_IS_UBWC(layout->format);
+
+	/* Per-format logic for verifying active planes */
+	if (DPU_FORMAT_IS_YUV(layout->format)) {
+		/************************************************/
+		/*      UBWC            **                      */
+		/*      buffer          **      DPU PLANE       */
+		/*      format          **                      */
+		/************************************************/
+		/* -------------------  ** -------------------- */
+		/* |      Y meta     |  ** |    Y bitstream   | */
+		/* |       data      |  ** |       plane      | */
+		/* -------------------  ** -------------------- */
+		/* |    Y bitstream  |  ** |  CbCr bitstream  | */
+		/* |       data      |  ** |       plane      | */
+		/* -------------------  ** -------------------- */
+		/* |   Cbcr metadata |  ** |       Y meta     | */
+		/* |       data      |  ** |       plane      | */
+		/* -------------------  ** -------------------- */
+		/* |  CbCr bitstream |  ** |     CbCr meta    | */
+		/* |       data      |  ** |       plane      | */
+		/* -------------------  ** -------------------- */
+		/************************************************/
+
+		/* configure Y bitstream plane */
+		layout->plane_addr[0] = base_addr + layout->plane_size[2];
+
+		/* configure CbCr bitstream plane */
+		layout->plane_addr[1] = base_addr + layout->plane_size[0]
+			+ layout->plane_size[2] + layout->plane_size[3];
+
+		if (!meta)
+			goto done;
+
+		/* configure Y metadata plane */
+		layout->plane_addr[2] = base_addr;
+
+		/* configure CbCr metadata plane */
+		layout->plane_addr[3] = base_addr + layout->plane_size[0]
+			+ layout->plane_size[2];
+
+	} else {
+		/************************************************/
+		/*      UBWC            **                      */
+		/*      buffer          **      DPU PLANE       */
+		/*      format          **                      */
+		/************************************************/
+		/* -------------------  ** -------------------- */
+		/* |      RGB meta   |  ** |   RGB bitstream  | */
+		/* |       data      |  ** |       plane      | */
+		/* -------------------  ** -------------------- */
+		/* |  RGB bitstream  |  ** |       NONE       | */
+		/* |       data      |  ** |                  | */
+		/* -------------------  ** -------------------- */
+		/*                      ** |     RGB meta     | */
+		/*                      ** |       plane      | */
+		/*                      ** -------------------- */
+		/************************************************/
+
+		layout->plane_addr[0] = base_addr + layout->plane_size[2];
+		layout->plane_addr[1] = 0;
+
+		if (!meta)
+			goto done;
+
+		layout->plane_addr[2] = base_addr;
+		layout->plane_addr[3] = 0;
+	}
+done:
+	return 0;
+}
+
+static int _dpu_format_populate_addrs_linear(
+		struct msm_gem_address_space *aspace,
+		struct drm_framebuffer *fb,
+		struct dpu_hw_fmt_layout *layout)
+{
+	unsigned int i;
+
+	/* Can now check the pitches given vs pitches expected */
+	for (i = 0; i < layout->num_planes; ++i) {
+		if (layout->plane_pitch[i] > fb->pitches[i]) {
+			DRM_ERROR("plane %u expected pitch %u, fb %u\n",
+				i, layout->plane_pitch[i], fb->pitches[i]);
+			return -EINVAL;
+		}
+	}
+
+	/* Populate addresses for simple formats here */
+	for (i = 0; i < layout->num_planes; ++i) {
+		if (aspace)
+			layout->plane_addr[i] =
+				msm_framebuffer_iova(fb, aspace, i);
+		if (!layout->plane_addr[i]) {
+			DRM_ERROR("failed to retrieve base addr\n");
+			return -EFAULT;
+		}
+	}
+
+	return 0;
+}
+
+int dpu_format_populate_layout(
+		struct msm_gem_address_space *aspace,
+		struct drm_framebuffer *fb,
+		struct dpu_hw_fmt_layout *layout)
+{
+	uint32_t plane_addr[DPU_MAX_PLANES];
+	int i, ret;
+
+	if (!fb || !layout) {
+		DRM_ERROR("invalid arguments\n");
+		return -EINVAL;
+	}
+
+	if ((fb->width > DPU_MAX_IMG_WIDTH) ||
+			(fb->height > DPU_MAX_IMG_HEIGHT)) {
+		DRM_ERROR("image dimensions outside max range\n");
+		return -ERANGE;
+	}
+
+	layout->format = to_dpu_format(msm_framebuffer_format(fb));
+
+	/* Populate the plane sizes etc via get_format */
+	ret = dpu_format_get_plane_sizes(layout->format, fb->width, fb->height,
+			layout, fb->pitches);
+	if (ret)
+		return ret;
+
+	for (i = 0; i < DPU_MAX_PLANES; ++i)
+		plane_addr[i] = layout->plane_addr[i];
+
+	/* Populate the addresses given the fb */
+	if (DPU_FORMAT_IS_UBWC(layout->format) ||
+			DPU_FORMAT_IS_TILE(layout->format))
+		ret = _dpu_format_populate_addrs_ubwc(aspace, fb, layout);
+	else
+		ret = _dpu_format_populate_addrs_linear(aspace, fb, layout);
+
+	/* check if anything changed */
+	if (!ret && !memcmp(plane_addr, layout->plane_addr, sizeof(plane_addr)))
+		ret = -EAGAIN;
+
+	return ret;
+}
+
+int dpu_format_check_modified_format(
+		const struct msm_kms *kms,
+		const struct msm_format *msm_fmt,
+		const struct drm_mode_fb_cmd2 *cmd,
+		struct drm_gem_object **bos)
+{
+	int ret, i, num_base_fmt_planes;
+	const struct dpu_format *fmt;
+	struct dpu_hw_fmt_layout layout;
+	uint32_t bos_total_size = 0;
+
+	if (!msm_fmt || !cmd || !bos) {
+		DRM_ERROR("invalid arguments\n");
+		return -EINVAL;
+	}
+
+	fmt = to_dpu_format(msm_fmt);
+	num_base_fmt_planes = drm_format_num_planes(fmt->base.pixel_format);
+
+	ret = dpu_format_get_plane_sizes(fmt, cmd->width, cmd->height,
+			&layout, cmd->pitches);
+	if (ret)
+		return ret;
+
+	for (i = 0; i < num_base_fmt_planes; i++) {
+		if (!bos[i]) {
+			DRM_ERROR("invalid handle for plane %d\n", i);
+			return -EINVAL;
+		}
+		if ((i == 0) || (bos[i] != bos[0]))
+			bos_total_size += bos[i]->size;
+	}
+
+	if (bos_total_size < layout.total_size) {
+		DRM_ERROR("buffers total size too small %u expected %u\n",
+				bos_total_size, layout.total_size);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+const struct dpu_format *dpu_get_dpu_format_ext(
+		const uint32_t format,
+		const uint64_t modifier)
+{
+	uint32_t i = 0;
+	const struct dpu_format *fmt = NULL;
+	const struct dpu_format *map = NULL;
+	ssize_t map_size = 0;
+
+	/*
+	 * Currently only support exactly zero or one modifier.
+	 * All planes use the same modifier.
+	 */
+	DPU_DEBUG("plane format modifier 0x%llX\n", modifier);
+
+	switch (modifier) {
+	case 0:
+		map = dpu_format_map;
+		map_size = ARRAY_SIZE(dpu_format_map);
+		break;
+	case DRM_FORMAT_MOD_QCOM_COMPRESSED:
+		map = dpu_format_map_ubwc;
+		map_size = ARRAY_SIZE(dpu_format_map_ubwc);
+		DPU_DEBUG("found fmt: %4.4s  DRM_FORMAT_MOD_QCOM_COMPRESSED\n",
+				(char *)&format);
+		break;
+	default:
+		DPU_ERROR("unsupported format modifier %llX\n", modifier);
+		return NULL;
+	}
+
+	for (i = 0; i < map_size; i++) {
+		if (format == map[i].base.pixel_format) {
+			fmt = &map[i];
+			break;
+		}
+	}
+
+	if (fmt == NULL)
+		DPU_ERROR("unsupported fmt: %4.4s modifier 0x%llX\n",
+			(char *)&format, modifier);
+	else
+		DPU_DEBUG("fmt %4.4s mod 0x%llX ubwc %d yuv %d\n",
+				(char *)&format, modifier,
+				DPU_FORMAT_IS_UBWC(fmt),
+				DPU_FORMAT_IS_YUV(fmt));
+
+	return fmt;
+}
+
+const struct msm_format *dpu_get_msm_format(
+		struct msm_kms *kms,
+		const uint32_t format,
+		const uint64_t modifiers)
+{
+	const struct dpu_format *fmt = dpu_get_dpu_format_ext(format,
+			modifiers);
+	if (fmt)
+		return &fmt->base;
+	return NULL;
+}
+
+uint32_t dpu_populate_formats(
+		const struct dpu_format_extended *format_list,
+		uint32_t *pixel_formats,
+		uint64_t *pixel_modifiers,
+		uint32_t pixel_formats_max)
+{
+	uint32_t i, fourcc_format;
+
+	if (!format_list || !pixel_formats)
+		return 0;
+
+	for (i = 0, fourcc_format = 0;
+			format_list->fourcc_format && i < pixel_formats_max;
+			++format_list) {
+		/* verify if listed format is in dpu_format_map? */
+
+		/* optionally return modified formats */
+		if (pixel_modifiers) {
+			/* assume same modifier for all fb planes */
+			pixel_formats[i] = format_list->fourcc_format;
+			pixel_modifiers[i++] = format_list->modifier;
+		} else {
+			/* assume base formats grouped together */
+			if (fourcc_format != format_list->fourcc_format) {
+				fourcc_format = format_list->fourcc_format;
+				pixel_formats[i++] = fourcc_format;
+			}
+		}
+	}
+
+	return i;
+}
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_formats.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_formats.h
new file mode 100644
index 000000000000..a54451d8d011
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_formats.h
@@ -0,0 +1,88 @@
+/* Copyright (c) 2015-2018, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef _DPU_FORMATS_H
+#define _DPU_FORMATS_H
+
+#include <drm/drm_fourcc.h>
+#include "msm_gem.h"
+#include "dpu_hw_mdss.h"
+
+/**
+ * dpu_get_dpu_format_ext() - Returns dpu format structure pointer.
+ * @format:          DRM FourCC Code
+ * @modifiers:       format modifier array from client, one per plane
+ */
+const struct dpu_format *dpu_get_dpu_format_ext(
+		const uint32_t format,
+		const uint64_t modifier);
+
+#define dpu_get_dpu_format(f) dpu_get_dpu_format_ext(f, 0)
+
+/**
+ * dpu_get_msm_format - get an dpu_format by its msm_format base
+ *                     callback function registers with the msm_kms layer
+ * @kms:             kms driver
+ * @format:          DRM FourCC Code
+ * @modifiers:       data layout modifier
+ */
+const struct msm_format *dpu_get_msm_format(
+		struct msm_kms *kms,
+		const uint32_t format,
+		const uint64_t modifiers);
+
+/**
+ * dpu_populate_formats - populate the given array with fourcc codes supported
+ * @format_list:       pointer to list of possible formats
+ * @pixel_formats:     array to populate with fourcc codes
+ * @pixel_modifiers:   array to populate with drm modifiers, can be NULL
+ * @pixel_formats_max: length of pixel formats array
+ * Return: number of elements populated
+ */
+uint32_t dpu_populate_formats(
+		const struct dpu_format_extended *format_list,
+		uint32_t *pixel_formats,
+		uint64_t *pixel_modifiers,
+		uint32_t pixel_formats_max);
+
+/**
+ * dpu_format_check_modified_format - validate format and buffers for
+ *                   dpu non-standard, i.e. modified format
+ * @kms:             kms driver
+ * @msm_fmt:         pointer to the msm_fmt base pointer of an dpu_format
+ * @cmd:             fb_cmd2 structure user request
+ * @bos:             gem buffer object list
+ *
+ * Return: error code on failure, 0 on success
+ */
+int dpu_format_check_modified_format(
+		const struct msm_kms *kms,
+		const struct msm_format *msm_fmt,
+		const struct drm_mode_fb_cmd2 *cmd,
+		struct drm_gem_object **bos);
+
+/**
+ * dpu_format_populate_layout - populate the given format layout based on
+ *                     mmu, fb, and format found in the fb
+ * @aspace:            address space pointer
+ * @fb:                framebuffer pointer
+ * @fmtl:              format layout structure to populate
+ *
+ * Return: error code on failure, -EAGAIN if success but the addresses
+ *         are the same as before or 0 if new addresses were populated
+ */
+int dpu_format_populate_layout(
+		struct msm_gem_address_space *aspace,
+		struct drm_framebuffer *fb,
+		struct dpu_hw_fmt_layout *fmtl);
+
+#endif /*_DPU_FORMATS_H */
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_blk.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_blk.c
new file mode 100644
index 000000000000..58d29e43faef
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_blk.c
@@ -0,0 +1,155 @@
+/* Copyright (c) 2017-2018, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#define pr_fmt(fmt)	"[drm:%s:%d] " fmt, __func__, __LINE__
+
+#include <linux/mutex.h>
+#include <linux/errno.h>
+#include <linux/slab.h>
+
+#include "dpu_hw_mdss.h"
+#include "dpu_hw_blk.h"
+
+/* Serialization lock for dpu_hw_blk_list */
+static DEFINE_MUTEX(dpu_hw_blk_lock);
+
+/* List of all hw block objects */
+static LIST_HEAD(dpu_hw_blk_list);
+
+/**
+ * dpu_hw_blk_init - initialize hw block object
+ * @type: hw block type - enum dpu_hw_blk_type
+ * @id: instance id of the hw block
+ * @ops: Pointer to block operations
+ * return: 0 if success; error code otherwise
+ */
+int dpu_hw_blk_init(struct dpu_hw_blk *hw_blk, u32 type, int id,
+		struct dpu_hw_blk_ops *ops)
+{
+	if (!hw_blk) {
+		pr_err("invalid parameters\n");
+		return -EINVAL;
+	}
+
+	INIT_LIST_HEAD(&hw_blk->list);
+	hw_blk->type = type;
+	hw_blk->id = id;
+	atomic_set(&hw_blk->refcount, 0);
+
+	if (ops)
+		hw_blk->ops = *ops;
+
+	mutex_lock(&dpu_hw_blk_lock);
+	list_add(&hw_blk->list, &dpu_hw_blk_list);
+	mutex_unlock(&dpu_hw_blk_lock);
+
+	return 0;
+}
+
+/**
+ * dpu_hw_blk_destroy - destroy hw block object.
+ * @hw_blk:  pointer to hw block object
+ * return: none
+ */
+void dpu_hw_blk_destroy(struct dpu_hw_blk *hw_blk)
+{
+	if (!hw_blk) {
+		pr_err("invalid parameters\n");
+		return;
+	}
+
+	if (atomic_read(&hw_blk->refcount))
+		pr_err("hw_blk:%d.%d invalid refcount\n", hw_blk->type,
+				hw_blk->id);
+
+	mutex_lock(&dpu_hw_blk_lock);
+	list_del(&hw_blk->list);
+	mutex_unlock(&dpu_hw_blk_lock);
+}
+
+/**
+ * dpu_hw_blk_get - get hw_blk from free pool
+ * @hw_blk: if specified, increment reference count only
+ * @type: if hw_blk is not specified, allocate the next available of this type
+ * @id: if specified (>= 0), allocate the given instance of the above type
+ * return: pointer to hw block object
+ */
+struct dpu_hw_blk *dpu_hw_blk_get(struct dpu_hw_blk *hw_blk, u32 type, int id)
+{
+	struct dpu_hw_blk *curr;
+	int rc, refcount;
+
+	if (!hw_blk) {
+		mutex_lock(&dpu_hw_blk_lock);
+		list_for_each_entry(curr, &dpu_hw_blk_list, list) {
+			if ((curr->type != type) ||
+					(id >= 0 && curr->id != id) ||
+					(id < 0 &&
+						atomic_read(&curr->refcount)))
+				continue;
+
+			hw_blk = curr;
+			break;
+		}
+		mutex_unlock(&dpu_hw_blk_lock);
+	}
+
+	if (!hw_blk) {
+		pr_debug("no hw_blk:%d\n", type);
+		return NULL;
+	}
+
+	refcount = atomic_inc_return(&hw_blk->refcount);
+
+	if (refcount == 1 && hw_blk->ops.start) {
+		rc = hw_blk->ops.start(hw_blk);
+		if (rc) {
+			pr_err("failed to start  hw_blk:%d rc:%d\n", type, rc);
+			goto error_start;
+		}
+	}
+
+	pr_debug("hw_blk:%d.%d refcount:%d\n", hw_blk->type,
+			hw_blk->id, refcount);
+	return hw_blk;
+
+error_start:
+	dpu_hw_blk_put(hw_blk);
+	return ERR_PTR(rc);
+}
+
+/**
+ * dpu_hw_blk_put - put hw_blk to free pool if decremented refcount is zero
+ * @hw_blk: hw block to be freed
+ * @free_blk: function to be called when reference count goes to zero
+ */
+void dpu_hw_blk_put(struct dpu_hw_blk *hw_blk)
+{
+	if (!hw_blk) {
+		pr_err("invalid parameters\n");
+		return;
+	}
+
+	pr_debug("hw_blk:%d.%d refcount:%d\n", hw_blk->type, hw_blk->id,
+			atomic_read(&hw_blk->refcount));
+
+	if (!atomic_read(&hw_blk->refcount)) {
+		pr_err("hw_blk:%d.%d invalid put\n", hw_blk->type, hw_blk->id);
+		return;
+	}
+
+	if (atomic_dec_return(&hw_blk->refcount))
+		return;
+
+	if (hw_blk->ops.stop)
+		hw_blk->ops.stop(hw_blk);
+}
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_blk.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_blk.h
new file mode 100644
index 000000000000..0f4ca8af1ec5
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_blk.h
@@ -0,0 +1,53 @@
+/* Copyright (c) 2017-2018, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef _DPU_HW_BLK_H
+#define _DPU_HW_BLK_H
+
+#include <linux/types.h>
+#include <linux/list.h>
+#include <linux/atomic.h>
+
+struct dpu_hw_blk;
+
+/**
+ * struct dpu_hw_blk_ops - common hardware block operations
+ * @start: start operation on first get
+ * @stop: stop operation on last put
+ */
+struct dpu_hw_blk_ops {
+	int (*start)(struct dpu_hw_blk *);
+	void (*stop)(struct dpu_hw_blk *);
+};
+
+/**
+ * struct dpu_hw_blk - definition of hardware block object
+ * @list: list of hardware blocks
+ * @type: hardware block type
+ * @id: instance id
+ * @refcount: reference/usage count
+ */
+struct dpu_hw_blk {
+	struct list_head list;
+	u32 type;
+	int id;
+	atomic_t refcount;
+	struct dpu_hw_blk_ops ops;
+};
+
+int dpu_hw_blk_init(struct dpu_hw_blk *hw_blk, u32 type, int id,
+		struct dpu_hw_blk_ops *ops);
+void dpu_hw_blk_destroy(struct dpu_hw_blk *hw_blk);
+
+struct dpu_hw_blk *dpu_hw_blk_get(struct dpu_hw_blk *hw_blk, u32 type, int id);
+void dpu_hw_blk_put(struct dpu_hw_blk *hw_blk);
+#endif /*_DPU_HW_BLK_H */
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.c
new file mode 100644
index 000000000000..44ee06398b1d
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.c
@@ -0,0 +1,511 @@
+/* Copyright (c) 2015-2018, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#define pr_fmt(fmt)	"[drm:%s:%d] " fmt, __func__, __LINE__
+#include <linux/slab.h>
+#include <linux/of_address.h>
+#include <linux/platform_device.h>
+#include "dpu_hw_mdss.h"
+#include "dpu_hw_catalog.h"
+#include "dpu_hw_catalog_format.h"
+#include "dpu_kms.h"
+
+#define VIG_SDM845_MASK \
+	(BIT(DPU_SSPP_SRC) | BIT(DPU_SSPP_SCALER_QSEED3) | BIT(DPU_SSPP_QOS) |\
+	BIT(DPU_SSPP_CSC_10BIT) | BIT(DPU_SSPP_CDP) | BIT(DPU_SSPP_QOS_8LVL) |\
+	BIT(DPU_SSPP_TS_PREFILL) | BIT(DPU_SSPP_EXCL_RECT))
+
+#define DMA_SDM845_MASK \
+	(BIT(DPU_SSPP_SRC) | BIT(DPU_SSPP_QOS) | BIT(DPU_SSPP_QOS_8LVL) |\
+	BIT(DPU_SSPP_TS_PREFILL) | BIT(DPU_SSPP_TS_PREFILL_REC1) |\
+	BIT(DPU_SSPP_CDP) | BIT(DPU_SSPP_EXCL_RECT))
+
+#define MIXER_SDM845_MASK \
+	(BIT(DPU_MIXER_SOURCESPLIT) | BIT(DPU_DIM_LAYER))
+
+#define PINGPONG_SDM845_MASK BIT(DPU_PINGPONG_DITHER)
+
+#define PINGPONG_SDM845_SPLIT_MASK \
+	(PINGPONG_SDM845_MASK | BIT(DPU_PINGPONG_TE2))
+
+#define DEFAULT_PIXEL_RAM_SIZE		(50 * 1024)
+#define DEFAULT_DPU_LINE_WIDTH		2048
+#define DEFAULT_DPU_OUTPUT_LINE_WIDTH	2560
+
+#define MAX_HORZ_DECIMATION	4
+#define MAX_VERT_DECIMATION	4
+
+#define MAX_UPSCALE_RATIO	20
+#define MAX_DOWNSCALE_RATIO	4
+#define SSPP_UNITY_SCALE	1
+
+#define STRCAT(X, Y) (X Y)
+
+/*************************************************************
+ * DPU sub blocks config
+ *************************************************************/
+/* DPU top level caps */
+static const struct dpu_caps sdm845_dpu_caps = {
+	.max_mixer_width = DEFAULT_DPU_OUTPUT_LINE_WIDTH,
+	.max_mixer_blendstages = 0xb,
+	.qseed_type = DPU_SSPP_SCALER_QSEED3,
+	.smart_dma_rev = DPU_SSPP_SMART_DMA_V2,
+	.ubwc_version = DPU_HW_UBWC_VER_20,
+	.has_src_split = true,
+	.has_dim_layer = true,
+	.has_idle_pc = true,
+};
+
+static struct dpu_mdp_cfg sdm845_mdp[] = {
+	{
+	.name = "top_0", .id = MDP_TOP,
+	.base = 0x0, .len = 0x45C,
+	.features = 0,
+	.highest_bank_bit = 0x2,
+	.has_dest_scaler = true,
+	.clk_ctrls[DPU_CLK_CTRL_VIG0] = {
+			.reg_off = 0x2AC, .bit_off = 0},
+	.clk_ctrls[DPU_CLK_CTRL_VIG1] = {
+			.reg_off = 0x2B4, .bit_off = 0},
+	.clk_ctrls[DPU_CLK_CTRL_VIG2] = {
+			.reg_off = 0x2BC, .bit_off = 0},
+	.clk_ctrls[DPU_CLK_CTRL_VIG3] = {
+			.reg_off = 0x2C4, .bit_off = 0},
+	.clk_ctrls[DPU_CLK_CTRL_DMA0] = {
+			.reg_off = 0x2AC, .bit_off = 8},
+	.clk_ctrls[DPU_CLK_CTRL_DMA1] = {
+			.reg_off = 0x2B4, .bit_off = 8},
+	.clk_ctrls[DPU_CLK_CTRL_CURSOR0] = {
+			.reg_off = 0x2BC, .bit_off = 8},
+	.clk_ctrls[DPU_CLK_CTRL_CURSOR1] = {
+			.reg_off = 0x2C4, .bit_off = 8},
+	},
+};
+
+/*************************************************************
+ * CTL sub blocks config
+ *************************************************************/
+static struct dpu_ctl_cfg sdm845_ctl[] = {
+	{
+	.name = "ctl_0", .id = CTL_0,
+	.base = 0x1000, .len = 0xE4,
+	.features = BIT(DPU_CTL_SPLIT_DISPLAY)
+	},
+	{
+	.name = "ctl_1", .id = CTL_1,
+	.base = 0x1200, .len = 0xE4,
+	.features = BIT(DPU_CTL_SPLIT_DISPLAY)
+	},
+	{
+	.name = "ctl_2", .id = CTL_2,
+	.base = 0x1400, .len = 0xE4,
+	.features = 0
+	},
+	{
+	.name = "ctl_3", .id = CTL_3,
+	.base = 0x1600, .len = 0xE4,
+	.features = 0
+	},
+	{
+	.name = "ctl_4", .id = CTL_4,
+	.base = 0x1800, .len = 0xE4,
+	.features = 0
+	},
+};
+
+/*************************************************************
+ * SSPP sub blocks config
+ *************************************************************/
+
+/* SSPP common configuration */
+static const struct dpu_sspp_blks_common sdm845_sspp_common = {
+	.maxlinewidth = DEFAULT_DPU_OUTPUT_LINE_WIDTH,
+	.pixel_ram_size = DEFAULT_PIXEL_RAM_SIZE,
+	.maxhdeciexp = MAX_HORZ_DECIMATION,
+	.maxvdeciexp = MAX_VERT_DECIMATION,
+};
+
+#define _VIG_SBLK(num, sdma_pri) \
+	{ \
+	.common = &sdm845_sspp_common, \
+	.maxdwnscale = MAX_DOWNSCALE_RATIO, \
+	.maxupscale = MAX_UPSCALE_RATIO, \
+	.smart_dma_priority = sdma_pri, \
+	.src_blk = {.name = STRCAT("sspp_src_", num), \
+		.id = DPU_SSPP_SRC, .base = 0x00, .len = 0x150,}, \
+	.scaler_blk = {.name = STRCAT("sspp_scaler", num), \
+		.id = DPU_SSPP_SCALER_QSEED3, \
+		.base = 0xa00, .len = 0xa0,}, \
+	.csc_blk = {.name = STRCAT("sspp_csc", num), \
+		.id = DPU_SSPP_CSC_10BIT, \
+		.base = 0x1a00, .len = 0x100,}, \
+	.format_list = plane_formats_yuv, \
+	.virt_format_list = plane_formats, \
+	}
+
+#define _DMA_SBLK(num, sdma_pri) \
+	{ \
+	.common = &sdm845_sspp_common, \
+	.maxdwnscale = SSPP_UNITY_SCALE, \
+	.maxupscale = SSPP_UNITY_SCALE, \
+	.smart_dma_priority = sdma_pri, \
+	.src_blk = {.name = STRCAT("sspp_src_", num), \
+		.id = DPU_SSPP_SRC, .base = 0x00, .len = 0x150,}, \
+	.format_list = plane_formats, \
+	.virt_format_list = plane_formats, \
+	}
+
+static const struct dpu_sspp_sub_blks sdm845_vig_sblk_0 = _VIG_SBLK("0", 5);
+static const struct dpu_sspp_sub_blks sdm845_vig_sblk_1 = _VIG_SBLK("1", 6);
+static const struct dpu_sspp_sub_blks sdm845_vig_sblk_2 = _VIG_SBLK("2", 7);
+static const struct dpu_sspp_sub_blks sdm845_vig_sblk_3 = _VIG_SBLK("3", 8);
+
+static const struct dpu_sspp_sub_blks sdm845_dma_sblk_0 = _DMA_SBLK("8", 1);
+static const struct dpu_sspp_sub_blks sdm845_dma_sblk_1 = _DMA_SBLK("9", 2);
+static const struct dpu_sspp_sub_blks sdm845_dma_sblk_2 = _DMA_SBLK("10", 3);
+static const struct dpu_sspp_sub_blks sdm845_dma_sblk_3 = _DMA_SBLK("11", 4);
+
+#define SSPP_VIG_BLK(_name, _id, _base, _sblk, _xinid, _clkctrl) \
+	{ \
+	.name = _name, .id = _id, \
+	.base = _base, .len = 0x1c8, \
+	.features = VIG_SDM845_MASK, \
+	.sblk = &_sblk, \
+	.xin_id = _xinid, \
+	.type = SSPP_TYPE_VIG, \
+	.clk_ctrl = _clkctrl \
+	}
+
+#define SSPP_DMA_BLK(_name, _id, _base, _sblk, _xinid, _clkctrl) \
+	{ \
+	.name = _name, .id = _id, \
+	.base = _base, .len = 0x1c8, \
+	.features = DMA_SDM845_MASK, \
+	.sblk = &_sblk, \
+	.xin_id = _xinid, \
+	.type = SSPP_TYPE_DMA, \
+	.clk_ctrl = _clkctrl \
+	}
+
+static struct dpu_sspp_cfg sdm845_sspp[] = {
+	SSPP_VIG_BLK("sspp_0", SSPP_VIG0, 0x4000,
+		sdm845_vig_sblk_0, 0, DPU_CLK_CTRL_VIG0),
+	SSPP_VIG_BLK("sspp_1", SSPP_VIG1, 0x6000,
+		sdm845_vig_sblk_1, 4, DPU_CLK_CTRL_VIG1),
+	SSPP_VIG_BLK("sspp_2", SSPP_VIG2, 0x8000,
+		sdm845_vig_sblk_2, 8, DPU_CLK_CTRL_VIG2),
+	SSPP_VIG_BLK("sspp_3", SSPP_VIG3, 0xa000,
+		sdm845_vig_sblk_3, 12, DPU_CLK_CTRL_VIG3),
+	SSPP_DMA_BLK("sspp_8", SSPP_DMA0, 0x24000,
+		sdm845_dma_sblk_0, 1, DPU_CLK_CTRL_DMA0),
+	SSPP_DMA_BLK("sspp_9", SSPP_DMA1, 0x26000,
+		sdm845_dma_sblk_1, 5, DPU_CLK_CTRL_DMA1),
+	SSPP_DMA_BLK("sspp_10", SSPP_DMA2, 0x28000,
+		sdm845_dma_sblk_2, 9, DPU_CLK_CTRL_CURSOR0),
+	SSPP_DMA_BLK("sspp_11", SSPP_DMA3, 0x2a000,
+		sdm845_dma_sblk_3, 13, DPU_CLK_CTRL_CURSOR1),
+};
+
+/*************************************************************
+ * MIXER sub blocks config
+ *************************************************************/
+static const struct dpu_lm_sub_blks sdm845_lm_sblk = {
+	.maxwidth = DEFAULT_DPU_OUTPUT_LINE_WIDTH,
+	.maxblendstages = 11, /* excluding base layer */
+	.blendstage_base = { /* offsets relative to mixer base */
+		0x20, 0x38, 0x50, 0x68, 0x80, 0x98,
+		0xb0, 0xc8, 0xe0, 0xf8, 0x110
+	},
+};
+
+#define LM_BLK(_name, _id, _base, _ds, _pp, _lmpair) \
+	{ \
+	.name = _name, .id = _id, \
+	.base = _base, .len = 0x320, \
+	.features = MIXER_SDM845_MASK, \
+	.sblk = &sdm845_lm_sblk, \
+	.ds = _ds, \
+	.pingpong = _pp, \
+	.lm_pair_mask = (1 << _lmpair) \
+	}
+
+static struct dpu_lm_cfg sdm845_lm[] = {
+	LM_BLK("lm_0", LM_0, 0x44000, DS_0, PINGPONG_0, LM_1),
+	LM_BLK("lm_1", LM_1, 0x45000, DS_1, PINGPONG_1, LM_0),
+	LM_BLK("lm_2", LM_2, 0x46000, DS_MAX, PINGPONG_2, LM_5),
+	LM_BLK("lm_3", LM_3, 0x0, DS_MAX, PINGPONG_MAX, 0),
+	LM_BLK("lm_4", LM_4, 0x0, DS_MAX, PINGPONG_MAX, 0),
+	LM_BLK("lm_5", LM_5, 0x49000, DS_MAX, PINGPONG_3, LM_2),
+};
+
+/*************************************************************
+ * DS sub blocks config
+ *************************************************************/
+static const struct dpu_ds_top_cfg sdm845_ds_top = {
+	.name = "ds_top_0", .id = DS_TOP,
+	.base = 0x60000, .len = 0xc,
+	.maxinputwidth = DEFAULT_DPU_LINE_WIDTH,
+	.maxoutputwidth = DEFAULT_DPU_OUTPUT_LINE_WIDTH,
+	.maxupscale = MAX_UPSCALE_RATIO,
+};
+
+#define DS_BLK(_name, _id, _base) \
+	{\
+	.name = _name, .id = _id, \
+	.base = _base, .len = 0x800, \
+	.features = DPU_SSPP_SCALER_QSEED3, \
+	.top = &sdm845_ds_top \
+	}
+
+static struct dpu_ds_cfg sdm845_ds[] = {
+	DS_BLK("ds_0", DS_0, 0x800),
+	DS_BLK("ds_1", DS_1, 0x1000),
+};
+
+/*************************************************************
+ * PINGPONG sub blocks config
+ *************************************************************/
+static const struct dpu_pingpong_sub_blks sdm845_pp_sblk_te = {
+	.te2 = {.id = DPU_PINGPONG_TE2, .base = 0x2000, .len = 0x0,
+		.version = 0x1},
+	.dither = {.id = DPU_PINGPONG_DITHER, .base = 0x30e0,
+		.len = 0x20, .version = 0x10000},
+};
+
+static const struct dpu_pingpong_sub_blks sdm845_pp_sblk = {
+	.dither = {.id = DPU_PINGPONG_DITHER, .base = 0x30e0,
+		.len = 0x20, .version = 0x10000},
+};
+
+#define PP_BLK_TE(_name, _id, _base) \
+	{\
+	.name = _name, .id = _id, \
+	.base = _base, .len = 0xd4, \
+	.features = PINGPONG_SDM845_SPLIT_MASK, \
+	.sblk = &sdm845_pp_sblk_te \
+	}
+#define PP_BLK(_name, _id, _base) \
+	{\
+	.name = _name, .id = _id, \
+	.base = _base, .len = 0xd4, \
+	.features = PINGPONG_SDM845_MASK, \
+	.sblk = &sdm845_pp_sblk \
+	}
+
+static struct dpu_pingpong_cfg sdm845_pp[] = {
+	PP_BLK_TE("pingpong_0", PINGPONG_0, 0x70000),
+	PP_BLK_TE("pingpong_1", PINGPONG_1, 0x70800),
+	PP_BLK("pingpong_2", PINGPONG_2, 0x71000),
+	PP_BLK("pingpong_3", PINGPONG_3, 0x71800),
+};
+
+/*************************************************************
+ * INTF sub blocks config
+ *************************************************************/
+#define INTF_BLK(_name, _id, _base, _type, _ctrl_id) \
+	{\
+	.name = _name, .id = _id, \
+	.base = _base, .len = 0x280, \
+	.type = _type, \
+	.controller_id = _ctrl_id, \
+	.prog_fetch_lines_worst_case = 24 \
+	}
+
+static struct dpu_intf_cfg sdm845_intf[] = {
+	INTF_BLK("intf_0", INTF_0, 0x6A000, INTF_DP, 0),
+	INTF_BLK("intf_1", INTF_1, 0x6A800, INTF_DSI, 0),
+	INTF_BLK("intf_2", INTF_2, 0x6B000, INTF_DSI, 1),
+	INTF_BLK("intf_3", INTF_3, 0x6B800, INTF_DP, 1),
+};
+
+/*************************************************************
+ * CDM sub blocks config
+ *************************************************************/
+static struct dpu_cdm_cfg sdm845_cdm[] = {
+	{
+	.name = "cdm_0", .id = CDM_0,
+	.base = 0x79200, .len = 0x224,
+	.features = 0,
+	.intf_connect = BIT(INTF_3),
+	},
+};
+
+/*************************************************************
+ * VBIF sub blocks config
+ *************************************************************/
+/* VBIF QOS remap */
+static u32 sdm845_rt_pri_lvl[] = {3, 3, 4, 4, 5, 5, 6, 6};
+static u32 sdm845_nrt_pri_lvl[] = {3, 3, 3, 3, 3, 3, 3, 3};
+
+static struct dpu_vbif_cfg sdm845_vbif[] = {
+	{
+	.name = "vbif_0", .id = VBIF_0,
+	.base = 0, .len = 0x1040,
+	.features = BIT(DPU_VBIF_QOS_REMAP),
+	.xin_halt_timeout = 0x4000,
+	.qos_rt_tbl = {
+		.npriority_lvl = ARRAY_SIZE(sdm845_rt_pri_lvl),
+		.priority_lvl = sdm845_rt_pri_lvl,
+		},
+	.qos_nrt_tbl = {
+		.npriority_lvl = ARRAY_SIZE(sdm845_nrt_pri_lvl),
+		.priority_lvl = sdm845_nrt_pri_lvl,
+		},
+	.memtype_count = 14,
+	.memtype = {3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3},
+	},
+};
+
+static struct dpu_reg_dma_cfg sdm845_regdma = {
+	.base = 0x0, .version = 0x1, .trigger_sel_off = 0x119c
+};
+
+/*************************************************************
+ * PERF data config
+ *************************************************************/
+
+/* SSPP QOS LUTs */
+static struct dpu_qos_lut_entry sdm845_qos_linear[] = {
+	{.fl = 4, .lut = 0x357},
+	{.fl = 5, .lut = 0x3357},
+	{.fl = 6, .lut = 0x23357},
+	{.fl = 7, .lut = 0x223357},
+	{.fl = 8, .lut = 0x2223357},
+	{.fl = 9, .lut = 0x22223357},
+	{.fl = 10, .lut = 0x222223357},
+	{.fl = 11, .lut = 0x2222223357},
+	{.fl = 12, .lut = 0x22222223357},
+	{.fl = 13, .lut = 0x222222223357},
+	{.fl = 14, .lut = 0x1222222223357},
+	{.fl = 0, .lut = 0x11222222223357}
+};
+
+static struct dpu_qos_lut_entry sdm845_qos_macrotile[] = {
+	{.fl = 10, .lut = 0x344556677},
+	{.fl = 11, .lut = 0x3344556677},
+	{.fl = 12, .lut = 0x23344556677},
+	{.fl = 13, .lut = 0x223344556677},
+	{.fl = 14, .lut = 0x1223344556677},
+	{.fl = 0, .lut = 0x112233344556677},
+};
+
+static struct dpu_qos_lut_entry sdm845_qos_nrt[] = {
+	{.fl = 0, .lut = 0x0},
+};
+
+static struct dpu_perf_cfg sdm845_perf_data = {
+	.max_bw_low = 6800000,
+	.max_bw_high = 6800000,
+	.min_core_ib = 2400000,
+	.min_llcc_ib = 800000,
+	.min_dram_ib = 800000,
+	.core_ib_ff = "6.0",
+	.core_clk_ff = "1.0",
+	.comp_ratio_rt =
+	"NV12/5/1/1.23 AB24/5/1/1.23 XB24/5/1/1.23",
+	.comp_ratio_nrt =
+	"NV12/5/1/1.25 AB24/5/1/1.25 XB24/5/1/1.25",
+	.undersized_prefill_lines = 2,
+	.xtra_prefill_lines = 2,
+	.dest_scale_prefill_lines = 3,
+	.macrotile_prefill_lines = 4,
+	.yuv_nv12_prefill_lines = 8,
+	.linear_prefill_lines = 1,
+	.downscaling_prefill_lines = 1,
+	.amortizable_threshold = 25,
+	.min_prefill_lines = 24,
+	.danger_lut_tbl = {0xf, 0xffff, 0x0},
+	.qos_lut_tbl = {
+		{.nentry = ARRAY_SIZE(sdm845_qos_linear),
+		.entries = sdm845_qos_linear
+		},
+		{.nentry = ARRAY_SIZE(sdm845_qos_macrotile),
+		.entries = sdm845_qos_macrotile
+		},
+		{.nentry = ARRAY_SIZE(sdm845_qos_nrt),
+		.entries = sdm845_qos_nrt
+		},
+	},
+	.cdp_cfg = {
+		{.rd_enable = 1, .wr_enable = 1},
+		{.rd_enable = 1, .wr_enable = 0}
+	},
+};
+
+/*************************************************************
+ * Hardware catalog init
+ *************************************************************/
+
+/*
+ * sdm845_cfg_init(): populate sdm845 dpu sub-blocks reg offsets
+ * and instance counts.
+ */
+static void sdm845_cfg_init(struct dpu_mdss_cfg *dpu_cfg)
+{
+	*dpu_cfg = (struct dpu_mdss_cfg){
+		.caps = &sdm845_dpu_caps,
+		.mdp_count = ARRAY_SIZE(sdm845_mdp),
+		.mdp = sdm845_mdp,
+		.ctl_count = ARRAY_SIZE(sdm845_ctl),
+		.ctl = sdm845_ctl,
+		.sspp_count = ARRAY_SIZE(sdm845_sspp),
+		.sspp = sdm845_sspp,
+		.mixer_count = ARRAY_SIZE(sdm845_lm),
+		.mixer = sdm845_lm,
+		.ds_count = ARRAY_SIZE(sdm845_ds),
+		.ds = sdm845_ds,
+		.pingpong_count = ARRAY_SIZE(sdm845_pp),
+		.pingpong = sdm845_pp,
+		.cdm_count = ARRAY_SIZE(sdm845_cdm),
+		.cdm = sdm845_cdm,
+		.intf_count = ARRAY_SIZE(sdm845_intf),
+		.intf = sdm845_intf,
+		.vbif_count = ARRAY_SIZE(sdm845_vbif),
+		.vbif = sdm845_vbif,
+		.reg_dma_count = 1,
+		.dma_cfg = sdm845_regdma,
+		.perf = sdm845_perf_data,
+	};
+}
+
+static struct dpu_mdss_hw_cfg_handler cfg_handler[] = {
+	{ .hw_rev = DPU_HW_VER_400, .cfg_init = sdm845_cfg_init},
+	{ .hw_rev = DPU_HW_VER_401, .cfg_init = sdm845_cfg_init},
+};
+
+void dpu_hw_catalog_deinit(struct dpu_mdss_cfg *dpu_cfg)
+{
+	kfree(dpu_cfg);
+}
+
+struct dpu_mdss_cfg *dpu_hw_catalog_init(u32 hw_rev)
+{
+	int i;
+	struct dpu_mdss_cfg *dpu_cfg;
+
+	dpu_cfg = kzalloc(sizeof(*dpu_cfg), GFP_KERNEL);
+	if (!dpu_cfg)
+		return ERR_PTR(-ENOMEM);
+
+	for (i = 0; i < ARRAY_SIZE(cfg_handler); i++) {
+		if (cfg_handler[i].hw_rev == hw_rev) {
+			cfg_handler[i].cfg_init(dpu_cfg);
+			dpu_cfg->hwversion = hw_rev;
+			return dpu_cfg;
+		}
+	}
+
+	DPU_ERROR("unsupported chipset id:%X\n", hw_rev);
+	dpu_hw_catalog_deinit(dpu_cfg);
+	return ERR_PTR(-ENODEV);
+}
+
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.h
new file mode 100644
index 000000000000..f0cb0d4fc80e
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.h
@@ -0,0 +1,804 @@
+/* Copyright (c) 2015-2018, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef _DPU_HW_CATALOG_H
+#define _DPU_HW_CATALOG_H
+
+#include <linux/kernel.h>
+#include <linux/bug.h>
+#include <linux/bitmap.h>
+#include <linux/err.h>
+#include <drm/drmP.h>
+
+/**
+ * Max hardware block count: For ex: max 12 SSPP pipes or
+ * 5 ctl paths. In all cases, it can have max 12 hardware blocks
+ * based on current design
+ */
+#define MAX_BLOCKS    12
+
+#define DPU_HW_VER(MAJOR, MINOR, STEP) (((MAJOR & 0xF) << 28)    |\
+		((MINOR & 0xFFF) << 16)  |\
+		(STEP & 0xFFFF))
+
+#define DPU_HW_MAJOR(rev)		((rev) >> 28)
+#define DPU_HW_MINOR(rev)		(((rev) >> 16) & 0xFFF)
+#define DPU_HW_STEP(rev)		((rev) & 0xFFFF)
+#define DPU_HW_MAJOR_MINOR(rev)		((rev) >> 16)
+
+#define IS_DPU_MAJOR_MINOR_SAME(rev1, rev2)   \
+	(DPU_HW_MAJOR_MINOR((rev1)) == DPU_HW_MAJOR_MINOR((rev2)))
+
+#define DPU_HW_VER_170	DPU_HW_VER(1, 7, 0) /* 8996 v1.0 */
+#define DPU_HW_VER_171	DPU_HW_VER(1, 7, 1) /* 8996 v2.0 */
+#define DPU_HW_VER_172	DPU_HW_VER(1, 7, 2) /* 8996 v3.0 */
+#define DPU_HW_VER_300	DPU_HW_VER(3, 0, 0) /* 8998 v1.0 */
+#define DPU_HW_VER_301	DPU_HW_VER(3, 0, 1) /* 8998 v1.1 */
+#define DPU_HW_VER_400	DPU_HW_VER(4, 0, 0) /* sdm845 v1.0 */
+#define DPU_HW_VER_401	DPU_HW_VER(4, 0, 1) /* sdm845 v2.0 */
+#define DPU_HW_VER_410	DPU_HW_VER(4, 1, 0) /* sdm670 v1.0 */
+#define DPU_HW_VER_500	DPU_HW_VER(5, 0, 0) /* sdm855 v1.0 */
+
+
+#define IS_MSM8996_TARGET(rev) IS_DPU_MAJOR_MINOR_SAME((rev), DPU_HW_VER_170)
+#define IS_MSM8998_TARGET(rev) IS_DPU_MAJOR_MINOR_SAME((rev), DPU_HW_VER_300)
+#define IS_SDM845_TARGET(rev) IS_DPU_MAJOR_MINOR_SAME((rev), DPU_HW_VER_400)
+#define IS_SDM670_TARGET(rev) IS_DPU_MAJOR_MINOR_SAME((rev), DPU_HW_VER_410)
+#define IS_SDM855_TARGET(rev) IS_DPU_MAJOR_MINOR_SAME((rev), DPU_HW_VER_500)
+
+
+#define DPU_HW_BLK_NAME_LEN	16
+
+#define MAX_IMG_WIDTH 0x3fff
+#define MAX_IMG_HEIGHT 0x3fff
+
+#define CRTC_DUAL_MIXERS	2
+
+#define MAX_XIN_COUNT 16
+
+/**
+ * Supported UBWC feature versions
+ */
+enum {
+	DPU_HW_UBWC_VER_10 = 0x100,
+	DPU_HW_UBWC_VER_20 = 0x200,
+	DPU_HW_UBWC_VER_30 = 0x300,
+};
+
+#define IS_UBWC_20_SUPPORTED(rev)       ((rev) >= DPU_HW_UBWC_VER_20)
+
+/**
+ * MDP TOP BLOCK features
+ * @DPU_MDP_PANIC_PER_PIPE Panic configuration needs to be be done per pipe
+ * @DPU_MDP_10BIT_SUPPORT, Chipset supports 10 bit pixel formats
+ * @DPU_MDP_BWC,           MDSS HW supports Bandwidth compression.
+ * @DPU_MDP_UBWC_1_0,      This chipsets supports Universal Bandwidth
+ *                         compression initial revision
+ * @DPU_MDP_UBWC_1_5,      Universal Bandwidth compression version 1.5
+ * @DPU_MDP_MAX            Maximum value
+
+ */
+enum {
+	DPU_MDP_PANIC_PER_PIPE = 0x1,
+	DPU_MDP_10BIT_SUPPORT,
+	DPU_MDP_BWC,
+	DPU_MDP_UBWC_1_0,
+	DPU_MDP_UBWC_1_5,
+	DPU_MDP_MAX
+};
+
+/**
+ * SSPP sub-blocks/features
+ * @DPU_SSPP_SRC             Src and fetch part of the pipes,
+ * @DPU_SSPP_SCALER_QSEED2,  QSEED2 algorithm support
+ * @DPU_SSPP_SCALER_QSEED3,  QSEED3 alogorithm support
+ * @DPU_SSPP_SCALER_RGB,     RGB Scaler, supported by RGB pipes
+ * @DPU_SSPP_CSC,            Support of Color space converion
+ * @DPU_SSPP_CSC_10BIT,      Support of 10-bit Color space conversion
+ * @DPU_SSPP_CURSOR,         SSPP can be used as a cursor layer
+ * @DPU_SSPP_QOS,            SSPP support QoS control, danger/safe/creq
+ * @DPU_SSPP_QOS_8LVL,       SSPP support 8-level QoS control
+ * @DPU_SSPP_EXCL_RECT,      SSPP supports exclusion rect
+ * @DPU_SSPP_SMART_DMA_V1,   SmartDMA 1.0 support
+ * @DPU_SSPP_SMART_DMA_V2,   SmartDMA 2.0 support
+ * @DPU_SSPP_TS_PREFILL      Supports prefill with traffic shaper
+ * @DPU_SSPP_TS_PREFILL_REC1 Supports prefill with traffic shaper multirec
+ * @DPU_SSPP_CDP             Supports client driven prefetch
+ * @DPU_SSPP_MAX             maximum value
+ */
+enum {
+	DPU_SSPP_SRC = 0x1,
+	DPU_SSPP_SCALER_QSEED2,
+	DPU_SSPP_SCALER_QSEED3,
+	DPU_SSPP_SCALER_RGB,
+	DPU_SSPP_CSC,
+	DPU_SSPP_CSC_10BIT,
+	DPU_SSPP_CURSOR,
+	DPU_SSPP_QOS,
+	DPU_SSPP_QOS_8LVL,
+	DPU_SSPP_EXCL_RECT,
+	DPU_SSPP_SMART_DMA_V1,
+	DPU_SSPP_SMART_DMA_V2,
+	DPU_SSPP_TS_PREFILL,
+	DPU_SSPP_TS_PREFILL_REC1,
+	DPU_SSPP_CDP,
+	DPU_SSPP_MAX
+};
+
+/*
+ * MIXER sub-blocks/features
+ * @DPU_MIXER_LAYER           Layer mixer layer blend configuration,
+ * @DPU_MIXER_SOURCESPLIT     Layer mixer supports source-split configuration
+ * @DPU_MIXER_GC              Gamma correction block
+ * @DPU_DIM_LAYER             Layer mixer supports dim layer
+ * @DPU_MIXER_MAX             maximum value
+ */
+enum {
+	DPU_MIXER_LAYER = 0x1,
+	DPU_MIXER_SOURCESPLIT,
+	DPU_MIXER_GC,
+	DPU_DIM_LAYER,
+	DPU_MIXER_MAX
+};
+
+/**
+ * PINGPONG sub-blocks
+ * @DPU_PINGPONG_TE         Tear check block
+ * @DPU_PINGPONG_TE2        Additional tear check block for split pipes
+ * @DPU_PINGPONG_SPLIT      PP block supports split fifo
+ * @DPU_PINGPONG_SLAVE      PP block is a suitable slave for split fifo
+ * @DPU_PINGPONG_DITHER,    Dither blocks
+ * @DPU_PINGPONG_MAX
+ */
+enum {
+	DPU_PINGPONG_TE = 0x1,
+	DPU_PINGPONG_TE2,
+	DPU_PINGPONG_SPLIT,
+	DPU_PINGPONG_SLAVE,
+	DPU_PINGPONG_DITHER,
+	DPU_PINGPONG_MAX
+};
+
+/**
+ * CTL sub-blocks
+ * @DPU_CTL_SPLIT_DISPLAY       CTL supports video mode split display
+ * @DPU_CTL_MAX
+ */
+enum {
+	DPU_CTL_SPLIT_DISPLAY = 0x1,
+	DPU_CTL_MAX
+};
+
+/**
+ * VBIF sub-blocks and features
+ * @DPU_VBIF_QOS_OTLIM        VBIF supports OT Limit
+ * @DPU_VBIF_QOS_REMAP        VBIF supports QoS priority remap
+ * @DPU_VBIF_MAX              maximum value
+ */
+enum {
+	DPU_VBIF_QOS_OTLIM = 0x1,
+	DPU_VBIF_QOS_REMAP,
+	DPU_VBIF_MAX
+};
+
+/**
+ * MACRO DPU_HW_BLK_INFO - information of HW blocks inside DPU
+ * @name:              string name for debug purposes
+ * @id:                enum identifying this block
+ * @base:              register base offset to mdss
+ * @len:               length of hardware block
+ * @features           bit mask identifying sub-blocks/features
+ */
+#define DPU_HW_BLK_INFO \
+	char name[DPU_HW_BLK_NAME_LEN]; \
+	u32 id; \
+	u32 base; \
+	u32 len; \
+	unsigned long features
+
+/**
+ * MACRO DPU_HW_SUBBLK_INFO - information of HW sub-block inside DPU
+ * @name:              string name for debug purposes
+ * @id:                enum identifying this sub-block
+ * @base:              offset of this sub-block relative to the block
+ *                     offset
+ * @len                register block length of this sub-block
+ */
+#define DPU_HW_SUBBLK_INFO \
+	char name[DPU_HW_BLK_NAME_LEN]; \
+	u32 id; \
+	u32 base; \
+	u32 len
+
+/**
+ * struct dpu_src_blk: SSPP part of the source pipes
+ * @info:   HW register and features supported by this sub-blk
+ */
+struct dpu_src_blk {
+	DPU_HW_SUBBLK_INFO;
+};
+
+/**
+ * struct dpu_scaler_blk: Scaler information
+ * @info:   HW register and features supported by this sub-blk
+ * @version: qseed block revision
+ */
+struct dpu_scaler_blk {
+	DPU_HW_SUBBLK_INFO;
+	u32 version;
+};
+
+struct dpu_csc_blk {
+	DPU_HW_SUBBLK_INFO;
+};
+
+/**
+ * struct dpu_pp_blk : Pixel processing sub-blk information
+ * @info:   HW register and features supported by this sub-blk
+ * @version: HW Algorithm version
+ */
+struct dpu_pp_blk {
+	DPU_HW_SUBBLK_INFO;
+	u32 version;
+};
+
+/**
+ * struct dpu_format_extended - define dpu specific pixel format+modifier
+ * @fourcc_format: Base FOURCC pixel format code
+ * @modifier: 64-bit drm format modifier, same modifier must be applied to all
+ *            framebuffer planes
+ */
+struct dpu_format_extended {
+	uint32_t fourcc_format;
+	uint64_t modifier;
+};
+
+/**
+ * enum dpu_qos_lut_usage - define QoS LUT use cases
+ */
+enum dpu_qos_lut_usage {
+	DPU_QOS_LUT_USAGE_LINEAR,
+	DPU_QOS_LUT_USAGE_MACROTILE,
+	DPU_QOS_LUT_USAGE_NRT,
+	DPU_QOS_LUT_USAGE_MAX,
+};
+
+/**
+ * struct dpu_qos_lut_entry - define QoS LUT table entry
+ * @fl: fill level, or zero on last entry to indicate default lut
+ * @lut: lut to use if equal to or less than fill level
+ */
+struct dpu_qos_lut_entry {
+	u32 fl;
+	u64 lut;
+};
+
+/**
+ * struct dpu_qos_lut_tbl - define QoS LUT table
+ * @nentry: number of entry in this table
+ * @entries: Pointer to table entries
+ */
+struct dpu_qos_lut_tbl {
+	u32 nentry;
+	struct dpu_qos_lut_entry *entries;
+};
+
+/**
+ * struct dpu_caps - define DPU capabilities
+ * @max_mixer_width    max layer mixer line width support.
+ * @max_mixer_blendstages max layer mixer blend stages or
+ *                       supported z order
+ * @qseed_type         qseed2 or qseed3 support.
+ * @smart_dma_rev      Supported version of SmartDMA feature.
+ * @ubwc_version       UBWC feature version (0x0 for not supported)
+ * @has_src_split      source split feature status
+ * @has_dim_layer      dim layer feature status
+ * @has_idle_pc        indicate if idle power collapse feature is supported
+ */
+struct dpu_caps {
+	u32 max_mixer_width;
+	u32 max_mixer_blendstages;
+	u32 qseed_type;
+	u32 smart_dma_rev;
+	u32 ubwc_version;
+	bool has_src_split;
+	bool has_dim_layer;
+	bool has_idle_pc;
+};
+
+/**
+ * struct dpu_sspp_blks_common : SSPP sub-blocks common configuration
+ * @maxwidth: max pixelwidth supported by this pipe
+ * @pixel_ram_size: size of latency hiding and de-tiling buffer in bytes
+ * @maxhdeciexp: max horizontal decimation supported by this pipe
+ *				(max is 2^value)
+ * @maxvdeciexp: max vertical decimation supported by this pipe
+ *				(max is 2^value)
+ */
+struct dpu_sspp_blks_common {
+	u32 maxlinewidth;
+	u32 pixel_ram_size;
+	u32 maxhdeciexp;
+	u32 maxvdeciexp;
+};
+
+/**
+ * struct dpu_sspp_sub_blks : SSPP sub-blocks
+ * common: Pointer to common configurations shared by sub blocks
+ * @creq_vblank: creq priority during vertical blanking
+ * @danger_vblank: danger priority during vertical blanking
+ * @maxdwnscale: max downscale ratio supported(without DECIMATION)
+ * @maxupscale:  maxupscale ratio supported
+ * @smart_dma_priority: hw priority of rect1 of multirect pipe
+ * @max_per_pipe_bw: maximum allowable bandwidth of this pipe in kBps
+ * @src_blk:
+ * @scaler_blk:
+ * @csc_blk:
+ * @hsic:
+ * @memcolor:
+ * @pcc_blk:
+ * @igc_blk:
+ * @format_list: Pointer to list of supported formats
+ * @virt_format_list: Pointer to list of supported formats for virtual planes
+ */
+struct dpu_sspp_sub_blks {
+	const struct dpu_sspp_blks_common *common;
+	u32 creq_vblank;
+	u32 danger_vblank;
+	u32 maxdwnscale;
+	u32 maxupscale;
+	u32 smart_dma_priority;
+	u32 max_per_pipe_bw;
+	struct dpu_src_blk src_blk;
+	struct dpu_scaler_blk scaler_blk;
+	struct dpu_pp_blk csc_blk;
+	struct dpu_pp_blk hsic_blk;
+	struct dpu_pp_blk memcolor_blk;
+	struct dpu_pp_blk pcc_blk;
+	struct dpu_pp_blk igc_blk;
+
+	const struct dpu_format_extended *format_list;
+	const struct dpu_format_extended *virt_format_list;
+};
+
+/**
+ * struct dpu_lm_sub_blks:      information of mixer block
+ * @maxwidth:               Max pixel width supported by this mixer
+ * @maxblendstages:         Max number of blend-stages supported
+ * @blendstage_base:        Blend-stage register base offset
+ * @gc: gamma correction block
+ */
+struct dpu_lm_sub_blks {
+	u32 maxwidth;
+	u32 maxblendstages;
+	u32 blendstage_base[MAX_BLOCKS];
+	struct dpu_pp_blk gc;
+};
+
+struct dpu_pingpong_sub_blks {
+	struct dpu_pp_blk te;
+	struct dpu_pp_blk te2;
+	struct dpu_pp_blk dither;
+};
+
+/**
+ * dpu_clk_ctrl_type - Defines top level clock control signals
+ */
+enum dpu_clk_ctrl_type {
+	DPU_CLK_CTRL_NONE,
+	DPU_CLK_CTRL_VIG0,
+	DPU_CLK_CTRL_VIG1,
+	DPU_CLK_CTRL_VIG2,
+	DPU_CLK_CTRL_VIG3,
+	DPU_CLK_CTRL_VIG4,
+	DPU_CLK_CTRL_RGB0,
+	DPU_CLK_CTRL_RGB1,
+	DPU_CLK_CTRL_RGB2,
+	DPU_CLK_CTRL_RGB3,
+	DPU_CLK_CTRL_DMA0,
+	DPU_CLK_CTRL_DMA1,
+	DPU_CLK_CTRL_CURSOR0,
+	DPU_CLK_CTRL_CURSOR1,
+	DPU_CLK_CTRL_INLINE_ROT0_SSPP,
+	DPU_CLK_CTRL_MAX,
+};
+
+/* struct dpu_clk_ctrl_reg : Clock control register
+ * @reg_off:           register offset
+ * @bit_off:           bit offset
+ */
+struct dpu_clk_ctrl_reg {
+	u32 reg_off;
+	u32 bit_off;
+};
+
+/* struct dpu_mdp_cfg : MDP TOP-BLK instance info
+ * @id:                index identifying this block
+ * @base:              register base offset to mdss
+ * @features           bit mask identifying sub-blocks/features
+ * @highest_bank_bit:  UBWC parameter
+ * @ubwc_static:       ubwc static configuration
+ * @ubwc_swizzle:      ubwc default swizzle setting
+ * @has_dest_scaler:   indicates support of destination scaler
+ * @clk_ctrls          clock control register definition
+ */
+struct dpu_mdp_cfg {
+	DPU_HW_BLK_INFO;
+	u32 highest_bank_bit;
+	u32 ubwc_static;
+	u32 ubwc_swizzle;
+	bool has_dest_scaler;
+	struct dpu_clk_ctrl_reg clk_ctrls[DPU_CLK_CTRL_MAX];
+};
+
+/* struct dpu_mdp_cfg : MDP TOP-BLK instance info
+ * @id:                index identifying this block
+ * @base:              register base offset to mdss
+ * @features           bit mask identifying sub-blocks/features
+ */
+struct dpu_ctl_cfg {
+	DPU_HW_BLK_INFO;
+};
+
+/**
+ * struct dpu_sspp_cfg - information of source pipes
+ * @id:                index identifying this block
+ * @base               register offset of this block
+ * @features           bit mask identifying sub-blocks/features
+ * @sblk:              SSPP sub-blocks information
+ * @xin_id:            bus client identifier
+ * @clk_ctrl           clock control identifier
+ * @type               sspp type identifier
+ */
+struct dpu_sspp_cfg {
+	DPU_HW_BLK_INFO;
+	const struct dpu_sspp_sub_blks *sblk;
+	u32 xin_id;
+	enum dpu_clk_ctrl_type clk_ctrl;
+	u32 type;
+};
+
+/**
+ * struct dpu_lm_cfg - information of layer mixer blocks
+ * @id:                index identifying this block
+ * @base               register offset of this block
+ * @features           bit mask identifying sub-blocks/features
+ * @sblk:              LM Sub-blocks information
+ * @pingpong:          ID of connected PingPong, PINGPONG_MAX if unsupported
+ * @ds:                ID of connected DS, DS_MAX if unsupported
+ * @lm_pair_mask:      Bitmask of LMs that can be controlled by same CTL
+ */
+struct dpu_lm_cfg {
+	DPU_HW_BLK_INFO;
+	const struct dpu_lm_sub_blks *sblk;
+	u32 pingpong;
+	u32 ds;
+	unsigned long lm_pair_mask;
+};
+
+/**
+ * struct dpu_ds_top_cfg - information of dest scaler top
+ * @id               enum identifying this block
+ * @base             register offset of this block
+ * @features         bit mask identifying features
+ * @version          hw version of dest scaler
+ * @maxinputwidth    maximum input line width
+ * @maxoutputwidth   maximum output line width
+ * @maxupscale       maximum upscale ratio
+ */
+struct dpu_ds_top_cfg {
+	DPU_HW_BLK_INFO;
+	u32 version;
+	u32 maxinputwidth;
+	u32 maxoutputwidth;
+	u32 maxupscale;
+};
+
+/**
+ * struct dpu_ds_cfg - information of dest scaler blocks
+ * @id          enum identifying this block
+ * @base        register offset wrt DS top offset
+ * @features    bit mask identifying features
+ * @version     hw version of the qseed block
+ * @top         DS top information
+ */
+struct dpu_ds_cfg {
+	DPU_HW_BLK_INFO;
+	u32 version;
+	const struct dpu_ds_top_cfg *top;
+};
+
+/**
+ * struct dpu_pingpong_cfg - information of PING-PONG blocks
+ * @id                 enum identifying this block
+ * @base               register offset of this block
+ * @features           bit mask identifying sub-blocks/features
+ * @sblk               sub-blocks information
+ */
+struct dpu_pingpong_cfg  {
+	DPU_HW_BLK_INFO;
+	const struct dpu_pingpong_sub_blks *sblk;
+};
+
+/**
+ * struct dpu_cdm_cfg - information of chroma down blocks
+ * @id                 enum identifying this block
+ * @base               register offset of this block
+ * @features           bit mask identifying sub-blocks/features
+ * @intf_connect       Bitmask of INTF IDs this CDM can connect to
+ */
+struct dpu_cdm_cfg   {
+	DPU_HW_BLK_INFO;
+	unsigned long intf_connect;
+};
+
+/**
+ * struct dpu_intf_cfg - information of timing engine blocks
+ * @id                 enum identifying this block
+ * @base               register offset of this block
+ * @features           bit mask identifying sub-blocks/features
+ * @type:              Interface type(DSI, DP, HDMI)
+ * @controller_id:     Controller Instance ID in case of multiple of intf type
+ * @prog_fetch_lines_worst_case	Worst case latency num lines needed to prefetch
+ */
+struct dpu_intf_cfg  {
+	DPU_HW_BLK_INFO;
+	u32 type;   /* interface type*/
+	u32 controller_id;
+	u32 prog_fetch_lines_worst_case;
+};
+
+/**
+ * struct dpu_vbif_dynamic_ot_cfg - dynamic OT setting
+ * @pps                pixel per seconds
+ * @ot_limit           OT limit to use up to specified pixel per second
+ */
+struct dpu_vbif_dynamic_ot_cfg {
+	u64 pps;
+	u32 ot_limit;
+};
+
+/**
+ * struct dpu_vbif_dynamic_ot_tbl - dynamic OT setting table
+ * @count              length of cfg
+ * @cfg                pointer to array of configuration settings with
+ *                     ascending requirements
+ */
+struct dpu_vbif_dynamic_ot_tbl {
+	u32 count;
+	struct dpu_vbif_dynamic_ot_cfg *cfg;
+};
+
+/**
+ * struct dpu_vbif_qos_tbl - QoS priority table
+ * @npriority_lvl      num of priority level
+ * @priority_lvl       pointer to array of priority level in ascending order
+ */
+struct dpu_vbif_qos_tbl {
+	u32 npriority_lvl;
+	u32 *priority_lvl;
+};
+
+/**
+ * struct dpu_vbif_cfg - information of VBIF blocks
+ * @id                 enum identifying this block
+ * @base               register offset of this block
+ * @features           bit mask identifying sub-blocks/features
+ * @ot_rd_limit        default OT read limit
+ * @ot_wr_limit        default OT write limit
+ * @xin_halt_timeout   maximum time (in usec) for xin to halt
+ * @dynamic_ot_rd_tbl  dynamic OT read configuration table
+ * @dynamic_ot_wr_tbl  dynamic OT write configuration table
+ * @qos_rt_tbl         real-time QoS priority table
+ * @qos_nrt_tbl        non-real-time QoS priority table
+ * @memtype_count      number of defined memtypes
+ * @memtype            array of xin memtype definitions
+ */
+struct dpu_vbif_cfg {
+	DPU_HW_BLK_INFO;
+	u32 default_ot_rd_limit;
+	u32 default_ot_wr_limit;
+	u32 xin_halt_timeout;
+	struct dpu_vbif_dynamic_ot_tbl dynamic_ot_rd_tbl;
+	struct dpu_vbif_dynamic_ot_tbl dynamic_ot_wr_tbl;
+	struct dpu_vbif_qos_tbl qos_rt_tbl;
+	struct dpu_vbif_qos_tbl qos_nrt_tbl;
+	u32 memtype_count;
+	u32 memtype[MAX_XIN_COUNT];
+};
+/**
+ * struct dpu_reg_dma_cfg - information of lut dma blocks
+ * @id                 enum identifying this block
+ * @base               register offset of this block
+ * @features           bit mask identifying sub-blocks/features
+ * @version            version of lutdma hw block
+ * @trigger_sel_off    offset to trigger select registers of lutdma
+ */
+struct dpu_reg_dma_cfg {
+	DPU_HW_BLK_INFO;
+	u32 version;
+	u32 trigger_sel_off;
+};
+
+/**
+ * Define CDP use cases
+ * @DPU_PERF_CDP_UDAGE_RT: real-time use cases
+ * @DPU_PERF_CDP_USAGE_NRT: non real-time use cases such as WFD
+ */
+enum {
+	DPU_PERF_CDP_USAGE_RT,
+	DPU_PERF_CDP_USAGE_NRT,
+	DPU_PERF_CDP_USAGE_MAX
+};
+
+/**
+ * struct dpu_perf_cdp_cfg - define CDP use case configuration
+ * @rd_enable: true if read pipe CDP is enabled
+ * @wr_enable: true if write pipe CDP is enabled
+ */
+struct dpu_perf_cdp_cfg {
+	bool rd_enable;
+	bool wr_enable;
+};
+
+/**
+ * struct dpu_perf_cfg - performance control settings
+ * @max_bw_low         low threshold of maximum bandwidth (kbps)
+ * @max_bw_high        high threshold of maximum bandwidth (kbps)
+ * @min_core_ib        minimum bandwidth for core (kbps)
+ * @min_core_ib        minimum mnoc ib vote in kbps
+ * @min_llcc_ib        minimum llcc ib vote in kbps
+ * @min_dram_ib        minimum dram ib vote in kbps
+ * @core_ib_ff         core instantaneous bandwidth fudge factor
+ * @core_clk_ff        core clock fudge factor
+ * @comp_ratio_rt      string of 0 or more of <fourcc>/<ven>/<mod>/<comp ratio>
+ * @comp_ratio_nrt     string of 0 or more of <fourcc>/<ven>/<mod>/<comp ratio>
+ * @undersized_prefill_lines   undersized prefill in lines
+ * @xtra_prefill_lines         extra prefill latency in lines
+ * @dest_scale_prefill_lines   destination scaler latency in lines
+ * @macrotile_perfill_lines    macrotile latency in lines
+ * @yuv_nv12_prefill_lines     yuv_nv12 latency in lines
+ * @linear_prefill_lines       linear latency in lines
+ * @downscaling_prefill_lines  downscaling latency in lines
+ * @amortizable_theshold minimum y position for traffic shaping prefill
+ * @min_prefill_lines  minimum pipeline latency in lines
+ * @safe_lut_tbl: LUT tables for safe signals
+ * @danger_lut_tbl: LUT tables for danger signals
+ * @qos_lut_tbl: LUT tables for QoS signals
+ * @cdp_cfg            cdp use case configurations
+ */
+struct dpu_perf_cfg {
+	u32 max_bw_low;
+	u32 max_bw_high;
+	u32 min_core_ib;
+	u32 min_llcc_ib;
+	u32 min_dram_ib;
+	const char *core_ib_ff;
+	const char *core_clk_ff;
+	const char *comp_ratio_rt;
+	const char *comp_ratio_nrt;
+	u32 undersized_prefill_lines;
+	u32 xtra_prefill_lines;
+	u32 dest_scale_prefill_lines;
+	u32 macrotile_prefill_lines;
+	u32 yuv_nv12_prefill_lines;
+	u32 linear_prefill_lines;
+	u32 downscaling_prefill_lines;
+	u32 amortizable_threshold;
+	u32 min_prefill_lines;
+	u32 safe_lut_tbl[DPU_QOS_LUT_USAGE_MAX];
+	u32 danger_lut_tbl[DPU_QOS_LUT_USAGE_MAX];
+	struct dpu_qos_lut_tbl qos_lut_tbl[DPU_QOS_LUT_USAGE_MAX];
+	struct dpu_perf_cdp_cfg cdp_cfg[DPU_PERF_CDP_USAGE_MAX];
+};
+
+/**
+ * struct dpu_mdss_cfg - information of MDSS HW
+ * This is the main catalog data structure representing
+ * this HW version. Contains number of instances,
+ * register offsets, capabilities of the all MDSS HW sub-blocks.
+ *
+ * @dma_formats        Supported formats for dma pipe
+ * @cursor_formats     Supported formats for cursor pipe
+ * @vig_formats        Supported formats for vig pipe
+ */
+struct dpu_mdss_cfg {
+	u32 hwversion;
+
+	const struct dpu_caps *caps;
+
+	u32 mdp_count;
+	struct dpu_mdp_cfg *mdp;
+
+	u32 ctl_count;
+	struct dpu_ctl_cfg *ctl;
+
+	u32 sspp_count;
+	struct dpu_sspp_cfg *sspp;
+
+	u32 mixer_count;
+	struct dpu_lm_cfg *mixer;
+
+	u32 ds_count;
+	struct dpu_ds_cfg *ds;
+
+	u32 pingpong_count;
+	struct dpu_pingpong_cfg *pingpong;
+
+	u32 cdm_count;
+	struct dpu_cdm_cfg *cdm;
+
+	u32 intf_count;
+	struct dpu_intf_cfg *intf;
+
+	u32 vbif_count;
+	struct dpu_vbif_cfg *vbif;
+
+	u32 reg_dma_count;
+	struct dpu_reg_dma_cfg dma_cfg;
+
+	u32 ad_count;
+
+	/* Add additional block data structures here */
+
+	struct dpu_perf_cfg perf;
+	struct dpu_format_extended *dma_formats;
+	struct dpu_format_extended *cursor_formats;
+	struct dpu_format_extended *vig_formats;
+};
+
+struct dpu_mdss_hw_cfg_handler {
+	u32 hw_rev;
+	void (*cfg_init)(struct dpu_mdss_cfg *dpu_cfg);
+};
+
+/*
+ * Access Macros
+ */
+#define BLK_MDP(s) ((s)->mdp)
+#define BLK_CTL(s) ((s)->ctl)
+#define BLK_VIG(s) ((s)->vig)
+#define BLK_RGB(s) ((s)->rgb)
+#define BLK_DMA(s) ((s)->dma)
+#define BLK_CURSOR(s) ((s)->cursor)
+#define BLK_MIXER(s) ((s)->mixer)
+#define BLK_DS(s) ((s)->ds)
+#define BLK_PINGPONG(s) ((s)->pingpong)
+#define BLK_CDM(s) ((s)->cdm)
+#define BLK_INTF(s) ((s)->intf)
+#define BLK_AD(s) ((s)->ad)
+
+/**
+ * dpu_hw_catalog_init - dpu hardware catalog init API retrieves
+ * hardcoded target specific catalog information in config structure
+ * @hw_rev:       caller needs provide the hardware revision.
+ *
+ * Return: dpu config structure
+ */
+struct dpu_mdss_cfg *dpu_hw_catalog_init(u32 hw_rev);
+
+/**
+ * dpu_hw_catalog_deinit - dpu hardware catalog cleanup
+ * @dpu_cfg:      pointer returned from init function
+ */
+void dpu_hw_catalog_deinit(struct dpu_mdss_cfg *dpu_cfg);
+
+/**
+ * dpu_hw_sspp_multirect_enabled - check multirect enabled for the sspp
+ * @cfg:          pointer to sspp cfg
+ */
+static inline bool dpu_hw_sspp_multirect_enabled(const struct dpu_sspp_cfg *cfg)
+{
+	return test_bit(DPU_SSPP_SMART_DMA_V1, &cfg->features) ||
+			 test_bit(DPU_SSPP_SMART_DMA_V2, &cfg->features);
+}
+#endif /* _DPU_HW_CATALOG_H */
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog_format.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog_format.h
new file mode 100644
index 000000000000..3c9f028628ef
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog_format.h
@@ -0,0 +1,168 @@
+/* Copyright (c) 2015-2018, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include "dpu_hw_mdss.h"
+
+static const struct dpu_format_extended plane_formats[] = {
+	{DRM_FORMAT_ARGB8888, 0},
+	{DRM_FORMAT_ABGR8888, 0},
+	{DRM_FORMAT_RGBA8888, 0},
+	{DRM_FORMAT_ABGR8888, DRM_FORMAT_MOD_QCOM_COMPRESSED},
+	{DRM_FORMAT_BGRA8888, 0},
+	{DRM_FORMAT_XRGB8888, 0},
+	{DRM_FORMAT_RGBX8888, 0},
+	{DRM_FORMAT_BGRX8888, 0},
+	{DRM_FORMAT_XBGR8888, 0},
+	{DRM_FORMAT_XBGR8888, DRM_FORMAT_MOD_QCOM_COMPRESSED},
+	{DRM_FORMAT_RGB888, 0},
+	{DRM_FORMAT_BGR888, 0},
+	{DRM_FORMAT_RGB565, 0},
+	{DRM_FORMAT_BGR565, DRM_FORMAT_MOD_QCOM_COMPRESSED},
+	{DRM_FORMAT_BGR565, 0},
+	{DRM_FORMAT_ARGB1555, 0},
+	{DRM_FORMAT_ABGR1555, 0},
+	{DRM_FORMAT_RGBA5551, 0},
+	{DRM_FORMAT_BGRA5551, 0},
+	{DRM_FORMAT_XRGB1555, 0},
+	{DRM_FORMAT_XBGR1555, 0},
+	{DRM_FORMAT_RGBX5551, 0},
+	{DRM_FORMAT_BGRX5551, 0},
+	{DRM_FORMAT_ARGB4444, 0},
+	{DRM_FORMAT_ABGR4444, 0},
+	{DRM_FORMAT_RGBA4444, 0},
+	{DRM_FORMAT_BGRA4444, 0},
+	{DRM_FORMAT_XRGB4444, 0},
+	{DRM_FORMAT_XBGR4444, 0},
+	{DRM_FORMAT_RGBX4444, 0},
+	{DRM_FORMAT_BGRX4444, 0},
+	{0, 0},
+};
+
+static const struct dpu_format_extended plane_formats_yuv[] = {
+	{DRM_FORMAT_ARGB8888, 0},
+	{DRM_FORMAT_ABGR8888, 0},
+	{DRM_FORMAT_RGBA8888, 0},
+	{DRM_FORMAT_BGRX8888, 0},
+	{DRM_FORMAT_ABGR8888, DRM_FORMAT_MOD_QCOM_COMPRESSED},
+	{DRM_FORMAT_BGRA8888, 0},
+	{DRM_FORMAT_XRGB8888, 0},
+	{DRM_FORMAT_XBGR8888, 0},
+	{DRM_FORMAT_RGBX8888, 0},
+	{DRM_FORMAT_XBGR8888, DRM_FORMAT_MOD_QCOM_COMPRESSED},
+	{DRM_FORMAT_RGB888, 0},
+	{DRM_FORMAT_BGR888, 0},
+	{DRM_FORMAT_RGB565, 0},
+	{DRM_FORMAT_BGR565, DRM_FORMAT_MOD_QCOM_COMPRESSED},
+	{DRM_FORMAT_BGR565, 0},
+	{DRM_FORMAT_ARGB1555, 0},
+	{DRM_FORMAT_ABGR1555, 0},
+	{DRM_FORMAT_RGBA5551, 0},
+	{DRM_FORMAT_BGRA5551, 0},
+	{DRM_FORMAT_XRGB1555, 0},
+	{DRM_FORMAT_XBGR1555, 0},
+	{DRM_FORMAT_RGBX5551, 0},
+	{DRM_FORMAT_BGRX5551, 0},
+	{DRM_FORMAT_ARGB4444, 0},
+	{DRM_FORMAT_ABGR4444, 0},
+	{DRM_FORMAT_RGBA4444, 0},
+	{DRM_FORMAT_BGRA4444, 0},
+	{DRM_FORMAT_XRGB4444, 0},
+	{DRM_FORMAT_XBGR4444, 0},
+	{DRM_FORMAT_RGBX4444, 0},
+	{DRM_FORMAT_BGRX4444, 0},
+
+	{DRM_FORMAT_NV12, 0},
+	{DRM_FORMAT_NV12, DRM_FORMAT_MOD_QCOM_COMPRESSED},
+	{DRM_FORMAT_NV21, 0},
+	{DRM_FORMAT_NV16, 0},
+	{DRM_FORMAT_NV61, 0},
+	{DRM_FORMAT_VYUY, 0},
+	{DRM_FORMAT_UYVY, 0},
+	{DRM_FORMAT_YUYV, 0},
+	{DRM_FORMAT_YVYU, 0},
+	{DRM_FORMAT_YUV420, 0},
+	{DRM_FORMAT_YVU420, 0},
+	{0, 0},
+};
+
+static const struct dpu_format_extended cursor_formats[] = {
+	{DRM_FORMAT_ARGB8888, 0},
+	{DRM_FORMAT_ABGR8888, 0},
+	{DRM_FORMAT_RGBA8888, 0},
+	{DRM_FORMAT_BGRA8888, 0},
+	{DRM_FORMAT_XRGB8888, 0},
+	{DRM_FORMAT_ARGB1555, 0},
+	{DRM_FORMAT_ABGR1555, 0},
+	{DRM_FORMAT_RGBA5551, 0},
+	{DRM_FORMAT_BGRA5551, 0},
+	{DRM_FORMAT_ARGB4444, 0},
+	{DRM_FORMAT_ABGR4444, 0},
+	{DRM_FORMAT_RGBA4444, 0},
+	{DRM_FORMAT_BGRA4444, 0},
+	{0, 0},
+};
+
+static const struct dpu_format_extended wb2_formats[] = {
+	{DRM_FORMAT_RGB565, 0},
+	{DRM_FORMAT_BGR565, DRM_FORMAT_MOD_QCOM_COMPRESSED},
+	{DRM_FORMAT_RGB888, 0},
+	{DRM_FORMAT_ARGB8888, 0},
+	{DRM_FORMAT_RGBA8888, 0},
+	{DRM_FORMAT_ABGR8888, DRM_FORMAT_MOD_QCOM_COMPRESSED},
+	{DRM_FORMAT_XRGB8888, 0},
+	{DRM_FORMAT_RGBX8888, 0},
+	{DRM_FORMAT_XBGR8888, DRM_FORMAT_MOD_QCOM_COMPRESSED},
+	{DRM_FORMAT_ARGB1555, 0},
+	{DRM_FORMAT_RGBA5551, 0},
+	{DRM_FORMAT_XRGB1555, 0},
+	{DRM_FORMAT_RGBX5551, 0},
+	{DRM_FORMAT_ARGB4444, 0},
+	{DRM_FORMAT_RGBA4444, 0},
+	{DRM_FORMAT_RGBX4444, 0},
+	{DRM_FORMAT_XRGB4444, 0},
+
+	{DRM_FORMAT_BGR565, 0},
+	{DRM_FORMAT_BGR888, 0},
+	{DRM_FORMAT_ABGR8888, 0},
+	{DRM_FORMAT_BGRA8888, 0},
+	{DRM_FORMAT_BGRX8888, 0},
+	{DRM_FORMAT_XBGR8888, 0},
+	{DRM_FORMAT_ABGR1555, 0},
+	{DRM_FORMAT_BGRA5551, 0},
+	{DRM_FORMAT_XBGR1555, 0},
+	{DRM_FORMAT_BGRX5551, 0},
+	{DRM_FORMAT_ABGR4444, 0},
+	{DRM_FORMAT_BGRA4444, 0},
+	{DRM_FORMAT_BGRX4444, 0},
+	{DRM_FORMAT_XBGR4444, 0},
+
+	{DRM_FORMAT_YUV420, 0},
+	{DRM_FORMAT_NV12, 0},
+	{DRM_FORMAT_NV12, DRM_FORMAT_MOD_QCOM_COMPRESSED},
+	{DRM_FORMAT_NV16, 0},
+	{DRM_FORMAT_YUYV, 0},
+
+	{0, 0},
+};
+
+static const struct dpu_format_extended rgb_10bit_formats[] = {
+	{DRM_FORMAT_BGRA1010102, 0},
+	{DRM_FORMAT_BGRX1010102, 0},
+	{DRM_FORMAT_RGBA1010102, 0},
+	{DRM_FORMAT_RGBX1010102, 0},
+	{DRM_FORMAT_ABGR2101010, 0},
+	{DRM_FORMAT_ABGR2101010, DRM_FORMAT_MOD_QCOM_COMPRESSED},
+	{DRM_FORMAT_XBGR2101010, 0},
+	{DRM_FORMAT_XBGR2101010, DRM_FORMAT_MOD_QCOM_COMPRESSED},
+	{DRM_FORMAT_ARGB2101010, 0},
+	{DRM_FORMAT_XRGB2101010, 0},
+};
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_cdm.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_cdm.c
new file mode 100644
index 000000000000..554874ba0c3b
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_cdm.c
@@ -0,0 +1,323 @@
+/* Copyright (c) 2015-2018, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include "dpu_hw_mdss.h"
+#include "dpu_hwio.h"
+#include "dpu_hw_catalog.h"
+#include "dpu_hw_cdm.h"
+#include "dpu_dbg.h"
+#include "dpu_kms.h"
+
+#define CDM_CSC_10_OPMODE                  0x000
+#define CDM_CSC_10_BASE                    0x004
+
+#define CDM_CDWN2_OP_MODE                  0x100
+#define CDM_CDWN2_CLAMP_OUT                0x104
+#define CDM_CDWN2_PARAMS_3D_0              0x108
+#define CDM_CDWN2_PARAMS_3D_1              0x10C
+#define CDM_CDWN2_COEFF_COSITE_H_0         0x110
+#define CDM_CDWN2_COEFF_COSITE_H_1         0x114
+#define CDM_CDWN2_COEFF_COSITE_H_2         0x118
+#define CDM_CDWN2_COEFF_OFFSITE_H_0        0x11C
+#define CDM_CDWN2_COEFF_OFFSITE_H_1        0x120
+#define CDM_CDWN2_COEFF_OFFSITE_H_2        0x124
+#define CDM_CDWN2_COEFF_COSITE_V           0x128
+#define CDM_CDWN2_COEFF_OFFSITE_V          0x12C
+#define CDM_CDWN2_OUT_SIZE                 0x130
+
+#define CDM_HDMI_PACK_OP_MODE              0x200
+#define CDM_CSC_10_MATRIX_COEFF_0          0x004
+
+/**
+ * Horizontal coefficients for cosite chroma downscale
+ * s13 representation of coefficients
+ */
+static u32 cosite_h_coeff[] = {0x00000016, 0x000001cc, 0x0100009e};
+
+/**
+ * Horizontal coefficients for offsite chroma downscale
+ */
+static u32 offsite_h_coeff[] = {0x000b0005, 0x01db01eb, 0x00e40046};
+
+/**
+ * Vertical coefficients for cosite chroma downscale
+ */
+static u32 cosite_v_coeff[] = {0x00080004};
+/**
+ * Vertical coefficients for offsite chroma downscale
+ */
+static u32 offsite_v_coeff[] = {0x00060002};
+
+/* Limited Range rgb2yuv coeff with clamp and bias values for CSC 10 module */
+static struct dpu_csc_cfg rgb2yuv_cfg = {
+	{
+		0x0083, 0x0102, 0x0032,
+		0x1fb5, 0x1f6c, 0x00e1,
+		0x00e1, 0x1f45, 0x1fdc
+	},
+	{ 0x00, 0x00, 0x00 },
+	{ 0x0040, 0x0200, 0x0200 },
+	{ 0x000, 0x3ff, 0x000, 0x3ff, 0x000, 0x3ff },
+	{ 0x040, 0x3ac, 0x040, 0x3c0, 0x040, 0x3c0 },
+};
+
+static struct dpu_cdm_cfg *_cdm_offset(enum dpu_cdm cdm,
+		struct dpu_mdss_cfg *m,
+		void __iomem *addr,
+		struct dpu_hw_blk_reg_map *b)
+{
+	int i;
+
+	for (i = 0; i < m->cdm_count; i++) {
+		if (cdm == m->cdm[i].id) {
+			b->base_off = addr;
+			b->blk_off = m->cdm[i].base;
+			b->length = m->cdm[i].len;
+			b->hwversion = m->hwversion;
+			b->log_mask = DPU_DBG_MASK_CDM;
+			return &m->cdm[i];
+		}
+	}
+
+	return ERR_PTR(-EINVAL);
+}
+
+static int dpu_hw_cdm_setup_csc_10bit(struct dpu_hw_cdm *ctx,
+		struct dpu_csc_cfg *data)
+{
+	dpu_hw_csc_setup(&ctx->hw, CDM_CSC_10_MATRIX_COEFF_0, data, true);
+
+	return 0;
+}
+
+static int dpu_hw_cdm_setup_cdwn(struct dpu_hw_cdm *ctx,
+		struct dpu_hw_cdm_cfg *cfg)
+{
+	struct dpu_hw_blk_reg_map *c = &ctx->hw;
+	u32 opmode = 0;
+	u32 out_size = 0;
+
+	if (cfg->output_bit_depth == CDM_CDWN_OUTPUT_10BIT)
+		opmode &= ~BIT(7);
+	else
+		opmode |= BIT(7);
+
+	/* ENABLE DWNS_H bit */
+	opmode |= BIT(1);
+
+	switch (cfg->h_cdwn_type) {
+	case CDM_CDWN_DISABLE:
+		/* CLEAR METHOD_H field */
+		opmode &= ~(0x18);
+		/* CLEAR DWNS_H bit */
+		opmode &= ~BIT(1);
+		break;
+	case CDM_CDWN_PIXEL_DROP:
+		/* Clear METHOD_H field (pixel drop is 0) */
+		opmode &= ~(0x18);
+		break;
+	case CDM_CDWN_AVG:
+		/* Clear METHOD_H field (Average is 0x1) */
+		opmode &= ~(0x18);
+		opmode |= (0x1 << 0x3);
+		break;
+	case CDM_CDWN_COSITE:
+		/* Clear METHOD_H field (Average is 0x2) */
+		opmode &= ~(0x18);
+		opmode |= (0x2 << 0x3);
+		/* Co-site horizontal coefficients */
+		DPU_REG_WRITE(c, CDM_CDWN2_COEFF_COSITE_H_0,
+				cosite_h_coeff[0]);
+		DPU_REG_WRITE(c, CDM_CDWN2_COEFF_COSITE_H_1,
+				cosite_h_coeff[1]);
+		DPU_REG_WRITE(c, CDM_CDWN2_COEFF_COSITE_H_2,
+				cosite_h_coeff[2]);
+		break;
+	case CDM_CDWN_OFFSITE:
+		/* Clear METHOD_H field (Average is 0x3) */
+		opmode &= ~(0x18);
+		opmode |= (0x3 << 0x3);
+
+		/* Off-site horizontal coefficients */
+		DPU_REG_WRITE(c, CDM_CDWN2_COEFF_OFFSITE_H_0,
+				offsite_h_coeff[0]);
+		DPU_REG_WRITE(c, CDM_CDWN2_COEFF_OFFSITE_H_1,
+				offsite_h_coeff[1]);
+		DPU_REG_WRITE(c, CDM_CDWN2_COEFF_OFFSITE_H_2,
+				offsite_h_coeff[2]);
+		break;
+	default:
+		pr_err("%s invalid horz down sampling type\n", __func__);
+		return -EINVAL;
+	}
+
+	/* ENABLE DWNS_V bit */
+	opmode |= BIT(2);
+
+	switch (cfg->v_cdwn_type) {
+	case CDM_CDWN_DISABLE:
+		/* CLEAR METHOD_V field */
+		opmode &= ~(0x60);
+		/* CLEAR DWNS_V bit */
+		opmode &= ~BIT(2);
+		break;
+	case CDM_CDWN_PIXEL_DROP:
+		/* Clear METHOD_V field (pixel drop is 0) */
+		opmode &= ~(0x60);
+		break;
+	case CDM_CDWN_AVG:
+		/* Clear METHOD_V field (Average is 0x1) */
+		opmode &= ~(0x60);
+		opmode |= (0x1 << 0x5);
+		break;
+	case CDM_CDWN_COSITE:
+		/* Clear METHOD_V field (Average is 0x2) */
+		opmode &= ~(0x60);
+		opmode |= (0x2 << 0x5);
+		/* Co-site vertical coefficients */
+		DPU_REG_WRITE(c,
+				CDM_CDWN2_COEFF_COSITE_V,
+				cosite_v_coeff[0]);
+		break;
+	case CDM_CDWN_OFFSITE:
+		/* Clear METHOD_V field (Average is 0x3) */
+		opmode &= ~(0x60);
+		opmode |= (0x3 << 0x5);
+
+		/* Off-site vertical coefficients */
+		DPU_REG_WRITE(c,
+				CDM_CDWN2_COEFF_OFFSITE_V,
+				offsite_v_coeff[0]);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	if (cfg->v_cdwn_type || cfg->h_cdwn_type)
+		opmode |= BIT(0); /* EN CDWN module */
+	else
+		opmode &= ~BIT(0);
+
+	out_size = (cfg->output_width & 0xFFFF) |
+		((cfg->output_height & 0xFFFF) << 16);
+	DPU_REG_WRITE(c, CDM_CDWN2_OUT_SIZE, out_size);
+	DPU_REG_WRITE(c, CDM_CDWN2_OP_MODE, opmode);
+	DPU_REG_WRITE(c, CDM_CDWN2_CLAMP_OUT,
+			((0x3FF << 16) | 0x0));
+
+	return 0;
+}
+
+static int dpu_hw_cdm_enable(struct dpu_hw_cdm *ctx,
+		struct dpu_hw_cdm_cfg *cdm)
+{
+	struct dpu_hw_blk_reg_map *c = &ctx->hw;
+	const struct dpu_format *fmt = cdm->output_fmt;
+	struct cdm_output_cfg cdm_cfg = { 0 };
+	u32 opmode = 0;
+	u32 csc = 0;
+
+	if (!DPU_FORMAT_IS_YUV(fmt))
+		return -EINVAL;
+
+	if (cdm->output_type == CDM_CDWN_OUTPUT_HDMI) {
+		if (fmt->chroma_sample != DPU_CHROMA_H1V2)
+			return -EINVAL; /*unsupported format */
+		opmode = BIT(0);
+		opmode |= (fmt->chroma_sample << 1);
+		cdm_cfg.intf_en = true;
+	}
+
+	csc |= BIT(2);
+	csc &= ~BIT(1);
+	csc |= BIT(0);
+
+	if (ctx->hw_mdp && ctx->hw_mdp->ops.setup_cdm_output)
+		ctx->hw_mdp->ops.setup_cdm_output(ctx->hw_mdp, &cdm_cfg);
+
+	DPU_REG_WRITE(c, CDM_CSC_10_OPMODE, csc);
+	DPU_REG_WRITE(c, CDM_HDMI_PACK_OP_MODE, opmode);
+	return 0;
+}
+
+static void dpu_hw_cdm_disable(struct dpu_hw_cdm *ctx)
+{
+	struct cdm_output_cfg cdm_cfg = { 0 };
+
+	if (ctx->hw_mdp && ctx->hw_mdp->ops.setup_cdm_output)
+		ctx->hw_mdp->ops.setup_cdm_output(ctx->hw_mdp, &cdm_cfg);
+}
+
+static void _setup_cdm_ops(struct dpu_hw_cdm_ops *ops,
+	unsigned long features)
+{
+	ops->setup_csc_data = dpu_hw_cdm_setup_csc_10bit;
+	ops->setup_cdwn = dpu_hw_cdm_setup_cdwn;
+	ops->enable = dpu_hw_cdm_enable;
+	ops->disable = dpu_hw_cdm_disable;
+}
+
+static struct dpu_hw_blk_ops dpu_hw_ops = {
+	.start = NULL,
+	.stop = NULL,
+};
+
+struct dpu_hw_cdm *dpu_hw_cdm_init(enum dpu_cdm idx,
+		void __iomem *addr,
+		struct dpu_mdss_cfg *m,
+		struct dpu_hw_mdp *hw_mdp)
+{
+	struct dpu_hw_cdm *c;
+	struct dpu_cdm_cfg *cfg;
+	int rc;
+
+	c = kzalloc(sizeof(*c), GFP_KERNEL);
+	if (!c)
+		return ERR_PTR(-ENOMEM);
+
+	cfg = _cdm_offset(idx, m, addr, &c->hw);
+	if (IS_ERR_OR_NULL(cfg)) {
+		kfree(c);
+		return ERR_PTR(-EINVAL);
+	}
+
+	c->idx = idx;
+	c->caps = cfg;
+	_setup_cdm_ops(&c->ops, c->caps->features);
+	c->hw_mdp = hw_mdp;
+
+	rc = dpu_hw_blk_init(&c->base, DPU_HW_BLK_CDM, idx, &dpu_hw_ops);
+	if (rc) {
+		DPU_ERROR("failed to init hw blk %d\n", rc);
+		goto blk_init_error;
+	}
+
+	/*
+	 * Perform any default initialization for the chroma down module
+	 * @setup default csc coefficients
+	 */
+	dpu_hw_cdm_setup_csc_10bit(c, &rgb2yuv_cfg);
+
+	return c;
+
+blk_init_error:
+	kzfree(c);
+
+	return ERR_PTR(rc);
+}
+
+void dpu_hw_cdm_destroy(struct dpu_hw_cdm *cdm)
+{
+	if (cdm)
+		dpu_hw_blk_destroy(&cdm->base);
+	kfree(cdm);
+}
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_cdm.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_cdm.h
new file mode 100644
index 000000000000..5cceb1ecb8e0
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_cdm.h
@@ -0,0 +1,139 @@
+/* Copyright (c) 2015-2018, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef _DPU_HW_CDM_H
+#define _DPU_HW_CDM_H
+
+#include "dpu_hw_mdss.h"
+#include "dpu_hw_top.h"
+#include "dpu_hw_blk.h"
+
+struct dpu_hw_cdm;
+
+struct dpu_hw_cdm_cfg {
+	u32 output_width;
+	u32 output_height;
+	u32 output_bit_depth;
+	u32 h_cdwn_type;
+	u32 v_cdwn_type;
+	const struct dpu_format *output_fmt;
+	u32 output_type;
+	int flags;
+};
+
+enum dpu_hw_cdwn_type {
+	CDM_CDWN_DISABLE,
+	CDM_CDWN_PIXEL_DROP,
+	CDM_CDWN_AVG,
+	CDM_CDWN_COSITE,
+	CDM_CDWN_OFFSITE,
+};
+
+enum dpu_hw_cdwn_output_type {
+	CDM_CDWN_OUTPUT_HDMI,
+	CDM_CDWN_OUTPUT_WB,
+};
+
+enum dpu_hw_cdwn_output_bit_depth {
+	CDM_CDWN_OUTPUT_8BIT,
+	CDM_CDWN_OUTPUT_10BIT,
+};
+
+/**
+ * struct dpu_hw_cdm_ops : Interface to the chroma down Hw driver functions
+ *                         Assumption is these functions will be called after
+ *                         clocks are enabled
+ *  @setup_csc:            Programs the csc matrix
+ *  @setup_cdwn:           Sets up the chroma down sub module
+ *  @enable:               Enables the output to interface and programs the
+ *                         output packer
+ *  @disable:              Puts the cdm in bypass mode
+ */
+struct dpu_hw_cdm_ops {
+	/**
+	 * Programs the CSC matrix for conversion from RGB space to YUV space,
+	 * it is optional to call this function as this matrix is automatically
+	 * set during initialization, user should call this if it wants
+	 * to program a different matrix than default matrix.
+	 * @cdm:          Pointer to the chroma down context structure
+	 * @data          Pointer to CSC configuration data
+	 * return:        0 if success; error code otherwise
+	 */
+	int (*setup_csc_data)(struct dpu_hw_cdm *cdm,
+			struct dpu_csc_cfg *data);
+
+	/**
+	 * Programs the Chroma downsample part.
+	 * @cdm         Pointer to chroma down context
+	 */
+	int (*setup_cdwn)(struct dpu_hw_cdm *cdm,
+	struct dpu_hw_cdm_cfg *cfg);
+
+	/**
+	 * Enable the CDM module
+	 * @cdm         Pointer to chroma down context
+	 */
+	int (*enable)(struct dpu_hw_cdm *cdm,
+	struct dpu_hw_cdm_cfg *cfg);
+
+	/**
+	 * Disable the CDM module
+	 * @cdm         Pointer to chroma down context
+	 */
+	void (*disable)(struct dpu_hw_cdm *cdm);
+};
+
+struct dpu_hw_cdm {
+	struct dpu_hw_blk base;
+	struct dpu_hw_blk_reg_map hw;
+
+	/* chroma down */
+	const struct dpu_cdm_cfg *caps;
+	enum  dpu_cdm  idx;
+
+	/* mdp top hw driver */
+	struct dpu_hw_mdp *hw_mdp;
+
+	/* ops */
+	struct dpu_hw_cdm_ops ops;
+};
+
+/**
+ * dpu_hw_cdm - convert base object dpu_hw_base to container
+ * @hw: Pointer to base hardware block
+ * return: Pointer to hardware block container
+ */
+static inline struct dpu_hw_cdm *to_dpu_hw_cdm(struct dpu_hw_blk *hw)
+{
+	return container_of(hw, struct dpu_hw_cdm, base);
+}
+
+/**
+ * dpu_hw_cdm_init - initializes the cdm hw driver object.
+ * should be called once before accessing every cdm.
+ * @idx:  cdm index for which driver object is required
+ * @addr: mapped register io address of MDP
+ * @m :   pointer to mdss catalog data
+ * @hw_mdp:  pointer to mdp top hw driver object
+ */
+struct dpu_hw_cdm *dpu_hw_cdm_init(enum dpu_cdm idx,
+		void __iomem *addr,
+		struct dpu_mdss_cfg *m,
+		struct dpu_hw_mdp *hw_mdp);
+
+/**
+ * dpu_hw_cdm_destroy - destroys CDM driver context
+ * @cdm:   pointer to CDM driver context
+ */
+void dpu_hw_cdm_destroy(struct dpu_hw_cdm *cdm);
+
+#endif /*_DPU_HW_CDM_H */
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.c
new file mode 100644
index 000000000000..06be7cf7ce50
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.c
@@ -0,0 +1,540 @@
+/* Copyright (c) 2015-2018, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/delay.h>
+#include "dpu_hwio.h"
+#include "dpu_hw_ctl.h"
+#include "dpu_dbg.h"
+#include "dpu_kms.h"
+
+#define   CTL_LAYER(lm)                 \
+	(((lm) == LM_5) ? (0x024) : (((lm) - LM_0) * 0x004))
+#define   CTL_LAYER_EXT(lm)             \
+	(0x40 + (((lm) - LM_0) * 0x004))
+#define   CTL_LAYER_EXT2(lm)             \
+	(0x70 + (((lm) - LM_0) * 0x004))
+#define   CTL_LAYER_EXT3(lm)             \
+	(0xA0 + (((lm) - LM_0) * 0x004))
+#define   CTL_TOP                       0x014
+#define   CTL_FLUSH                     0x018
+#define   CTL_START                     0x01C
+#define   CTL_PREPARE                   0x0d0
+#define   CTL_SW_RESET                  0x030
+#define   CTL_LAYER_EXTN_OFFSET         0x40
+
+#define CTL_MIXER_BORDER_OUT            BIT(24)
+#define CTL_FLUSH_MASK_CTL              BIT(17)
+
+#define DPU_REG_RESET_TIMEOUT_US        2000
+
+static struct dpu_ctl_cfg *_ctl_offset(enum dpu_ctl ctl,
+		struct dpu_mdss_cfg *m,
+		void __iomem *addr,
+		struct dpu_hw_blk_reg_map *b)
+{
+	int i;
+
+	for (i = 0; i < m->ctl_count; i++) {
+		if (ctl == m->ctl[i].id) {
+			b->base_off = addr;
+			b->blk_off = m->ctl[i].base;
+			b->length = m->ctl[i].len;
+			b->hwversion = m->hwversion;
+			b->log_mask = DPU_DBG_MASK_CTL;
+			return &m->ctl[i];
+		}
+	}
+	return ERR_PTR(-ENOMEM);
+}
+
+static int _mixer_stages(const struct dpu_lm_cfg *mixer, int count,
+		enum dpu_lm lm)
+{
+	int i;
+	int stages = -EINVAL;
+
+	for (i = 0; i < count; i++) {
+		if (lm == mixer[i].id) {
+			stages = mixer[i].sblk->maxblendstages;
+			break;
+		}
+	}
+
+	return stages;
+}
+
+static inline void dpu_hw_ctl_trigger_start(struct dpu_hw_ctl *ctx)
+{
+	DPU_REG_WRITE(&ctx->hw, CTL_START, 0x1);
+}
+
+static inline void dpu_hw_ctl_trigger_pending(struct dpu_hw_ctl *ctx)
+{
+	DPU_REG_WRITE(&ctx->hw, CTL_PREPARE, 0x1);
+}
+
+static inline void dpu_hw_ctl_clear_pending_flush(struct dpu_hw_ctl *ctx)
+{
+	ctx->pending_flush_mask = 0x0;
+}
+
+static inline void dpu_hw_ctl_update_pending_flush(struct dpu_hw_ctl *ctx,
+		u32 flushbits)
+{
+	ctx->pending_flush_mask |= flushbits;
+}
+
+static u32 dpu_hw_ctl_get_pending_flush(struct dpu_hw_ctl *ctx)
+{
+	if (!ctx)
+		return 0x0;
+
+	return ctx->pending_flush_mask;
+}
+
+static inline void dpu_hw_ctl_trigger_flush(struct dpu_hw_ctl *ctx)
+{
+
+	DPU_REG_WRITE(&ctx->hw, CTL_FLUSH, ctx->pending_flush_mask);
+}
+
+static inline u32 dpu_hw_ctl_get_flush_register(struct dpu_hw_ctl *ctx)
+{
+	struct dpu_hw_blk_reg_map *c = &ctx->hw;
+
+	return DPU_REG_READ(c, CTL_FLUSH);
+}
+
+static inline uint32_t dpu_hw_ctl_get_bitmask_sspp(struct dpu_hw_ctl *ctx,
+	enum dpu_sspp sspp)
+{
+	uint32_t flushbits = 0;
+
+	switch (sspp) {
+	case SSPP_VIG0:
+		flushbits =  BIT(0);
+		break;
+	case SSPP_VIG1:
+		flushbits = BIT(1);
+		break;
+	case SSPP_VIG2:
+		flushbits = BIT(2);
+		break;
+	case SSPP_VIG3:
+		flushbits = BIT(18);
+		break;
+	case SSPP_RGB0:
+		flushbits = BIT(3);
+		break;
+	case SSPP_RGB1:
+		flushbits = BIT(4);
+		break;
+	case SSPP_RGB2:
+		flushbits = BIT(5);
+		break;
+	case SSPP_RGB3:
+		flushbits = BIT(19);
+		break;
+	case SSPP_DMA0:
+		flushbits = BIT(11);
+		break;
+	case SSPP_DMA1:
+		flushbits = BIT(12);
+		break;
+	case SSPP_DMA2:
+		flushbits = BIT(24);
+		break;
+	case SSPP_DMA3:
+		flushbits = BIT(25);
+		break;
+	case SSPP_CURSOR0:
+		flushbits = BIT(22);
+		break;
+	case SSPP_CURSOR1:
+		flushbits = BIT(23);
+		break;
+	default:
+		break;
+	}
+
+	return flushbits;
+}
+
+static inline uint32_t dpu_hw_ctl_get_bitmask_mixer(struct dpu_hw_ctl *ctx,
+	enum dpu_lm lm)
+{
+	uint32_t flushbits = 0;
+
+	switch (lm) {
+	case LM_0:
+		flushbits = BIT(6);
+		break;
+	case LM_1:
+		flushbits = BIT(7);
+		break;
+	case LM_2:
+		flushbits = BIT(8);
+		break;
+	case LM_3:
+		flushbits = BIT(9);
+		break;
+	case LM_4:
+		flushbits = BIT(10);
+		break;
+	case LM_5:
+		flushbits = BIT(20);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	flushbits |= CTL_FLUSH_MASK_CTL;
+
+	return flushbits;
+}
+
+static inline int dpu_hw_ctl_get_bitmask_intf(struct dpu_hw_ctl *ctx,
+		u32 *flushbits, enum dpu_intf intf)
+{
+	switch (intf) {
+	case INTF_0:
+		*flushbits |= BIT(31);
+		break;
+	case INTF_1:
+		*flushbits |= BIT(30);
+		break;
+	case INTF_2:
+		*flushbits |= BIT(29);
+		break;
+	case INTF_3:
+		*flushbits |= BIT(28);
+		break;
+	default:
+		return -EINVAL;
+	}
+	return 0;
+}
+
+static inline int dpu_hw_ctl_get_bitmask_cdm(struct dpu_hw_ctl *ctx,
+		u32 *flushbits, enum dpu_cdm cdm)
+{
+	switch (cdm) {
+	case CDM_0:
+		*flushbits |= BIT(26);
+		break;
+	default:
+		return -EINVAL;
+	}
+	return 0;
+}
+
+static u32 dpu_hw_ctl_poll_reset_status(struct dpu_hw_ctl *ctx, u32 timeout_us)
+{
+	struct dpu_hw_blk_reg_map *c = &ctx->hw;
+	ktime_t timeout;
+	u32 status;
+
+	timeout = ktime_add_us(ktime_get(), timeout_us);
+
+	/*
+	 * it takes around 30us to have mdp finish resetting its ctl path
+	 * poll every 50us so that reset should be completed at 1st poll
+	 */
+	do {
+		status = DPU_REG_READ(c, CTL_SW_RESET);
+		status &= 0x1;
+		if (status)
+			usleep_range(20, 50);
+	} while (status && ktime_compare_safe(ktime_get(), timeout) < 0);
+
+	return status;
+}
+
+static int dpu_hw_ctl_reset_control(struct dpu_hw_ctl *ctx)
+{
+	struct dpu_hw_blk_reg_map *c = &ctx->hw;
+
+	pr_debug("issuing hw ctl reset for ctl:%d\n", ctx->idx);
+	DPU_REG_WRITE(c, CTL_SW_RESET, 0x1);
+	if (dpu_hw_ctl_poll_reset_status(ctx, DPU_REG_RESET_TIMEOUT_US))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int dpu_hw_ctl_wait_reset_status(struct dpu_hw_ctl *ctx)
+{
+	struct dpu_hw_blk_reg_map *c = &ctx->hw;
+	u32 status;
+
+	status = DPU_REG_READ(c, CTL_SW_RESET);
+	status &= 0x01;
+	if (!status)
+		return 0;
+
+	pr_debug("hw ctl reset is set for ctl:%d\n", ctx->idx);
+	if (dpu_hw_ctl_poll_reset_status(ctx, DPU_REG_RESET_TIMEOUT_US)) {
+		pr_err("hw recovery is not complete for ctl:%d\n", ctx->idx);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static void dpu_hw_ctl_clear_all_blendstages(struct dpu_hw_ctl *ctx)
+{
+	struct dpu_hw_blk_reg_map *c = &ctx->hw;
+	int i;
+
+	for (i = 0; i < ctx->mixer_count; i++) {
+		DPU_REG_WRITE(c, CTL_LAYER(LM_0 + i), 0);
+		DPU_REG_WRITE(c, CTL_LAYER_EXT(LM_0 + i), 0);
+		DPU_REG_WRITE(c, CTL_LAYER_EXT2(LM_0 + i), 0);
+		DPU_REG_WRITE(c, CTL_LAYER_EXT3(LM_0 + i), 0);
+	}
+}
+
+static void dpu_hw_ctl_setup_blendstage(struct dpu_hw_ctl *ctx,
+	enum dpu_lm lm, struct dpu_hw_stage_cfg *stage_cfg)
+{
+	struct dpu_hw_blk_reg_map *c = &ctx->hw;
+	u32 mixercfg = 0, mixercfg_ext = 0, mix, ext;
+	u32 mixercfg_ext2 = 0, mixercfg_ext3 = 0;
+	int i, j;
+	u8 stages;
+	int pipes_per_stage;
+
+	stages = _mixer_stages(ctx->mixer_hw_caps, ctx->mixer_count, lm);
+	if (stages < 0)
+		return;
+
+	if (test_bit(DPU_MIXER_SOURCESPLIT,
+		&ctx->mixer_hw_caps->features))
+		pipes_per_stage = PIPES_PER_STAGE;
+	else
+		pipes_per_stage = 1;
+
+	mixercfg = CTL_MIXER_BORDER_OUT; /* always set BORDER_OUT */
+
+	if (!stage_cfg)
+		goto exit;
+
+	for (i = 0; i <= stages; i++) {
+		/* overflow to ext register if 'i + 1 > 7' */
+		mix = (i + 1) & 0x7;
+		ext = i >= 7;
+
+		for (j = 0 ; j < pipes_per_stage; j++) {
+			enum dpu_sspp_multirect_index rect_index =
+				stage_cfg->multirect_index[i][j];
+
+			switch (stage_cfg->stage[i][j]) {
+			case SSPP_VIG0:
+				if (rect_index == DPU_SSPP_RECT_1) {
+					mixercfg_ext3 |= ((i + 1) & 0xF) << 0;
+				} else {
+					mixercfg |= mix << 0;
+					mixercfg_ext |= ext << 0;
+				}
+				break;
+			case SSPP_VIG1:
+				if (rect_index == DPU_SSPP_RECT_1) {
+					mixercfg_ext3 |= ((i + 1) & 0xF) << 4;
+				} else {
+					mixercfg |= mix << 3;
+					mixercfg_ext |= ext << 2;
+				}
+				break;
+			case SSPP_VIG2:
+				if (rect_index == DPU_SSPP_RECT_1) {
+					mixercfg_ext3 |= ((i + 1) & 0xF) << 8;
+				} else {
+					mixercfg |= mix << 6;
+					mixercfg_ext |= ext << 4;
+				}
+				break;
+			case SSPP_VIG3:
+				if (rect_index == DPU_SSPP_RECT_1) {
+					mixercfg_ext3 |= ((i + 1) & 0xF) << 12;
+				} else {
+					mixercfg |= mix << 26;
+					mixercfg_ext |= ext << 6;
+				}
+				break;
+			case SSPP_RGB0:
+				mixercfg |= mix << 9;
+				mixercfg_ext |= ext << 8;
+				break;
+			case SSPP_RGB1:
+				mixercfg |= mix << 12;
+				mixercfg_ext |= ext << 10;
+				break;
+			case SSPP_RGB2:
+				mixercfg |= mix << 15;
+				mixercfg_ext |= ext << 12;
+				break;
+			case SSPP_RGB3:
+				mixercfg |= mix << 29;
+				mixercfg_ext |= ext << 14;
+				break;
+			case SSPP_DMA0:
+				if (rect_index == DPU_SSPP_RECT_1) {
+					mixercfg_ext2 |= ((i + 1) & 0xF) << 8;
+				} else {
+					mixercfg |= mix << 18;
+					mixercfg_ext |= ext << 16;
+				}
+				break;
+			case SSPP_DMA1:
+				if (rect_index == DPU_SSPP_RECT_1) {
+					mixercfg_ext2 |= ((i + 1) & 0xF) << 12;
+				} else {
+					mixercfg |= mix << 21;
+					mixercfg_ext |= ext << 18;
+				}
+				break;
+			case SSPP_DMA2:
+				if (rect_index == DPU_SSPP_RECT_1) {
+					mixercfg_ext2 |= ((i + 1) & 0xF) << 16;
+				} else {
+					mix |= (i + 1) & 0xF;
+					mixercfg_ext2 |= mix << 0;
+				}
+				break;
+			case SSPP_DMA3:
+				if (rect_index == DPU_SSPP_RECT_1) {
+					mixercfg_ext2 |= ((i + 1) & 0xF) << 20;
+				} else {
+					mix |= (i + 1) & 0xF;
+					mixercfg_ext2 |= mix << 4;
+				}
+				break;
+			case SSPP_CURSOR0:
+				mixercfg_ext |= ((i + 1) & 0xF) << 20;
+				break;
+			case SSPP_CURSOR1:
+				mixercfg_ext |= ((i + 1) & 0xF) << 26;
+				break;
+			default:
+				break;
+			}
+		}
+	}
+
+exit:
+	DPU_REG_WRITE(c, CTL_LAYER(lm), mixercfg);
+	DPU_REG_WRITE(c, CTL_LAYER_EXT(lm), mixercfg_ext);
+	DPU_REG_WRITE(c, CTL_LAYER_EXT2(lm), mixercfg_ext2);
+	DPU_REG_WRITE(c, CTL_LAYER_EXT3(lm), mixercfg_ext3);
+}
+
+static void dpu_hw_ctl_intf_cfg(struct dpu_hw_ctl *ctx,
+		struct dpu_hw_intf_cfg *cfg)
+{
+	struct dpu_hw_blk_reg_map *c = &ctx->hw;
+	u32 intf_cfg = 0;
+
+	intf_cfg |= (cfg->intf & 0xF) << 4;
+
+	if (cfg->mode_3d) {
+		intf_cfg |= BIT(19);
+		intf_cfg |= (cfg->mode_3d - 0x1) << 20;
+	}
+
+	switch (cfg->intf_mode_sel) {
+	case DPU_CTL_MODE_SEL_VID:
+		intf_cfg &= ~BIT(17);
+		intf_cfg &= ~(0x3 << 15);
+		break;
+	case DPU_CTL_MODE_SEL_CMD:
+		intf_cfg |= BIT(17);
+		intf_cfg |= ((cfg->stream_sel & 0x3) << 15);
+		break;
+	default:
+		pr_err("unknown interface type %d\n", cfg->intf_mode_sel);
+		return;
+	}
+
+	DPU_REG_WRITE(c, CTL_TOP, intf_cfg);
+}
+
+static void _setup_ctl_ops(struct dpu_hw_ctl_ops *ops,
+		unsigned long cap)
+{
+	ops->clear_pending_flush = dpu_hw_ctl_clear_pending_flush;
+	ops->update_pending_flush = dpu_hw_ctl_update_pending_flush;
+	ops->get_pending_flush = dpu_hw_ctl_get_pending_flush;
+	ops->trigger_flush = dpu_hw_ctl_trigger_flush;
+	ops->get_flush_register = dpu_hw_ctl_get_flush_register;
+	ops->trigger_start = dpu_hw_ctl_trigger_start;
+	ops->trigger_pending = dpu_hw_ctl_trigger_pending;
+	ops->setup_intf_cfg = dpu_hw_ctl_intf_cfg;
+	ops->reset = dpu_hw_ctl_reset_control;
+	ops->wait_reset_status = dpu_hw_ctl_wait_reset_status;
+	ops->clear_all_blendstages = dpu_hw_ctl_clear_all_blendstages;
+	ops->setup_blendstage = dpu_hw_ctl_setup_blendstage;
+	ops->get_bitmask_sspp = dpu_hw_ctl_get_bitmask_sspp;
+	ops->get_bitmask_mixer = dpu_hw_ctl_get_bitmask_mixer;
+	ops->get_bitmask_intf = dpu_hw_ctl_get_bitmask_intf;
+	ops->get_bitmask_cdm = dpu_hw_ctl_get_bitmask_cdm;
+};
+
+static struct dpu_hw_blk_ops dpu_hw_ops = {
+	.start = NULL,
+	.stop = NULL,
+};
+
+struct dpu_hw_ctl *dpu_hw_ctl_init(enum dpu_ctl idx,
+		void __iomem *addr,
+		struct dpu_mdss_cfg *m)
+{
+	struct dpu_hw_ctl *c;
+	struct dpu_ctl_cfg *cfg;
+	int rc;
+
+	c = kzalloc(sizeof(*c), GFP_KERNEL);
+	if (!c)
+		return ERR_PTR(-ENOMEM);
+
+	cfg = _ctl_offset(idx, m, addr, &c->hw);
+	if (IS_ERR_OR_NULL(cfg)) {
+		kfree(c);
+		pr_err("failed to create dpu_hw_ctl %d\n", idx);
+		return ERR_PTR(-EINVAL);
+	}
+
+	c->caps = cfg;
+	_setup_ctl_ops(&c->ops, c->caps->features);
+	c->idx = idx;
+	c->mixer_count = m->mixer_count;
+	c->mixer_hw_caps = m->mixer;
+
+	rc = dpu_hw_blk_init(&c->base, DPU_HW_BLK_CTL, idx, &dpu_hw_ops);
+	if (rc) {
+		DPU_ERROR("failed to init hw blk %d\n", rc);
+		goto blk_init_error;
+	}
+
+	return c;
+
+blk_init_error:
+	kzfree(c);
+
+	return ERR_PTR(rc);
+}
+
+void dpu_hw_ctl_destroy(struct dpu_hw_ctl *ctx)
+{
+	if (ctx)
+		dpu_hw_blk_destroy(&ctx->base);
+	kfree(ctx);
+}
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.h
new file mode 100644
index 000000000000..c66a71f8b839
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.h
@@ -0,0 +1,218 @@
+/* Copyright (c) 2015-2018, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef _DPU_HW_CTL_H
+#define _DPU_HW_CTL_H
+
+#include "dpu_hw_mdss.h"
+#include "dpu_hw_util.h"
+#include "dpu_hw_catalog.h"
+#include "dpu_hw_sspp.h"
+#include "dpu_hw_blk.h"
+
+/**
+ * dpu_ctl_mode_sel: Interface mode selection
+ * DPU_CTL_MODE_SEL_VID:    Video mode interface
+ * DPU_CTL_MODE_SEL_CMD:    Command mode interface
+ */
+enum dpu_ctl_mode_sel {
+	DPU_CTL_MODE_SEL_VID = 0,
+	DPU_CTL_MODE_SEL_CMD
+};
+
+struct dpu_hw_ctl;
+/**
+ * struct dpu_hw_stage_cfg - blending stage cfg
+ * @stage : SSPP_ID at each stage
+ * @multirect_index: index of the rectangle of SSPP.
+ */
+struct dpu_hw_stage_cfg {
+	enum dpu_sspp stage[DPU_STAGE_MAX][PIPES_PER_STAGE];
+	enum dpu_sspp_multirect_index multirect_index
+					[DPU_STAGE_MAX][PIPES_PER_STAGE];
+};
+
+/**
+ * struct dpu_hw_intf_cfg :Describes how the DPU writes data to output interface
+ * @intf :                 Interface id
+ * @mode_3d:               3d mux configuration
+ * @intf_mode_sel:         Interface mode, cmd / vid
+ * @stream_sel:            Stream selection for multi-stream interfaces
+ */
+struct dpu_hw_intf_cfg {
+	enum dpu_intf intf;
+	enum dpu_3d_blend_mode mode_3d;
+	enum dpu_ctl_mode_sel intf_mode_sel;
+	int stream_sel;
+};
+
+/**
+ * struct dpu_hw_ctl_ops - Interface to the wb Hw driver functions
+ * Assumption is these functions will be called after clocks are enabled
+ */
+struct dpu_hw_ctl_ops {
+	/**
+	 * kickoff hw operation for Sw controlled interfaces
+	 * DSI cmd mode and WB interface are SW controlled
+	 * @ctx       : ctl path ctx pointer
+	 */
+	void (*trigger_start)(struct dpu_hw_ctl *ctx);
+
+	/**
+	 * kickoff prepare is in progress hw operation for sw
+	 * controlled interfaces: DSI cmd mode and WB interface
+	 * are SW controlled
+	 * @ctx       : ctl path ctx pointer
+	 */
+	void (*trigger_pending)(struct dpu_hw_ctl *ctx);
+
+	/**
+	 * Clear the value of the cached pending_flush_mask
+	 * No effect on hardware
+	 * @ctx       : ctl path ctx pointer
+	 */
+	void (*clear_pending_flush)(struct dpu_hw_ctl *ctx);
+
+	/**
+	 * Query the value of the cached pending_flush_mask
+	 * No effect on hardware
+	 * @ctx       : ctl path ctx pointer
+	 */
+	u32 (*get_pending_flush)(struct dpu_hw_ctl *ctx);
+
+	/**
+	 * OR in the given flushbits to the cached pending_flush_mask
+	 * No effect on hardware
+	 * @ctx       : ctl path ctx pointer
+	 * @flushbits : module flushmask
+	 */
+	void (*update_pending_flush)(struct dpu_hw_ctl *ctx,
+		u32 flushbits);
+
+	/**
+	 * Write the value of the pending_flush_mask to hardware
+	 * @ctx       : ctl path ctx pointer
+	 */
+	void (*trigger_flush)(struct dpu_hw_ctl *ctx);
+
+	/**
+	 * Read the value of the flush register
+	 * @ctx       : ctl path ctx pointer
+	 * @Return: value of the ctl flush register.
+	 */
+	u32 (*get_flush_register)(struct dpu_hw_ctl *ctx);
+
+	/**
+	 * Setup ctl_path interface config
+	 * @ctx
+	 * @cfg    : interface config structure pointer
+	 */
+	void (*setup_intf_cfg)(struct dpu_hw_ctl *ctx,
+		struct dpu_hw_intf_cfg *cfg);
+
+	int (*reset)(struct dpu_hw_ctl *c);
+
+	/*
+	 * wait_reset_status - checks ctl reset status
+	 * @ctx       : ctl path ctx pointer
+	 *
+	 * This function checks the ctl reset status bit.
+	 * If the reset bit is set, it keeps polling the status till the hw
+	 * reset is complete.
+	 * Returns: 0 on success or -error if reset incomplete within interval
+	 */
+	int (*wait_reset_status)(struct dpu_hw_ctl *ctx);
+
+	uint32_t (*get_bitmask_sspp)(struct dpu_hw_ctl *ctx,
+		enum dpu_sspp blk);
+
+	uint32_t (*get_bitmask_mixer)(struct dpu_hw_ctl *ctx,
+		enum dpu_lm blk);
+
+	int (*get_bitmask_intf)(struct dpu_hw_ctl *ctx,
+		u32 *flushbits,
+		enum dpu_intf blk);
+
+	int (*get_bitmask_cdm)(struct dpu_hw_ctl *ctx,
+		u32 *flushbits,
+		enum dpu_cdm blk);
+
+	/**
+	 * Set all blend stages to disabled
+	 * @ctx       : ctl path ctx pointer
+	 */
+	void (*clear_all_blendstages)(struct dpu_hw_ctl *ctx);
+
+	/**
+	 * Configure layer mixer to pipe configuration
+	 * @ctx       : ctl path ctx pointer
+	 * @lm        : layer mixer enumeration
+	 * @cfg       : blend stage configuration
+	 */
+	void (*setup_blendstage)(struct dpu_hw_ctl *ctx,
+		enum dpu_lm lm, struct dpu_hw_stage_cfg *cfg);
+};
+
+/**
+ * struct dpu_hw_ctl : CTL PATH driver object
+ * @base: hardware block base structure
+ * @hw: block register map object
+ * @idx: control path index
+ * @caps: control path capabilities
+ * @mixer_count: number of mixers
+ * @mixer_hw_caps: mixer hardware capabilities
+ * @pending_flush_mask: storage for pending ctl_flush managed via ops
+ * @ops: operation list
+ */
+struct dpu_hw_ctl {
+	struct dpu_hw_blk base;
+	struct dpu_hw_blk_reg_map hw;
+
+	/* ctl path */
+	int idx;
+	const struct dpu_ctl_cfg *caps;
+	int mixer_count;
+	const struct dpu_lm_cfg *mixer_hw_caps;
+	u32 pending_flush_mask;
+
+	/* ops */
+	struct dpu_hw_ctl_ops ops;
+};
+
+/**
+ * dpu_hw_ctl - convert base object dpu_hw_base to container
+ * @hw: Pointer to base hardware block
+ * return: Pointer to hardware block container
+ */
+static inline struct dpu_hw_ctl *to_dpu_hw_ctl(struct dpu_hw_blk *hw)
+{
+	return container_of(hw, struct dpu_hw_ctl, base);
+}
+
+/**
+ * dpu_hw_ctl_init(): Initializes the ctl_path hw driver object.
+ * should be called before accessing every ctl path registers.
+ * @idx:  ctl_path index for which driver object is required
+ * @addr: mapped register io address of MDP
+ * @m :   pointer to mdss catalog data
+ */
+struct dpu_hw_ctl *dpu_hw_ctl_init(enum dpu_ctl idx,
+		void __iomem *addr,
+		struct dpu_mdss_cfg *m);
+
+/**
+ * dpu_hw_ctl_destroy(): Destroys ctl driver context
+ * should be called to free the context
+ */
+void dpu_hw_ctl_destroy(struct dpu_hw_ctl *ctx);
+
+#endif /*_DPU_HW_CTL_H */
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_interrupts.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_interrupts.c
new file mode 100644
index 000000000000..c0b7f0049365
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_interrupts.c
@@ -0,0 +1,1183 @@
+/* Copyright (c) 2016-2018, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/bitops.h>
+#include <linux/slab.h>
+
+#include "dpu_kms.h"
+#include "dpu_hw_interrupts.h"
+#include "dpu_hw_util.h"
+#include "dpu_hw_mdss.h"
+
+/**
+ * Register offsets in MDSS register file for the interrupt registers
+ * w.r.t. to the MDP base
+ */
+#define MDP_SSPP_TOP0_OFF		0x0
+#define MDP_INTF_0_OFF			0x6A000
+#define MDP_INTF_1_OFF			0x6A800
+#define MDP_INTF_2_OFF			0x6B000
+#define MDP_INTF_3_OFF			0x6B800
+#define MDP_INTF_4_OFF			0x6C000
+#define MDP_AD4_0_OFF			0x7C000
+#define MDP_AD4_1_OFF			0x7D000
+#define MDP_AD4_INTR_EN_OFF		0x41c
+#define MDP_AD4_INTR_CLEAR_OFF		0x424
+#define MDP_AD4_INTR_STATUS_OFF		0x420
+
+/**
+ * WB interrupt status bit definitions
+ */
+#define DPU_INTR_WB_0_DONE BIT(0)
+#define DPU_INTR_WB_1_DONE BIT(1)
+#define DPU_INTR_WB_2_DONE BIT(4)
+
+/**
+ * WDOG timer interrupt status bit definitions
+ */
+#define DPU_INTR_WD_TIMER_0_DONE BIT(2)
+#define DPU_INTR_WD_TIMER_1_DONE BIT(3)
+#define DPU_INTR_WD_TIMER_2_DONE BIT(5)
+#define DPU_INTR_WD_TIMER_3_DONE BIT(6)
+#define DPU_INTR_WD_TIMER_4_DONE BIT(7)
+
+/**
+ * Pingpong interrupt status bit definitions
+ */
+#define DPU_INTR_PING_PONG_0_DONE BIT(8)
+#define DPU_INTR_PING_PONG_1_DONE BIT(9)
+#define DPU_INTR_PING_PONG_2_DONE BIT(10)
+#define DPU_INTR_PING_PONG_3_DONE BIT(11)
+#define DPU_INTR_PING_PONG_0_RD_PTR BIT(12)
+#define DPU_INTR_PING_PONG_1_RD_PTR BIT(13)
+#define DPU_INTR_PING_PONG_2_RD_PTR BIT(14)
+#define DPU_INTR_PING_PONG_3_RD_PTR BIT(15)
+#define DPU_INTR_PING_PONG_0_WR_PTR BIT(16)
+#define DPU_INTR_PING_PONG_1_WR_PTR BIT(17)
+#define DPU_INTR_PING_PONG_2_WR_PTR BIT(18)
+#define DPU_INTR_PING_PONG_3_WR_PTR BIT(19)
+#define DPU_INTR_PING_PONG_0_AUTOREFRESH_DONE BIT(20)
+#define DPU_INTR_PING_PONG_1_AUTOREFRESH_DONE BIT(21)
+#define DPU_INTR_PING_PONG_2_AUTOREFRESH_DONE BIT(22)
+#define DPU_INTR_PING_PONG_3_AUTOREFRESH_DONE BIT(23)
+
+/**
+ * Interface interrupt status bit definitions
+ */
+#define DPU_INTR_INTF_0_UNDERRUN BIT(24)
+#define DPU_INTR_INTF_1_UNDERRUN BIT(26)
+#define DPU_INTR_INTF_2_UNDERRUN BIT(28)
+#define DPU_INTR_INTF_3_UNDERRUN BIT(30)
+#define DPU_INTR_INTF_0_VSYNC BIT(25)
+#define DPU_INTR_INTF_1_VSYNC BIT(27)
+#define DPU_INTR_INTF_2_VSYNC BIT(29)
+#define DPU_INTR_INTF_3_VSYNC BIT(31)
+
+/**
+ * Pingpong Secondary interrupt status bit definitions
+ */
+#define DPU_INTR_PING_PONG_S0_AUTOREFRESH_DONE BIT(0)
+#define DPU_INTR_PING_PONG_S0_WR_PTR BIT(4)
+#define DPU_INTR_PING_PONG_S0_RD_PTR BIT(8)
+#define DPU_INTR_PING_PONG_S0_TEAR_DETECTED BIT(22)
+#define DPU_INTR_PING_PONG_S0_TE_DETECTED BIT(28)
+
+/**
+ * Pingpong TEAR detection interrupt status bit definitions
+ */
+#define DPU_INTR_PING_PONG_0_TEAR_DETECTED BIT(16)
+#define DPU_INTR_PING_PONG_1_TEAR_DETECTED BIT(17)
+#define DPU_INTR_PING_PONG_2_TEAR_DETECTED BIT(18)
+#define DPU_INTR_PING_PONG_3_TEAR_DETECTED BIT(19)
+
+/**
+ * Pingpong TE detection interrupt status bit definitions
+ */
+#define DPU_INTR_PING_PONG_0_TE_DETECTED BIT(24)
+#define DPU_INTR_PING_PONG_1_TE_DETECTED BIT(25)
+#define DPU_INTR_PING_PONG_2_TE_DETECTED BIT(26)
+#define DPU_INTR_PING_PONG_3_TE_DETECTED BIT(27)
+
+/**
+ * Ctl start interrupt status bit definitions
+ */
+#define DPU_INTR_CTL_0_START BIT(9)
+#define DPU_INTR_CTL_1_START BIT(10)
+#define DPU_INTR_CTL_2_START BIT(11)
+#define DPU_INTR_CTL_3_START BIT(12)
+#define DPU_INTR_CTL_4_START BIT(13)
+
+/**
+ * Concurrent WB overflow interrupt status bit definitions
+ */
+#define DPU_INTR_CWB_2_OVERFLOW BIT(14)
+#define DPU_INTR_CWB_3_OVERFLOW BIT(15)
+
+/**
+ * Histogram VIG done interrupt status bit definitions
+ */
+#define DPU_INTR_HIST_VIG_0_DONE BIT(0)
+#define DPU_INTR_HIST_VIG_1_DONE BIT(4)
+#define DPU_INTR_HIST_VIG_2_DONE BIT(8)
+#define DPU_INTR_HIST_VIG_3_DONE BIT(10)
+
+/**
+ * Histogram VIG reset Sequence done interrupt status bit definitions
+ */
+#define DPU_INTR_HIST_VIG_0_RSTSEQ_DONE BIT(1)
+#define DPU_INTR_HIST_VIG_1_RSTSEQ_DONE BIT(5)
+#define DPU_INTR_HIST_VIG_2_RSTSEQ_DONE BIT(9)
+#define DPU_INTR_HIST_VIG_3_RSTSEQ_DONE BIT(11)
+
+/**
+ * Histogram DSPP done interrupt status bit definitions
+ */
+#define DPU_INTR_HIST_DSPP_0_DONE BIT(12)
+#define DPU_INTR_HIST_DSPP_1_DONE BIT(16)
+#define DPU_INTR_HIST_DSPP_2_DONE BIT(20)
+#define DPU_INTR_HIST_DSPP_3_DONE BIT(22)
+
+/**
+ * Histogram DSPP reset Sequence done interrupt status bit definitions
+ */
+#define DPU_INTR_HIST_DSPP_0_RSTSEQ_DONE BIT(13)
+#define DPU_INTR_HIST_DSPP_1_RSTSEQ_DONE BIT(17)
+#define DPU_INTR_HIST_DSPP_2_RSTSEQ_DONE BIT(21)
+#define DPU_INTR_HIST_DSPP_3_RSTSEQ_DONE BIT(23)
+
+/**
+ * INTF interrupt status bit definitions
+ */
+#define DPU_INTR_VIDEO_INTO_STATIC BIT(0)
+#define DPU_INTR_VIDEO_OUTOF_STATIC BIT(1)
+#define DPU_INTR_DSICMD_0_INTO_STATIC BIT(2)
+#define DPU_INTR_DSICMD_0_OUTOF_STATIC BIT(3)
+#define DPU_INTR_DSICMD_1_INTO_STATIC BIT(4)
+#define DPU_INTR_DSICMD_1_OUTOF_STATIC BIT(5)
+#define DPU_INTR_DSICMD_2_INTO_STATIC BIT(6)
+#define DPU_INTR_DSICMD_2_OUTOF_STATIC BIT(7)
+#define DPU_INTR_PROG_LINE BIT(8)
+
+/**
+ * AD4 interrupt status bit definitions
+ */
+#define DPU_INTR_BRIGHTPR_UPDATED BIT(4)
+#define DPU_INTR_DARKENH_UPDATED BIT(3)
+#define DPU_INTR_STREN_OUTROI_UPDATED BIT(2)
+#define DPU_INTR_STREN_INROI_UPDATED BIT(1)
+#define DPU_INTR_BACKLIGHT_UPDATED BIT(0)
+/**
+ * struct dpu_intr_reg - array of DPU register sets
+ * @clr_off:	offset to CLEAR reg
+ * @en_off:	offset to ENABLE reg
+ * @status_off:	offset to STATUS reg
+ */
+struct dpu_intr_reg {
+	u32 clr_off;
+	u32 en_off;
+	u32 status_off;
+};
+
+/**
+ * struct dpu_irq_type - maps each irq with i/f
+ * @intr_type:		type of interrupt listed in dpu_intr_type
+ * @instance_idx:	instance index of the associated HW block in DPU
+ * @irq_mask:		corresponding bit in the interrupt status reg
+ * @reg_idx:		which reg set to use
+ */
+struct dpu_irq_type {
+	u32 intr_type;
+	u32 instance_idx;
+	u32 irq_mask;
+	u32 reg_idx;
+};
+
+/**
+ * List of DPU interrupt registers
+ */
+static const struct dpu_intr_reg dpu_intr_set[] = {
+	{
+		MDP_SSPP_TOP0_OFF+INTR_CLEAR,
+		MDP_SSPP_TOP0_OFF+INTR_EN,
+		MDP_SSPP_TOP0_OFF+INTR_STATUS
+	},
+	{
+		MDP_SSPP_TOP0_OFF+INTR2_CLEAR,
+		MDP_SSPP_TOP0_OFF+INTR2_EN,
+		MDP_SSPP_TOP0_OFF+INTR2_STATUS
+	},
+	{
+		MDP_SSPP_TOP0_OFF+HIST_INTR_CLEAR,
+		MDP_SSPP_TOP0_OFF+HIST_INTR_EN,
+		MDP_SSPP_TOP0_OFF+HIST_INTR_STATUS
+	},
+	{
+		MDP_INTF_0_OFF+INTF_INTR_CLEAR,
+		MDP_INTF_0_OFF+INTF_INTR_EN,
+		MDP_INTF_0_OFF+INTF_INTR_STATUS
+	},
+	{
+		MDP_INTF_1_OFF+INTF_INTR_CLEAR,
+		MDP_INTF_1_OFF+INTF_INTR_EN,
+		MDP_INTF_1_OFF+INTF_INTR_STATUS
+	},
+	{
+		MDP_INTF_2_OFF+INTF_INTR_CLEAR,
+		MDP_INTF_2_OFF+INTF_INTR_EN,
+		MDP_INTF_2_OFF+INTF_INTR_STATUS
+	},
+	{
+		MDP_INTF_3_OFF+INTF_INTR_CLEAR,
+		MDP_INTF_3_OFF+INTF_INTR_EN,
+		MDP_INTF_3_OFF+INTF_INTR_STATUS
+	},
+	{
+		MDP_INTF_4_OFF+INTF_INTR_CLEAR,
+		MDP_INTF_4_OFF+INTF_INTR_EN,
+		MDP_INTF_4_OFF+INTF_INTR_STATUS
+	},
+	{
+		MDP_AD4_0_OFF + MDP_AD4_INTR_CLEAR_OFF,
+		MDP_AD4_0_OFF + MDP_AD4_INTR_EN_OFF,
+		MDP_AD4_0_OFF + MDP_AD4_INTR_STATUS_OFF,
+	},
+	{
+		MDP_AD4_1_OFF + MDP_AD4_INTR_CLEAR_OFF,
+		MDP_AD4_1_OFF + MDP_AD4_INTR_EN_OFF,
+		MDP_AD4_1_OFF + MDP_AD4_INTR_STATUS_OFF,
+	}
+};
+
+/**
+ * IRQ mapping table - use for lookup an irq_idx in this table that have
+ *                     a matching interface type and instance index.
+ */
+static const struct dpu_irq_type dpu_irq_map[] = {
+	/* BEGIN MAP_RANGE: 0-31, INTR */
+	/* irq_idx: 0-3 */
+	{ DPU_IRQ_TYPE_WB_ROT_COMP, WB_0, DPU_INTR_WB_0_DONE, 0},
+	{ DPU_IRQ_TYPE_WB_ROT_COMP, WB_1, DPU_INTR_WB_1_DONE, 0},
+	{ DPU_IRQ_TYPE_WD_TIMER, WD_TIMER_0, DPU_INTR_WD_TIMER_0_DONE, 0},
+	{ DPU_IRQ_TYPE_WD_TIMER, WD_TIMER_1, DPU_INTR_WD_TIMER_1_DONE, 0},
+	/* irq_idx: 4-7 */
+	{ DPU_IRQ_TYPE_WB_WFD_COMP, WB_2, DPU_INTR_WB_2_DONE, 0},
+	{ DPU_IRQ_TYPE_WD_TIMER, WD_TIMER_2, DPU_INTR_WD_TIMER_2_DONE, 0},
+	{ DPU_IRQ_TYPE_WD_TIMER, WD_TIMER_3, DPU_INTR_WD_TIMER_3_DONE, 0},
+	{ DPU_IRQ_TYPE_WD_TIMER, WD_TIMER_4, DPU_INTR_WD_TIMER_4_DONE, 0},
+	/* irq_idx: 8-11 */
+	{ DPU_IRQ_TYPE_PING_PONG_COMP, PINGPONG_0,
+		DPU_INTR_PING_PONG_0_DONE, 0},
+	{ DPU_IRQ_TYPE_PING_PONG_COMP, PINGPONG_1,
+		DPU_INTR_PING_PONG_1_DONE, 0},
+	{ DPU_IRQ_TYPE_PING_PONG_COMP, PINGPONG_2,
+		DPU_INTR_PING_PONG_2_DONE, 0},
+	{ DPU_IRQ_TYPE_PING_PONG_COMP, PINGPONG_3,
+		DPU_INTR_PING_PONG_3_DONE, 0},
+	/* irq_idx: 12-15 */
+	{ DPU_IRQ_TYPE_PING_PONG_RD_PTR, PINGPONG_0,
+		DPU_INTR_PING_PONG_0_RD_PTR, 0},
+	{ DPU_IRQ_TYPE_PING_PONG_RD_PTR, PINGPONG_1,
+		DPU_INTR_PING_PONG_1_RD_PTR, 0},
+	{ DPU_IRQ_TYPE_PING_PONG_RD_PTR, PINGPONG_2,
+		DPU_INTR_PING_PONG_2_RD_PTR, 0},
+	{ DPU_IRQ_TYPE_PING_PONG_RD_PTR, PINGPONG_3,
+		DPU_INTR_PING_PONG_3_RD_PTR, 0},
+	/* irq_idx: 16-19 */
+	{ DPU_IRQ_TYPE_PING_PONG_WR_PTR, PINGPONG_0,
+		DPU_INTR_PING_PONG_0_WR_PTR, 0},
+	{ DPU_IRQ_TYPE_PING_PONG_WR_PTR, PINGPONG_1,
+		DPU_INTR_PING_PONG_1_WR_PTR, 0},
+	{ DPU_IRQ_TYPE_PING_PONG_WR_PTR, PINGPONG_2,
+		DPU_INTR_PING_PONG_2_WR_PTR, 0},
+	{ DPU_IRQ_TYPE_PING_PONG_WR_PTR, PINGPONG_3,
+		DPU_INTR_PING_PONG_3_WR_PTR, 0},
+	/* irq_idx: 20-23 */
+	{ DPU_IRQ_TYPE_PING_PONG_AUTO_REF, PINGPONG_0,
+		DPU_INTR_PING_PONG_0_AUTOREFRESH_DONE, 0},
+	{ DPU_IRQ_TYPE_PING_PONG_AUTO_REF, PINGPONG_1,
+		DPU_INTR_PING_PONG_1_AUTOREFRESH_DONE, 0},
+	{ DPU_IRQ_TYPE_PING_PONG_AUTO_REF, PINGPONG_2,
+		DPU_INTR_PING_PONG_2_AUTOREFRESH_DONE, 0},
+	{ DPU_IRQ_TYPE_PING_PONG_AUTO_REF, PINGPONG_3,
+		DPU_INTR_PING_PONG_3_AUTOREFRESH_DONE, 0},
+	/* irq_idx: 24-27 */
+	{ DPU_IRQ_TYPE_INTF_UNDER_RUN, INTF_0, DPU_INTR_INTF_0_UNDERRUN, 0},
+	{ DPU_IRQ_TYPE_INTF_VSYNC, INTF_0, DPU_INTR_INTF_0_VSYNC, 0},
+	{ DPU_IRQ_TYPE_INTF_UNDER_RUN, INTF_1, DPU_INTR_INTF_1_UNDERRUN, 0},
+	{ DPU_IRQ_TYPE_INTF_VSYNC, INTF_1, DPU_INTR_INTF_1_VSYNC, 0},
+	/* irq_idx: 28-31 */
+	{ DPU_IRQ_TYPE_INTF_UNDER_RUN, INTF_2, DPU_INTR_INTF_2_UNDERRUN, 0},
+	{ DPU_IRQ_TYPE_INTF_VSYNC, INTF_2, DPU_INTR_INTF_2_VSYNC, 0},
+	{ DPU_IRQ_TYPE_INTF_UNDER_RUN, INTF_3, DPU_INTR_INTF_3_UNDERRUN, 0},
+	{ DPU_IRQ_TYPE_INTF_VSYNC, INTF_3, DPU_INTR_INTF_3_VSYNC, 0},
+
+	/* BEGIN MAP_RANGE: 32-64, INTR2 */
+	/* irq_idx: 32-35 */
+	{ DPU_IRQ_TYPE_PING_PONG_AUTO_REF, PINGPONG_S0,
+		DPU_INTR_PING_PONG_S0_AUTOREFRESH_DONE, 1},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 1},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 1},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 1},
+	/* irq_idx: 36-39 */
+	{ DPU_IRQ_TYPE_PING_PONG_WR_PTR, PINGPONG_S0,
+		DPU_INTR_PING_PONG_S0_WR_PTR, 1},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 1},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 1},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 1},
+	/* irq_idx: 40 */
+	{ DPU_IRQ_TYPE_PING_PONG_RD_PTR, PINGPONG_S0,
+		DPU_INTR_PING_PONG_S0_RD_PTR, 1},
+	/* irq_idx: 41-45 */
+	{ DPU_IRQ_TYPE_CTL_START, CTL_0,
+		DPU_INTR_CTL_0_START, 1},
+	{ DPU_IRQ_TYPE_CTL_START, CTL_1,
+		DPU_INTR_CTL_1_START, 1},
+	{ DPU_IRQ_TYPE_CTL_START, CTL_2,
+		DPU_INTR_CTL_2_START, 1},
+	{ DPU_IRQ_TYPE_CTL_START, CTL_3,
+		DPU_INTR_CTL_3_START, 1},
+	{ DPU_IRQ_TYPE_CTL_START, CTL_4,
+		DPU_INTR_CTL_4_START, 1},
+	/* irq_idx: 46-47 */
+	{ DPU_IRQ_TYPE_CWB_OVERFLOW, CWB_2, DPU_INTR_CWB_2_OVERFLOW, 1},
+	{ DPU_IRQ_TYPE_CWB_OVERFLOW, CWB_3, DPU_INTR_CWB_3_OVERFLOW, 1},
+	/* irq_idx: 48-51 */
+	{ DPU_IRQ_TYPE_PING_PONG_TEAR_CHECK, PINGPONG_0,
+		DPU_INTR_PING_PONG_0_TEAR_DETECTED, 1},
+	{ DPU_IRQ_TYPE_PING_PONG_TEAR_CHECK, PINGPONG_1,
+		DPU_INTR_PING_PONG_1_TEAR_DETECTED, 1},
+	{ DPU_IRQ_TYPE_PING_PONG_TEAR_CHECK, PINGPONG_2,
+		DPU_INTR_PING_PONG_2_TEAR_DETECTED, 1},
+	{ DPU_IRQ_TYPE_PING_PONG_TEAR_CHECK, PINGPONG_3,
+		DPU_INTR_PING_PONG_3_TEAR_DETECTED, 1},
+	/* irq_idx: 52-55 */
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 1},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 1},
+	{ DPU_IRQ_TYPE_PING_PONG_TEAR_CHECK, PINGPONG_S0,
+		DPU_INTR_PING_PONG_S0_TEAR_DETECTED, 1},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 1},
+	/* irq_idx: 56-59 */
+	{ DPU_IRQ_TYPE_PING_PONG_TE_CHECK, PINGPONG_0,
+		DPU_INTR_PING_PONG_0_TE_DETECTED, 1},
+	{ DPU_IRQ_TYPE_PING_PONG_TE_CHECK, PINGPONG_1,
+		DPU_INTR_PING_PONG_1_TE_DETECTED, 1},
+	{ DPU_IRQ_TYPE_PING_PONG_TE_CHECK, PINGPONG_2,
+		DPU_INTR_PING_PONG_2_TE_DETECTED, 1},
+	{ DPU_IRQ_TYPE_PING_PONG_TE_CHECK, PINGPONG_3,
+		DPU_INTR_PING_PONG_3_TE_DETECTED, 1},
+	/* irq_idx: 60-63 */
+	{ DPU_IRQ_TYPE_PING_PONG_TE_CHECK, PINGPONG_S0,
+		DPU_INTR_PING_PONG_S0_TE_DETECTED, 1},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 1},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 1},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 1},
+
+	/* BEGIN MAP_RANGE: 64-95 HIST */
+	/* irq_idx: 64-67 */
+	{ DPU_IRQ_TYPE_HIST_VIG_DONE, SSPP_VIG0, DPU_INTR_HIST_VIG_0_DONE, 2},
+	{ DPU_IRQ_TYPE_HIST_VIG_RSTSEQ, SSPP_VIG0,
+		DPU_INTR_HIST_VIG_0_RSTSEQ_DONE, 2},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 2},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 2},
+	/* irq_idx: 68-71 */
+	{ DPU_IRQ_TYPE_HIST_VIG_DONE, SSPP_VIG1, DPU_INTR_HIST_VIG_1_DONE, 2},
+	{ DPU_IRQ_TYPE_HIST_VIG_RSTSEQ, SSPP_VIG1,
+		DPU_INTR_HIST_VIG_1_RSTSEQ_DONE, 2},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 2},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 2},
+	/* irq_idx: 72-75 */
+	{ DPU_IRQ_TYPE_HIST_VIG_DONE, SSPP_VIG2, DPU_INTR_HIST_VIG_2_DONE, 2},
+	{ DPU_IRQ_TYPE_HIST_VIG_RSTSEQ, SSPP_VIG2,
+		DPU_INTR_HIST_VIG_2_RSTSEQ_DONE, 2},
+	{ DPU_IRQ_TYPE_HIST_VIG_DONE, SSPP_VIG3, DPU_INTR_HIST_VIG_3_DONE, 2},
+	{ DPU_IRQ_TYPE_HIST_VIG_RSTSEQ, SSPP_VIG3,
+		DPU_INTR_HIST_VIG_3_RSTSEQ_DONE, 2},
+	/* irq_idx: 76-79 */
+	{ DPU_IRQ_TYPE_HIST_DSPP_DONE, DSPP_0, DPU_INTR_HIST_DSPP_0_DONE, 2},
+	{ DPU_IRQ_TYPE_HIST_DSPP_RSTSEQ, DSPP_0,
+		DPU_INTR_HIST_DSPP_0_RSTSEQ_DONE, 2},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 2},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 2},
+	/* irq_idx: 80-83 */
+	{ DPU_IRQ_TYPE_HIST_DSPP_DONE, DSPP_1, DPU_INTR_HIST_DSPP_1_DONE, 2},
+	{ DPU_IRQ_TYPE_HIST_DSPP_RSTSEQ, DSPP_1,
+		DPU_INTR_HIST_DSPP_1_RSTSEQ_DONE, 2},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 2},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 2},
+	/* irq_idx: 84-87 */
+	{ DPU_IRQ_TYPE_HIST_DSPP_DONE, DSPP_2, DPU_INTR_HIST_DSPP_2_DONE, 2},
+	{ DPU_IRQ_TYPE_HIST_DSPP_RSTSEQ, DSPP_2,
+		DPU_INTR_HIST_DSPP_2_RSTSEQ_DONE, 2},
+	{ DPU_IRQ_TYPE_HIST_DSPP_DONE, DSPP_3, DPU_INTR_HIST_DSPP_3_DONE, 2},
+	{ DPU_IRQ_TYPE_HIST_DSPP_RSTSEQ, DSPP_3,
+		DPU_INTR_HIST_DSPP_3_RSTSEQ_DONE, 2},
+	/* irq_idx: 88-91 */
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 2},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 2},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 2},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 2},
+	/* irq_idx: 92-95 */
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 2},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 2},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 2},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 2},
+
+	/* BEGIN MAP_RANGE: 96-127 INTF_0_INTR */
+	/* irq_idx: 96-99 */
+	{ DPU_IRQ_TYPE_SFI_VIDEO_IN, INTF_0,
+		DPU_INTR_VIDEO_INTO_STATIC, 3},
+	{ DPU_IRQ_TYPE_SFI_VIDEO_OUT, INTF_0,
+		DPU_INTR_VIDEO_OUTOF_STATIC, 3},
+	{ DPU_IRQ_TYPE_SFI_CMD_0_IN, INTF_0,
+		DPU_INTR_DSICMD_0_INTO_STATIC, 3},
+	{ DPU_IRQ_TYPE_SFI_CMD_0_OUT, INTF_0,
+		DPU_INTR_DSICMD_0_OUTOF_STATIC, 3},
+	/* irq_idx: 100-103 */
+	{ DPU_IRQ_TYPE_SFI_CMD_1_IN, INTF_0,
+		DPU_INTR_DSICMD_1_INTO_STATIC, 3},
+	{ DPU_IRQ_TYPE_SFI_CMD_1_OUT, INTF_0,
+		DPU_INTR_DSICMD_1_OUTOF_STATIC, 3},
+	{ DPU_IRQ_TYPE_SFI_CMD_2_IN, INTF_0,
+		DPU_INTR_DSICMD_2_INTO_STATIC, 3},
+	{ DPU_IRQ_TYPE_SFI_CMD_2_OUT, INTF_0,
+		DPU_INTR_DSICMD_2_OUTOF_STATIC, 3},
+	/* irq_idx: 104-107 */
+	{ DPU_IRQ_TYPE_PROG_LINE, INTF_0, DPU_INTR_PROG_LINE, 3},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 3},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 3},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 3},
+	/* irq_idx: 108-111 */
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 3},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 3},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 3},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 3},
+	/* irq_idx: 112-115 */
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 3},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 3},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 3},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 3},
+	/* irq_idx: 116-119 */
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 3},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 3},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 3},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 3},
+	/* irq_idx: 120-123 */
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 3},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 3},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 3},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 3},
+	/* irq_idx: 124-127 */
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 3},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 3},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 3},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 3},
+
+	/* BEGIN MAP_RANGE: 128-159 INTF_1_INTR */
+	/* irq_idx: 128-131 */
+	{ DPU_IRQ_TYPE_SFI_VIDEO_IN, INTF_1,
+		DPU_INTR_VIDEO_INTO_STATIC, 4},
+	{ DPU_IRQ_TYPE_SFI_VIDEO_OUT, INTF_1,
+		DPU_INTR_VIDEO_OUTOF_STATIC, 4},
+	{ DPU_IRQ_TYPE_SFI_CMD_0_IN, INTF_1,
+		DPU_INTR_DSICMD_0_INTO_STATIC, 4},
+	{ DPU_IRQ_TYPE_SFI_CMD_0_OUT, INTF_1,
+		DPU_INTR_DSICMD_0_OUTOF_STATIC, 4},
+	/* irq_idx: 132-135 */
+	{ DPU_IRQ_TYPE_SFI_CMD_1_IN, INTF_1,
+		DPU_INTR_DSICMD_1_INTO_STATIC, 4},
+	{ DPU_IRQ_TYPE_SFI_CMD_1_OUT, INTF_1,
+		DPU_INTR_DSICMD_1_OUTOF_STATIC, 4},
+	{ DPU_IRQ_TYPE_SFI_CMD_2_IN, INTF_1,
+		DPU_INTR_DSICMD_2_INTO_STATIC, 4},
+	{ DPU_IRQ_TYPE_SFI_CMD_2_OUT, INTF_1,
+		DPU_INTR_DSICMD_2_OUTOF_STATIC, 4},
+	/* irq_idx: 136-139 */
+	{ DPU_IRQ_TYPE_PROG_LINE, INTF_1, DPU_INTR_PROG_LINE, 4},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 4},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 4},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 4},
+	/* irq_idx: 140-143 */
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 4},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 4},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 4},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 4},
+	/* irq_idx: 144-147 */
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 4},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 4},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 4},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 4},
+	/* irq_idx: 148-151 */
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 4},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 4},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 4},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 4},
+	/* irq_idx: 152-155 */
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 4},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 4},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 4},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 4},
+	/* irq_idx: 156-159 */
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 4},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 4},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 4},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 4},
+
+	/* BEGIN MAP_RANGE: 160-191 INTF_2_INTR */
+	/* irq_idx: 160-163 */
+	{ DPU_IRQ_TYPE_SFI_VIDEO_IN, INTF_2,
+		DPU_INTR_VIDEO_INTO_STATIC, 5},
+	{ DPU_IRQ_TYPE_SFI_VIDEO_OUT, INTF_2,
+		DPU_INTR_VIDEO_OUTOF_STATIC, 5},
+	{ DPU_IRQ_TYPE_SFI_CMD_0_IN, INTF_2,
+		DPU_INTR_DSICMD_0_INTO_STATIC, 5},
+	{ DPU_IRQ_TYPE_SFI_CMD_0_OUT, INTF_2,
+		DPU_INTR_DSICMD_0_OUTOF_STATIC, 5},
+	/* irq_idx: 164-167 */
+	{ DPU_IRQ_TYPE_SFI_CMD_1_IN, INTF_2,
+		DPU_INTR_DSICMD_1_INTO_STATIC, 5},
+	{ DPU_IRQ_TYPE_SFI_CMD_1_OUT, INTF_2,
+		DPU_INTR_DSICMD_1_OUTOF_STATIC, 5},
+	{ DPU_IRQ_TYPE_SFI_CMD_2_IN, INTF_2,
+		DPU_INTR_DSICMD_2_INTO_STATIC, 5},
+	{ DPU_IRQ_TYPE_SFI_CMD_2_OUT, INTF_2,
+		DPU_INTR_DSICMD_2_OUTOF_STATIC, 5},
+	/* irq_idx: 168-171 */
+	{ DPU_IRQ_TYPE_PROG_LINE, INTF_2, DPU_INTR_PROG_LINE, 5},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 5},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 5},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 5},
+	/* irq_idx: 172-175 */
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 5},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 5},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 5},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 5},
+	/* irq_idx: 176-179 */
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 5},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 5},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 5},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 5},
+	/* irq_idx: 180-183 */
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 5},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 5},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 5},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 5},
+	/* irq_idx: 184-187 */
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 5},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 5},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 5},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 5},
+	/* irq_idx: 188-191 */
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 5},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 5},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 5},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 5},
+
+	/* BEGIN MAP_RANGE: 192-223 INTF_3_INTR */
+	/* irq_idx: 192-195 */
+	{ DPU_IRQ_TYPE_SFI_VIDEO_IN, INTF_3,
+		DPU_INTR_VIDEO_INTO_STATIC, 6},
+	{ DPU_IRQ_TYPE_SFI_VIDEO_OUT, INTF_3,
+		DPU_INTR_VIDEO_OUTOF_STATIC, 6},
+	{ DPU_IRQ_TYPE_SFI_CMD_0_IN, INTF_3,
+		DPU_INTR_DSICMD_0_INTO_STATIC, 6},
+	{ DPU_IRQ_TYPE_SFI_CMD_0_OUT, INTF_3,
+		DPU_INTR_DSICMD_0_OUTOF_STATIC, 6},
+	/* irq_idx: 196-199 */
+	{ DPU_IRQ_TYPE_SFI_CMD_1_IN, INTF_3,
+		DPU_INTR_DSICMD_1_INTO_STATIC, 6},
+	{ DPU_IRQ_TYPE_SFI_CMD_1_OUT, INTF_3,
+		DPU_INTR_DSICMD_1_OUTOF_STATIC, 6},
+	{ DPU_IRQ_TYPE_SFI_CMD_2_IN, INTF_3,
+		DPU_INTR_DSICMD_2_INTO_STATIC, 6},
+	{ DPU_IRQ_TYPE_SFI_CMD_2_OUT, INTF_3,
+		DPU_INTR_DSICMD_2_OUTOF_STATIC, 6},
+	/* irq_idx: 200-203 */
+	{ DPU_IRQ_TYPE_PROG_LINE, INTF_3, DPU_INTR_PROG_LINE, 6},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 6},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 6},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 6},
+	/* irq_idx: 204-207 */
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 6},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 6},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 6},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 6},
+	/* irq_idx: 208-211 */
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 6},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 6},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 6},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 6},
+	/* irq_idx: 212-215 */
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 6},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 6},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 6},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 6},
+	/* irq_idx: 216-219 */
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 6},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 6},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 6},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 6},
+	/* irq_idx: 220-223 */
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 6},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 6},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 6},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 6},
+
+	/* BEGIN MAP_RANGE: 224-255 INTF_4_INTR */
+	/* irq_idx: 224-227 */
+	{ DPU_IRQ_TYPE_SFI_VIDEO_IN, INTF_4,
+		DPU_INTR_VIDEO_INTO_STATIC, 7},
+	{ DPU_IRQ_TYPE_SFI_VIDEO_OUT, INTF_4,
+		DPU_INTR_VIDEO_OUTOF_STATIC, 7},
+	{ DPU_IRQ_TYPE_SFI_CMD_0_IN, INTF_4,
+		DPU_INTR_DSICMD_0_INTO_STATIC, 7},
+	{ DPU_IRQ_TYPE_SFI_CMD_0_OUT, INTF_4,
+		DPU_INTR_DSICMD_0_OUTOF_STATIC, 7},
+	/* irq_idx: 228-231 */
+	{ DPU_IRQ_TYPE_SFI_CMD_1_IN, INTF_4,
+		DPU_INTR_DSICMD_1_INTO_STATIC, 7},
+	{ DPU_IRQ_TYPE_SFI_CMD_1_OUT, INTF_4,
+		DPU_INTR_DSICMD_1_OUTOF_STATIC, 7},
+	{ DPU_IRQ_TYPE_SFI_CMD_2_IN, INTF_4,
+		DPU_INTR_DSICMD_2_INTO_STATIC, 7},
+	{ DPU_IRQ_TYPE_SFI_CMD_2_OUT, INTF_4,
+		DPU_INTR_DSICMD_2_OUTOF_STATIC, 7},
+	/* irq_idx: 232-235 */
+	{ DPU_IRQ_TYPE_PROG_LINE, INTF_4, DPU_INTR_PROG_LINE, 7},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 7},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 7},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 7},
+	/* irq_idx: 236-239 */
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 7},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 7},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 7},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 7},
+	/* irq_idx: 240-243 */
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 7},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 7},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 7},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 7},
+	/* irq_idx: 244-247 */
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 7},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 7},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 7},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 7},
+	/* irq_idx: 248-251 */
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 7},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 7},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 7},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 7},
+	/* irq_idx: 252-255 */
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 7},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 7},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 7},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 7},
+
+	/* BEGIN MAP_RANGE: 256-287 AD4_0_INTR */
+	/* irq_idx: 256-259 */
+	{ DPU_IRQ_TYPE_AD4_BL_DONE, DSPP_0, DPU_INTR_BACKLIGHT_UPDATED, 8},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 8},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 8},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 8},
+	/* irq_idx: 260-263 */
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 8},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 8},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 8},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 8},
+	/* irq_idx: 264-267 */
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 8},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 8},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 8},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 8},
+	/* irq_idx: 268-271 */
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 8},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 8},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 8},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 8},
+	/* irq_idx: 272-275 */
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 8},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 8},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 8},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 8},
+	/* irq_idx: 276-279 */
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 8},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 8},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 8},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 8},
+	/* irq_idx: 280-283 */
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 8},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 8},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 8},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 8},
+	/* irq_idx: 284-287 */
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 8},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 8},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 8},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 8},
+
+	/* BEGIN MAP_RANGE: 288-319 AD4_1_INTR */
+	/* irq_idx: 288-291 */
+	{ DPU_IRQ_TYPE_AD4_BL_DONE, DSPP_1, DPU_INTR_BACKLIGHT_UPDATED, 9},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 9},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 9},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 9},
+	/* irq_idx: 292-295 */
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 9},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 9},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 9},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 9},
+	/* irq_idx: 296-299 */
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 9},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 9},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 9},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 9},
+	/* irq_idx: 300-303 */
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 9},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 9},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 9},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 9},
+	/* irq_idx: 304-307 */
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 9},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 9},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 9},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 9},
+	/* irq_idx: 308-311 */
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 9},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 9},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 9},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 9},
+	/* irq_idx: 312-315 */
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 9},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 9},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 9},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 9},
+	/* irq_idx: 315-319 */
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 9},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 9},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 9},
+	{ DPU_IRQ_TYPE_RESERVED, 0, 0, 9},
+};
+
+static int dpu_hw_intr_irqidx_lookup(enum dpu_intr_type intr_type,
+		u32 instance_idx)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(dpu_irq_map); i++) {
+		if (intr_type == dpu_irq_map[i].intr_type &&
+			instance_idx == dpu_irq_map[i].instance_idx)
+			return i;
+	}
+
+	pr_debug("IRQ lookup fail!! intr_type=%d, instance_idx=%d\n",
+			intr_type, instance_idx);
+	return -EINVAL;
+}
+
+static void dpu_hw_intr_set_mask(struct dpu_hw_intr *intr, uint32_t reg_off,
+		uint32_t mask)
+{
+	if (!intr)
+		return;
+
+	DPU_REG_WRITE(&intr->hw, reg_off, mask);
+
+	/* ensure register writes go through */
+	wmb();
+}
+
+static void dpu_hw_intr_dispatch_irq(struct dpu_hw_intr *intr,
+		void (*cbfunc)(void *, int),
+		void *arg)
+{
+	int reg_idx;
+	int irq_idx;
+	int start_idx;
+	int end_idx;
+	u32 irq_status;
+	unsigned long irq_flags;
+
+	if (!intr)
+		return;
+
+	/*
+	 * The dispatcher will save the IRQ status before calling here.
+	 * Now need to go through each IRQ status and find matching
+	 * irq lookup index.
+	 */
+	spin_lock_irqsave(&intr->irq_lock, irq_flags);
+	for (reg_idx = 0; reg_idx < ARRAY_SIZE(dpu_intr_set); reg_idx++) {
+		irq_status = intr->save_irq_status[reg_idx];
+
+		/*
+		 * Each Interrupt register has a range of 32 indexes, and
+		 * that is static for dpu_irq_map.
+		 */
+		start_idx = reg_idx * 32;
+		end_idx = start_idx + 32;
+
+		if (start_idx >= ARRAY_SIZE(dpu_irq_map) ||
+				end_idx > ARRAY_SIZE(dpu_irq_map))
+			continue;
+
+		/*
+		 * Search through matching intr status from irq map.
+		 * start_idx and end_idx defined the search range in
+		 * the dpu_irq_map.
+		 */
+		for (irq_idx = start_idx;
+				(irq_idx < end_idx) && irq_status;
+				irq_idx++)
+			if ((irq_status & dpu_irq_map[irq_idx].irq_mask) &&
+				(dpu_irq_map[irq_idx].reg_idx == reg_idx)) {
+				/*
+				 * Once a match on irq mask, perform a callback
+				 * to the given cbfunc. cbfunc will take care
+				 * the interrupt status clearing. If cbfunc is
+				 * not provided, then the interrupt clearing
+				 * is here.
+				 */
+				if (cbfunc)
+					cbfunc(arg, irq_idx);
+				else
+					intr->ops.clear_intr_status_nolock(
+							intr, irq_idx);
+
+				/*
+				 * When callback finish, clear the irq_status
+				 * with the matching mask. Once irq_status
+				 * is all cleared, the search can be stopped.
+				 */
+				irq_status &= ~dpu_irq_map[irq_idx].irq_mask;
+			}
+	}
+	spin_unlock_irqrestore(&intr->irq_lock, irq_flags);
+}
+
+static int dpu_hw_intr_enable_irq(struct dpu_hw_intr *intr, int irq_idx)
+{
+	int reg_idx;
+	unsigned long irq_flags;
+	const struct dpu_intr_reg *reg;
+	const struct dpu_irq_type *irq;
+	const char *dbgstr = NULL;
+	uint32_t cache_irq_mask;
+
+	if (!intr)
+		return -EINVAL;
+
+	if (irq_idx < 0 || irq_idx >= ARRAY_SIZE(dpu_irq_map)) {
+		pr_err("invalid IRQ index: [%d]\n", irq_idx);
+		return -EINVAL;
+	}
+
+	irq = &dpu_irq_map[irq_idx];
+	reg_idx = irq->reg_idx;
+	reg = &dpu_intr_set[reg_idx];
+
+	spin_lock_irqsave(&intr->irq_lock, irq_flags);
+	cache_irq_mask = intr->cache_irq_mask[reg_idx];
+	if (cache_irq_mask & irq->irq_mask) {
+		dbgstr = "DPU IRQ already set:";
+	} else {
+		dbgstr = "DPU IRQ enabled:";
+
+		cache_irq_mask |= irq->irq_mask;
+		/* Cleaning any pending interrupt */
+		DPU_REG_WRITE(&intr->hw, reg->clr_off, irq->irq_mask);
+		/* Enabling interrupts with the new mask */
+		DPU_REG_WRITE(&intr->hw, reg->en_off, cache_irq_mask);
+
+		/* ensure register write goes through */
+		wmb();
+
+		intr->cache_irq_mask[reg_idx] = cache_irq_mask;
+	}
+	spin_unlock_irqrestore(&intr->irq_lock, irq_flags);
+
+	pr_debug("%s MASK:0x%.8x, CACHE-MASK:0x%.8x\n", dbgstr,
+			irq->irq_mask, cache_irq_mask);
+
+	return 0;
+}
+
+static int dpu_hw_intr_disable_irq_nolock(struct dpu_hw_intr *intr, int irq_idx)
+{
+	int reg_idx;
+	const struct dpu_intr_reg *reg;
+	const struct dpu_irq_type *irq;
+	const char *dbgstr = NULL;
+	uint32_t cache_irq_mask;
+
+	if (!intr)
+		return -EINVAL;
+
+	if (irq_idx < 0 || irq_idx >= ARRAY_SIZE(dpu_irq_map)) {
+		pr_err("invalid IRQ index: [%d]\n", irq_idx);
+		return -EINVAL;
+	}
+
+	irq = &dpu_irq_map[irq_idx];
+	reg_idx = irq->reg_idx;
+	reg = &dpu_intr_set[reg_idx];
+
+	cache_irq_mask = intr->cache_irq_mask[reg_idx];
+	if ((cache_irq_mask & irq->irq_mask) == 0) {
+		dbgstr = "DPU IRQ is already cleared:";
+	} else {
+		dbgstr = "DPU IRQ mask disable:";
+
+		cache_irq_mask &= ~irq->irq_mask;
+		/* Disable interrupts based on the new mask */
+		DPU_REG_WRITE(&intr->hw, reg->en_off, cache_irq_mask);
+		/* Cleaning any pending interrupt */
+		DPU_REG_WRITE(&intr->hw, reg->clr_off, irq->irq_mask);
+
+		/* ensure register write goes through */
+		wmb();
+
+		intr->cache_irq_mask[reg_idx] = cache_irq_mask;
+	}
+
+	pr_debug("%s MASK:0x%.8x, CACHE-MASK:0x%.8x\n", dbgstr,
+			irq->irq_mask, cache_irq_mask);
+
+	return 0;
+}
+
+static int dpu_hw_intr_disable_irq(struct dpu_hw_intr *intr, int irq_idx)
+{
+	unsigned long irq_flags;
+
+	if (!intr)
+		return -EINVAL;
+
+	if (irq_idx < 0 || irq_idx >= ARRAY_SIZE(dpu_irq_map)) {
+		pr_err("invalid IRQ index: [%d]\n", irq_idx);
+		return -EINVAL;
+	}
+
+	spin_lock_irqsave(&intr->irq_lock, irq_flags);
+	dpu_hw_intr_disable_irq_nolock(intr, irq_idx);
+	spin_unlock_irqrestore(&intr->irq_lock, irq_flags);
+
+	return 0;
+}
+
+static int dpu_hw_intr_clear_irqs(struct dpu_hw_intr *intr)
+{
+	int i;
+
+	if (!intr)
+		return -EINVAL;
+
+	for (i = 0; i < ARRAY_SIZE(dpu_intr_set); i++)
+		DPU_REG_WRITE(&intr->hw, dpu_intr_set[i].clr_off, 0xffffffff);
+
+	/* ensure register writes go through */
+	wmb();
+
+	return 0;
+}
+
+static int dpu_hw_intr_disable_irqs(struct dpu_hw_intr *intr)
+{
+	int i;
+
+	if (!intr)
+		return -EINVAL;
+
+	for (i = 0; i < ARRAY_SIZE(dpu_intr_set); i++)
+		DPU_REG_WRITE(&intr->hw, dpu_intr_set[i].en_off, 0x00000000);
+
+	/* ensure register writes go through */
+	wmb();
+
+	return 0;
+}
+
+static int dpu_hw_intr_get_valid_interrupts(struct dpu_hw_intr *intr,
+		uint32_t *mask)
+{
+	if (!intr || !mask)
+		return -EINVAL;
+
+	*mask = IRQ_SOURCE_MDP | IRQ_SOURCE_DSI0 | IRQ_SOURCE_DSI1
+		| IRQ_SOURCE_HDMI | IRQ_SOURCE_EDP;
+
+	return 0;
+}
+
+static void dpu_hw_intr_get_interrupt_statuses(struct dpu_hw_intr *intr)
+{
+	int i;
+	u32 enable_mask;
+	unsigned long irq_flags;
+
+	if (!intr)
+		return;
+
+	spin_lock_irqsave(&intr->irq_lock, irq_flags);
+	for (i = 0; i < ARRAY_SIZE(dpu_intr_set); i++) {
+		/* Read interrupt status */
+		intr->save_irq_status[i] = DPU_REG_READ(&intr->hw,
+				dpu_intr_set[i].status_off);
+
+		/* Read enable mask */
+		enable_mask = DPU_REG_READ(&intr->hw, dpu_intr_set[i].en_off);
+
+		/* and clear the interrupt */
+		if (intr->save_irq_status[i])
+			DPU_REG_WRITE(&intr->hw, dpu_intr_set[i].clr_off,
+					intr->save_irq_status[i]);
+
+		/* Finally update IRQ status based on enable mask */
+		intr->save_irq_status[i] &= enable_mask;
+	}
+
+	/* ensure register writes go through */
+	wmb();
+
+	spin_unlock_irqrestore(&intr->irq_lock, irq_flags);
+}
+
+static void dpu_hw_intr_clear_intr_status_nolock(struct dpu_hw_intr *intr,
+		int irq_idx)
+{
+	int reg_idx;
+
+	if (!intr)
+		return;
+
+	reg_idx = dpu_irq_map[irq_idx].reg_idx;
+	DPU_REG_WRITE(&intr->hw, dpu_intr_set[reg_idx].clr_off,
+			dpu_irq_map[irq_idx].irq_mask);
+
+	/* ensure register writes go through */
+	wmb();
+}
+
+static void dpu_hw_intr_clear_interrupt_status(struct dpu_hw_intr *intr,
+		int irq_idx)
+{
+	unsigned long irq_flags;
+
+	if (!intr)
+		return;
+
+	spin_lock_irqsave(&intr->irq_lock, irq_flags);
+	dpu_hw_intr_clear_intr_status_nolock(intr, irq_idx);
+	spin_unlock_irqrestore(&intr->irq_lock, irq_flags);
+}
+
+static u32 dpu_hw_intr_get_interrupt_status(struct dpu_hw_intr *intr,
+		int irq_idx, bool clear)
+{
+	int reg_idx;
+	unsigned long irq_flags;
+	u32 intr_status;
+
+	if (!intr)
+		return 0;
+
+	if (irq_idx >= ARRAY_SIZE(dpu_irq_map) || irq_idx < 0) {
+		pr_err("invalid IRQ index: [%d]\n", irq_idx);
+		return 0;
+	}
+
+	spin_lock_irqsave(&intr->irq_lock, irq_flags);
+
+	reg_idx = dpu_irq_map[irq_idx].reg_idx;
+	intr_status = DPU_REG_READ(&intr->hw,
+			dpu_intr_set[reg_idx].status_off) &
+					dpu_irq_map[irq_idx].irq_mask;
+	if (intr_status && clear)
+		DPU_REG_WRITE(&intr->hw, dpu_intr_set[reg_idx].clr_off,
+				intr_status);
+
+	/* ensure register writes go through */
+	wmb();
+
+	spin_unlock_irqrestore(&intr->irq_lock, irq_flags);
+
+	return intr_status;
+}
+
+static void __setup_intr_ops(struct dpu_hw_intr_ops *ops)
+{
+	ops->set_mask = dpu_hw_intr_set_mask;
+	ops->irq_idx_lookup = dpu_hw_intr_irqidx_lookup;
+	ops->enable_irq = dpu_hw_intr_enable_irq;
+	ops->disable_irq = dpu_hw_intr_disable_irq;
+	ops->dispatch_irqs = dpu_hw_intr_dispatch_irq;
+	ops->clear_all_irqs = dpu_hw_intr_clear_irqs;
+	ops->disable_all_irqs = dpu_hw_intr_disable_irqs;
+	ops->get_valid_interrupts = dpu_hw_intr_get_valid_interrupts;
+	ops->get_interrupt_statuses = dpu_hw_intr_get_interrupt_statuses;
+	ops->clear_interrupt_status = dpu_hw_intr_clear_interrupt_status;
+	ops->clear_intr_status_nolock = dpu_hw_intr_clear_intr_status_nolock;
+	ops->get_interrupt_status = dpu_hw_intr_get_interrupt_status;
+}
+
+static void __intr_offset(struct dpu_mdss_cfg *m,
+		void __iomem *addr, struct dpu_hw_blk_reg_map *hw)
+{
+	hw->base_off = addr;
+	hw->blk_off = m->mdp[0].base;
+	hw->hwversion = m->hwversion;
+}
+
+struct dpu_hw_intr *dpu_hw_intr_init(void __iomem *addr,
+		struct dpu_mdss_cfg *m)
+{
+	struct dpu_hw_intr *intr;
+
+	if (!addr || !m)
+		return ERR_PTR(-EINVAL);
+
+	intr = kzalloc(sizeof(*intr), GFP_KERNEL);
+	if (!intr)
+		return ERR_PTR(-ENOMEM);
+
+	__intr_offset(m, addr, &intr->hw);
+	__setup_intr_ops(&intr->ops);
+
+	intr->irq_idx_tbl_size = ARRAY_SIZE(dpu_irq_map);
+
+	intr->cache_irq_mask = kcalloc(ARRAY_SIZE(dpu_intr_set), sizeof(u32),
+			GFP_KERNEL);
+	if (intr->cache_irq_mask == NULL) {
+		kfree(intr);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	intr->save_irq_status = kcalloc(ARRAY_SIZE(dpu_intr_set), sizeof(u32),
+			GFP_KERNEL);
+	if (intr->save_irq_status == NULL) {
+		kfree(intr->cache_irq_mask);
+		kfree(intr);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	spin_lock_init(&intr->irq_lock);
+
+	return intr;
+}
+
+void dpu_hw_intr_destroy(struct dpu_hw_intr *intr)
+{
+	if (intr) {
+		kfree(intr->cache_irq_mask);
+		kfree(intr->save_irq_status);
+		kfree(intr);
+	}
+}
+
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_interrupts.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_interrupts.h
new file mode 100644
index 000000000000..61e4cba36562
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_interrupts.h
@@ -0,0 +1,257 @@
+/* Copyright (c) 2016-2018, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef _DPU_HW_INTERRUPTS_H
+#define _DPU_HW_INTERRUPTS_H
+
+#include <linux/types.h>
+
+#include "dpu_hwio.h"
+#include "dpu_hw_catalog.h"
+#include "dpu_hw_util.h"
+#include "dpu_hw_mdss.h"
+
+#define IRQ_SOURCE_MDP		BIT(0)
+#define IRQ_SOURCE_DSI0		BIT(4)
+#define IRQ_SOURCE_DSI1		BIT(5)
+#define IRQ_SOURCE_HDMI		BIT(8)
+#define IRQ_SOURCE_EDP		BIT(12)
+#define IRQ_SOURCE_MHL		BIT(16)
+
+/**
+ * dpu_intr_type - HW Interrupt Type
+ * @DPU_IRQ_TYPE_WB_ROT_COMP:		WB rotator done
+ * @DPU_IRQ_TYPE_WB_WFD_COMP:		WB WFD done
+ * @DPU_IRQ_TYPE_PING_PONG_COMP:	PingPong done
+ * @DPU_IRQ_TYPE_PING_PONG_RD_PTR:	PingPong read pointer
+ * @DPU_IRQ_TYPE_PING_PONG_WR_PTR:	PingPong write pointer
+ * @DPU_IRQ_TYPE_PING_PONG_AUTO_REF:	PingPong auto refresh
+ * @DPU_IRQ_TYPE_PING_PONG_TEAR_CHECK:	PingPong Tear check
+ * @DPU_IRQ_TYPE_PING_PONG_TE_CHECK:	PingPong TE detection
+ * @DPU_IRQ_TYPE_INTF_UNDER_RUN:	INTF underrun
+ * @DPU_IRQ_TYPE_INTF_VSYNC:		INTF VSYNC
+ * @DPU_IRQ_TYPE_CWB_OVERFLOW:		Concurrent WB overflow
+ * @DPU_IRQ_TYPE_HIST_VIG_DONE:		VIG Histogram done
+ * @DPU_IRQ_TYPE_HIST_VIG_RSTSEQ:	VIG Histogram reset
+ * @DPU_IRQ_TYPE_HIST_DSPP_DONE:	DSPP Histogram done
+ * @DPU_IRQ_TYPE_HIST_DSPP_RSTSEQ:	DSPP Histogram reset
+ * @DPU_IRQ_TYPE_WD_TIMER:		Watchdog timer
+ * @DPU_IRQ_TYPE_SFI_VIDEO_IN:		Video static frame INTR into static
+ * @DPU_IRQ_TYPE_SFI_VIDEO_OUT:		Video static frame INTR out-of static
+ * @DPU_IRQ_TYPE_SFI_CMD_0_IN:		DSI CMD0 static frame INTR into static
+ * @DPU_IRQ_TYPE_SFI_CMD_0_OUT:		DSI CMD0 static frame INTR out-of static
+ * @DPU_IRQ_TYPE_SFI_CMD_1_IN:		DSI CMD1 static frame INTR into static
+ * @DPU_IRQ_TYPE_SFI_CMD_1_OUT:		DSI CMD1 static frame INTR out-of static
+ * @DPU_IRQ_TYPE_SFI_CMD_2_IN:		DSI CMD2 static frame INTR into static
+ * @DPU_IRQ_TYPE_SFI_CMD_2_OUT:		DSI CMD2 static frame INTR out-of static
+ * @DPU_IRQ_TYPE_PROG_LINE:		Programmable Line interrupt
+ * @DPU_IRQ_TYPE_AD4_BL_DONE:		AD4 backlight
+ * @DPU_IRQ_TYPE_CTL_START:		Control start
+ * @DPU_IRQ_TYPE_RESERVED:		Reserved for expansion
+ */
+enum dpu_intr_type {
+	DPU_IRQ_TYPE_WB_ROT_COMP,
+	DPU_IRQ_TYPE_WB_WFD_COMP,
+	DPU_IRQ_TYPE_PING_PONG_COMP,
+	DPU_IRQ_TYPE_PING_PONG_RD_PTR,
+	DPU_IRQ_TYPE_PING_PONG_WR_PTR,
+	DPU_IRQ_TYPE_PING_PONG_AUTO_REF,
+	DPU_IRQ_TYPE_PING_PONG_TEAR_CHECK,
+	DPU_IRQ_TYPE_PING_PONG_TE_CHECK,
+	DPU_IRQ_TYPE_INTF_UNDER_RUN,
+	DPU_IRQ_TYPE_INTF_VSYNC,
+	DPU_IRQ_TYPE_CWB_OVERFLOW,
+	DPU_IRQ_TYPE_HIST_VIG_DONE,
+	DPU_IRQ_TYPE_HIST_VIG_RSTSEQ,
+	DPU_IRQ_TYPE_HIST_DSPP_DONE,
+	DPU_IRQ_TYPE_HIST_DSPP_RSTSEQ,
+	DPU_IRQ_TYPE_WD_TIMER,
+	DPU_IRQ_TYPE_SFI_VIDEO_IN,
+	DPU_IRQ_TYPE_SFI_VIDEO_OUT,
+	DPU_IRQ_TYPE_SFI_CMD_0_IN,
+	DPU_IRQ_TYPE_SFI_CMD_0_OUT,
+	DPU_IRQ_TYPE_SFI_CMD_1_IN,
+	DPU_IRQ_TYPE_SFI_CMD_1_OUT,
+	DPU_IRQ_TYPE_SFI_CMD_2_IN,
+	DPU_IRQ_TYPE_SFI_CMD_2_OUT,
+	DPU_IRQ_TYPE_PROG_LINE,
+	DPU_IRQ_TYPE_AD4_BL_DONE,
+	DPU_IRQ_TYPE_CTL_START,
+	DPU_IRQ_TYPE_RESERVED,
+};
+
+struct dpu_hw_intr;
+
+/**
+ * Interrupt operations.
+ */
+struct dpu_hw_intr_ops {
+	/**
+	 * set_mask - Programs the given interrupt register with the
+	 *            given interrupt mask. Register value will get overwritten.
+	 * @intr:	HW interrupt handle
+	 * @reg_off:	MDSS HW register offset
+	 * @irqmask:	IRQ mask value
+	 */
+	void (*set_mask)(
+			struct dpu_hw_intr *intr,
+			uint32_t reg,
+			uint32_t irqmask);
+
+	/**
+	 * irq_idx_lookup - Lookup IRQ index on the HW interrupt type
+	 *                 Used for all irq related ops
+	 * @intr_type:		Interrupt type defined in dpu_intr_type
+	 * @instance_idx:	HW interrupt block instance
+	 * @return:		irq_idx or -EINVAL for lookup fail
+	 */
+	int (*irq_idx_lookup)(
+			enum dpu_intr_type intr_type,
+			u32 instance_idx);
+
+	/**
+	 * enable_irq - Enable IRQ based on lookup IRQ index
+	 * @intr:	HW interrupt handle
+	 * @irq_idx:	Lookup irq index return from irq_idx_lookup
+	 * @return:	0 for success, otherwise failure
+	 */
+	int (*enable_irq)(
+			struct dpu_hw_intr *intr,
+			int irq_idx);
+
+	/**
+	 * disable_irq - Disable IRQ based on lookup IRQ index
+	 * @intr:	HW interrupt handle
+	 * @irq_idx:	Lookup irq index return from irq_idx_lookup
+	 * @return:	0 for success, otherwise failure
+	 */
+	int (*disable_irq)(
+			struct dpu_hw_intr *intr,
+			int irq_idx);
+
+	/**
+	 * clear_all_irqs - Clears all the interrupts (i.e. acknowledges
+	 *                  any asserted IRQs). Useful during reset.
+	 * @intr:	HW interrupt handle
+	 * @return:	0 for success, otherwise failure
+	 */
+	int (*clear_all_irqs)(
+			struct dpu_hw_intr *intr);
+
+	/**
+	 * disable_all_irqs - Disables all the interrupts. Useful during reset.
+	 * @intr:	HW interrupt handle
+	 * @return:	0 for success, otherwise failure
+	 */
+	int (*disable_all_irqs)(
+			struct dpu_hw_intr *intr);
+
+	/**
+	 * dispatch_irqs - IRQ dispatcher will call the given callback
+	 *                 function when a matching interrupt status bit is
+	 *                 found in the irq mapping table.
+	 * @intr:	HW interrupt handle
+	 * @cbfunc:	Callback function pointer
+	 * @arg:	Argument to pass back during callback
+	 */
+	void (*dispatch_irqs)(
+			struct dpu_hw_intr *intr,
+			void (*cbfunc)(void *arg, int irq_idx),
+			void *arg);
+
+	/**
+	 * get_interrupt_statuses - Gets and store value from all interrupt
+	 *                          status registers that are currently fired.
+	 * @intr:	HW interrupt handle
+	 */
+	void (*get_interrupt_statuses)(
+			struct dpu_hw_intr *intr);
+
+	/**
+	 * clear_interrupt_status - Clears HW interrupt status based on given
+	 *                          lookup IRQ index.
+	 * @intr:	HW interrupt handle
+	 * @irq_idx:	Lookup irq index return from irq_idx_lookup
+	 */
+	void (*clear_interrupt_status)(
+			struct dpu_hw_intr *intr,
+			int irq_idx);
+
+	/**
+	 * clear_intr_status_nolock() - clears the HW interrupts without lock
+	 * @intr:	HW interrupt handle
+	 * @irq_idx:	Lookup irq index return from irq_idx_lookup
+	 */
+	void (*clear_intr_status_nolock)(
+			struct dpu_hw_intr *intr,
+			int irq_idx);
+
+	/**
+	 * get_interrupt_status - Gets HW interrupt status, and clear if set,
+	 *                        based on given lookup IRQ index.
+	 * @intr:	HW interrupt handle
+	 * @irq_idx:	Lookup irq index return from irq_idx_lookup
+	 * @clear:	True to clear irq after read
+	 */
+	u32 (*get_interrupt_status)(
+			struct dpu_hw_intr *intr,
+			int irq_idx,
+			bool clear);
+
+	/**
+	 * get_valid_interrupts - Gets a mask of all valid interrupt sources
+	 *                        within DPU. These are actually status bits
+	 *                        within interrupt registers that specify the
+	 *                        source of the interrupt in IRQs. For example,
+	 *                        valid interrupt sources can be MDP, DSI,
+	 *                        HDMI etc.
+	 * @intr:	HW interrupt handle
+	 * @mask:	Returning the interrupt source MASK
+	 * @return:	0 for success, otherwise failure
+	 */
+	int (*get_valid_interrupts)(
+			struct dpu_hw_intr *intr,
+			uint32_t *mask);
+};
+
+/**
+ * struct dpu_hw_intr: hw interrupts handling data structure
+ * @hw:               virtual address mapping
+ * @ops:              function pointer mapping for IRQ handling
+ * @cache_irq_mask:   array of IRQ enable masks reg storage created during init
+ * @save_irq_status:  array of IRQ status reg storage created during init
+ * @irq_idx_tbl_size: total number of irq_idx mapped in the hw_interrupts
+ * @irq_lock:         spinlock for accessing IRQ resources
+ */
+struct dpu_hw_intr {
+	struct dpu_hw_blk_reg_map hw;
+	struct dpu_hw_intr_ops ops;
+	u32 *cache_irq_mask;
+	u32 *save_irq_status;
+	u32 irq_idx_tbl_size;
+	spinlock_t irq_lock;
+};
+
+/**
+ * dpu_hw_intr_init(): Initializes the interrupts hw object
+ * @addr: mapped register io address of MDP
+ * @m :   pointer to mdss catalog data
+ */
+struct dpu_hw_intr *dpu_hw_intr_init(void __iomem *addr,
+		struct dpu_mdss_cfg *m);
+
+/**
+ * dpu_hw_intr_destroy(): Cleanup interrutps hw object
+ * @intr: pointer to interrupts hw object
+ */
+void dpu_hw_intr_destroy(struct dpu_hw_intr *intr);
+#endif
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_intf.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_intf.c
new file mode 100644
index 000000000000..d280df5613c9
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_intf.c
@@ -0,0 +1,349 @@
+/* Copyright (c) 2015-2018, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include "dpu_hwio.h"
+#include "dpu_hw_catalog.h"
+#include "dpu_hw_intf.h"
+#include "dpu_dbg.h"
+#include "dpu_kms.h"
+
+#define INTF_TIMING_ENGINE_EN           0x000
+#define INTF_CONFIG                     0x004
+#define INTF_HSYNC_CTL                  0x008
+#define INTF_VSYNC_PERIOD_F0            0x00C
+#define INTF_VSYNC_PERIOD_F1            0x010
+#define INTF_VSYNC_PULSE_WIDTH_F0       0x014
+#define INTF_VSYNC_PULSE_WIDTH_F1       0x018
+#define INTF_DISPLAY_V_START_F0         0x01C
+#define INTF_DISPLAY_V_START_F1         0x020
+#define INTF_DISPLAY_V_END_F0           0x024
+#define INTF_DISPLAY_V_END_F1           0x028
+#define INTF_ACTIVE_V_START_F0          0x02C
+#define INTF_ACTIVE_V_START_F1          0x030
+#define INTF_ACTIVE_V_END_F0            0x034
+#define INTF_ACTIVE_V_END_F1            0x038
+#define INTF_DISPLAY_HCTL               0x03C
+#define INTF_ACTIVE_HCTL                0x040
+#define INTF_BORDER_COLOR               0x044
+#define INTF_UNDERFLOW_COLOR            0x048
+#define INTF_HSYNC_SKEW                 0x04C
+#define INTF_POLARITY_CTL               0x050
+#define INTF_TEST_CTL                   0x054
+#define INTF_TP_COLOR0                  0x058
+#define INTF_TP_COLOR1                  0x05C
+#define INTF_FRAME_LINE_COUNT_EN        0x0A8
+#define INTF_FRAME_COUNT                0x0AC
+#define   INTF_LINE_COUNT               0x0B0
+
+#define   INTF_DEFLICKER_CONFIG         0x0F0
+#define   INTF_DEFLICKER_STRNG_COEFF    0x0F4
+#define   INTF_DEFLICKER_WEAK_COEFF     0x0F8
+
+#define   INTF_DSI_CMD_MODE_TRIGGER_EN  0x084
+#define   INTF_PANEL_FORMAT             0x090
+#define   INTF_TPG_ENABLE               0x100
+#define   INTF_TPG_MAIN_CONTROL         0x104
+#define   INTF_TPG_VIDEO_CONFIG         0x108
+#define   INTF_TPG_COMPONENT_LIMITS     0x10C
+#define   INTF_TPG_RECTANGLE            0x110
+#define   INTF_TPG_INITIAL_VALUE        0x114
+#define   INTF_TPG_BLK_WHITE_PATTERN_FRAMES   0x118
+#define   INTF_TPG_RGB_MAPPING          0x11C
+#define   INTF_PROG_FETCH_START         0x170
+#define   INTF_PROG_ROT_START           0x174
+
+#define   INTF_FRAME_LINE_COUNT_EN      0x0A8
+#define   INTF_FRAME_COUNT              0x0AC
+#define   INTF_LINE_COUNT               0x0B0
+
+#define INTF_MISR_CTRL			0x180
+#define INTF_MISR_SIGNATURE		0x184
+
+static struct dpu_intf_cfg *_intf_offset(enum dpu_intf intf,
+		struct dpu_mdss_cfg *m,
+		void __iomem *addr,
+		struct dpu_hw_blk_reg_map *b)
+{
+	int i;
+
+	for (i = 0; i < m->intf_count; i++) {
+		if ((intf == m->intf[i].id) &&
+		(m->intf[i].type != INTF_NONE)) {
+			b->base_off = addr;
+			b->blk_off = m->intf[i].base;
+			b->length = m->intf[i].len;
+			b->hwversion = m->hwversion;
+			b->log_mask = DPU_DBG_MASK_INTF;
+			return &m->intf[i];
+		}
+	}
+
+	return ERR_PTR(-EINVAL);
+}
+
+static void dpu_hw_intf_setup_timing_engine(struct dpu_hw_intf *ctx,
+		const struct intf_timing_params *p,
+		const struct dpu_format *fmt)
+{
+	struct dpu_hw_blk_reg_map *c = &ctx->hw;
+	u32 hsync_period, vsync_period;
+	u32 display_v_start, display_v_end;
+	u32 hsync_start_x, hsync_end_x;
+	u32 active_h_start, active_h_end;
+	u32 active_v_start, active_v_end;
+	u32 active_hctl, display_hctl, hsync_ctl;
+	u32 polarity_ctl, den_polarity, hsync_polarity, vsync_polarity;
+	u32 panel_format;
+	u32 intf_cfg;
+
+	/* read interface_cfg */
+	intf_cfg = DPU_REG_READ(c, INTF_CONFIG);
+	hsync_period = p->hsync_pulse_width + p->h_back_porch + p->width +
+	p->h_front_porch;
+	vsync_period = p->vsync_pulse_width + p->v_back_porch + p->height +
+	p->v_front_porch;
+
+	display_v_start = ((p->vsync_pulse_width + p->v_back_porch) *
+	hsync_period) + p->hsync_skew;
+	display_v_end = ((vsync_period - p->v_front_porch) * hsync_period) +
+	p->hsync_skew - 1;
+
+	if (ctx->cap->type == INTF_EDP || ctx->cap->type == INTF_DP) {
+		display_v_start += p->hsync_pulse_width + p->h_back_porch;
+		display_v_end -= p->h_front_porch;
+	}
+
+	hsync_start_x = p->h_back_porch + p->hsync_pulse_width;
+	hsync_end_x = hsync_period - p->h_front_porch - 1;
+
+	if (p->width != p->xres) {
+		active_h_start = hsync_start_x;
+		active_h_end = active_h_start + p->xres - 1;
+	} else {
+		active_h_start = 0;
+		active_h_end = 0;
+	}
+
+	if (p->height != p->yres) {
+		active_v_start = display_v_start;
+		active_v_end = active_v_start + (p->yres * hsync_period) - 1;
+	} else {
+		active_v_start = 0;
+		active_v_end = 0;
+	}
+
+	if (active_h_end) {
+		active_hctl = (active_h_end << 16) | active_h_start;
+		intf_cfg |= BIT(29);	/* ACTIVE_H_ENABLE */
+	} else {
+		active_hctl = 0;
+	}
+
+	if (active_v_end)
+		intf_cfg |= BIT(30); /* ACTIVE_V_ENABLE */
+
+	hsync_ctl = (hsync_period << 16) | p->hsync_pulse_width;
+	display_hctl = (hsync_end_x << 16) | hsync_start_x;
+
+	den_polarity = 0;
+	if (ctx->cap->type == INTF_HDMI) {
+		hsync_polarity = p->yres >= 720 ? 0 : 1;
+		vsync_polarity = p->yres >= 720 ? 0 : 1;
+	} else {
+		hsync_polarity = 0;
+		vsync_polarity = 0;
+	}
+	polarity_ctl = (den_polarity << 2) | /*  DEN Polarity  */
+		(vsync_polarity << 1) | /* VSYNC Polarity */
+		(hsync_polarity << 0);  /* HSYNC Polarity */
+
+	if (!DPU_FORMAT_IS_YUV(fmt))
+		panel_format = (fmt->bits[C0_G_Y] |
+				(fmt->bits[C1_B_Cb] << 2) |
+				(fmt->bits[C2_R_Cr] << 4) |
+				(0x21 << 8));
+	else
+		/* Interface treats all the pixel data in RGB888 format */
+		panel_format = (COLOR_8BIT |
+				(COLOR_8BIT << 2) |
+				(COLOR_8BIT << 4) |
+				(0x21 << 8));
+
+	DPU_REG_WRITE(c, INTF_HSYNC_CTL, hsync_ctl);
+	DPU_REG_WRITE(c, INTF_VSYNC_PERIOD_F0, vsync_period * hsync_period);
+	DPU_REG_WRITE(c, INTF_VSYNC_PULSE_WIDTH_F0,
+			p->vsync_pulse_width * hsync_period);
+	DPU_REG_WRITE(c, INTF_DISPLAY_HCTL, display_hctl);
+	DPU_REG_WRITE(c, INTF_DISPLAY_V_START_F0, display_v_start);
+	DPU_REG_WRITE(c, INTF_DISPLAY_V_END_F0, display_v_end);
+	DPU_REG_WRITE(c, INTF_ACTIVE_HCTL,  active_hctl);
+	DPU_REG_WRITE(c, INTF_ACTIVE_V_START_F0, active_v_start);
+	DPU_REG_WRITE(c, INTF_ACTIVE_V_END_F0, active_v_end);
+	DPU_REG_WRITE(c, INTF_BORDER_COLOR, p->border_clr);
+	DPU_REG_WRITE(c, INTF_UNDERFLOW_COLOR, p->underflow_clr);
+	DPU_REG_WRITE(c, INTF_HSYNC_SKEW, p->hsync_skew);
+	DPU_REG_WRITE(c, INTF_POLARITY_CTL, polarity_ctl);
+	DPU_REG_WRITE(c, INTF_FRAME_LINE_COUNT_EN, 0x3);
+	DPU_REG_WRITE(c, INTF_CONFIG, intf_cfg);
+	DPU_REG_WRITE(c, INTF_PANEL_FORMAT, panel_format);
+}
+
+static void dpu_hw_intf_enable_timing_engine(
+		struct dpu_hw_intf *intf,
+		u8 enable)
+{
+	struct dpu_hw_blk_reg_map *c = &intf->hw;
+	/* Note: Display interface select is handled in top block hw layer */
+	DPU_REG_WRITE(c, INTF_TIMING_ENGINE_EN, enable != 0);
+}
+
+static void dpu_hw_intf_setup_prg_fetch(
+		struct dpu_hw_intf *intf,
+		const struct intf_prog_fetch *fetch)
+{
+	struct dpu_hw_blk_reg_map *c = &intf->hw;
+	int fetch_enable;
+
+	/*
+	 * Fetch should always be outside the active lines. If the fetching
+	 * is programmed within active region, hardware behavior is unknown.
+	 */
+
+	fetch_enable = DPU_REG_READ(c, INTF_CONFIG);
+	if (fetch->enable) {
+		fetch_enable |= BIT(31);
+		DPU_REG_WRITE(c, INTF_PROG_FETCH_START,
+				fetch->fetch_start);
+	} else {
+		fetch_enable &= ~BIT(31);
+	}
+
+	DPU_REG_WRITE(c, INTF_CONFIG, fetch_enable);
+}
+
+static void dpu_hw_intf_get_status(
+		struct dpu_hw_intf *intf,
+		struct intf_status *s)
+{
+	struct dpu_hw_blk_reg_map *c = &intf->hw;
+
+	s->is_en = DPU_REG_READ(c, INTF_TIMING_ENGINE_EN);
+	if (s->is_en) {
+		s->frame_count = DPU_REG_READ(c, INTF_FRAME_COUNT);
+		s->line_count = DPU_REG_READ(c, INTF_LINE_COUNT);
+	} else {
+		s->line_count = 0;
+		s->frame_count = 0;
+	}
+}
+
+static void dpu_hw_intf_setup_misr(struct dpu_hw_intf *intf,
+						bool enable, u32 frame_count)
+{
+	struct dpu_hw_blk_reg_map *c = &intf->hw;
+	u32 config = 0;
+
+	DPU_REG_WRITE(c, INTF_MISR_CTRL, MISR_CTRL_STATUS_CLEAR);
+	/* clear misr data */
+	wmb();
+
+	if (enable)
+		config = (frame_count & MISR_FRAME_COUNT_MASK) |
+			MISR_CTRL_ENABLE | INTF_MISR_CTRL_FREE_RUN_MASK;
+
+	DPU_REG_WRITE(c, INTF_MISR_CTRL, config);
+}
+
+static u32 dpu_hw_intf_collect_misr(struct dpu_hw_intf *intf)
+{
+	struct dpu_hw_blk_reg_map *c = &intf->hw;
+
+	return DPU_REG_READ(c, INTF_MISR_SIGNATURE);
+}
+
+static u32 dpu_hw_intf_get_line_count(struct dpu_hw_intf *intf)
+{
+	struct dpu_hw_blk_reg_map *c;
+
+	if (!intf)
+		return 0;
+
+	c = &intf->hw;
+
+	return DPU_REG_READ(c, INTF_LINE_COUNT);
+}
+
+static void _setup_intf_ops(struct dpu_hw_intf_ops *ops,
+		unsigned long cap)
+{
+	ops->setup_timing_gen = dpu_hw_intf_setup_timing_engine;
+	ops->setup_prg_fetch  = dpu_hw_intf_setup_prg_fetch;
+	ops->get_status = dpu_hw_intf_get_status;
+	ops->enable_timing = dpu_hw_intf_enable_timing_engine;
+	ops->setup_misr = dpu_hw_intf_setup_misr;
+	ops->collect_misr = dpu_hw_intf_collect_misr;
+	ops->get_line_count = dpu_hw_intf_get_line_count;
+}
+
+static struct dpu_hw_blk_ops dpu_hw_ops = {
+	.start = NULL,
+	.stop = NULL,
+};
+
+struct dpu_hw_intf *dpu_hw_intf_init(enum dpu_intf idx,
+		void __iomem *addr,
+		struct dpu_mdss_cfg *m)
+{
+	struct dpu_hw_intf *c;
+	struct dpu_intf_cfg *cfg;
+	int rc;
+
+	c = kzalloc(sizeof(*c), GFP_KERNEL);
+	if (!c)
+		return ERR_PTR(-ENOMEM);
+
+	cfg = _intf_offset(idx, m, addr, &c->hw);
+	if (IS_ERR_OR_NULL(cfg)) {
+		kfree(c);
+		pr_err("failed to create dpu_hw_intf %d\n", idx);
+		return ERR_PTR(-EINVAL);
+	}
+
+	/*
+	 * Assign ops
+	 */
+	c->idx = idx;
+	c->cap = cfg;
+	c->mdss = m;
+	_setup_intf_ops(&c->ops, c->cap->features);
+
+	rc = dpu_hw_blk_init(&c->base, DPU_HW_BLK_INTF, idx, &dpu_hw_ops);
+	if (rc) {
+		DPU_ERROR("failed to init hw blk %d\n", rc);
+		goto blk_init_error;
+	}
+
+	return c;
+
+blk_init_error:
+	kzfree(c);
+
+	return ERR_PTR(rc);
+}
+
+void dpu_hw_intf_destroy(struct dpu_hw_intf *intf)
+{
+	if (intf)
+		dpu_hw_blk_destroy(&intf->base);
+	kfree(intf);
+}
+
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_intf.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_intf.h
new file mode 100644
index 000000000000..a79d735da68d
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_intf.h
@@ -0,0 +1,128 @@
+/* Copyright (c) 2015-2018, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef _DPU_HW_INTF_H
+#define _DPU_HW_INTF_H
+
+#include "dpu_hw_catalog.h"
+#include "dpu_hw_mdss.h"
+#include "dpu_hw_util.h"
+#include "dpu_hw_blk.h"
+
+struct dpu_hw_intf;
+
+/* intf timing settings */
+struct intf_timing_params {
+	u32 width;		/* active width */
+	u32 height;		/* active height */
+	u32 xres;		/* Display panel width */
+	u32 yres;		/* Display panel height */
+
+	u32 h_back_porch;
+	u32 h_front_porch;
+	u32 v_back_porch;
+	u32 v_front_porch;
+	u32 hsync_pulse_width;
+	u32 vsync_pulse_width;
+	u32 hsync_polarity;
+	u32 vsync_polarity;
+	u32 border_clr;
+	u32 underflow_clr;
+	u32 hsync_skew;
+};
+
+struct intf_prog_fetch {
+	u8 enable;
+	/* vsync counter for the front porch pixel line */
+	u32 fetch_start;
+};
+
+struct intf_status {
+	u8 is_en;		/* interface timing engine is enabled or not */
+	u32 frame_count;	/* frame count since timing engine enabled */
+	u32 line_count;		/* current line count including blanking */
+};
+
+/**
+ * struct dpu_hw_intf_ops : Interface to the interface Hw driver functions
+ *  Assumption is these functions will be called after clocks are enabled
+ * @ setup_timing_gen : programs the timing engine
+ * @ setup_prog_fetch : enables/disables the programmable fetch logic
+ * @ enable_timing: enable/disable timing engine
+ * @ get_status: returns if timing engine is enabled or not
+ * @ setup_misr: enables/disables MISR in HW register
+ * @ collect_misr: reads and stores MISR data from HW register
+ * @ get_line_count: reads current vertical line counter
+ */
+struct dpu_hw_intf_ops {
+	void (*setup_timing_gen)(struct dpu_hw_intf *intf,
+			const struct intf_timing_params *p,
+			const struct dpu_format *fmt);
+
+	void (*setup_prg_fetch)(struct dpu_hw_intf *intf,
+			const struct intf_prog_fetch *fetch);
+
+	void (*enable_timing)(struct dpu_hw_intf *intf,
+			u8 enable);
+
+	void (*get_status)(struct dpu_hw_intf *intf,
+			struct intf_status *status);
+
+	void (*setup_misr)(struct dpu_hw_intf *intf,
+			bool enable, u32 frame_count);
+
+	u32 (*collect_misr)(struct dpu_hw_intf *intf);
+
+	u32 (*get_line_count)(struct dpu_hw_intf *intf);
+};
+
+struct dpu_hw_intf {
+	struct dpu_hw_blk base;
+	struct dpu_hw_blk_reg_map hw;
+
+	/* intf */
+	enum dpu_intf idx;
+	const struct dpu_intf_cfg *cap;
+	const struct dpu_mdss_cfg *mdss;
+
+	/* ops */
+	struct dpu_hw_intf_ops ops;
+};
+
+/**
+ * to_dpu_hw_intf - convert base object dpu_hw_base to container
+ * @hw: Pointer to base hardware block
+ * return: Pointer to hardware block container
+ */
+static inline struct dpu_hw_intf *to_dpu_hw_intf(struct dpu_hw_blk *hw)
+{
+	return container_of(hw, struct dpu_hw_intf, base);
+}
+
+/**
+ * dpu_hw_intf_init(): Initializes the intf driver for the passed
+ * interface idx.
+ * @idx:  interface index for which driver object is required
+ * @addr: mapped register io address of MDP
+ * @m :   pointer to mdss catalog data
+ */
+struct dpu_hw_intf *dpu_hw_intf_init(enum dpu_intf idx,
+		void __iomem *addr,
+		struct dpu_mdss_cfg *m);
+
+/**
+ * dpu_hw_intf_destroy(): Destroys INTF driver context
+ * @intf:   Pointer to INTF driver context
+ */
+void dpu_hw_intf_destroy(struct dpu_hw_intf *intf);
+
+#endif /*_DPU_HW_INTF_H */
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_lm.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_lm.c
new file mode 100644
index 000000000000..4ab72b0f07a5
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_lm.c
@@ -0,0 +1,261 @@
+/* Copyright (c) 2015-2018, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include "dpu_kms.h"
+#include "dpu_hw_catalog.h"
+#include "dpu_hwio.h"
+#include "dpu_hw_lm.h"
+#include "dpu_hw_mdss.h"
+#include "dpu_dbg.h"
+#include "dpu_kms.h"
+
+#define LM_OP_MODE                        0x00
+#define LM_OUT_SIZE                       0x04
+#define LM_BORDER_COLOR_0                 0x08
+#define LM_BORDER_COLOR_1                 0x010
+
+/* These register are offset to mixer base + stage base */
+#define LM_BLEND0_OP                     0x00
+#define LM_BLEND0_CONST_ALPHA            0x04
+#define LM_FG_COLOR_FILL_COLOR_0         0x08
+#define LM_FG_COLOR_FILL_COLOR_1         0x0C
+#define LM_FG_COLOR_FILL_SIZE            0x10
+#define LM_FG_COLOR_FILL_XY              0x14
+
+#define LM_BLEND0_FG_ALPHA               0x04
+#define LM_BLEND0_BG_ALPHA               0x08
+
+#define LM_MISR_CTRL			0x310
+#define LM_MISR_SIGNATURE		0x314
+
+static struct dpu_lm_cfg *_lm_offset(enum dpu_lm mixer,
+		struct dpu_mdss_cfg *m,
+		void __iomem *addr,
+		struct dpu_hw_blk_reg_map *b)
+{
+	int i;
+
+	for (i = 0; i < m->mixer_count; i++) {
+		if (mixer == m->mixer[i].id) {
+			b->base_off = addr;
+			b->blk_off = m->mixer[i].base;
+			b->length = m->mixer[i].len;
+			b->hwversion = m->hwversion;
+			b->log_mask = DPU_DBG_MASK_LM;
+			return &m->mixer[i];
+		}
+	}
+
+	return ERR_PTR(-ENOMEM);
+}
+
+/**
+ * _stage_offset(): returns the relative offset of the blend registers
+ * for the stage to be setup
+ * @c:     mixer ctx contains the mixer to be programmed
+ * @stage: stage index to setup
+ */
+static inline int _stage_offset(struct dpu_hw_mixer *ctx, enum dpu_stage stage)
+{
+	const struct dpu_lm_sub_blks *sblk = ctx->cap->sblk;
+	int rc;
+
+	if (stage == DPU_STAGE_BASE)
+		rc = -EINVAL;
+	else if (stage <= sblk->maxblendstages)
+		rc = sblk->blendstage_base[stage - DPU_STAGE_0];
+	else
+		rc = -EINVAL;
+
+	return rc;
+}
+
+static void dpu_hw_lm_setup_out(struct dpu_hw_mixer *ctx,
+		struct dpu_hw_mixer_cfg *mixer)
+{
+	struct dpu_hw_blk_reg_map *c = &ctx->hw;
+	u32 outsize;
+	u32 op_mode;
+
+	op_mode = DPU_REG_READ(c, LM_OP_MODE);
+
+	outsize = mixer->out_height << 16 | mixer->out_width;
+	DPU_REG_WRITE(c, LM_OUT_SIZE, outsize);
+
+	/* SPLIT_LEFT_RIGHT */
+	if (mixer->right_mixer)
+		op_mode |= BIT(31);
+	else
+		op_mode &= ~BIT(31);
+	DPU_REG_WRITE(c, LM_OP_MODE, op_mode);
+}
+
+static void dpu_hw_lm_setup_border_color(struct dpu_hw_mixer *ctx,
+		struct dpu_mdss_color *color,
+		u8 border_en)
+{
+	struct dpu_hw_blk_reg_map *c = &ctx->hw;
+
+	if (border_en) {
+		DPU_REG_WRITE(c, LM_BORDER_COLOR_0,
+			(color->color_0 & 0xFFF) |
+			((color->color_1 & 0xFFF) << 0x10));
+		DPU_REG_WRITE(c, LM_BORDER_COLOR_1,
+			(color->color_2 & 0xFFF) |
+			((color->color_3 & 0xFFF) << 0x10));
+	}
+}
+
+static void dpu_hw_lm_setup_blend_config_sdm845(struct dpu_hw_mixer *ctx,
+	u32 stage, u32 fg_alpha, u32 bg_alpha, u32 blend_op)
+{
+	struct dpu_hw_blk_reg_map *c = &ctx->hw;
+	int stage_off;
+	u32 const_alpha;
+
+	if (stage == DPU_STAGE_BASE)
+		return;
+
+	stage_off = _stage_offset(ctx, stage);
+	if (WARN_ON(stage_off < 0))
+		return;
+
+	const_alpha = (bg_alpha & 0xFF) | ((fg_alpha & 0xFF) << 16);
+	DPU_REG_WRITE(c, LM_BLEND0_CONST_ALPHA + stage_off, const_alpha);
+	DPU_REG_WRITE(c, LM_BLEND0_OP + stage_off, blend_op);
+}
+
+static void dpu_hw_lm_setup_blend_config(struct dpu_hw_mixer *ctx,
+	u32 stage, u32 fg_alpha, u32 bg_alpha, u32 blend_op)
+{
+	struct dpu_hw_blk_reg_map *c = &ctx->hw;
+	int stage_off;
+
+	if (stage == DPU_STAGE_BASE)
+		return;
+
+	stage_off = _stage_offset(ctx, stage);
+	if (WARN_ON(stage_off < 0))
+		return;
+
+	DPU_REG_WRITE(c, LM_BLEND0_FG_ALPHA + stage_off, fg_alpha);
+	DPU_REG_WRITE(c, LM_BLEND0_BG_ALPHA + stage_off, bg_alpha);
+	DPU_REG_WRITE(c, LM_BLEND0_OP + stage_off, blend_op);
+}
+
+static void dpu_hw_lm_setup_color3(struct dpu_hw_mixer *ctx,
+	uint32_t mixer_op_mode)
+{
+	struct dpu_hw_blk_reg_map *c = &ctx->hw;
+	int op_mode;
+
+	/* read the existing op_mode configuration */
+	op_mode = DPU_REG_READ(c, LM_OP_MODE);
+
+	op_mode = (op_mode & (BIT(31) | BIT(30))) | mixer_op_mode;
+
+	DPU_REG_WRITE(c, LM_OP_MODE, op_mode);
+}
+
+static void dpu_hw_lm_gc(struct dpu_hw_mixer *mixer,
+			void *cfg)
+{
+}
+
+static void dpu_hw_lm_setup_misr(struct dpu_hw_mixer *ctx,
+				bool enable, u32 frame_count)
+{
+	struct dpu_hw_blk_reg_map *c = &ctx->hw;
+	u32 config = 0;
+
+	DPU_REG_WRITE(c, LM_MISR_CTRL, MISR_CTRL_STATUS_CLEAR);
+	/* clear misr data */
+	wmb();
+
+	if (enable)
+		config = (frame_count & MISR_FRAME_COUNT_MASK) |
+			MISR_CTRL_ENABLE | INTF_MISR_CTRL_FREE_RUN_MASK;
+
+	DPU_REG_WRITE(c, LM_MISR_CTRL, config);
+}
+
+static u32 dpu_hw_lm_collect_misr(struct dpu_hw_mixer *ctx)
+{
+	struct dpu_hw_blk_reg_map *c = &ctx->hw;
+
+	return DPU_REG_READ(c, LM_MISR_SIGNATURE);
+}
+
+static void _setup_mixer_ops(struct dpu_mdss_cfg *m,
+		struct dpu_hw_lm_ops *ops,
+		unsigned long features)
+{
+	ops->setup_mixer_out = dpu_hw_lm_setup_out;
+	if (IS_SDM845_TARGET(m->hwversion) || IS_SDM670_TARGET(m->hwversion))
+		ops->setup_blend_config = dpu_hw_lm_setup_blend_config_sdm845;
+	else
+		ops->setup_blend_config = dpu_hw_lm_setup_blend_config;
+	ops->setup_alpha_out = dpu_hw_lm_setup_color3;
+	ops->setup_border_color = dpu_hw_lm_setup_border_color;
+	ops->setup_gc = dpu_hw_lm_gc;
+	ops->setup_misr = dpu_hw_lm_setup_misr;
+	ops->collect_misr = dpu_hw_lm_collect_misr;
+};
+
+static struct dpu_hw_blk_ops dpu_hw_ops = {
+	.start = NULL,
+	.stop = NULL,
+};
+
+struct dpu_hw_mixer *dpu_hw_lm_init(enum dpu_lm idx,
+		void __iomem *addr,
+		struct dpu_mdss_cfg *m)
+{
+	struct dpu_hw_mixer *c;
+	struct dpu_lm_cfg *cfg;
+	int rc;
+
+	c = kzalloc(sizeof(*c), GFP_KERNEL);
+	if (!c)
+		return ERR_PTR(-ENOMEM);
+
+	cfg = _lm_offset(idx, m, addr, &c->hw);
+	if (IS_ERR_OR_NULL(cfg)) {
+		kfree(c);
+		return ERR_PTR(-EINVAL);
+	}
+
+	/* Assign ops */
+	c->idx = idx;
+	c->cap = cfg;
+	_setup_mixer_ops(m, &c->ops, c->cap->features);
+
+	rc = dpu_hw_blk_init(&c->base, DPU_HW_BLK_LM, idx, &dpu_hw_ops);
+	if (rc) {
+		DPU_ERROR("failed to init hw blk %d\n", rc);
+		goto blk_init_error;
+	}
+
+	return c;
+
+blk_init_error:
+	kzfree(c);
+
+	return ERR_PTR(rc);
+}
+
+void dpu_hw_lm_destroy(struct dpu_hw_mixer *lm)
+{
+	if (lm)
+		dpu_hw_blk_destroy(&lm->base);
+	kfree(lm);
+}
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_lm.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_lm.h
new file mode 100644
index 000000000000..e29e5dab31bf
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_lm.h
@@ -0,0 +1,122 @@
+/* Copyright (c) 2015-2018, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef _DPU_HW_LM_H
+#define _DPU_HW_LM_H
+
+#include "dpu_hw_mdss.h"
+#include "dpu_hw_util.h"
+#include "dpu_hw_blk.h"
+
+struct dpu_hw_mixer;
+
+struct dpu_hw_mixer_cfg {
+	u32 out_width;
+	u32 out_height;
+	bool right_mixer;
+	int flags;
+};
+
+struct dpu_hw_color3_cfg {
+	u8 keep_fg[DPU_STAGE_MAX];
+};
+
+/**
+ *
+ * struct dpu_hw_lm_ops : Interface to the mixer Hw driver functions
+ *  Assumption is these functions will be called after clocks are enabled
+ */
+struct dpu_hw_lm_ops {
+	/*
+	 * Sets up mixer output width and height
+	 * and border color if enabled
+	 */
+	void (*setup_mixer_out)(struct dpu_hw_mixer *ctx,
+		struct dpu_hw_mixer_cfg *cfg);
+
+	/*
+	 * Alpha blending configuration
+	 * for the specified stage
+	 */
+	void (*setup_blend_config)(struct dpu_hw_mixer *ctx, uint32_t stage,
+		uint32_t fg_alpha, uint32_t bg_alpha, uint32_t blend_op);
+
+	/*
+	 * Alpha color component selection from either fg or bg
+	 */
+	void (*setup_alpha_out)(struct dpu_hw_mixer *ctx, uint32_t mixer_op);
+
+	/**
+	 * setup_border_color : enable/disable border color
+	 */
+	void (*setup_border_color)(struct dpu_hw_mixer *ctx,
+		struct dpu_mdss_color *color,
+		u8 border_en);
+	/**
+	 * setup_gc : enable/disable gamma correction feature
+	 */
+	void (*setup_gc)(struct dpu_hw_mixer *mixer,
+			void *cfg);
+
+	/* setup_misr: enables/disables MISR in HW register */
+	void (*setup_misr)(struct dpu_hw_mixer *ctx,
+			bool enable, u32 frame_count);
+
+	/* collect_misr: reads and stores MISR data from HW register */
+	u32 (*collect_misr)(struct dpu_hw_mixer *ctx);
+};
+
+struct dpu_hw_mixer {
+	struct dpu_hw_blk base;
+	struct dpu_hw_blk_reg_map hw;
+
+	/* lm */
+	enum dpu_lm  idx;
+	const struct dpu_lm_cfg   *cap;
+	const struct dpu_mdp_cfg  *mdp;
+	const struct dpu_ctl_cfg  *ctl;
+
+	/* ops */
+	struct dpu_hw_lm_ops ops;
+
+	/* store mixer info specific to display */
+	struct dpu_hw_mixer_cfg cfg;
+};
+
+/**
+ * to_dpu_hw_mixer - convert base object dpu_hw_base to container
+ * @hw: Pointer to base hardware block
+ * return: Pointer to hardware block container
+ */
+static inline struct dpu_hw_mixer *to_dpu_hw_mixer(struct dpu_hw_blk *hw)
+{
+	return container_of(hw, struct dpu_hw_mixer, base);
+}
+
+/**
+ * dpu_hw_lm_init(): Initializes the mixer hw driver object.
+ * should be called once before accessing every mixer.
+ * @idx:  mixer index for which driver object is required
+ * @addr: mapped register io address of MDP
+ * @m :   pointer to mdss catalog data
+ */
+struct dpu_hw_mixer *dpu_hw_lm_init(enum dpu_lm idx,
+		void __iomem *addr,
+		struct dpu_mdss_cfg *m);
+
+/**
+ * dpu_hw_lm_destroy(): Destroys layer mixer driver context
+ * @lm:   Pointer to LM driver context
+ */
+void dpu_hw_lm_destroy(struct dpu_hw_mixer *lm);
+
+#endif /*_DPU_HW_LM_H */
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_mdss.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_mdss.h
new file mode 100644
index 000000000000..35e6bf930924
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_mdss.h
@@ -0,0 +1,465 @@
+/* Copyright (c) 2015-2018, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef _DPU_HW_MDSS_H
+#define _DPU_HW_MDSS_H
+
+#include <linux/kernel.h>
+#include <linux/err.h>
+
+#include "msm_drv.h"
+
+#define DPU_DBG_NAME			"dpu"
+
+#define DPU_NONE                        0
+
+#ifndef DPU_CSC_MATRIX_COEFF_SIZE
+#define DPU_CSC_MATRIX_COEFF_SIZE	9
+#endif
+
+#ifndef DPU_CSC_CLAMP_SIZE
+#define DPU_CSC_CLAMP_SIZE		6
+#endif
+
+#ifndef DPU_CSC_BIAS_SIZE
+#define DPU_CSC_BIAS_SIZE		3
+#endif
+
+#ifndef DPU_MAX_PLANES
+#define DPU_MAX_PLANES			4
+#endif
+
+#define PIPES_PER_STAGE			2
+#ifndef DPU_MAX_DE_CURVES
+#define DPU_MAX_DE_CURVES		3
+#endif
+
+enum dpu_format_flags {
+	DPU_FORMAT_FLAG_YUV_BIT,
+	DPU_FORMAT_FLAG_DX_BIT,
+	DPU_FORMAT_FLAG_COMPRESSED_BIT,
+	DPU_FORMAT_FLAG_BIT_MAX,
+};
+
+#define DPU_FORMAT_FLAG_YUV		BIT(DPU_FORMAT_FLAG_YUV_BIT)
+#define DPU_FORMAT_FLAG_DX		BIT(DPU_FORMAT_FLAG_DX_BIT)
+#define DPU_FORMAT_FLAG_COMPRESSED	BIT(DPU_FORMAT_FLAG_COMPRESSED_BIT)
+#define DPU_FORMAT_IS_YUV(X)		\
+	(test_bit(DPU_FORMAT_FLAG_YUV_BIT, (X)->flag))
+#define DPU_FORMAT_IS_DX(X)		\
+	(test_bit(DPU_FORMAT_FLAG_DX_BIT, (X)->flag))
+#define DPU_FORMAT_IS_LINEAR(X)		((X)->fetch_mode == DPU_FETCH_LINEAR)
+#define DPU_FORMAT_IS_TILE(X) \
+	(((X)->fetch_mode == DPU_FETCH_UBWC) && \
+			!test_bit(DPU_FORMAT_FLAG_COMPRESSED_BIT, (X)->flag))
+#define DPU_FORMAT_IS_UBWC(X) \
+	(((X)->fetch_mode == DPU_FETCH_UBWC) && \
+			test_bit(DPU_FORMAT_FLAG_COMPRESSED_BIT, (X)->flag))
+
+#define DPU_BLEND_FG_ALPHA_FG_CONST	(0 << 0)
+#define DPU_BLEND_FG_ALPHA_BG_CONST	(1 << 0)
+#define DPU_BLEND_FG_ALPHA_FG_PIXEL	(2 << 0)
+#define DPU_BLEND_FG_ALPHA_BG_PIXEL	(3 << 0)
+#define DPU_BLEND_FG_INV_ALPHA		(1 << 2)
+#define DPU_BLEND_FG_MOD_ALPHA		(1 << 3)
+#define DPU_BLEND_FG_INV_MOD_ALPHA	(1 << 4)
+#define DPU_BLEND_FG_TRANSP_EN		(1 << 5)
+#define DPU_BLEND_BG_ALPHA_FG_CONST	(0 << 8)
+#define DPU_BLEND_BG_ALPHA_BG_CONST	(1 << 8)
+#define DPU_BLEND_BG_ALPHA_FG_PIXEL	(2 << 8)
+#define DPU_BLEND_BG_ALPHA_BG_PIXEL	(3 << 8)
+#define DPU_BLEND_BG_INV_ALPHA		(1 << 10)
+#define DPU_BLEND_BG_MOD_ALPHA		(1 << 11)
+#define DPU_BLEND_BG_INV_MOD_ALPHA	(1 << 12)
+#define DPU_BLEND_BG_TRANSP_EN		(1 << 13)
+
+#define DPU_VSYNC0_SOURCE_GPIO		0
+#define DPU_VSYNC1_SOURCE_GPIO		1
+#define DPU_VSYNC2_SOURCE_GPIO		2
+#define DPU_VSYNC_SOURCE_INTF_0		3
+#define DPU_VSYNC_SOURCE_INTF_1		4
+#define DPU_VSYNC_SOURCE_INTF_2		5
+#define DPU_VSYNC_SOURCE_INTF_3		6
+#define DPU_VSYNC_SOURCE_WD_TIMER_4	11
+#define DPU_VSYNC_SOURCE_WD_TIMER_3	12
+#define DPU_VSYNC_SOURCE_WD_TIMER_2	13
+#define DPU_VSYNC_SOURCE_WD_TIMER_1	14
+#define DPU_VSYNC_SOURCE_WD_TIMER_0	15
+
+enum dpu_hw_blk_type {
+	DPU_HW_BLK_TOP = 0,
+	DPU_HW_BLK_SSPP,
+	DPU_HW_BLK_LM,
+	DPU_HW_BLK_CTL,
+	DPU_HW_BLK_CDM,
+	DPU_HW_BLK_PINGPONG,
+	DPU_HW_BLK_INTF,
+	DPU_HW_BLK_WB,
+	DPU_HW_BLK_MAX,
+};
+
+enum dpu_mdp {
+	MDP_TOP = 0x1,
+	MDP_MAX,
+};
+
+enum dpu_sspp {
+	SSPP_NONE,
+	SSPP_VIG0,
+	SSPP_VIG1,
+	SSPP_VIG2,
+	SSPP_VIG3,
+	SSPP_RGB0,
+	SSPP_RGB1,
+	SSPP_RGB2,
+	SSPP_RGB3,
+	SSPP_DMA0,
+	SSPP_DMA1,
+	SSPP_DMA2,
+	SSPP_DMA3,
+	SSPP_CURSOR0,
+	SSPP_CURSOR1,
+	SSPP_MAX
+};
+
+enum dpu_sspp_type {
+	SSPP_TYPE_VIG,
+	SSPP_TYPE_RGB,
+	SSPP_TYPE_DMA,
+	SSPP_TYPE_CURSOR,
+	SSPP_TYPE_MAX
+};
+
+enum dpu_lm {
+	LM_0 = 1,
+	LM_1,
+	LM_2,
+	LM_3,
+	LM_4,
+	LM_5,
+	LM_6,
+	LM_MAX
+};
+
+enum dpu_stage {
+	DPU_STAGE_BASE = 0,
+	DPU_STAGE_0,
+	DPU_STAGE_1,
+	DPU_STAGE_2,
+	DPU_STAGE_3,
+	DPU_STAGE_4,
+	DPU_STAGE_5,
+	DPU_STAGE_6,
+	DPU_STAGE_7,
+	DPU_STAGE_8,
+	DPU_STAGE_9,
+	DPU_STAGE_10,
+	DPU_STAGE_MAX
+};
+enum dpu_dspp {
+	DSPP_0 = 1,
+	DSPP_1,
+	DSPP_2,
+	DSPP_3,
+	DSPP_MAX
+};
+
+enum dpu_ds {
+	DS_TOP,
+	DS_0,
+	DS_1,
+	DS_MAX
+};
+
+enum dpu_ctl {
+	CTL_0 = 1,
+	CTL_1,
+	CTL_2,
+	CTL_3,
+	CTL_4,
+	CTL_MAX
+};
+
+enum dpu_cdm {
+	CDM_0 = 1,
+	CDM_1,
+	CDM_MAX
+};
+
+enum dpu_pingpong {
+	PINGPONG_0 = 1,
+	PINGPONG_1,
+	PINGPONG_2,
+	PINGPONG_3,
+	PINGPONG_4,
+	PINGPONG_S0,
+	PINGPONG_MAX
+};
+
+enum dpu_intf {
+	INTF_0 = 1,
+	INTF_1,
+	INTF_2,
+	INTF_3,
+	INTF_4,
+	INTF_5,
+	INTF_6,
+	INTF_MAX
+};
+
+enum dpu_intf_type {
+	INTF_NONE = 0x0,
+	INTF_DSI = 0x1,
+	INTF_HDMI = 0x3,
+	INTF_LCDC = 0x5,
+	INTF_EDP = 0x9,
+	INTF_DP = 0xa,
+	INTF_TYPE_MAX,
+
+	/* virtual interfaces */
+	INTF_WB = 0x100,
+};
+
+enum dpu_intf_mode {
+	INTF_MODE_NONE = 0,
+	INTF_MODE_CMD,
+	INTF_MODE_VIDEO,
+	INTF_MODE_WB_BLOCK,
+	INTF_MODE_WB_LINE,
+	INTF_MODE_MAX
+};
+
+enum dpu_wb {
+	WB_0 = 1,
+	WB_1,
+	WB_2,
+	WB_3,
+	WB_MAX
+};
+
+enum dpu_ad {
+	AD_0 = 0x1,
+	AD_1,
+	AD_MAX
+};
+
+enum dpu_cwb {
+	CWB_0 = 0x1,
+	CWB_1,
+	CWB_2,
+	CWB_3,
+	CWB_MAX
+};
+
+enum dpu_wd_timer {
+	WD_TIMER_0 = 0x1,
+	WD_TIMER_1,
+	WD_TIMER_2,
+	WD_TIMER_3,
+	WD_TIMER_4,
+	WD_TIMER_5,
+	WD_TIMER_MAX
+};
+
+enum dpu_vbif {
+	VBIF_0,
+	VBIF_1,
+	VBIF_MAX,
+	VBIF_RT = VBIF_0,
+	VBIF_NRT = VBIF_1
+};
+
+enum dpu_iommu_domain {
+	DPU_IOMMU_DOMAIN_UNSECURE,
+	DPU_IOMMU_DOMAIN_SECURE,
+	DPU_IOMMU_DOMAIN_MAX
+};
+
+/**
+ * DPU HW,Component order color map
+ */
+enum {
+	C0_G_Y = 0,
+	C1_B_Cb = 1,
+	C2_R_Cr = 2,
+	C3_ALPHA = 3
+};
+
+/**
+ * enum dpu_plane_type - defines how the color component pixel packing
+ * @DPU_PLANE_INTERLEAVED   : Color components in single plane
+ * @DPU_PLANE_PLANAR        : Color component in separate planes
+ * @DPU_PLANE_PSEUDO_PLANAR : Chroma components interleaved in separate plane
+ */
+enum dpu_plane_type {
+	DPU_PLANE_INTERLEAVED,
+	DPU_PLANE_PLANAR,
+	DPU_PLANE_PSEUDO_PLANAR,
+};
+
+/**
+ * enum dpu_chroma_samp_type - chroma sub-samplng type
+ * @DPU_CHROMA_RGB   : No chroma subsampling
+ * @DPU_CHROMA_H2V1  : Chroma pixels are horizontally subsampled
+ * @DPU_CHROMA_H1V2  : Chroma pixels are vertically subsampled
+ * @DPU_CHROMA_420   : 420 subsampling
+ */
+enum dpu_chroma_samp_type {
+	DPU_CHROMA_RGB,
+	DPU_CHROMA_H2V1,
+	DPU_CHROMA_H1V2,
+	DPU_CHROMA_420
+};
+
+/**
+ * dpu_fetch_type - Defines How DPU HW fetches data
+ * @DPU_FETCH_LINEAR   : fetch is line by line
+ * @DPU_FETCH_TILE     : fetches data in Z order from a tile
+ * @DPU_FETCH_UBWC     : fetch and decompress data
+ */
+enum dpu_fetch_type {
+	DPU_FETCH_LINEAR,
+	DPU_FETCH_TILE,
+	DPU_FETCH_UBWC
+};
+
+/**
+ * Value of enum chosen to fit the number of bits
+ * expected by the HW programming.
+ */
+enum {
+	COLOR_ALPHA_1BIT = 0,
+	COLOR_ALPHA_4BIT = 1,
+	COLOR_4BIT = 0,
+	COLOR_5BIT = 1, /* No 5-bit Alpha */
+	COLOR_6BIT = 2, /* 6-Bit Alpha also = 2 */
+	COLOR_8BIT = 3, /* 8-Bit Alpha also = 3 */
+};
+
+/**
+ * enum dpu_3d_blend_mode
+ * Desribes how the 3d data is blended
+ * @BLEND_3D_NONE      : 3d blending not enabled
+ * @BLEND_3D_FRAME_INT : Frame interleaving
+ * @BLEND_3D_H_ROW_INT : Horizontal row interleaving
+ * @BLEND_3D_V_ROW_INT : vertical row interleaving
+ * @BLEND_3D_COL_INT   : column interleaving
+ * @BLEND_3D_MAX       :
+ */
+enum dpu_3d_blend_mode {
+	BLEND_3D_NONE = 0,
+	BLEND_3D_FRAME_INT,
+	BLEND_3D_H_ROW_INT,
+	BLEND_3D_V_ROW_INT,
+	BLEND_3D_COL_INT,
+	BLEND_3D_MAX
+};
+
+/** struct dpu_format - defines the format configuration which
+ * allows DPU HW to correctly fetch and decode the format
+ * @base: base msm_format struture containing fourcc code
+ * @fetch_planes: how the color components are packed in pixel format
+ * @element: element color ordering
+ * @bits: element bit widths
+ * @chroma_sample: chroma sub-samplng type
+ * @unpack_align_msb: unpack aligned, 0 to LSB, 1 to MSB
+ * @unpack_tight: 0 for loose, 1 for tight
+ * @unpack_count: 0 = 1 component, 1 = 2 component
+ * @bpp: bytes per pixel
+ * @alpha_enable: whether the format has an alpha channel
+ * @num_planes: number of planes (including meta data planes)
+ * @fetch_mode: linear, tiled, or ubwc hw fetch behavior
+ * @is_yuv: is format a yuv variant
+ * @flag: usage bit flags
+ * @tile_width: format tile width
+ * @tile_height: format tile height
+ */
+struct dpu_format {
+	struct msm_format base;
+	enum dpu_plane_type fetch_planes;
+	u8 element[DPU_MAX_PLANES];
+	u8 bits[DPU_MAX_PLANES];
+	enum dpu_chroma_samp_type chroma_sample;
+	u8 unpack_align_msb;
+	u8 unpack_tight;
+	u8 unpack_count;
+	u8 bpp;
+	u8 alpha_enable;
+	u8 num_planes;
+	enum dpu_fetch_type fetch_mode;
+	DECLARE_BITMAP(flag, DPU_FORMAT_FLAG_BIT_MAX);
+	u16 tile_width;
+	u16 tile_height;
+};
+#define to_dpu_format(x) container_of(x, struct dpu_format, base)
+
+/**
+ * struct dpu_hw_fmt_layout - format information of the source pixel data
+ * @format: pixel format parameters
+ * @num_planes: number of planes (including meta data planes)
+ * @width: image width
+ * @height: image height
+ * @total_size: total size in bytes
+ * @plane_addr: address of each plane
+ * @plane_size: length of each plane
+ * @plane_pitch: pitch of each plane
+ */
+struct dpu_hw_fmt_layout {
+	const struct dpu_format *format;
+	uint32_t num_planes;
+	uint32_t width;
+	uint32_t height;
+	uint32_t total_size;
+	uint32_t plane_addr[DPU_MAX_PLANES];
+	uint32_t plane_size[DPU_MAX_PLANES];
+	uint32_t plane_pitch[DPU_MAX_PLANES];
+};
+
+struct dpu_csc_cfg {
+	/* matrix coefficients in S15.16 format */
+	uint32_t csc_mv[DPU_CSC_MATRIX_COEFF_SIZE];
+	uint32_t csc_pre_bv[DPU_CSC_BIAS_SIZE];
+	uint32_t csc_post_bv[DPU_CSC_BIAS_SIZE];
+	uint32_t csc_pre_lv[DPU_CSC_CLAMP_SIZE];
+	uint32_t csc_post_lv[DPU_CSC_CLAMP_SIZE];
+};
+
+/**
+ * struct dpu_mdss_color - mdss color description
+ * color 0 : green
+ * color 1 : blue
+ * color 2 : red
+ * color 3 : alpha
+ */
+struct dpu_mdss_color {
+	u32 color_0;
+	u32 color_1;
+	u32 color_2;
+	u32 color_3;
+};
+
+/*
+ * Define bit masks for h/w logging.
+ */
+#define DPU_DBG_MASK_NONE     (1 << 0)
+#define DPU_DBG_MASK_CDM      (1 << 1)
+#define DPU_DBG_MASK_INTF     (1 << 2)
+#define DPU_DBG_MASK_LM       (1 << 3)
+#define DPU_DBG_MASK_CTL      (1 << 4)
+#define DPU_DBG_MASK_PINGPONG (1 << 5)
+#define DPU_DBG_MASK_SSPP     (1 << 6)
+#define DPU_DBG_MASK_WB       (1 << 7)
+#define DPU_DBG_MASK_TOP      (1 << 8)
+#define DPU_DBG_MASK_VBIF     (1 << 9)
+#define DPU_DBG_MASK_ROT      (1 << 10)
+
+#endif  /* _DPU_HW_MDSS_H */
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_pingpong.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_pingpong.c
new file mode 100644
index 000000000000..cc3a623903f4
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_pingpong.c
@@ -0,0 +1,250 @@
+/* Copyright (c) 2015-2018, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/iopoll.h>
+
+#include "dpu_hw_mdss.h"
+#include "dpu_hwio.h"
+#include "dpu_hw_catalog.h"
+#include "dpu_hw_pingpong.h"
+#include "dpu_dbg.h"
+#include "dpu_kms.h"
+#include "dpu_trace.h"
+
+#define PP_TEAR_CHECK_EN                0x000
+#define PP_SYNC_CONFIG_VSYNC            0x004
+#define PP_SYNC_CONFIG_HEIGHT           0x008
+#define PP_SYNC_WRCOUNT                 0x00C
+#define PP_VSYNC_INIT_VAL               0x010
+#define PP_INT_COUNT_VAL                0x014
+#define PP_SYNC_THRESH                  0x018
+#define PP_START_POS                    0x01C
+#define PP_RD_PTR_IRQ                   0x020
+#define PP_WR_PTR_IRQ                   0x024
+#define PP_OUT_LINE_COUNT               0x028
+#define PP_LINE_COUNT                   0x02C
+
+#define PP_FBC_MODE                     0x034
+#define PP_FBC_BUDGET_CTL               0x038
+#define PP_FBC_LOSSY_MODE               0x03C
+
+static struct dpu_pingpong_cfg *_pingpong_offset(enum dpu_pingpong pp,
+		struct dpu_mdss_cfg *m,
+		void __iomem *addr,
+		struct dpu_hw_blk_reg_map *b)
+{
+	int i;
+
+	for (i = 0; i < m->pingpong_count; i++) {
+		if (pp == m->pingpong[i].id) {
+			b->base_off = addr;
+			b->blk_off = m->pingpong[i].base;
+			b->length = m->pingpong[i].len;
+			b->hwversion = m->hwversion;
+			b->log_mask = DPU_DBG_MASK_PINGPONG;
+			return &m->pingpong[i];
+		}
+	}
+
+	return ERR_PTR(-EINVAL);
+}
+
+static int dpu_hw_pp_setup_te_config(struct dpu_hw_pingpong *pp,
+		struct dpu_hw_tear_check *te)
+{
+	struct dpu_hw_blk_reg_map *c;
+	int cfg;
+
+	if (!pp || !te)
+		return -EINVAL;
+	c = &pp->hw;
+
+	cfg = BIT(19); /*VSYNC_COUNTER_EN */
+	if (te->hw_vsync_mode)
+		cfg |= BIT(20);
+
+	cfg |= te->vsync_count;
+
+	DPU_REG_WRITE(c, PP_SYNC_CONFIG_VSYNC, cfg);
+	DPU_REG_WRITE(c, PP_SYNC_CONFIG_HEIGHT, te->sync_cfg_height);
+	DPU_REG_WRITE(c, PP_VSYNC_INIT_VAL, te->vsync_init_val);
+	DPU_REG_WRITE(c, PP_RD_PTR_IRQ, te->rd_ptr_irq);
+	DPU_REG_WRITE(c, PP_START_POS, te->start_pos);
+	DPU_REG_WRITE(c, PP_SYNC_THRESH,
+			((te->sync_threshold_continue << 16) |
+			 te->sync_threshold_start));
+	DPU_REG_WRITE(c, PP_SYNC_WRCOUNT,
+			(te->start_pos + te->sync_threshold_start + 1));
+
+	return 0;
+}
+
+static int dpu_hw_pp_poll_timeout_wr_ptr(struct dpu_hw_pingpong *pp,
+		u32 timeout_us)
+{
+	struct dpu_hw_blk_reg_map *c;
+	u32 val;
+	int rc;
+
+	if (!pp)
+		return -EINVAL;
+
+	c = &pp->hw;
+	rc = readl_poll_timeout(c->base_off + c->blk_off + PP_LINE_COUNT,
+			val, (val & 0xffff) >= 1, 10, timeout_us);
+
+	return rc;
+}
+
+static int dpu_hw_pp_enable_te(struct dpu_hw_pingpong *pp, bool enable)
+{
+	struct dpu_hw_blk_reg_map *c;
+
+	if (!pp)
+		return -EINVAL;
+	c = &pp->hw;
+
+	DPU_REG_WRITE(c, PP_TEAR_CHECK_EN, enable);
+	return 0;
+}
+
+static int dpu_hw_pp_connect_external_te(struct dpu_hw_pingpong *pp,
+		bool enable_external_te)
+{
+	struct dpu_hw_blk_reg_map *c = &pp->hw;
+	u32 cfg;
+	int orig;
+
+	if (!pp)
+		return -EINVAL;
+
+	c = &pp->hw;
+	cfg = DPU_REG_READ(c, PP_SYNC_CONFIG_VSYNC);
+	orig = (bool)(cfg & BIT(20));
+	if (enable_external_te)
+		cfg |= BIT(20);
+	else
+		cfg &= ~BIT(20);
+	DPU_REG_WRITE(c, PP_SYNC_CONFIG_VSYNC, cfg);
+	trace_dpu_pp_connect_ext_te(pp->idx - PINGPONG_0, cfg);
+
+	return orig;
+}
+
+static int dpu_hw_pp_get_vsync_info(struct dpu_hw_pingpong *pp,
+		struct dpu_hw_pp_vsync_info *info)
+{
+	struct dpu_hw_blk_reg_map *c;
+	u32 val;
+
+	if (!pp || !info)
+		return -EINVAL;
+	c = &pp->hw;
+
+	val = DPU_REG_READ(c, PP_VSYNC_INIT_VAL);
+	info->rd_ptr_init_val = val & 0xffff;
+
+	val = DPU_REG_READ(c, PP_INT_COUNT_VAL);
+	info->rd_ptr_frame_count = (val & 0xffff0000) >> 16;
+	info->rd_ptr_line_count = val & 0xffff;
+
+	val = DPU_REG_READ(c, PP_LINE_COUNT);
+	info->wr_ptr_line_count = val & 0xffff;
+
+	return 0;
+}
+
+static u32 dpu_hw_pp_get_line_count(struct dpu_hw_pingpong *pp)
+{
+	struct dpu_hw_blk_reg_map *c = &pp->hw;
+	u32 height, init;
+	u32 line = 0xFFFF;
+
+	if (!pp)
+		return 0;
+	c = &pp->hw;
+
+	init = DPU_REG_READ(c, PP_VSYNC_INIT_VAL) & 0xFFFF;
+	height = DPU_REG_READ(c, PP_SYNC_CONFIG_HEIGHT) & 0xFFFF;
+
+	if (height < init)
+		goto line_count_exit;
+
+	line = DPU_REG_READ(c, PP_INT_COUNT_VAL) & 0xFFFF;
+
+	if (line < init)
+		line += (0xFFFF - init);
+	else
+		line -= init;
+
+line_count_exit:
+	return line;
+}
+
+static void _setup_pingpong_ops(struct dpu_hw_pingpong_ops *ops,
+	const struct dpu_pingpong_cfg *hw_cap)
+{
+	ops->setup_tearcheck = dpu_hw_pp_setup_te_config;
+	ops->enable_tearcheck = dpu_hw_pp_enable_te;
+	ops->connect_external_te = dpu_hw_pp_connect_external_te;
+	ops->get_vsync_info = dpu_hw_pp_get_vsync_info;
+	ops->poll_timeout_wr_ptr = dpu_hw_pp_poll_timeout_wr_ptr;
+	ops->get_line_count = dpu_hw_pp_get_line_count;
+};
+
+static struct dpu_hw_blk_ops dpu_hw_ops = {
+	.start = NULL,
+	.stop = NULL,
+};
+
+struct dpu_hw_pingpong *dpu_hw_pingpong_init(enum dpu_pingpong idx,
+		void __iomem *addr,
+		struct dpu_mdss_cfg *m)
+{
+	struct dpu_hw_pingpong *c;
+	struct dpu_pingpong_cfg *cfg;
+	int rc;
+
+	c = kzalloc(sizeof(*c), GFP_KERNEL);
+	if (!c)
+		return ERR_PTR(-ENOMEM);
+
+	cfg = _pingpong_offset(idx, m, addr, &c->hw);
+	if (IS_ERR_OR_NULL(cfg)) {
+		kfree(c);
+		return ERR_PTR(-EINVAL);
+	}
+
+	c->idx = idx;
+	c->caps = cfg;
+	_setup_pingpong_ops(&c->ops, c->caps);
+
+	rc = dpu_hw_blk_init(&c->base, DPU_HW_BLK_PINGPONG, idx, &dpu_hw_ops);
+	if (rc) {
+		DPU_ERROR("failed to init hw blk %d\n", rc);
+		goto blk_init_error;
+	}
+
+	return c;
+
+blk_init_error:
+	kzfree(c);
+
+	return ERR_PTR(rc);
+}
+
+void dpu_hw_pingpong_destroy(struct dpu_hw_pingpong *pp)
+{
+	if (pp)
+		dpu_hw_blk_destroy(&pp->base);
+	kfree(pp);
+}
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_pingpong.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_pingpong.h
new file mode 100644
index 000000000000..3caccd7d6a3e
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_pingpong.h
@@ -0,0 +1,136 @@
+/* Copyright (c) 2015-2018, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef _DPU_HW_PINGPONG_H
+#define _DPU_HW_PINGPONG_H
+
+#include "dpu_hw_catalog.h"
+#include "dpu_hw_mdss.h"
+#include "dpu_hw_util.h"
+#include "dpu_hw_blk.h"
+
+struct dpu_hw_pingpong;
+
+struct dpu_hw_tear_check {
+	/*
+	 * This is ratio of MDP VSYNC clk freq(Hz) to
+	 * refresh rate divided by no of lines
+	 */
+	u32 vsync_count;
+	u32 sync_cfg_height;
+	u32 vsync_init_val;
+	u32 sync_threshold_start;
+	u32 sync_threshold_continue;
+	u32 start_pos;
+	u32 rd_ptr_irq;
+	u8 hw_vsync_mode;
+};
+
+struct dpu_hw_pp_vsync_info {
+	u32 rd_ptr_init_val;	/* value of rd pointer at vsync edge */
+	u32 rd_ptr_frame_count;	/* num frames sent since enabling interface */
+	u32 rd_ptr_line_count;	/* current line on panel (rd ptr) */
+	u32 wr_ptr_line_count;	/* current line within pp fifo (wr ptr) */
+};
+
+/**
+ *
+ * struct dpu_hw_pingpong_ops : Interface to the pingpong Hw driver functions
+ *  Assumption is these functions will be called after clocks are enabled
+ *  @setup_tearcheck : program tear check values
+ *  @enable_tearcheck : enables tear check
+ *  @get_vsync_info : retries timing info of the panel
+ *  @setup_dither : function to program the dither hw block
+ *  @get_line_count: obtain current vertical line counter
+ */
+struct dpu_hw_pingpong_ops {
+	/**
+	 * enables vysnc generation and sets up init value of
+	 * read pointer and programs the tear check cofiguration
+	 */
+	int (*setup_tearcheck)(struct dpu_hw_pingpong *pp,
+			struct dpu_hw_tear_check *cfg);
+
+	/**
+	 * enables tear check block
+	 */
+	int (*enable_tearcheck)(struct dpu_hw_pingpong *pp,
+			bool enable);
+
+	/**
+	 * read, modify, write to either set or clear listening to external TE
+	 * @Return: 1 if TE was originally connected, 0 if not, or -ERROR
+	 */
+	int (*connect_external_te)(struct dpu_hw_pingpong *pp,
+			bool enable_external_te);
+
+	/**
+	 * provides the programmed and current
+	 * line_count
+	 */
+	int (*get_vsync_info)(struct dpu_hw_pingpong *pp,
+			struct dpu_hw_pp_vsync_info  *info);
+
+	/**
+	 * poll until write pointer transmission starts
+	 * @Return: 0 on success, -ETIMEDOUT on timeout
+	 */
+	int (*poll_timeout_wr_ptr)(struct dpu_hw_pingpong *pp, u32 timeout_us);
+
+	/**
+	 * Obtain current vertical line counter
+	 */
+	u32 (*get_line_count)(struct dpu_hw_pingpong *pp);
+};
+
+struct dpu_hw_pingpong {
+	struct dpu_hw_blk base;
+	struct dpu_hw_blk_reg_map hw;
+
+	/* pingpong */
+	enum dpu_pingpong idx;
+	const struct dpu_pingpong_cfg *caps;
+
+	/* ops */
+	struct dpu_hw_pingpong_ops ops;
+};
+
+/**
+ * dpu_hw_pingpong - convert base object dpu_hw_base to container
+ * @hw: Pointer to base hardware block
+ * return: Pointer to hardware block container
+ */
+static inline struct dpu_hw_pingpong *to_dpu_hw_pingpong(struct dpu_hw_blk *hw)
+{
+	return container_of(hw, struct dpu_hw_pingpong, base);
+}
+
+/**
+ * dpu_hw_pingpong_init - initializes the pingpong driver for the passed
+ *	pingpong idx.
+ * @idx:  Pingpong index for which driver object is required
+ * @addr: Mapped register io address of MDP
+ * @m:    Pointer to mdss catalog data
+ * Returns: Error code or allocated dpu_hw_pingpong context
+ */
+struct dpu_hw_pingpong *dpu_hw_pingpong_init(enum dpu_pingpong idx,
+		void __iomem *addr,
+		struct dpu_mdss_cfg *m);
+
+/**
+ * dpu_hw_pingpong_destroy - destroys pingpong driver context
+ *	should be called to free the context
+ * @pp:   Pointer to PP driver context returned by dpu_hw_pingpong_init
+ */
+void dpu_hw_pingpong_destroy(struct dpu_hw_pingpong *pp);
+
+#endif /*_DPU_HW_PINGPONG_H */
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.c
new file mode 100644
index 000000000000..c25b52a6b219
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.c
@@ -0,0 +1,753 @@
+/* Copyright (c) 2015-2018, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include "dpu_hwio.h"
+#include "dpu_hw_catalog.h"
+#include "dpu_hw_lm.h"
+#include "dpu_hw_sspp.h"
+#include "dpu_dbg.h"
+#include "dpu_kms.h"
+
+#define DPU_FETCH_CONFIG_RESET_VALUE   0x00000087
+
+/* DPU_SSPP_SRC */
+#define SSPP_SRC_SIZE                      0x00
+#define SSPP_SRC_XY                        0x08
+#define SSPP_OUT_SIZE                      0x0c
+#define SSPP_OUT_XY                        0x10
+#define SSPP_SRC0_ADDR                     0x14
+#define SSPP_SRC1_ADDR                     0x18
+#define SSPP_SRC2_ADDR                     0x1C
+#define SSPP_SRC3_ADDR                     0x20
+#define SSPP_SRC_YSTRIDE0                  0x24
+#define SSPP_SRC_YSTRIDE1                  0x28
+#define SSPP_SRC_FORMAT                    0x30
+#define SSPP_SRC_UNPACK_PATTERN            0x34
+#define SSPP_SRC_OP_MODE                   0x38
+
+/* SSPP_MULTIRECT*/
+#define SSPP_SRC_SIZE_REC1                 0x16C
+#define SSPP_SRC_XY_REC1                   0x168
+#define SSPP_OUT_SIZE_REC1                 0x160
+#define SSPP_OUT_XY_REC1                   0x164
+#define SSPP_SRC_FORMAT_REC1               0x174
+#define SSPP_SRC_UNPACK_PATTERN_REC1       0x178
+#define SSPP_SRC_OP_MODE_REC1              0x17C
+#define SSPP_MULTIRECT_OPMODE              0x170
+#define SSPP_SRC_CONSTANT_COLOR_REC1       0x180
+#define SSPP_EXCL_REC_SIZE_REC1            0x184
+#define SSPP_EXCL_REC_XY_REC1              0x188
+
+#define MDSS_MDP_OP_DEINTERLACE            BIT(22)
+#define MDSS_MDP_OP_DEINTERLACE_ODD        BIT(23)
+#define MDSS_MDP_OP_IGC_ROM_1              BIT(18)
+#define MDSS_MDP_OP_IGC_ROM_0              BIT(17)
+#define MDSS_MDP_OP_IGC_EN                 BIT(16)
+#define MDSS_MDP_OP_FLIP_UD                BIT(14)
+#define MDSS_MDP_OP_FLIP_LR                BIT(13)
+#define MDSS_MDP_OP_BWC_EN                 BIT(0)
+#define MDSS_MDP_OP_PE_OVERRIDE            BIT(31)
+#define MDSS_MDP_OP_BWC_LOSSLESS           (0 << 1)
+#define MDSS_MDP_OP_BWC_Q_HIGH             (1 << 1)
+#define MDSS_MDP_OP_BWC_Q_MED              (2 << 1)
+
+#define SSPP_SRC_CONSTANT_COLOR            0x3c
+#define SSPP_EXCL_REC_CTL                  0x40
+#define SSPP_UBWC_STATIC_CTRL              0x44
+#define SSPP_FETCH_CONFIG                  0x048
+#define SSPP_DANGER_LUT                    0x60
+#define SSPP_SAFE_LUT                      0x64
+#define SSPP_CREQ_LUT                      0x68
+#define SSPP_QOS_CTRL                      0x6C
+#define SSPP_DECIMATION_CONFIG             0xB4
+#define SSPP_SRC_ADDR_SW_STATUS            0x70
+#define SSPP_CREQ_LUT_0                    0x74
+#define SSPP_CREQ_LUT_1                    0x78
+#define SSPP_SW_PIX_EXT_C0_LR              0x100
+#define SSPP_SW_PIX_EXT_C0_TB              0x104
+#define SSPP_SW_PIX_EXT_C0_REQ_PIXELS      0x108
+#define SSPP_SW_PIX_EXT_C1C2_LR            0x110
+#define SSPP_SW_PIX_EXT_C1C2_TB            0x114
+#define SSPP_SW_PIX_EXT_C1C2_REQ_PIXELS    0x118
+#define SSPP_SW_PIX_EXT_C3_LR              0x120
+#define SSPP_SW_PIX_EXT_C3_TB              0x124
+#define SSPP_SW_PIX_EXT_C3_REQ_PIXELS      0x128
+#define SSPP_TRAFFIC_SHAPER                0x130
+#define SSPP_CDP_CNTL                      0x134
+#define SSPP_UBWC_ERROR_STATUS             0x138
+#define SSPP_TRAFFIC_SHAPER_PREFILL        0x150
+#define SSPP_TRAFFIC_SHAPER_REC1_PREFILL   0x154
+#define SSPP_TRAFFIC_SHAPER_REC1           0x158
+#define SSPP_EXCL_REC_SIZE                 0x1B4
+#define SSPP_EXCL_REC_XY                   0x1B8
+#define SSPP_VIG_OP_MODE                   0x0
+#define SSPP_VIG_CSC_10_OP_MODE            0x0
+#define SSPP_TRAFFIC_SHAPER_BPC_MAX        0xFF
+
+/* SSPP_QOS_CTRL */
+#define SSPP_QOS_CTRL_VBLANK_EN            BIT(16)
+#define SSPP_QOS_CTRL_DANGER_SAFE_EN       BIT(0)
+#define SSPP_QOS_CTRL_DANGER_VBLANK_MASK   0x3
+#define SSPP_QOS_CTRL_DANGER_VBLANK_OFF    4
+#define SSPP_QOS_CTRL_CREQ_VBLANK_MASK     0x3
+#define SSPP_QOS_CTRL_CREQ_VBLANK_OFF      20
+
+/* DPU_SSPP_SCALER_QSEED2 */
+#define SCALE_CONFIG                       0x04
+#define COMP0_3_PHASE_STEP_X               0x10
+#define COMP0_3_PHASE_STEP_Y               0x14
+#define COMP1_2_PHASE_STEP_X               0x18
+#define COMP1_2_PHASE_STEP_Y               0x1c
+#define COMP0_3_INIT_PHASE_X               0x20
+#define COMP0_3_INIT_PHASE_Y               0x24
+#define COMP1_2_INIT_PHASE_X               0x28
+#define COMP1_2_INIT_PHASE_Y               0x2C
+#define VIG_0_QSEED2_SHARP                 0x30
+
+/*
+ * Definitions for ViG op modes
+ */
+#define VIG_OP_CSC_DST_DATAFMT BIT(19)
+#define VIG_OP_CSC_SRC_DATAFMT BIT(18)
+#define VIG_OP_CSC_EN          BIT(17)
+#define VIG_OP_MEM_PROT_CONT   BIT(15)
+#define VIG_OP_MEM_PROT_VAL    BIT(14)
+#define VIG_OP_MEM_PROT_SAT    BIT(13)
+#define VIG_OP_MEM_PROT_HUE    BIT(12)
+#define VIG_OP_HIST            BIT(8)
+#define VIG_OP_SKY_COL         BIT(7)
+#define VIG_OP_FOIL            BIT(6)
+#define VIG_OP_SKIN_COL        BIT(5)
+#define VIG_OP_PA_EN           BIT(4)
+#define VIG_OP_PA_SAT_ZERO_EXP BIT(2)
+#define VIG_OP_MEM_PROT_BLEND  BIT(1)
+
+/*
+ * Definitions for CSC 10 op modes
+ */
+#define VIG_CSC_10_SRC_DATAFMT BIT(1)
+#define VIG_CSC_10_EN          BIT(0)
+#define CSC_10BIT_OFFSET       4
+
+/* traffic shaper clock in Hz */
+#define TS_CLK			19200000
+
+static inline int _sspp_subblk_offset(struct dpu_hw_pipe *ctx,
+		int s_id,
+		u32 *idx)
+{
+	int rc = 0;
+	const struct dpu_sspp_sub_blks *sblk = ctx->cap->sblk;
+
+	if (!ctx)
+		return -EINVAL;
+
+	switch (s_id) {
+	case DPU_SSPP_SRC:
+		*idx = sblk->src_blk.base;
+		break;
+	case DPU_SSPP_SCALER_QSEED2:
+	case DPU_SSPP_SCALER_QSEED3:
+	case DPU_SSPP_SCALER_RGB:
+		*idx = sblk->scaler_blk.base;
+		break;
+	case DPU_SSPP_CSC:
+	case DPU_SSPP_CSC_10BIT:
+		*idx = sblk->csc_blk.base;
+		break;
+	default:
+		rc = -EINVAL;
+	}
+
+	return rc;
+}
+
+static void dpu_hw_sspp_setup_multirect(struct dpu_hw_pipe *ctx,
+		enum dpu_sspp_multirect_index index,
+		enum dpu_sspp_multirect_mode mode)
+{
+	u32 mode_mask;
+	u32 idx;
+
+	if (_sspp_subblk_offset(ctx, DPU_SSPP_SRC, &idx))
+		return;
+
+	if (index == DPU_SSPP_RECT_SOLO) {
+		/**
+		 * if rect index is RECT_SOLO, we cannot expect a
+		 * virtual plane sharing the same SSPP id. So we go
+		 * and disable multirect
+		 */
+		mode_mask = 0;
+	} else {
+		mode_mask = DPU_REG_READ(&ctx->hw, SSPP_MULTIRECT_OPMODE + idx);
+		mode_mask |= index;
+		if (mode == DPU_SSPP_MULTIRECT_TIME_MX)
+			mode_mask |= BIT(2);
+		else
+			mode_mask &= ~BIT(2);
+	}
+
+	DPU_REG_WRITE(&ctx->hw, SSPP_MULTIRECT_OPMODE + idx, mode_mask);
+}
+
+static void _sspp_setup_opmode(struct dpu_hw_pipe *ctx,
+		u32 mask, u8 en)
+{
+	u32 idx;
+	u32 opmode;
+
+	if (!test_bit(DPU_SSPP_SCALER_QSEED2, &ctx->cap->features) ||
+		_sspp_subblk_offset(ctx, DPU_SSPP_SCALER_QSEED2, &idx) ||
+		!test_bit(DPU_SSPP_CSC, &ctx->cap->features))
+		return;
+
+	opmode = DPU_REG_READ(&ctx->hw, SSPP_VIG_OP_MODE + idx);
+
+	if (en)
+		opmode |= mask;
+	else
+		opmode &= ~mask;
+
+	DPU_REG_WRITE(&ctx->hw, SSPP_VIG_OP_MODE + idx, opmode);
+}
+
+static void _sspp_setup_csc10_opmode(struct dpu_hw_pipe *ctx,
+		u32 mask, u8 en)
+{
+	u32 idx;
+	u32 opmode;
+
+	if (_sspp_subblk_offset(ctx, DPU_SSPP_CSC_10BIT, &idx))
+		return;
+
+	opmode = DPU_REG_READ(&ctx->hw, SSPP_VIG_CSC_10_OP_MODE + idx);
+	if (en)
+		opmode |= mask;
+	else
+		opmode &= ~mask;
+
+	DPU_REG_WRITE(&ctx->hw, SSPP_VIG_CSC_10_OP_MODE + idx, opmode);
+}
+
+/**
+ * Setup source pixel format, flip,
+ */
+static void dpu_hw_sspp_setup_format(struct dpu_hw_pipe *ctx,
+		const struct dpu_format *fmt, u32 flags,
+		enum dpu_sspp_multirect_index rect_mode)
+{
+	struct dpu_hw_blk_reg_map *c;
+	u32 chroma_samp, unpack, src_format;
+	u32 opmode = 0;
+	u32 fast_clear = 0;
+	u32 op_mode_off, unpack_pat_off, format_off;
+	u32 idx;
+
+	if (_sspp_subblk_offset(ctx, DPU_SSPP_SRC, &idx) || !fmt)
+		return;
+
+	if (rect_mode == DPU_SSPP_RECT_SOLO || rect_mode == DPU_SSPP_RECT_0) {
+		op_mode_off = SSPP_SRC_OP_MODE;
+		unpack_pat_off = SSPP_SRC_UNPACK_PATTERN;
+		format_off = SSPP_SRC_FORMAT;
+	} else {
+		op_mode_off = SSPP_SRC_OP_MODE_REC1;
+		unpack_pat_off = SSPP_SRC_UNPACK_PATTERN_REC1;
+		format_off = SSPP_SRC_FORMAT_REC1;
+	}
+
+	c = &ctx->hw;
+	opmode = DPU_REG_READ(c, op_mode_off + idx);
+	opmode &= ~(MDSS_MDP_OP_FLIP_LR | MDSS_MDP_OP_FLIP_UD |
+			MDSS_MDP_OP_BWC_EN | MDSS_MDP_OP_PE_OVERRIDE);
+
+	if (flags & DPU_SSPP_FLIP_LR)
+		opmode |= MDSS_MDP_OP_FLIP_LR;
+	if (flags & DPU_SSPP_FLIP_UD)
+		opmode |= MDSS_MDP_OP_FLIP_UD;
+
+	chroma_samp = fmt->chroma_sample;
+	if (flags & DPU_SSPP_SOURCE_ROTATED_90) {
+		if (chroma_samp == DPU_CHROMA_H2V1)
+			chroma_samp = DPU_CHROMA_H1V2;
+		else if (chroma_samp == DPU_CHROMA_H1V2)
+			chroma_samp = DPU_CHROMA_H2V1;
+	}
+
+	src_format = (chroma_samp << 23) | (fmt->fetch_planes << 19) |
+		(fmt->bits[C3_ALPHA] << 6) | (fmt->bits[C2_R_Cr] << 4) |
+		(fmt->bits[C1_B_Cb] << 2) | (fmt->bits[C0_G_Y] << 0);
+
+	if (flags & DPU_SSPP_ROT_90)
+		src_format |= BIT(11); /* ROT90 */
+
+	if (fmt->alpha_enable && fmt->fetch_planes == DPU_PLANE_INTERLEAVED)
+		src_format |= BIT(8); /* SRCC3_EN */
+
+	if (flags & DPU_SSPP_SOLID_FILL)
+		src_format |= BIT(22);
+
+	unpack = (fmt->element[3] << 24) | (fmt->element[2] << 16) |
+		(fmt->element[1] << 8) | (fmt->element[0] << 0);
+	src_format |= ((fmt->unpack_count - 1) << 12) |
+		(fmt->unpack_tight << 17) |
+		(fmt->unpack_align_msb << 18) |
+		((fmt->bpp - 1) << 9);
+
+	if (fmt->fetch_mode != DPU_FETCH_LINEAR) {
+		if (DPU_FORMAT_IS_UBWC(fmt))
+			opmode |= MDSS_MDP_OP_BWC_EN;
+		src_format |= (fmt->fetch_mode & 3) << 30; /*FRAME_FORMAT */
+		DPU_REG_WRITE(c, SSPP_FETCH_CONFIG,
+			DPU_FETCH_CONFIG_RESET_VALUE |
+			ctx->mdp->highest_bank_bit << 18);
+		if (IS_UBWC_20_SUPPORTED(ctx->catalog->caps->ubwc_version)) {
+			fast_clear = fmt->alpha_enable ? BIT(31) : 0;
+			DPU_REG_WRITE(c, SSPP_UBWC_STATIC_CTRL,
+					fast_clear | (ctx->mdp->ubwc_swizzle) |
+					(ctx->mdp->highest_bank_bit << 4));
+		}
+	}
+
+	opmode |= MDSS_MDP_OP_PE_OVERRIDE;
+
+	/* if this is YUV pixel format, enable CSC */
+	if (DPU_FORMAT_IS_YUV(fmt))
+		src_format |= BIT(15);
+
+	if (DPU_FORMAT_IS_DX(fmt))
+		src_format |= BIT(14);
+
+	/* update scaler opmode, if appropriate */
+	if (test_bit(DPU_SSPP_CSC, &ctx->cap->features))
+		_sspp_setup_opmode(ctx, VIG_OP_CSC_EN | VIG_OP_CSC_SRC_DATAFMT,
+			DPU_FORMAT_IS_YUV(fmt));
+	else if (test_bit(DPU_SSPP_CSC_10BIT, &ctx->cap->features))
+		_sspp_setup_csc10_opmode(ctx,
+			VIG_CSC_10_EN | VIG_CSC_10_SRC_DATAFMT,
+			DPU_FORMAT_IS_YUV(fmt));
+
+	DPU_REG_WRITE(c, format_off + idx, src_format);
+	DPU_REG_WRITE(c, unpack_pat_off + idx, unpack);
+	DPU_REG_WRITE(c, op_mode_off + idx, opmode);
+
+	/* clear previous UBWC error */
+	DPU_REG_WRITE(c, SSPP_UBWC_ERROR_STATUS + idx, BIT(31));
+}
+
+static void dpu_hw_sspp_setup_pe_config(struct dpu_hw_pipe *ctx,
+		struct dpu_hw_pixel_ext *pe_ext)
+{
+	struct dpu_hw_blk_reg_map *c;
+	u8 color;
+	u32 lr_pe[4], tb_pe[4], tot_req_pixels[4];
+	const u32 bytemask = 0xff;
+	const u32 shortmask = 0xffff;
+	u32 idx;
+
+	if (_sspp_subblk_offset(ctx, DPU_SSPP_SRC, &idx) || !pe_ext)
+		return;
+
+	c = &ctx->hw;
+
+	/* program SW pixel extension override for all pipes*/
+	for (color = 0; color < DPU_MAX_PLANES; color++) {
+		/* color 2 has the same set of registers as color 1 */
+		if (color == 2)
+			continue;
+
+		lr_pe[color] = ((pe_ext->right_ftch[color] & bytemask) << 24)|
+			((pe_ext->right_rpt[color] & bytemask) << 16)|
+			((pe_ext->left_ftch[color] & bytemask) << 8)|
+			(pe_ext->left_rpt[color] & bytemask);
+
+		tb_pe[color] = ((pe_ext->btm_ftch[color] & bytemask) << 24)|
+			((pe_ext->btm_rpt[color] & bytemask) << 16)|
+			((pe_ext->top_ftch[color] & bytemask) << 8)|
+			(pe_ext->top_rpt[color] & bytemask);
+
+		tot_req_pixels[color] = (((pe_ext->roi_h[color] +
+			pe_ext->num_ext_pxls_top[color] +
+			pe_ext->num_ext_pxls_btm[color]) & shortmask) << 16) |
+			((pe_ext->roi_w[color] +
+			pe_ext->num_ext_pxls_left[color] +
+			pe_ext->num_ext_pxls_right[color]) & shortmask);
+	}
+
+	/* color 0 */
+	DPU_REG_WRITE(c, SSPP_SW_PIX_EXT_C0_LR + idx, lr_pe[0]);
+	DPU_REG_WRITE(c, SSPP_SW_PIX_EXT_C0_TB + idx, tb_pe[0]);
+	DPU_REG_WRITE(c, SSPP_SW_PIX_EXT_C0_REQ_PIXELS + idx,
+			tot_req_pixels[0]);
+
+	/* color 1 and color 2 */
+	DPU_REG_WRITE(c, SSPP_SW_PIX_EXT_C1C2_LR + idx, lr_pe[1]);
+	DPU_REG_WRITE(c, SSPP_SW_PIX_EXT_C1C2_TB + idx, tb_pe[1]);
+	DPU_REG_WRITE(c, SSPP_SW_PIX_EXT_C1C2_REQ_PIXELS + idx,
+			tot_req_pixels[1]);
+
+	/* color 3 */
+	DPU_REG_WRITE(c, SSPP_SW_PIX_EXT_C3_LR + idx, lr_pe[3]);
+	DPU_REG_WRITE(c, SSPP_SW_PIX_EXT_C3_TB + idx, lr_pe[3]);
+	DPU_REG_WRITE(c, SSPP_SW_PIX_EXT_C3_REQ_PIXELS + idx,
+			tot_req_pixels[3]);
+}
+
+static void _dpu_hw_sspp_setup_scaler3(struct dpu_hw_pipe *ctx,
+		struct dpu_hw_pipe_cfg *sspp,
+		struct dpu_hw_pixel_ext *pe,
+		void *scaler_cfg)
+{
+	u32 idx;
+	struct dpu_hw_scaler3_cfg *scaler3_cfg = scaler_cfg;
+
+	(void)pe;
+	if (_sspp_subblk_offset(ctx, DPU_SSPP_SCALER_QSEED3, &idx) || !sspp
+		|| !scaler3_cfg || !ctx || !ctx->cap || !ctx->cap->sblk)
+		return;
+
+	dpu_hw_setup_scaler3(&ctx->hw, scaler3_cfg, idx,
+			ctx->cap->sblk->scaler_blk.version,
+			sspp->layout.format);
+}
+
+static u32 _dpu_hw_sspp_get_scaler3_ver(struct dpu_hw_pipe *ctx)
+{
+	u32 idx;
+
+	if (!ctx || _sspp_subblk_offset(ctx, DPU_SSPP_SCALER_QSEED3, &idx))
+		return 0;
+
+	return dpu_hw_get_scaler3_ver(&ctx->hw, idx);
+}
+
+/**
+ * dpu_hw_sspp_setup_rects()
+ */
+static void dpu_hw_sspp_setup_rects(struct dpu_hw_pipe *ctx,
+		struct dpu_hw_pipe_cfg *cfg,
+		enum dpu_sspp_multirect_index rect_index)
+{
+	struct dpu_hw_blk_reg_map *c;
+	u32 src_size, src_xy, dst_size, dst_xy, ystride0, ystride1;
+	u32 src_size_off, src_xy_off, out_size_off, out_xy_off;
+	u32 idx;
+
+	if (_sspp_subblk_offset(ctx, DPU_SSPP_SRC, &idx) || !cfg)
+		return;
+
+	c = &ctx->hw;
+
+	if (rect_index == DPU_SSPP_RECT_SOLO || rect_index == DPU_SSPP_RECT_0) {
+		src_size_off = SSPP_SRC_SIZE;
+		src_xy_off = SSPP_SRC_XY;
+		out_size_off = SSPP_OUT_SIZE;
+		out_xy_off = SSPP_OUT_XY;
+	} else {
+		src_size_off = SSPP_SRC_SIZE_REC1;
+		src_xy_off = SSPP_SRC_XY_REC1;
+		out_size_off = SSPP_OUT_SIZE_REC1;
+		out_xy_off = SSPP_OUT_XY_REC1;
+	}
+
+
+	/* src and dest rect programming */
+	src_xy = (cfg->src_rect.y1 << 16) | cfg->src_rect.x1;
+	src_size = (drm_rect_height(&cfg->src_rect) << 16) |
+		   drm_rect_width(&cfg->src_rect);
+	dst_xy = (cfg->dst_rect.y1 << 16) | cfg->dst_rect.x1;
+	dst_size = (drm_rect_height(&cfg->dst_rect) << 16) |
+		drm_rect_width(&cfg->dst_rect);
+
+	if (rect_index == DPU_SSPP_RECT_SOLO) {
+		ystride0 = (cfg->layout.plane_pitch[0]) |
+			(cfg->layout.plane_pitch[1] << 16);
+		ystride1 = (cfg->layout.plane_pitch[2]) |
+			(cfg->layout.plane_pitch[3] << 16);
+	} else {
+		ystride0 = DPU_REG_READ(c, SSPP_SRC_YSTRIDE0 + idx);
+		ystride1 = DPU_REG_READ(c, SSPP_SRC_YSTRIDE1 + idx);
+
+		if (rect_index == DPU_SSPP_RECT_0) {
+			ystride0 = (ystride0 & 0xFFFF0000) |
+				(cfg->layout.plane_pitch[0] & 0x0000FFFF);
+			ystride1 = (ystride1 & 0xFFFF0000)|
+				(cfg->layout.plane_pitch[2] & 0x0000FFFF);
+		} else {
+			ystride0 = (ystride0 & 0x0000FFFF) |
+				((cfg->layout.plane_pitch[0] << 16) &
+				 0xFFFF0000);
+			ystride1 = (ystride1 & 0x0000FFFF) |
+				((cfg->layout.plane_pitch[2] << 16) &
+				 0xFFFF0000);
+		}
+	}
+
+	/* rectangle register programming */
+	DPU_REG_WRITE(c, src_size_off + idx, src_size);
+	DPU_REG_WRITE(c, src_xy_off + idx, src_xy);
+	DPU_REG_WRITE(c, out_size_off + idx, dst_size);
+	DPU_REG_WRITE(c, out_xy_off + idx, dst_xy);
+
+	DPU_REG_WRITE(c, SSPP_SRC_YSTRIDE0 + idx, ystride0);
+	DPU_REG_WRITE(c, SSPP_SRC_YSTRIDE1 + idx, ystride1);
+}
+
+static void dpu_hw_sspp_setup_sourceaddress(struct dpu_hw_pipe *ctx,
+		struct dpu_hw_pipe_cfg *cfg,
+		enum dpu_sspp_multirect_index rect_mode)
+{
+	int i;
+	u32 idx;
+
+	if (_sspp_subblk_offset(ctx, DPU_SSPP_SRC, &idx))
+		return;
+
+	if (rect_mode == DPU_SSPP_RECT_SOLO) {
+		for (i = 0; i < ARRAY_SIZE(cfg->layout.plane_addr); i++)
+			DPU_REG_WRITE(&ctx->hw, SSPP_SRC0_ADDR + idx + i * 0x4,
+					cfg->layout.plane_addr[i]);
+	} else if (rect_mode == DPU_SSPP_RECT_0) {
+		DPU_REG_WRITE(&ctx->hw, SSPP_SRC0_ADDR + idx,
+				cfg->layout.plane_addr[0]);
+		DPU_REG_WRITE(&ctx->hw, SSPP_SRC2_ADDR + idx,
+				cfg->layout.plane_addr[2]);
+	} else {
+		DPU_REG_WRITE(&ctx->hw, SSPP_SRC1_ADDR + idx,
+				cfg->layout.plane_addr[0]);
+		DPU_REG_WRITE(&ctx->hw, SSPP_SRC3_ADDR + idx,
+				cfg->layout.plane_addr[2]);
+	}
+}
+
+static void dpu_hw_sspp_setup_csc(struct dpu_hw_pipe *ctx,
+		struct dpu_csc_cfg *data)
+{
+	u32 idx;
+	bool csc10 = false;
+
+	if (_sspp_subblk_offset(ctx, DPU_SSPP_CSC, &idx) || !data)
+		return;
+
+	if (test_bit(DPU_SSPP_CSC_10BIT, &ctx->cap->features)) {
+		idx += CSC_10BIT_OFFSET;
+		csc10 = true;
+	}
+
+	dpu_hw_csc_setup(&ctx->hw, idx, data, csc10);
+}
+
+static void dpu_hw_sspp_setup_solidfill(struct dpu_hw_pipe *ctx, u32 color, enum
+		dpu_sspp_multirect_index rect_index)
+{
+	u32 idx;
+
+	if (_sspp_subblk_offset(ctx, DPU_SSPP_SRC, &idx))
+		return;
+
+	if (rect_index == DPU_SSPP_RECT_SOLO || rect_index == DPU_SSPP_RECT_0)
+		DPU_REG_WRITE(&ctx->hw, SSPP_SRC_CONSTANT_COLOR + idx, color);
+	else
+		DPU_REG_WRITE(&ctx->hw, SSPP_SRC_CONSTANT_COLOR_REC1 + idx,
+				color);
+}
+
+static void dpu_hw_sspp_setup_danger_safe_lut(struct dpu_hw_pipe *ctx,
+		struct dpu_hw_pipe_qos_cfg *cfg)
+{
+	u32 idx;
+
+	if (_sspp_subblk_offset(ctx, DPU_SSPP_SRC, &idx))
+		return;
+
+	DPU_REG_WRITE(&ctx->hw, SSPP_DANGER_LUT + idx, cfg->danger_lut);
+	DPU_REG_WRITE(&ctx->hw, SSPP_SAFE_LUT + idx, cfg->safe_lut);
+}
+
+static void dpu_hw_sspp_setup_creq_lut(struct dpu_hw_pipe *ctx,
+		struct dpu_hw_pipe_qos_cfg *cfg)
+{
+	u32 idx;
+
+	if (_sspp_subblk_offset(ctx, DPU_SSPP_SRC, &idx))
+		return;
+
+	if (ctx->cap && test_bit(DPU_SSPP_QOS_8LVL, &ctx->cap->features)) {
+		DPU_REG_WRITE(&ctx->hw, SSPP_CREQ_LUT_0 + idx, cfg->creq_lut);
+		DPU_REG_WRITE(&ctx->hw, SSPP_CREQ_LUT_1 + idx,
+				cfg->creq_lut >> 32);
+	} else {
+		DPU_REG_WRITE(&ctx->hw, SSPP_CREQ_LUT + idx, cfg->creq_lut);
+	}
+}
+
+static void dpu_hw_sspp_setup_qos_ctrl(struct dpu_hw_pipe *ctx,
+		struct dpu_hw_pipe_qos_cfg *cfg)
+{
+	u32 idx;
+	u32 qos_ctrl = 0;
+
+	if (_sspp_subblk_offset(ctx, DPU_SSPP_SRC, &idx))
+		return;
+
+	if (cfg->vblank_en) {
+		qos_ctrl |= ((cfg->creq_vblank &
+				SSPP_QOS_CTRL_CREQ_VBLANK_MASK) <<
+				SSPP_QOS_CTRL_CREQ_VBLANK_OFF);
+		qos_ctrl |= ((cfg->danger_vblank &
+				SSPP_QOS_CTRL_DANGER_VBLANK_MASK) <<
+				SSPP_QOS_CTRL_DANGER_VBLANK_OFF);
+		qos_ctrl |= SSPP_QOS_CTRL_VBLANK_EN;
+	}
+
+	if (cfg->danger_safe_en)
+		qos_ctrl |= SSPP_QOS_CTRL_DANGER_SAFE_EN;
+
+	DPU_REG_WRITE(&ctx->hw, SSPP_QOS_CTRL + idx, qos_ctrl);
+}
+
+static void dpu_hw_sspp_setup_cdp(struct dpu_hw_pipe *ctx,
+		struct dpu_hw_pipe_cdp_cfg *cfg)
+{
+	u32 idx;
+	u32 cdp_cntl = 0;
+
+	if (!ctx || !cfg)
+		return;
+
+	if (_sspp_subblk_offset(ctx, DPU_SSPP_SRC, &idx))
+		return;
+
+	if (cfg->enable)
+		cdp_cntl |= BIT(0);
+	if (cfg->ubwc_meta_enable)
+		cdp_cntl |= BIT(1);
+	if (cfg->tile_amortize_enable)
+		cdp_cntl |= BIT(2);
+	if (cfg->preload_ahead == DPU_SSPP_CDP_PRELOAD_AHEAD_64)
+		cdp_cntl |= BIT(3);
+
+	DPU_REG_WRITE(&ctx->hw, SSPP_CDP_CNTL, cdp_cntl);
+}
+
+static void _setup_layer_ops(struct dpu_hw_pipe *c,
+		unsigned long features)
+{
+	if (test_bit(DPU_SSPP_SRC, &features)) {
+		c->ops.setup_format = dpu_hw_sspp_setup_format;
+		c->ops.setup_rects = dpu_hw_sspp_setup_rects;
+		c->ops.setup_sourceaddress = dpu_hw_sspp_setup_sourceaddress;
+		c->ops.setup_solidfill = dpu_hw_sspp_setup_solidfill;
+		c->ops.setup_pe = dpu_hw_sspp_setup_pe_config;
+	}
+
+	if (test_bit(DPU_SSPP_QOS, &features)) {
+		c->ops.setup_danger_safe_lut =
+			dpu_hw_sspp_setup_danger_safe_lut;
+		c->ops.setup_creq_lut = dpu_hw_sspp_setup_creq_lut;
+		c->ops.setup_qos_ctrl = dpu_hw_sspp_setup_qos_ctrl;
+	}
+
+	if (test_bit(DPU_SSPP_CSC, &features) ||
+		test_bit(DPU_SSPP_CSC_10BIT, &features))
+		c->ops.setup_csc = dpu_hw_sspp_setup_csc;
+
+	if (dpu_hw_sspp_multirect_enabled(c->cap))
+		c->ops.setup_multirect = dpu_hw_sspp_setup_multirect;
+
+	if (test_bit(DPU_SSPP_SCALER_QSEED3, &features)) {
+		c->ops.setup_scaler = _dpu_hw_sspp_setup_scaler3;
+		c->ops.get_scaler_ver = _dpu_hw_sspp_get_scaler3_ver;
+	}
+
+	if (test_bit(DPU_SSPP_CDP, &features))
+		c->ops.setup_cdp = dpu_hw_sspp_setup_cdp;
+}
+
+static struct dpu_sspp_cfg *_sspp_offset(enum dpu_sspp sspp,
+		void __iomem *addr,
+		struct dpu_mdss_cfg *catalog,
+		struct dpu_hw_blk_reg_map *b)
+{
+	int i;
+
+	if ((sspp < SSPP_MAX) && catalog && addr && b) {
+		for (i = 0; i < catalog->sspp_count; i++) {
+			if (sspp == catalog->sspp[i].id) {
+				b->base_off = addr;
+				b->blk_off = catalog->sspp[i].base;
+				b->length = catalog->sspp[i].len;
+				b->hwversion = catalog->hwversion;
+				b->log_mask = DPU_DBG_MASK_SSPP;
+				return &catalog->sspp[i];
+			}
+		}
+	}
+
+	return ERR_PTR(-ENOMEM);
+}
+
+static struct dpu_hw_blk_ops dpu_hw_ops = {
+	.start = NULL,
+	.stop = NULL,
+};
+
+struct dpu_hw_pipe *dpu_hw_sspp_init(enum dpu_sspp idx,
+		void __iomem *addr, struct dpu_mdss_cfg *catalog,
+		bool is_virtual_pipe)
+{
+	struct dpu_hw_pipe *hw_pipe;
+	struct dpu_sspp_cfg *cfg;
+	int rc;
+
+	if (!addr || !catalog)
+		return ERR_PTR(-EINVAL);
+
+	hw_pipe = kzalloc(sizeof(*hw_pipe), GFP_KERNEL);
+	if (!hw_pipe)
+		return ERR_PTR(-ENOMEM);
+
+	cfg = _sspp_offset(idx, addr, catalog, &hw_pipe->hw);
+	if (IS_ERR_OR_NULL(cfg)) {
+		kfree(hw_pipe);
+		return ERR_PTR(-EINVAL);
+	}
+
+	/* Assign ops */
+	hw_pipe->catalog = catalog;
+	hw_pipe->mdp = &catalog->mdp[0];
+	hw_pipe->idx = idx;
+	hw_pipe->cap = cfg;
+	_setup_layer_ops(hw_pipe, hw_pipe->cap->features);
+
+	rc = dpu_hw_blk_init(&hw_pipe->base, DPU_HW_BLK_SSPP, idx, &dpu_hw_ops);
+	if (rc) {
+		DPU_ERROR("failed to init hw blk %d\n", rc);
+		goto blk_init_error;
+	}
+
+	return hw_pipe;
+
+blk_init_error:
+	kzfree(hw_pipe);
+
+	return ERR_PTR(rc);
+}
+
+void dpu_hw_sspp_destroy(struct dpu_hw_pipe *ctx)
+{
+	if (ctx)
+		dpu_hw_blk_destroy(&ctx->base);
+	kfree(ctx);
+}
+
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.h
new file mode 100644
index 000000000000..4d81e5f5ce1b
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.h
@@ -0,0 +1,424 @@
+/* Copyright (c) 2015-2018, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef _DPU_HW_SSPP_H
+#define _DPU_HW_SSPP_H
+
+#include "dpu_hw_catalog.h"
+#include "dpu_hw_mdss.h"
+#include "dpu_hw_util.h"
+#include "dpu_hw_blk.h"
+#include "dpu_formats.h"
+
+struct dpu_hw_pipe;
+
+/**
+ * Flags
+ */
+#define DPU_SSPP_FLIP_LR		BIT(0)
+#define DPU_SSPP_FLIP_UD		BIT(1)
+#define DPU_SSPP_SOURCE_ROTATED_90	BIT(2)
+#define DPU_SSPP_ROT_90			BIT(3)
+#define DPU_SSPP_SOLID_FILL		BIT(4)
+
+/**
+ * Define all scaler feature bits in catalog
+ */
+#define DPU_SSPP_SCALER ((1UL << DPU_SSPP_SCALER_RGB) | \
+	(1UL << DPU_SSPP_SCALER_QSEED2) | \
+	(1UL << DPU_SSPP_SCALER_QSEED3))
+
+/**
+ * Component indices
+ */
+enum {
+	DPU_SSPP_COMP_0,
+	DPU_SSPP_COMP_1_2,
+	DPU_SSPP_COMP_2,
+	DPU_SSPP_COMP_3,
+
+	DPU_SSPP_COMP_MAX
+};
+
+/**
+ * DPU_SSPP_RECT_SOLO - multirect disabled
+ * DPU_SSPP_RECT_0 - rect0 of a multirect pipe
+ * DPU_SSPP_RECT_1 - rect1 of a multirect pipe
+ *
+ * Note: HW supports multirect with either RECT0 or
+ * RECT1. Considering no benefit of such configs over
+ * SOLO mode and to keep the plane management simple,
+ * we dont support single rect multirect configs.
+ */
+enum dpu_sspp_multirect_index {
+	DPU_SSPP_RECT_SOLO = 0,
+	DPU_SSPP_RECT_0,
+	DPU_SSPP_RECT_1,
+};
+
+enum dpu_sspp_multirect_mode {
+	DPU_SSPP_MULTIRECT_NONE = 0,
+	DPU_SSPP_MULTIRECT_PARALLEL,
+	DPU_SSPP_MULTIRECT_TIME_MX,
+};
+
+enum {
+	DPU_FRAME_LINEAR,
+	DPU_FRAME_TILE_A4X,
+	DPU_FRAME_TILE_A5X,
+};
+
+enum dpu_hw_filter {
+	DPU_SCALE_FILTER_NEAREST = 0,
+	DPU_SCALE_FILTER_BIL,
+	DPU_SCALE_FILTER_PCMN,
+	DPU_SCALE_FILTER_CA,
+	DPU_SCALE_FILTER_MAX
+};
+
+enum dpu_hw_filter_alpa {
+	DPU_SCALE_ALPHA_PIXEL_REP,
+	DPU_SCALE_ALPHA_BIL
+};
+
+enum dpu_hw_filter_yuv {
+	DPU_SCALE_2D_4X4,
+	DPU_SCALE_2D_CIR,
+	DPU_SCALE_1D_SEP,
+	DPU_SCALE_BIL
+};
+
+struct dpu_hw_sharp_cfg {
+	u32 strength;
+	u32 edge_thr;
+	u32 smooth_thr;
+	u32 noise_thr;
+};
+
+struct dpu_hw_pixel_ext {
+	/* scaling factors are enabled for this input layer */
+	uint8_t enable_pxl_ext;
+
+	int init_phase_x[DPU_MAX_PLANES];
+	int phase_step_x[DPU_MAX_PLANES];
+	int init_phase_y[DPU_MAX_PLANES];
+	int phase_step_y[DPU_MAX_PLANES];
+
+	/*
+	 * Number of pixels extension in left, right, top and bottom direction
+	 * for all color components. This pixel value for each color component
+	 * should be sum of fetch + repeat pixels.
+	 */
+	int num_ext_pxls_left[DPU_MAX_PLANES];
+	int num_ext_pxls_right[DPU_MAX_PLANES];
+	int num_ext_pxls_top[DPU_MAX_PLANES];
+	int num_ext_pxls_btm[DPU_MAX_PLANES];
+
+	/*
+	 * Number of pixels needs to be overfetched in left, right, top and
+	 * bottom directions from source image for scaling.
+	 */
+	int left_ftch[DPU_MAX_PLANES];
+	int right_ftch[DPU_MAX_PLANES];
+	int top_ftch[DPU_MAX_PLANES];
+	int btm_ftch[DPU_MAX_PLANES];
+
+	/*
+	 * Number of pixels needs to be repeated in left, right, top and
+	 * bottom directions for scaling.
+	 */
+	int left_rpt[DPU_MAX_PLANES];
+	int right_rpt[DPU_MAX_PLANES];
+	int top_rpt[DPU_MAX_PLANES];
+	int btm_rpt[DPU_MAX_PLANES];
+
+	uint32_t roi_w[DPU_MAX_PLANES];
+	uint32_t roi_h[DPU_MAX_PLANES];
+
+	/*
+	 * Filter type to be used for scaling in horizontal and vertical
+	 * directions
+	 */
+	enum dpu_hw_filter horz_filter[DPU_MAX_PLANES];
+	enum dpu_hw_filter vert_filter[DPU_MAX_PLANES];
+
+};
+
+/**
+ * struct dpu_hw_pipe_cfg : Pipe description
+ * @layout:    format layout information for programming buffer to hardware
+ * @src_rect:  src ROI, caller takes into account the different operations
+ *             such as decimation, flip etc to program this field
+ * @dest_rect: destination ROI.
+ * @index:     index of the rectangle of SSPP
+ * @mode:      parallel or time multiplex multirect mode
+ */
+struct dpu_hw_pipe_cfg {
+	struct dpu_hw_fmt_layout layout;
+	struct drm_rect src_rect;
+	struct drm_rect dst_rect;
+	enum dpu_sspp_multirect_index index;
+	enum dpu_sspp_multirect_mode mode;
+};
+
+/**
+ * struct dpu_hw_pipe_qos_cfg : Source pipe QoS configuration
+ * @danger_lut: LUT for generate danger level based on fill level
+ * @safe_lut: LUT for generate safe level based on fill level
+ * @creq_lut: LUT for generate creq level based on fill level
+ * @creq_vblank: creq value generated to vbif during vertical blanking
+ * @danger_vblank: danger value generated during vertical blanking
+ * @vblank_en: enable creq_vblank and danger_vblank during vblank
+ * @danger_safe_en: enable danger safe generation
+ */
+struct dpu_hw_pipe_qos_cfg {
+	u32 danger_lut;
+	u32 safe_lut;
+	u64 creq_lut;
+	u32 creq_vblank;
+	u32 danger_vblank;
+	bool vblank_en;
+	bool danger_safe_en;
+};
+
+/**
+ * enum CDP preload ahead address size
+ */
+enum {
+	DPU_SSPP_CDP_PRELOAD_AHEAD_32,
+	DPU_SSPP_CDP_PRELOAD_AHEAD_64
+};
+
+/**
+ * struct dpu_hw_pipe_cdp_cfg : CDP configuration
+ * @enable: true to enable CDP
+ * @ubwc_meta_enable: true to enable ubwc metadata preload
+ * @tile_amortize_enable: true to enable amortization control for tile format
+ * @preload_ahead: number of request to preload ahead
+ *	DPU_SSPP_CDP_PRELOAD_AHEAD_32,
+ *	DPU_SSPP_CDP_PRELOAD_AHEAD_64
+ */
+struct dpu_hw_pipe_cdp_cfg {
+	bool enable;
+	bool ubwc_meta_enable;
+	bool tile_amortize_enable;
+	u32 preload_ahead;
+};
+
+/**
+ * struct dpu_hw_pipe_ts_cfg - traffic shaper configuration
+ * @size: size to prefill in bytes, or zero to disable
+ * @time: time to prefill in usec, or zero to disable
+ */
+struct dpu_hw_pipe_ts_cfg {
+	u64 size;
+	u64 time;
+};
+
+/**
+ * struct dpu_hw_sspp_ops - interface to the SSPP Hw driver functions
+ * Caller must call the init function to get the pipe context for each pipe
+ * Assumption is these functions will be called after clocks are enabled
+ */
+struct dpu_hw_sspp_ops {
+	/**
+	 * setup_format - setup pixel format cropping rectangle, flip
+	 * @ctx: Pointer to pipe context
+	 * @cfg: Pointer to pipe config structure
+	 * @flags: Extra flags for format config
+	 * @index: rectangle index in multirect
+	 */
+	void (*setup_format)(struct dpu_hw_pipe *ctx,
+			const struct dpu_format *fmt, u32 flags,
+			enum dpu_sspp_multirect_index index);
+
+	/**
+	 * setup_rects - setup pipe ROI rectangles
+	 * @ctx: Pointer to pipe context
+	 * @cfg: Pointer to pipe config structure
+	 * @index: rectangle index in multirect
+	 */
+	void (*setup_rects)(struct dpu_hw_pipe *ctx,
+			struct dpu_hw_pipe_cfg *cfg,
+			enum dpu_sspp_multirect_index index);
+
+	/**
+	 * setup_pe - setup pipe pixel extension
+	 * @ctx: Pointer to pipe context
+	 * @pe_ext: Pointer to pixel ext settings
+	 */
+	void (*setup_pe)(struct dpu_hw_pipe *ctx,
+			struct dpu_hw_pixel_ext *pe_ext);
+
+	/**
+	 * setup_sourceaddress - setup pipe source addresses
+	 * @ctx: Pointer to pipe context
+	 * @cfg: Pointer to pipe config structure
+	 * @index: rectangle index in multirect
+	 */
+	void (*setup_sourceaddress)(struct dpu_hw_pipe *ctx,
+			struct dpu_hw_pipe_cfg *cfg,
+			enum dpu_sspp_multirect_index index);
+
+	/**
+	 * setup_csc - setup color space coversion
+	 * @ctx: Pointer to pipe context
+	 * @data: Pointer to config structure
+	 */
+	void (*setup_csc)(struct dpu_hw_pipe *ctx, struct dpu_csc_cfg *data);
+
+	/**
+	 * setup_solidfill - enable/disable colorfill
+	 * @ctx: Pointer to pipe context
+	 * @const_color: Fill color value
+	 * @flags: Pipe flags
+	 * @index: rectangle index in multirect
+	 */
+	void (*setup_solidfill)(struct dpu_hw_pipe *ctx, u32 color,
+			enum dpu_sspp_multirect_index index);
+
+	/**
+	 * setup_multirect - setup multirect configuration
+	 * @ctx: Pointer to pipe context
+	 * @index: rectangle index in multirect
+	 * @mode: parallel fetch / time multiplex multirect mode
+	 */
+
+	void (*setup_multirect)(struct dpu_hw_pipe *ctx,
+			enum dpu_sspp_multirect_index index,
+			enum dpu_sspp_multirect_mode mode);
+
+	/**
+	 * setup_sharpening - setup sharpening
+	 * @ctx: Pointer to pipe context
+	 * @cfg: Pointer to config structure
+	 */
+	void (*setup_sharpening)(struct dpu_hw_pipe *ctx,
+			struct dpu_hw_sharp_cfg *cfg);
+
+	/**
+	 * setup_danger_safe_lut - setup danger safe LUTs
+	 * @ctx: Pointer to pipe context
+	 * @cfg: Pointer to pipe QoS configuration
+	 *
+	 */
+	void (*setup_danger_safe_lut)(struct dpu_hw_pipe *ctx,
+			struct dpu_hw_pipe_qos_cfg *cfg);
+
+	/**
+	 * setup_creq_lut - setup CREQ LUT
+	 * @ctx: Pointer to pipe context
+	 * @cfg: Pointer to pipe QoS configuration
+	 *
+	 */
+	void (*setup_creq_lut)(struct dpu_hw_pipe *ctx,
+			struct dpu_hw_pipe_qos_cfg *cfg);
+
+	/**
+	 * setup_qos_ctrl - setup QoS control
+	 * @ctx: Pointer to pipe context
+	 * @cfg: Pointer to pipe QoS configuration
+	 *
+	 */
+	void (*setup_qos_ctrl)(struct dpu_hw_pipe *ctx,
+			struct dpu_hw_pipe_qos_cfg *cfg);
+
+	/**
+	 * setup_histogram - setup histograms
+	 * @ctx: Pointer to pipe context
+	 * @cfg: Pointer to histogram configuration
+	 */
+	void (*setup_histogram)(struct dpu_hw_pipe *ctx,
+			void *cfg);
+
+	/**
+	 * setup_scaler - setup scaler
+	 * @ctx: Pointer to pipe context
+	 * @pipe_cfg: Pointer to pipe configuration
+	 * @pe_cfg: Pointer to pixel extension configuration
+	 * @scaler_cfg: Pointer to scaler configuration
+	 */
+	void (*setup_scaler)(struct dpu_hw_pipe *ctx,
+		struct dpu_hw_pipe_cfg *pipe_cfg,
+		struct dpu_hw_pixel_ext *pe_cfg,
+		void *scaler_cfg);
+
+	/**
+	 * get_scaler_ver - get scaler h/w version
+	 * @ctx: Pointer to pipe context
+	 */
+	u32 (*get_scaler_ver)(struct dpu_hw_pipe *ctx);
+
+	/**
+	 * setup_cdp - setup client driven prefetch
+	 * @ctx: Pointer to pipe context
+	 * @cfg: Pointer to cdp configuration
+	 */
+	void (*setup_cdp)(struct dpu_hw_pipe *ctx,
+			struct dpu_hw_pipe_cdp_cfg *cfg);
+};
+
+/**
+ * struct dpu_hw_pipe - pipe description
+ * @base: hardware block base structure
+ * @hw: block hardware details
+ * @catalog: back pointer to catalog
+ * @mdp: pointer to associated mdp portion of the catalog
+ * @idx: pipe index
+ * @cap: pointer to layer_cfg
+ * @ops: pointer to operations possible for this pipe
+ */
+struct dpu_hw_pipe {
+	struct dpu_hw_blk base;
+	struct dpu_hw_blk_reg_map hw;
+	struct dpu_mdss_cfg *catalog;
+	struct dpu_mdp_cfg *mdp;
+
+	/* Pipe */
+	enum dpu_sspp idx;
+	const struct dpu_sspp_cfg *cap;
+
+	/* Ops */
+	struct dpu_hw_sspp_ops ops;
+};
+
+/**
+ * dpu_hw_pipe - convert base object dpu_hw_base to container
+ * @hw: Pointer to base hardware block
+ * return: Pointer to hardware block container
+ */
+static inline struct dpu_hw_pipe *to_dpu_hw_pipe(struct dpu_hw_blk *hw)
+{
+	return container_of(hw, struct dpu_hw_pipe, base);
+}
+
+/**
+ * dpu_hw_sspp_init - initializes the sspp hw driver object.
+ * Should be called once before accessing every pipe.
+ * @idx:  Pipe index for which driver object is required
+ * @addr: Mapped register io address of MDP
+ * @catalog : Pointer to mdss catalog data
+ * @is_virtual_pipe: is this pipe virtual pipe
+ */
+struct dpu_hw_pipe *dpu_hw_sspp_init(enum dpu_sspp idx,
+		void __iomem *addr, struct dpu_mdss_cfg *catalog,
+		bool is_virtual_pipe);
+
+/**
+ * dpu_hw_sspp_destroy(): Destroys SSPP driver context
+ * should be called during Hw pipe cleanup.
+ * @ctx:  Pointer to SSPP driver context returned by dpu_hw_sspp_init
+ */
+void dpu_hw_sspp_destroy(struct dpu_hw_pipe *ctx);
+
+#endif /*_DPU_HW_SSPP_H */
+
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_top.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_top.c
new file mode 100644
index 000000000000..db2798e862fc
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_top.c
@@ -0,0 +1,398 @@
+/* Copyright (c) 2015-2018, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include "dpu_hwio.h"
+#include "dpu_hw_catalog.h"
+#include "dpu_hw_top.h"
+#include "dpu_dbg.h"
+#include "dpu_kms.h"
+
+#define SSPP_SPARE                        0x28
+#define UBWC_STATIC                       0x144
+
+#define FLD_SPLIT_DISPLAY_CMD             BIT(1)
+#define FLD_SMART_PANEL_FREE_RUN          BIT(2)
+#define FLD_INTF_1_SW_TRG_MUX             BIT(4)
+#define FLD_INTF_2_SW_TRG_MUX             BIT(8)
+#define FLD_TE_LINE_INTER_WATERLEVEL_MASK 0xFFFF
+
+#define DANGER_STATUS                     0x360
+#define SAFE_STATUS                       0x364
+
+#define TE_LINE_INTERVAL                  0x3F4
+
+#define TRAFFIC_SHAPER_EN                 BIT(31)
+#define TRAFFIC_SHAPER_RD_CLIENT(num)     (0x030 + (num * 4))
+#define TRAFFIC_SHAPER_WR_CLIENT(num)     (0x060 + (num * 4))
+#define TRAFFIC_SHAPER_FIXPOINT_FACTOR    4
+
+#define MDP_WD_TIMER_0_CTL                0x380
+#define MDP_WD_TIMER_0_CTL2               0x384
+#define MDP_WD_TIMER_0_LOAD_VALUE         0x388
+#define MDP_WD_TIMER_1_CTL                0x390
+#define MDP_WD_TIMER_1_CTL2               0x394
+#define MDP_WD_TIMER_1_LOAD_VALUE         0x398
+#define MDP_WD_TIMER_2_CTL                0x420
+#define MDP_WD_TIMER_2_CTL2               0x424
+#define MDP_WD_TIMER_2_LOAD_VALUE         0x428
+#define MDP_WD_TIMER_3_CTL                0x430
+#define MDP_WD_TIMER_3_CTL2               0x434
+#define MDP_WD_TIMER_3_LOAD_VALUE         0x438
+#define MDP_WD_TIMER_4_CTL                0x440
+#define MDP_WD_TIMER_4_CTL2               0x444
+#define MDP_WD_TIMER_4_LOAD_VALUE         0x448
+
+#define MDP_TICK_COUNT                    16
+#define XO_CLK_RATE                       19200
+#define MS_TICKS_IN_SEC                   1000
+
+#define CALCULATE_WD_LOAD_VALUE(fps) \
+	((uint32_t)((MS_TICKS_IN_SEC * XO_CLK_RATE)/(MDP_TICK_COUNT * fps)))
+
+#define DCE_SEL                           0x450
+
+static void dpu_hw_setup_split_pipe(struct dpu_hw_mdp *mdp,
+		struct split_pipe_cfg *cfg)
+{
+	struct dpu_hw_blk_reg_map *c;
+	u32 upper_pipe = 0;
+	u32 lower_pipe = 0;
+
+	if (!mdp || !cfg)
+		return;
+
+	c = &mdp->hw;
+
+	if (cfg->en) {
+		if (cfg->mode == INTF_MODE_CMD) {
+			lower_pipe = FLD_SPLIT_DISPLAY_CMD;
+			/* interface controlling sw trigger */
+			if (cfg->intf == INTF_2)
+				lower_pipe |= FLD_INTF_1_SW_TRG_MUX;
+			else
+				lower_pipe |= FLD_INTF_2_SW_TRG_MUX;
+			upper_pipe = lower_pipe;
+		} else {
+			if (cfg->intf == INTF_2) {
+				lower_pipe = FLD_INTF_1_SW_TRG_MUX;
+				upper_pipe = FLD_INTF_2_SW_TRG_MUX;
+			} else {
+				lower_pipe = FLD_INTF_2_SW_TRG_MUX;
+				upper_pipe = FLD_INTF_1_SW_TRG_MUX;
+			}
+		}
+	}
+
+	DPU_REG_WRITE(c, SSPP_SPARE, cfg->split_flush_en ? 0x1 : 0x0);
+	DPU_REG_WRITE(c, SPLIT_DISPLAY_LOWER_PIPE_CTRL, lower_pipe);
+	DPU_REG_WRITE(c, SPLIT_DISPLAY_UPPER_PIPE_CTRL, upper_pipe);
+	DPU_REG_WRITE(c, SPLIT_DISPLAY_EN, cfg->en & 0x1);
+}
+
+static void dpu_hw_setup_cdm_output(struct dpu_hw_mdp *mdp,
+		struct cdm_output_cfg *cfg)
+{
+	struct dpu_hw_blk_reg_map *c;
+	u32 out_ctl = 0;
+
+	if (!mdp || !cfg)
+		return;
+
+	c = &mdp->hw;
+
+	if (cfg->intf_en)
+		out_ctl |= BIT(19);
+
+	DPU_REG_WRITE(c, MDP_OUT_CTL_0, out_ctl);
+}
+
+static bool dpu_hw_setup_clk_force_ctrl(struct dpu_hw_mdp *mdp,
+		enum dpu_clk_ctrl_type clk_ctrl, bool enable)
+{
+	struct dpu_hw_blk_reg_map *c;
+	u32 reg_off, bit_off;
+	u32 reg_val, new_val;
+	bool clk_forced_on;
+
+	if (!mdp)
+		return false;
+
+	c = &mdp->hw;
+
+	if (clk_ctrl <= DPU_CLK_CTRL_NONE || clk_ctrl >= DPU_CLK_CTRL_MAX)
+		return false;
+
+	reg_off = mdp->caps->clk_ctrls[clk_ctrl].reg_off;
+	bit_off = mdp->caps->clk_ctrls[clk_ctrl].bit_off;
+
+	reg_val = DPU_REG_READ(c, reg_off);
+
+	if (enable)
+		new_val = reg_val | BIT(bit_off);
+	else
+		new_val = reg_val & ~BIT(bit_off);
+
+	DPU_REG_WRITE(c, reg_off, new_val);
+
+	clk_forced_on = !(reg_val & BIT(bit_off));
+
+	return clk_forced_on;
+}
+
+
+static void dpu_hw_get_danger_status(struct dpu_hw_mdp *mdp,
+		struct dpu_danger_safe_status *status)
+{
+	struct dpu_hw_blk_reg_map *c;
+	u32 value;
+
+	if (!mdp || !status)
+		return;
+
+	c = &mdp->hw;
+
+	value = DPU_REG_READ(c, DANGER_STATUS);
+	status->mdp = (value >> 0) & 0x3;
+	status->sspp[SSPP_VIG0] = (value >> 4) & 0x3;
+	status->sspp[SSPP_VIG1] = (value >> 6) & 0x3;
+	status->sspp[SSPP_VIG2] = (value >> 8) & 0x3;
+	status->sspp[SSPP_VIG3] = (value >> 10) & 0x3;
+	status->sspp[SSPP_RGB0] = (value >> 12) & 0x3;
+	status->sspp[SSPP_RGB1] = (value >> 14) & 0x3;
+	status->sspp[SSPP_RGB2] = (value >> 16) & 0x3;
+	status->sspp[SSPP_RGB3] = (value >> 18) & 0x3;
+	status->sspp[SSPP_DMA0] = (value >> 20) & 0x3;
+	status->sspp[SSPP_DMA1] = (value >> 22) & 0x3;
+	status->sspp[SSPP_DMA2] = (value >> 28) & 0x3;
+	status->sspp[SSPP_DMA3] = (value >> 30) & 0x3;
+	status->sspp[SSPP_CURSOR0] = (value >> 24) & 0x3;
+	status->sspp[SSPP_CURSOR1] = (value >> 26) & 0x3;
+}
+
+static void dpu_hw_setup_vsync_source(struct dpu_hw_mdp *mdp,
+		struct dpu_vsync_source_cfg *cfg)
+{
+	struct dpu_hw_blk_reg_map *c;
+	u32 reg, wd_load_value, wd_ctl, wd_ctl2, i;
+	static const u32 pp_offset[PINGPONG_MAX] = {0xC, 0x8, 0x4, 0x13, 0x18};
+
+	if (!mdp || !cfg || (cfg->pp_count > ARRAY_SIZE(cfg->ppnumber)))
+		return;
+
+	c = &mdp->hw;
+	reg = DPU_REG_READ(c, MDP_VSYNC_SEL);
+	for (i = 0; i < cfg->pp_count; i++) {
+		int pp_idx = cfg->ppnumber[i] - PINGPONG_0;
+
+		if (pp_idx >= ARRAY_SIZE(pp_offset))
+			continue;
+
+		reg &= ~(0xf << pp_offset[pp_idx]);
+		reg |= (cfg->vsync_source & 0xf) << pp_offset[pp_idx];
+	}
+	DPU_REG_WRITE(c, MDP_VSYNC_SEL, reg);
+
+	if (cfg->vsync_source >= DPU_VSYNC_SOURCE_WD_TIMER_4 &&
+			cfg->vsync_source <= DPU_VSYNC_SOURCE_WD_TIMER_0) {
+		switch (cfg->vsync_source) {
+		case DPU_VSYNC_SOURCE_WD_TIMER_4:
+			wd_load_value = MDP_WD_TIMER_4_LOAD_VALUE;
+			wd_ctl = MDP_WD_TIMER_4_CTL;
+			wd_ctl2 = MDP_WD_TIMER_4_CTL2;
+			break;
+		case DPU_VSYNC_SOURCE_WD_TIMER_3:
+			wd_load_value = MDP_WD_TIMER_3_LOAD_VALUE;
+			wd_ctl = MDP_WD_TIMER_3_CTL;
+			wd_ctl2 = MDP_WD_TIMER_3_CTL2;
+			break;
+		case DPU_VSYNC_SOURCE_WD_TIMER_2:
+			wd_load_value = MDP_WD_TIMER_2_LOAD_VALUE;
+			wd_ctl = MDP_WD_TIMER_2_CTL;
+			wd_ctl2 = MDP_WD_TIMER_2_CTL2;
+			break;
+		case DPU_VSYNC_SOURCE_WD_TIMER_1:
+			wd_load_value = MDP_WD_TIMER_1_LOAD_VALUE;
+			wd_ctl = MDP_WD_TIMER_1_CTL;
+			wd_ctl2 = MDP_WD_TIMER_1_CTL2;
+			break;
+		case DPU_VSYNC_SOURCE_WD_TIMER_0:
+		default:
+			wd_load_value = MDP_WD_TIMER_0_LOAD_VALUE;
+			wd_ctl = MDP_WD_TIMER_0_CTL;
+			wd_ctl2 = MDP_WD_TIMER_0_CTL2;
+			break;
+		}
+
+		DPU_REG_WRITE(c, wd_load_value,
+			CALCULATE_WD_LOAD_VALUE(cfg->frame_rate));
+
+		DPU_REG_WRITE(c, wd_ctl, BIT(0)); /* clear timer */
+		reg = DPU_REG_READ(c, wd_ctl2);
+		reg |= BIT(8);		/* enable heartbeat timer */
+		reg |= BIT(0);		/* enable WD timer */
+		DPU_REG_WRITE(c, wd_ctl2, reg);
+
+		/* make sure that timers are enabled/disabled for vsync state */
+		wmb();
+	}
+}
+
+static void dpu_hw_get_safe_status(struct dpu_hw_mdp *mdp,
+		struct dpu_danger_safe_status *status)
+{
+	struct dpu_hw_blk_reg_map *c;
+	u32 value;
+
+	if (!mdp || !status)
+		return;
+
+	c = &mdp->hw;
+
+	value = DPU_REG_READ(c, SAFE_STATUS);
+	status->mdp = (value >> 0) & 0x1;
+	status->sspp[SSPP_VIG0] = (value >> 4) & 0x1;
+	status->sspp[SSPP_VIG1] = (value >> 6) & 0x1;
+	status->sspp[SSPP_VIG2] = (value >> 8) & 0x1;
+	status->sspp[SSPP_VIG3] = (value >> 10) & 0x1;
+	status->sspp[SSPP_RGB0] = (value >> 12) & 0x1;
+	status->sspp[SSPP_RGB1] = (value >> 14) & 0x1;
+	status->sspp[SSPP_RGB2] = (value >> 16) & 0x1;
+	status->sspp[SSPP_RGB3] = (value >> 18) & 0x1;
+	status->sspp[SSPP_DMA0] = (value >> 20) & 0x1;
+	status->sspp[SSPP_DMA1] = (value >> 22) & 0x1;
+	status->sspp[SSPP_DMA2] = (value >> 28) & 0x1;
+	status->sspp[SSPP_DMA3] = (value >> 30) & 0x1;
+	status->sspp[SSPP_CURSOR0] = (value >> 24) & 0x1;
+	status->sspp[SSPP_CURSOR1] = (value >> 26) & 0x1;
+}
+
+static void dpu_hw_reset_ubwc(struct dpu_hw_mdp *mdp, struct dpu_mdss_cfg *m)
+{
+	struct dpu_hw_blk_reg_map c;
+
+	if (!mdp || !m)
+		return;
+
+	if (!IS_UBWC_20_SUPPORTED(m->caps->ubwc_version))
+		return;
+
+	/* force blk offset to zero to access beginning of register region */
+	c = mdp->hw;
+	c.blk_off = 0x0;
+	DPU_REG_WRITE(&c, UBWC_STATIC, m->mdp[0].ubwc_static);
+}
+
+static void dpu_hw_intf_audio_select(struct dpu_hw_mdp *mdp)
+{
+	struct dpu_hw_blk_reg_map *c;
+
+	if (!mdp)
+		return;
+
+	c = &mdp->hw;
+
+	DPU_REG_WRITE(c, HDMI_DP_CORE_SELECT, 0x1);
+}
+
+static void _setup_mdp_ops(struct dpu_hw_mdp_ops *ops,
+		unsigned long cap)
+{
+	ops->setup_split_pipe = dpu_hw_setup_split_pipe;
+	ops->setup_cdm_output = dpu_hw_setup_cdm_output;
+	ops->setup_clk_force_ctrl = dpu_hw_setup_clk_force_ctrl;
+	ops->get_danger_status = dpu_hw_get_danger_status;
+	ops->setup_vsync_source = dpu_hw_setup_vsync_source;
+	ops->get_safe_status = dpu_hw_get_safe_status;
+	ops->reset_ubwc = dpu_hw_reset_ubwc;
+	ops->intf_audio_select = dpu_hw_intf_audio_select;
+}
+
+static const struct dpu_mdp_cfg *_top_offset(enum dpu_mdp mdp,
+		const struct dpu_mdss_cfg *m,
+		void __iomem *addr,
+		struct dpu_hw_blk_reg_map *b)
+{
+	int i;
+
+	if (!m || !addr || !b)
+		return ERR_PTR(-EINVAL);
+
+	for (i = 0; i < m->mdp_count; i++) {
+		if (mdp == m->mdp[i].id) {
+			b->base_off = addr;
+			b->blk_off = m->mdp[i].base;
+			b->length = m->mdp[i].len;
+			b->hwversion = m->hwversion;
+			b->log_mask = DPU_DBG_MASK_TOP;
+			return &m->mdp[i];
+		}
+	}
+
+	return ERR_PTR(-EINVAL);
+}
+
+static struct dpu_hw_blk_ops dpu_hw_ops = {
+	.start = NULL,
+	.stop = NULL,
+};
+
+struct dpu_hw_mdp *dpu_hw_mdptop_init(enum dpu_mdp idx,
+		void __iomem *addr,
+		const struct dpu_mdss_cfg *m)
+{
+	struct dpu_hw_mdp *mdp;
+	const struct dpu_mdp_cfg *cfg;
+	int rc;
+
+	if (!addr || !m)
+		return ERR_PTR(-EINVAL);
+
+	mdp = kzalloc(sizeof(*mdp), GFP_KERNEL);
+	if (!mdp)
+		return ERR_PTR(-ENOMEM);
+
+	cfg = _top_offset(idx, m, addr, &mdp->hw);
+	if (IS_ERR_OR_NULL(cfg)) {
+		kfree(mdp);
+		return ERR_PTR(-EINVAL);
+	}
+
+	/*
+	 * Assign ops
+	 */
+	mdp->idx = idx;
+	mdp->caps = cfg;
+	_setup_mdp_ops(&mdp->ops, mdp->caps->features);
+
+	rc = dpu_hw_blk_init(&mdp->base, DPU_HW_BLK_TOP, idx, &dpu_hw_ops);
+	if (rc) {
+		DPU_ERROR("failed to init hw blk %d\n", rc);
+		goto blk_init_error;
+	}
+
+	dpu_dbg_set_dpu_top_offset(mdp->hw.blk_off);
+
+	return mdp;
+
+blk_init_error:
+	kzfree(mdp);
+
+	return ERR_PTR(rc);
+}
+
+void dpu_hw_mdp_destroy(struct dpu_hw_mdp *mdp)
+{
+	if (mdp)
+		dpu_hw_blk_destroy(&mdp->base);
+	kfree(mdp);
+}
+
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_top.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_top.h
new file mode 100644
index 000000000000..899925aaa6d7
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_top.h
@@ -0,0 +1,202 @@
+/* Copyright (c) 2015-2018, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef _DPU_HW_TOP_H
+#define _DPU_HW_TOP_H
+
+#include "dpu_hw_catalog.h"
+#include "dpu_hw_mdss.h"
+#include "dpu_hw_util.h"
+#include "dpu_hw_blk.h"
+
+struct dpu_hw_mdp;
+
+/**
+ * struct traffic_shaper_cfg: traffic shaper configuration
+ * @en        : enable/disable traffic shaper
+ * @rd_client : true if read client; false if write client
+ * @client_id : client identifier
+ * @bpc_denom : denominator of byte per clk
+ * @bpc_numer : numerator of byte per clk
+ */
+struct traffic_shaper_cfg {
+	bool en;
+	bool rd_client;
+	u32 client_id;
+	u32 bpc_denom;
+	u64 bpc_numer;
+};
+
+/**
+ * struct split_pipe_cfg - pipe configuration for dual display panels
+ * @en        : Enable/disable dual pipe confguration
+ * @mode      : Panel interface mode
+ * @intf      : Interface id for main control path
+ * @split_flush_en: Allows both the paths to be flushed when master path is
+ *              flushed
+ */
+struct split_pipe_cfg {
+	bool en;
+	enum dpu_intf_mode mode;
+	enum dpu_intf intf;
+	bool split_flush_en;
+};
+
+/**
+ * struct cdm_output_cfg: output configuration for cdm
+ * @intf_en   : enable/disable interface output
+ */
+struct cdm_output_cfg {
+	bool intf_en;
+};
+
+/**
+ * struct dpu_danger_safe_status: danger and safe status signals
+ * @mdp: top level status
+ * @sspp: source pipe status
+ */
+struct dpu_danger_safe_status {
+	u8 mdp;
+	u8 sspp[SSPP_MAX];
+};
+
+/**
+ * struct dpu_vsync_source_cfg - configure vsync source and configure the
+ *                                    watchdog timers if required.
+ * @pp_count: number of ping pongs active
+ * @frame_rate: Display frame rate
+ * @ppnumber: ping pong index array
+ * @vsync_source: vsync source selection
+ */
+struct dpu_vsync_source_cfg {
+	u32 pp_count;
+	u32 frame_rate;
+	u32 ppnumber[PINGPONG_MAX];
+	u32 vsync_source;
+};
+
+/**
+ * struct dpu_hw_mdp_ops - interface to the MDP TOP Hw driver functions
+ * Assumption is these functions will be called after clocks are enabled.
+ * @setup_split_pipe : Programs the pipe control registers
+ * @setup_pp_split : Programs the pp split control registers
+ * @setup_cdm_output : programs cdm control
+ * @setup_traffic_shaper : programs traffic shaper control
+ */
+struct dpu_hw_mdp_ops {
+	/** setup_split_pipe() : Regsiters are not double buffered, thisk
+	 * function should be called before timing control enable
+	 * @mdp  : mdp top context driver
+	 * @cfg  : upper and lower part of pipe configuration
+	 */
+	void (*setup_split_pipe)(struct dpu_hw_mdp *mdp,
+			struct split_pipe_cfg *p);
+
+	/**
+	 * setup_cdm_output() : Setup selection control of the cdm data path
+	 * @mdp  : mdp top context driver
+	 * @cfg  : cdm output configuration
+	 */
+	void (*setup_cdm_output)(struct dpu_hw_mdp *mdp,
+			struct cdm_output_cfg *cfg);
+
+	/**
+	 * setup_traffic_shaper() : Setup traffic shaper control
+	 * @mdp  : mdp top context driver
+	 * @cfg  : traffic shaper configuration
+	 */
+	void (*setup_traffic_shaper)(struct dpu_hw_mdp *mdp,
+			struct traffic_shaper_cfg *cfg);
+
+	/**
+	 * setup_clk_force_ctrl - set clock force control
+	 * @mdp: mdp top context driver
+	 * @clk_ctrl: clock to be controlled
+	 * @enable: force on enable
+	 * @return: if the clock is forced-on by this function
+	 */
+	bool (*setup_clk_force_ctrl)(struct dpu_hw_mdp *mdp,
+			enum dpu_clk_ctrl_type clk_ctrl, bool enable);
+
+	/**
+	 * get_danger_status - get danger status
+	 * @mdp: mdp top context driver
+	 * @status: Pointer to danger safe status
+	 */
+	void (*get_danger_status)(struct dpu_hw_mdp *mdp,
+			struct dpu_danger_safe_status *status);
+
+	/**
+	 * setup_vsync_source - setup vsync source configuration details
+	 * @mdp: mdp top context driver
+	 * @cfg: vsync source selection configuration
+	 */
+	void (*setup_vsync_source)(struct dpu_hw_mdp *mdp,
+				struct dpu_vsync_source_cfg *cfg);
+
+	/**
+	 * get_safe_status - get safe status
+	 * @mdp: mdp top context driver
+	 * @status: Pointer to danger safe status
+	 */
+	void (*get_safe_status)(struct dpu_hw_mdp *mdp,
+			struct dpu_danger_safe_status *status);
+
+	/**
+	 * reset_ubwc - reset top level UBWC configuration
+	 * @mdp: mdp top context driver
+	 * @m: pointer to mdss catalog data
+	 */
+	void (*reset_ubwc)(struct dpu_hw_mdp *mdp, struct dpu_mdss_cfg *m);
+
+	/**
+	 * intf_audio_select - select the external interface for audio
+	 * @mdp: mdp top context driver
+	 */
+	void (*intf_audio_select)(struct dpu_hw_mdp *mdp);
+};
+
+struct dpu_hw_mdp {
+	struct dpu_hw_blk base;
+	struct dpu_hw_blk_reg_map hw;
+
+	/* top */
+	enum dpu_mdp idx;
+	const struct dpu_mdp_cfg *caps;
+
+	/* ops */
+	struct dpu_hw_mdp_ops ops;
+};
+
+/**
+ * to_dpu_hw_mdp - convert base object dpu_hw_base to container
+ * @hw: Pointer to base hardware block
+ * return: Pointer to hardware block container
+ */
+static inline struct dpu_hw_mdp *to_dpu_hw_mdp(struct dpu_hw_blk *hw)
+{
+	return container_of(hw, struct dpu_hw_mdp, base);
+}
+
+/**
+ * dpu_hw_mdptop_init - initializes the top driver for the passed idx
+ * @idx:  Interface index for which driver object is required
+ * @addr: Mapped register io address of MDP
+ * @m:    Pointer to mdss catalog data
+ */
+struct dpu_hw_mdp *dpu_hw_mdptop_init(enum dpu_mdp idx,
+		void __iomem *addr,
+		const struct dpu_mdss_cfg *m);
+
+void dpu_hw_mdp_destroy(struct dpu_hw_mdp *mdp);
+
+#endif /*_DPU_HW_TOP_H */
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_util.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_util.c
new file mode 100644
index 000000000000..4cabae480a7b
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_util.c
@@ -0,0 +1,368 @@
+/* Copyright (c) 2015-2018, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+#define pr_fmt(fmt)	"[drm:%s:%d] " fmt, __func__, __LINE__
+
+#include "msm_drv.h"
+#include "dpu_kms.h"
+#include "dpu_hw_mdss.h"
+#include "dpu_hw_util.h"
+
+/* using a file static variables for debugfs access */
+static u32 dpu_hw_util_log_mask = DPU_DBG_MASK_NONE;
+
+/* DPU_SCALER_QSEED3 */
+#define QSEED3_HW_VERSION                  0x00
+#define QSEED3_OP_MODE                     0x04
+#define QSEED3_RGB2Y_COEFF                 0x08
+#define QSEED3_PHASE_INIT                  0x0C
+#define QSEED3_PHASE_STEP_Y_H              0x10
+#define QSEED3_PHASE_STEP_Y_V              0x14
+#define QSEED3_PHASE_STEP_UV_H             0x18
+#define QSEED3_PHASE_STEP_UV_V             0x1C
+#define QSEED3_PRELOAD                     0x20
+#define QSEED3_DE_SHARPEN                  0x24
+#define QSEED3_DE_SHARPEN_CTL              0x28
+#define QSEED3_DE_SHAPE_CTL                0x2C
+#define QSEED3_DE_THRESHOLD                0x30
+#define QSEED3_DE_ADJUST_DATA_0            0x34
+#define QSEED3_DE_ADJUST_DATA_1            0x38
+#define QSEED3_DE_ADJUST_DATA_2            0x3C
+#define QSEED3_SRC_SIZE_Y_RGB_A            0x40
+#define QSEED3_SRC_SIZE_UV                 0x44
+#define QSEED3_DST_SIZE                    0x48
+#define QSEED3_COEF_LUT_CTRL               0x4C
+#define QSEED3_COEF_LUT_SWAP_BIT           0
+#define QSEED3_COEF_LUT_DIR_BIT            1
+#define QSEED3_COEF_LUT_Y_CIR_BIT          2
+#define QSEED3_COEF_LUT_UV_CIR_BIT         3
+#define QSEED3_COEF_LUT_Y_SEP_BIT          4
+#define QSEED3_COEF_LUT_UV_SEP_BIT         5
+#define QSEED3_BUFFER_CTRL                 0x50
+#define QSEED3_CLK_CTRL0                   0x54
+#define QSEED3_CLK_CTRL1                   0x58
+#define QSEED3_CLK_STATUS                  0x5C
+#define QSEED3_MISR_CTRL                   0x70
+#define QSEED3_MISR_SIGNATURE_0            0x74
+#define QSEED3_MISR_SIGNATURE_1            0x78
+#define QSEED3_PHASE_INIT_Y_H              0x90
+#define QSEED3_PHASE_INIT_Y_V              0x94
+#define QSEED3_PHASE_INIT_UV_H             0x98
+#define QSEED3_PHASE_INIT_UV_V             0x9C
+#define QSEED3_COEF_LUT                    0x100
+#define QSEED3_FILTERS                     5
+#define QSEED3_LUT_REGIONS                 4
+#define QSEED3_CIRCULAR_LUTS               9
+#define QSEED3_SEPARABLE_LUTS              10
+#define QSEED3_LUT_SIZE                    60
+#define QSEED3_ENABLE                      2
+#define QSEED3_DIR_LUT_SIZE                (200 * sizeof(u32))
+#define QSEED3_CIR_LUT_SIZE \
+	(QSEED3_LUT_SIZE * QSEED3_CIRCULAR_LUTS * sizeof(u32))
+#define QSEED3_SEP_LUT_SIZE \
+	(QSEED3_LUT_SIZE * QSEED3_SEPARABLE_LUTS * sizeof(u32))
+
+void dpu_reg_write(struct dpu_hw_blk_reg_map *c,
+		u32 reg_off,
+		u32 val,
+		const char *name)
+{
+	/* don't need to mutex protect this */
+	if (c->log_mask & dpu_hw_util_log_mask)
+		DPU_DEBUG_DRIVER("[%s:0x%X] <= 0x%X\n",
+				name, c->blk_off + reg_off, val);
+	writel_relaxed(val, c->base_off + c->blk_off + reg_off);
+}
+
+int dpu_reg_read(struct dpu_hw_blk_reg_map *c, u32 reg_off)
+{
+	return readl_relaxed(c->base_off + c->blk_off + reg_off);
+}
+
+u32 *dpu_hw_util_get_log_mask_ptr(void)
+{
+	return &dpu_hw_util_log_mask;
+}
+
+static void _dpu_hw_setup_scaler3_lut(struct dpu_hw_blk_reg_map *c,
+		struct dpu_hw_scaler3_cfg *scaler3_cfg, u32 offset)
+{
+	int i, j, filter;
+	int config_lut = 0x0;
+	unsigned long lut_flags;
+	u32 lut_addr, lut_offset, lut_len;
+	u32 *lut[QSEED3_FILTERS] = {NULL, NULL, NULL, NULL, NULL};
+	static const uint32_t off_tbl[QSEED3_FILTERS][QSEED3_LUT_REGIONS][2] = {
+		{{18, 0x000}, {12, 0x120}, {12, 0x1E0}, {8, 0x2A0} },
+		{{6, 0x320}, {3, 0x3E0}, {3, 0x440}, {3, 0x4A0} },
+		{{6, 0x500}, {3, 0x5c0}, {3, 0x620}, {3, 0x680} },
+		{{6, 0x380}, {3, 0x410}, {3, 0x470}, {3, 0x4d0} },
+		{{6, 0x560}, {3, 0x5f0}, {3, 0x650}, {3, 0x6b0} },
+	};
+
+	lut_flags = (unsigned long) scaler3_cfg->lut_flag;
+	if (test_bit(QSEED3_COEF_LUT_DIR_BIT, &lut_flags) &&
+		(scaler3_cfg->dir_len == QSEED3_DIR_LUT_SIZE)) {
+		lut[0] = scaler3_cfg->dir_lut;
+		config_lut = 1;
+	}
+	if (test_bit(QSEED3_COEF_LUT_Y_CIR_BIT, &lut_flags) &&
+		(scaler3_cfg->y_rgb_cir_lut_idx < QSEED3_CIRCULAR_LUTS) &&
+		(scaler3_cfg->cir_len == QSEED3_CIR_LUT_SIZE)) {
+		lut[1] = scaler3_cfg->cir_lut +
+			scaler3_cfg->y_rgb_cir_lut_idx * QSEED3_LUT_SIZE;
+		config_lut = 1;
+	}
+	if (test_bit(QSEED3_COEF_LUT_UV_CIR_BIT, &lut_flags) &&
+		(scaler3_cfg->uv_cir_lut_idx < QSEED3_CIRCULAR_LUTS) &&
+		(scaler3_cfg->cir_len == QSEED3_CIR_LUT_SIZE)) {
+		lut[2] = scaler3_cfg->cir_lut +
+			scaler3_cfg->uv_cir_lut_idx * QSEED3_LUT_SIZE;
+		config_lut = 1;
+	}
+	if (test_bit(QSEED3_COEF_LUT_Y_SEP_BIT, &lut_flags) &&
+		(scaler3_cfg->y_rgb_sep_lut_idx < QSEED3_SEPARABLE_LUTS) &&
+		(scaler3_cfg->sep_len == QSEED3_SEP_LUT_SIZE)) {
+		lut[3] = scaler3_cfg->sep_lut +
+			scaler3_cfg->y_rgb_sep_lut_idx * QSEED3_LUT_SIZE;
+		config_lut = 1;
+	}
+	if (test_bit(QSEED3_COEF_LUT_UV_SEP_BIT, &lut_flags) &&
+		(scaler3_cfg->uv_sep_lut_idx < QSEED3_SEPARABLE_LUTS) &&
+		(scaler3_cfg->sep_len == QSEED3_SEP_LUT_SIZE)) {
+		lut[4] = scaler3_cfg->sep_lut +
+			scaler3_cfg->uv_sep_lut_idx * QSEED3_LUT_SIZE;
+		config_lut = 1;
+	}
+
+	if (config_lut) {
+		for (filter = 0; filter < QSEED3_FILTERS; filter++) {
+			if (!lut[filter])
+				continue;
+			lut_offset = 0;
+			for (i = 0; i < QSEED3_LUT_REGIONS; i++) {
+				lut_addr = QSEED3_COEF_LUT + offset
+					+ off_tbl[filter][i][1];
+				lut_len = off_tbl[filter][i][0] << 2;
+				for (j = 0; j < lut_len; j++) {
+					DPU_REG_WRITE(c,
+						lut_addr,
+						(lut[filter])[lut_offset++]);
+					lut_addr += 4;
+				}
+			}
+		}
+	}
+
+	if (test_bit(QSEED3_COEF_LUT_SWAP_BIT, &lut_flags))
+		DPU_REG_WRITE(c, QSEED3_COEF_LUT_CTRL + offset, BIT(0));
+
+}
+
+static void _dpu_hw_setup_scaler3_de(struct dpu_hw_blk_reg_map *c,
+		struct dpu_hw_scaler3_de_cfg *de_cfg, u32 offset)
+{
+	u32 sharp_lvl, sharp_ctl, shape_ctl, de_thr;
+	u32 adjust_a, adjust_b, adjust_c;
+
+	if (!de_cfg->enable)
+		return;
+
+	sharp_lvl = (de_cfg->sharpen_level1 & 0x1FF) |
+		((de_cfg->sharpen_level2 & 0x1FF) << 16);
+
+	sharp_ctl = ((de_cfg->limit & 0xF) << 9) |
+		((de_cfg->prec_shift & 0x7) << 13) |
+		((de_cfg->clip & 0x7) << 16);
+
+	shape_ctl = (de_cfg->thr_quiet & 0xFF) |
+		((de_cfg->thr_dieout & 0x3FF) << 16);
+
+	de_thr = (de_cfg->thr_low & 0x3FF) |
+		((de_cfg->thr_high & 0x3FF) << 16);
+
+	adjust_a = (de_cfg->adjust_a[0] & 0x3FF) |
+		((de_cfg->adjust_a[1] & 0x3FF) << 10) |
+		((de_cfg->adjust_a[2] & 0x3FF) << 20);
+
+	adjust_b = (de_cfg->adjust_b[0] & 0x3FF) |
+		((de_cfg->adjust_b[1] & 0x3FF) << 10) |
+		((de_cfg->adjust_b[2] & 0x3FF) << 20);
+
+	adjust_c = (de_cfg->adjust_c[0] & 0x3FF) |
+		((de_cfg->adjust_c[1] & 0x3FF) << 10) |
+		((de_cfg->adjust_c[2] & 0x3FF) << 20);
+
+	DPU_REG_WRITE(c, QSEED3_DE_SHARPEN + offset, sharp_lvl);
+	DPU_REG_WRITE(c, QSEED3_DE_SHARPEN_CTL + offset, sharp_ctl);
+	DPU_REG_WRITE(c, QSEED3_DE_SHAPE_CTL + offset, shape_ctl);
+	DPU_REG_WRITE(c, QSEED3_DE_THRESHOLD + offset, de_thr);
+	DPU_REG_WRITE(c, QSEED3_DE_ADJUST_DATA_0 + offset, adjust_a);
+	DPU_REG_WRITE(c, QSEED3_DE_ADJUST_DATA_1 + offset, adjust_b);
+	DPU_REG_WRITE(c, QSEED3_DE_ADJUST_DATA_2 + offset, adjust_c);
+
+}
+
+void dpu_hw_setup_scaler3(struct dpu_hw_blk_reg_map *c,
+		struct dpu_hw_scaler3_cfg *scaler3_cfg,
+		u32 scaler_offset, u32 scaler_version,
+		const struct dpu_format *format)
+{
+	u32 op_mode = 0;
+	u32 phase_init, preload, src_y_rgb, src_uv, dst;
+
+	if (!scaler3_cfg->enable)
+		goto end;
+
+	op_mode |= BIT(0);
+	op_mode |= (scaler3_cfg->y_rgb_filter_cfg & 0x3) << 16;
+
+	if (format && DPU_FORMAT_IS_YUV(format)) {
+		op_mode |= BIT(12);
+		op_mode |= (scaler3_cfg->uv_filter_cfg & 0x3) << 24;
+	}
+
+	op_mode |= (scaler3_cfg->blend_cfg & 1) << 31;
+	op_mode |= (scaler3_cfg->dir_en) ? BIT(4) : 0;
+
+	preload =
+		((scaler3_cfg->preload_x[0] & 0x7F) << 0) |
+		((scaler3_cfg->preload_y[0] & 0x7F) << 8) |
+		((scaler3_cfg->preload_x[1] & 0x7F) << 16) |
+		((scaler3_cfg->preload_y[1] & 0x7F) << 24);
+
+	src_y_rgb = (scaler3_cfg->src_width[0] & 0x1FFFF) |
+		((scaler3_cfg->src_height[0] & 0x1FFFF) << 16);
+
+	src_uv = (scaler3_cfg->src_width[1] & 0x1FFFF) |
+		((scaler3_cfg->src_height[1] & 0x1FFFF) << 16);
+
+	dst = (scaler3_cfg->dst_width & 0x1FFFF) |
+		((scaler3_cfg->dst_height & 0x1FFFF) << 16);
+
+	if (scaler3_cfg->de.enable) {
+		_dpu_hw_setup_scaler3_de(c, &scaler3_cfg->de, scaler_offset);
+		op_mode |= BIT(8);
+	}
+
+	if (scaler3_cfg->lut_flag)
+		_dpu_hw_setup_scaler3_lut(c, scaler3_cfg,
+								scaler_offset);
+
+	if (scaler_version == 0x1002) {
+		phase_init =
+			((scaler3_cfg->init_phase_x[0] & 0x3F) << 0) |
+			((scaler3_cfg->init_phase_y[0] & 0x3F) << 8) |
+			((scaler3_cfg->init_phase_x[1] & 0x3F) << 16) |
+			((scaler3_cfg->init_phase_y[1] & 0x3F) << 24);
+		DPU_REG_WRITE(c, QSEED3_PHASE_INIT + scaler_offset, phase_init);
+	} else {
+		DPU_REG_WRITE(c, QSEED3_PHASE_INIT_Y_H + scaler_offset,
+			scaler3_cfg->init_phase_x[0] & 0x1FFFFF);
+		DPU_REG_WRITE(c, QSEED3_PHASE_INIT_Y_V + scaler_offset,
+			scaler3_cfg->init_phase_y[0] & 0x1FFFFF);
+		DPU_REG_WRITE(c, QSEED3_PHASE_INIT_UV_H + scaler_offset,
+			scaler3_cfg->init_phase_x[1] & 0x1FFFFF);
+		DPU_REG_WRITE(c, QSEED3_PHASE_INIT_UV_V + scaler_offset,
+			scaler3_cfg->init_phase_y[1] & 0x1FFFFF);
+	}
+
+	DPU_REG_WRITE(c, QSEED3_PHASE_STEP_Y_H + scaler_offset,
+		scaler3_cfg->phase_step_x[0] & 0xFFFFFF);
+
+	DPU_REG_WRITE(c, QSEED3_PHASE_STEP_Y_V + scaler_offset,
+		scaler3_cfg->phase_step_y[0] & 0xFFFFFF);
+
+	DPU_REG_WRITE(c, QSEED3_PHASE_STEP_UV_H + scaler_offset,
+		scaler3_cfg->phase_step_x[1] & 0xFFFFFF);
+
+	DPU_REG_WRITE(c, QSEED3_PHASE_STEP_UV_V + scaler_offset,
+		scaler3_cfg->phase_step_y[1] & 0xFFFFFF);
+
+	DPU_REG_WRITE(c, QSEED3_PRELOAD + scaler_offset, preload);
+
+	DPU_REG_WRITE(c, QSEED3_SRC_SIZE_Y_RGB_A + scaler_offset, src_y_rgb);
+
+	DPU_REG_WRITE(c, QSEED3_SRC_SIZE_UV + scaler_offset, src_uv);
+
+	DPU_REG_WRITE(c, QSEED3_DST_SIZE + scaler_offset, dst);
+
+end:
+	if (format && !DPU_FORMAT_IS_DX(format))
+		op_mode |= BIT(14);
+
+	if (format && format->alpha_enable) {
+		op_mode |= BIT(10);
+		if (scaler_version == 0x1002)
+			op_mode |= (scaler3_cfg->alpha_filter_cfg & 0x1) << 30;
+		else
+			op_mode |= (scaler3_cfg->alpha_filter_cfg & 0x3) << 29;
+	}
+
+	DPU_REG_WRITE(c, QSEED3_OP_MODE + scaler_offset, op_mode);
+}
+
+u32 dpu_hw_get_scaler3_ver(struct dpu_hw_blk_reg_map *c,
+			u32 scaler_offset)
+{
+	return DPU_REG_READ(c, QSEED3_HW_VERSION + scaler_offset);
+}
+
+void dpu_hw_csc_setup(struct dpu_hw_blk_reg_map *c,
+		u32 csc_reg_off,
+		struct dpu_csc_cfg *data, bool csc10)
+{
+	static const u32 matrix_shift = 7;
+	u32 clamp_shift = csc10 ? 16 : 8;
+	u32 val;
+
+	/* matrix coeff - convert S15.16 to S4.9 */
+	val = ((data->csc_mv[0] >> matrix_shift) & 0x1FFF) |
+		(((data->csc_mv[1] >> matrix_shift) & 0x1FFF) << 16);
+	DPU_REG_WRITE(c, csc_reg_off, val);
+	val = ((data->csc_mv[2] >> matrix_shift) & 0x1FFF) |
+		(((data->csc_mv[3] >> matrix_shift) & 0x1FFF) << 16);
+	DPU_REG_WRITE(c, csc_reg_off + 0x4, val);
+	val = ((data->csc_mv[4] >> matrix_shift) & 0x1FFF) |
+		(((data->csc_mv[5] >> matrix_shift) & 0x1FFF) << 16);
+	DPU_REG_WRITE(c, csc_reg_off + 0x8, val);
+	val = ((data->csc_mv[6] >> matrix_shift) & 0x1FFF) |
+		(((data->csc_mv[7] >> matrix_shift) & 0x1FFF) << 16);
+	DPU_REG_WRITE(c, csc_reg_off + 0xc, val);
+	val = (data->csc_mv[8] >> matrix_shift) & 0x1FFF;
+	DPU_REG_WRITE(c, csc_reg_off + 0x10, val);
+
+	/* Pre clamp */
+	val = (data->csc_pre_lv[0] << clamp_shift) | data->csc_pre_lv[1];
+	DPU_REG_WRITE(c, csc_reg_off + 0x14, val);
+	val = (data->csc_pre_lv[2] << clamp_shift) | data->csc_pre_lv[3];
+	DPU_REG_WRITE(c, csc_reg_off + 0x18, val);
+	val = (data->csc_pre_lv[4] << clamp_shift) | data->csc_pre_lv[5];
+	DPU_REG_WRITE(c, csc_reg_off + 0x1c, val);
+
+	/* Post clamp */
+	val = (data->csc_post_lv[0] << clamp_shift) | data->csc_post_lv[1];
+	DPU_REG_WRITE(c, csc_reg_off + 0x20, val);
+	val = (data->csc_post_lv[2] << clamp_shift) | data->csc_post_lv[3];
+	DPU_REG_WRITE(c, csc_reg_off + 0x24, val);
+	val = (data->csc_post_lv[4] << clamp_shift) | data->csc_post_lv[5];
+	DPU_REG_WRITE(c, csc_reg_off + 0x28, val);
+
+	/* Pre-Bias */
+	DPU_REG_WRITE(c, csc_reg_off + 0x2c, data->csc_pre_bv[0]);
+	DPU_REG_WRITE(c, csc_reg_off + 0x30, data->csc_pre_bv[1]);
+	DPU_REG_WRITE(c, csc_reg_off + 0x34, data->csc_pre_bv[2]);
+
+	/* Post-Bias */
+	DPU_REG_WRITE(c, csc_reg_off + 0x38, data->csc_post_bv[0]);
+	DPU_REG_WRITE(c, csc_reg_off + 0x3c, data->csc_post_bv[1]);
+	DPU_REG_WRITE(c, csc_reg_off + 0x40, data->csc_post_bv[2]);
+}
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_util.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_util.h
new file mode 100644
index 000000000000..1240f505ca53
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_util.h
@@ -0,0 +1,348 @@
+/* Copyright (c) 2015-2018, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef _DPU_HW_UTIL_H
+#define _DPU_HW_UTIL_H
+
+#include <linux/io.h>
+#include <linux/slab.h>
+#include "dpu_hw_mdss.h"
+
+#define REG_MASK(n)                     ((BIT(n)) - 1)
+struct dpu_format_extended;
+
+/*
+ * This is the common struct maintained by each sub block
+ * for mapping the register offsets in this block to the
+ * absoulute IO address
+ * @base_off:     mdp register mapped offset
+ * @blk_off:      pipe offset relative to mdss offset
+ * @length        length of register block offset
+ * @xin_id        xin id
+ * @hwversion     mdss hw version number
+ */
+struct dpu_hw_blk_reg_map {
+	void __iomem *base_off;
+	u32 blk_off;
+	u32 length;
+	u32 xin_id;
+	u32 hwversion;
+	u32 log_mask;
+};
+
+/**
+ * struct dpu_hw_scaler3_de_cfg : QSEEDv3 detail enhancer configuration
+ * @enable:         detail enhancer enable/disable
+ * @sharpen_level1: sharpening strength for noise
+ * @sharpen_level2: sharpening strength for signal
+ * @ clip:          clip shift
+ * @ limit:         limit value
+ * @ thr_quiet:     quiet threshold
+ * @ thr_dieout:    dieout threshold
+ * @ thr_high:      low threshold
+ * @ thr_high:      high threshold
+ * @ prec_shift:    precision shift
+ * @ adjust_a:      A-coefficients for mapping curve
+ * @ adjust_b:      B-coefficients for mapping curve
+ * @ adjust_c:      C-coefficients for mapping curve
+ */
+struct dpu_hw_scaler3_de_cfg {
+	u32 enable;
+	int16_t sharpen_level1;
+	int16_t sharpen_level2;
+	uint16_t clip;
+	uint16_t limit;
+	uint16_t thr_quiet;
+	uint16_t thr_dieout;
+	uint16_t thr_low;
+	uint16_t thr_high;
+	uint16_t prec_shift;
+	int16_t adjust_a[DPU_MAX_DE_CURVES];
+	int16_t adjust_b[DPU_MAX_DE_CURVES];
+	int16_t adjust_c[DPU_MAX_DE_CURVES];
+};
+
+
+/**
+ * struct dpu_hw_scaler3_cfg : QSEEDv3 configuration
+ * @enable:        scaler enable
+ * @dir_en:        direction detection block enable
+ * @ init_phase_x: horizontal initial phase
+ * @ phase_step_x: horizontal phase step
+ * @ init_phase_y: vertical initial phase
+ * @ phase_step_y: vertical phase step
+ * @ preload_x:    horizontal preload value
+ * @ preload_y:    vertical preload value
+ * @ src_width:    source width
+ * @ src_height:   source height
+ * @ dst_width:    destination width
+ * @ dst_height:   destination height
+ * @ y_rgb_filter_cfg: y/rgb plane filter configuration
+ * @ uv_filter_cfg: uv plane filter configuration
+ * @ alpha_filter_cfg: alpha filter configuration
+ * @ blend_cfg:    blend coefficients configuration
+ * @ lut_flag:     scaler LUT update flags
+ *                 0x1 swap LUT bank
+ *                 0x2 update 2D filter LUT
+ *                 0x4 update y circular filter LUT
+ *                 0x8 update uv circular filter LUT
+ *                 0x10 update y separable filter LUT
+ *                 0x20 update uv separable filter LUT
+ * @ dir_lut_idx:  2D filter LUT index
+ * @ y_rgb_cir_lut_idx: y circular filter LUT index
+ * @ uv_cir_lut_idx: uv circular filter LUT index
+ * @ y_rgb_sep_lut_idx: y circular filter LUT index
+ * @ uv_sep_lut_idx: uv separable filter LUT index
+ * @ dir_lut:      pointer to 2D LUT
+ * @ cir_lut:      pointer to circular filter LUT
+ * @ sep_lut:      pointer to separable filter LUT
+ * @ de: detail enhancer configuration
+ */
+struct dpu_hw_scaler3_cfg {
+	u32 enable;
+	u32 dir_en;
+	int32_t init_phase_x[DPU_MAX_PLANES];
+	int32_t phase_step_x[DPU_MAX_PLANES];
+	int32_t init_phase_y[DPU_MAX_PLANES];
+	int32_t phase_step_y[DPU_MAX_PLANES];
+
+	u32 preload_x[DPU_MAX_PLANES];
+	u32 preload_y[DPU_MAX_PLANES];
+	u32 src_width[DPU_MAX_PLANES];
+	u32 src_height[DPU_MAX_PLANES];
+
+	u32 dst_width;
+	u32 dst_height;
+
+	u32 y_rgb_filter_cfg;
+	u32 uv_filter_cfg;
+	u32 alpha_filter_cfg;
+	u32 blend_cfg;
+
+	u32 lut_flag;
+	u32 dir_lut_idx;
+
+	u32 y_rgb_cir_lut_idx;
+	u32 uv_cir_lut_idx;
+	u32 y_rgb_sep_lut_idx;
+	u32 uv_sep_lut_idx;
+	u32 *dir_lut;
+	size_t dir_len;
+	u32 *cir_lut;
+	size_t cir_len;
+	u32 *sep_lut;
+	size_t sep_len;
+
+	/*
+	 * Detail enhancer settings
+	 */
+	struct dpu_hw_scaler3_de_cfg de;
+};
+
+struct dpu_hw_scaler3_lut_cfg {
+	bool is_configured;
+	u32 *dir_lut;
+	size_t dir_len;
+	u32 *cir_lut;
+	size_t cir_len;
+	u32 *sep_lut;
+	size_t sep_len;
+};
+
+/**
+ * struct dpu_drm_pix_ext_v1 - version 1 of pixel ext structure
+ * @num_ext_pxls_lr: Number of total horizontal pixels
+ * @num_ext_pxls_tb: Number of total vertical lines
+ * @left_ftch:       Number of extra pixels to overfetch from left
+ * @right_ftch:      Number of extra pixels to overfetch from right
+ * @top_ftch:        Number of extra lines to overfetch from top
+ * @btm_ftch:        Number of extra lines to overfetch from bottom
+ * @left_rpt:        Number of extra pixels to repeat from left
+ * @right_rpt:       Number of extra pixels to repeat from right
+ * @top_rpt:         Number of extra lines to repeat from top
+ * @btm_rpt:         Number of extra lines to repeat from bottom
+ */
+struct dpu_drm_pix_ext_v1 {
+	/*
+	 * Number of pixels ext in left, right, top and bottom direction
+	 * for all color components.
+	 */
+	int32_t num_ext_pxls_lr[DPU_MAX_PLANES];
+	int32_t num_ext_pxls_tb[DPU_MAX_PLANES];
+
+	/*
+	 * Number of pixels needs to be overfetched in left, right, top
+	 * and bottom directions from source image for scaling.
+	 */
+	int32_t left_ftch[DPU_MAX_PLANES];
+	int32_t right_ftch[DPU_MAX_PLANES];
+	int32_t top_ftch[DPU_MAX_PLANES];
+	int32_t btm_ftch[DPU_MAX_PLANES];
+	/*
+	 * Number of pixels needs to be repeated in left, right, top and
+	 * bottom directions for scaling.
+	 */
+	int32_t left_rpt[DPU_MAX_PLANES];
+	int32_t right_rpt[DPU_MAX_PLANES];
+	int32_t top_rpt[DPU_MAX_PLANES];
+	int32_t btm_rpt[DPU_MAX_PLANES];
+
+};
+
+/**
+ * struct dpu_drm_de_v1 - version 1 of detail enhancer structure
+ * @enable:         Enables/disables detail enhancer
+ * @sharpen_level1: Sharpening strength for noise
+ * @sharpen_level2: Sharpening strength for context
+ * @clip:           Clip coefficient
+ * @limit:          Detail enhancer limit factor
+ * @thr_quiet:      Quite zone threshold
+ * @thr_dieout:     Die-out zone threshold
+ * @thr_low:        Linear zone left threshold
+ * @thr_high:       Linear zone right threshold
+ * @prec_shift:     Detail enhancer precision
+ * @adjust_a:       Mapping curves A coefficients
+ * @adjust_b:       Mapping curves B coefficients
+ * @adjust_c:       Mapping curves C coefficients
+ */
+struct dpu_drm_de_v1 {
+	uint32_t enable;
+	int16_t sharpen_level1;
+	int16_t sharpen_level2;
+	uint16_t clip;
+	uint16_t limit;
+	uint16_t thr_quiet;
+	uint16_t thr_dieout;
+	uint16_t thr_low;
+	uint16_t thr_high;
+	uint16_t prec_shift;
+	int16_t adjust_a[DPU_MAX_DE_CURVES];
+	int16_t adjust_b[DPU_MAX_DE_CURVES];
+	int16_t adjust_c[DPU_MAX_DE_CURVES];
+};
+
+/**
+ * struct dpu_drm_scaler_v2 - version 2 of struct dpu_drm_scaler
+ * @enable:            Scaler enable
+ * @dir_en:            Detail enhancer enable
+ * @pe:                Pixel extension settings
+ * @horz_decimate:     Horizontal decimation factor
+ * @vert_decimate:     Vertical decimation factor
+ * @init_phase_x:      Initial scaler phase values for x
+ * @phase_step_x:      Phase step values for x
+ * @init_phase_y:      Initial scaler phase values for y
+ * @phase_step_y:      Phase step values for y
+ * @preload_x:         Horizontal preload value
+ * @preload_y:         Vertical preload value
+ * @src_width:         Source width
+ * @src_height:        Source height
+ * @dst_width:         Destination width
+ * @dst_height:        Destination height
+ * @y_rgb_filter_cfg:  Y/RGB plane filter configuration
+ * @uv_filter_cfg:     UV plane filter configuration
+ * @alpha_filter_cfg:  Alpha filter configuration
+ * @blend_cfg:         Selection of blend coefficients
+ * @lut_flag:          LUT configuration flags
+ * @dir_lut_idx:       2d 4x4 LUT index
+ * @y_rgb_cir_lut_idx: Y/RGB circular LUT index
+ * @uv_cir_lut_idx:    UV circular LUT index
+ * @y_rgb_sep_lut_idx: Y/RGB separable LUT index
+ * @uv_sep_lut_idx:    UV separable LUT index
+ * @de:                Detail enhancer settings
+ */
+struct dpu_drm_scaler_v2 {
+	/*
+	 * General definitions
+	 */
+	uint32_t enable;
+	uint32_t dir_en;
+
+	/*
+	 * Pix ext settings
+	 */
+	struct dpu_drm_pix_ext_v1 pe;
+
+	/*
+	 * Decimation settings
+	 */
+	uint32_t horz_decimate;
+	uint32_t vert_decimate;
+
+	/*
+	 * Phase settings
+	 */
+	int32_t init_phase_x[DPU_MAX_PLANES];
+	int32_t phase_step_x[DPU_MAX_PLANES];
+	int32_t init_phase_y[DPU_MAX_PLANES];
+	int32_t phase_step_y[DPU_MAX_PLANES];
+
+	uint32_t preload_x[DPU_MAX_PLANES];
+	uint32_t preload_y[DPU_MAX_PLANES];
+	uint32_t src_width[DPU_MAX_PLANES];
+	uint32_t src_height[DPU_MAX_PLANES];
+
+	uint32_t dst_width;
+	uint32_t dst_height;
+
+	uint32_t y_rgb_filter_cfg;
+	uint32_t uv_filter_cfg;
+	uint32_t alpha_filter_cfg;
+	uint32_t blend_cfg;
+
+	uint32_t lut_flag;
+	uint32_t dir_lut_idx;
+
+	/* for Y(RGB) and UV planes*/
+	uint32_t y_rgb_cir_lut_idx;
+	uint32_t uv_cir_lut_idx;
+	uint32_t y_rgb_sep_lut_idx;
+	uint32_t uv_sep_lut_idx;
+
+	/*
+	 * Detail enhancer settings
+	 */
+	struct dpu_drm_de_v1 de;
+};
+
+
+u32 *dpu_hw_util_get_log_mask_ptr(void);
+
+void dpu_reg_write(struct dpu_hw_blk_reg_map *c,
+		u32 reg_off,
+		u32 val,
+		const char *name);
+int dpu_reg_read(struct dpu_hw_blk_reg_map *c, u32 reg_off);
+
+#define DPU_REG_WRITE(c, off, val) dpu_reg_write(c, off, val, #off)
+#define DPU_REG_READ(c, off) dpu_reg_read(c, off)
+
+#define MISR_FRAME_COUNT_MASK		0xFF
+#define MISR_CTRL_ENABLE		BIT(8)
+#define MISR_CTRL_STATUS		BIT(9)
+#define MISR_CTRL_STATUS_CLEAR		BIT(10)
+#define INTF_MISR_CTRL_FREE_RUN_MASK	BIT(31)
+
+void *dpu_hw_util_get_dir(void);
+
+void dpu_hw_setup_scaler3(struct dpu_hw_blk_reg_map *c,
+		struct dpu_hw_scaler3_cfg *scaler3_cfg,
+		u32 scaler_offset, u32 scaler_version,
+		const struct dpu_format *format);
+
+u32 dpu_hw_get_scaler3_ver(struct dpu_hw_blk_reg_map *c,
+		u32 scaler_offset);
+
+void dpu_hw_csc_setup(struct dpu_hw_blk_reg_map  *c,
+		u32 csc_reg_off,
+		struct dpu_csc_cfg *data, bool csc10);
+
+#endif /* _DPU_HW_UTIL_H */
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_vbif.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_vbif.c
new file mode 100644
index 000000000000..d43905525f92
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_vbif.c
@@ -0,0 +1,275 @@
+/* Copyright (c) 2015-2018, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include "dpu_hwio.h"
+#include "dpu_hw_catalog.h"
+#include "dpu_hw_vbif.h"
+#include "dpu_dbg.h"
+
+#define VBIF_VERSION			0x0000
+#define VBIF_CLK_FORCE_CTRL0		0x0008
+#define VBIF_CLK_FORCE_CTRL1		0x000C
+#define VBIF_QOS_REMAP_00		0x0020
+#define VBIF_QOS_REMAP_01		0x0024
+#define VBIF_QOS_REMAP_10		0x0028
+#define VBIF_QOS_REMAP_11		0x002C
+#define VBIF_WRITE_GATHER_EN		0x00AC
+#define VBIF_IN_RD_LIM_CONF0		0x00B0
+#define VBIF_IN_RD_LIM_CONF1		0x00B4
+#define VBIF_IN_RD_LIM_CONF2		0x00B8
+#define VBIF_IN_WR_LIM_CONF0		0x00C0
+#define VBIF_IN_WR_LIM_CONF1		0x00C4
+#define VBIF_IN_WR_LIM_CONF2		0x00C8
+#define VBIF_OUT_RD_LIM_CONF0		0x00D0
+#define VBIF_OUT_WR_LIM_CONF0		0x00D4
+#define VBIF_OUT_AXI_AMEMTYPE_CONF0	0x0160
+#define VBIF_OUT_AXI_AMEMTYPE_CONF1	0x0164
+#define VBIF_XIN_PND_ERR		0x0190
+#define VBIF_XIN_SRC_ERR		0x0194
+#define VBIF_XIN_CLR_ERR		0x019C
+#define VBIF_XIN_HALT_CTRL0		0x0200
+#define VBIF_XIN_HALT_CTRL1		0x0204
+#define VBIF_XINL_QOS_RP_REMAP_000	0x0550
+#define VBIF_XINL_QOS_LVL_REMAP_000	0x0590
+
+static void dpu_hw_clear_errors(struct dpu_hw_vbif *vbif,
+		u32 *pnd_errors, u32 *src_errors)
+{
+	struct dpu_hw_blk_reg_map *c;
+	u32 pnd, src;
+
+	if (!vbif)
+		return;
+	c = &vbif->hw;
+	pnd = DPU_REG_READ(c, VBIF_XIN_PND_ERR);
+	src = DPU_REG_READ(c, VBIF_XIN_SRC_ERR);
+
+	if (pnd_errors)
+		*pnd_errors = pnd;
+	if (src_errors)
+		*src_errors = src;
+
+	DPU_REG_WRITE(c, VBIF_XIN_CLR_ERR, pnd | src);
+}
+
+static void dpu_hw_set_mem_type(struct dpu_hw_vbif *vbif,
+		u32 xin_id, u32 value)
+{
+	struct dpu_hw_blk_reg_map *c;
+	u32 reg_off;
+	u32 bit_off;
+	u32 reg_val;
+
+	/*
+	 * Assume 4 bits per bit field, 8 fields per 32-bit register so
+	 * 16 bit fields maximum across two registers
+	 */
+	if (!vbif || xin_id >= MAX_XIN_COUNT || xin_id >= 16)
+		return;
+
+	c = &vbif->hw;
+
+	if (xin_id >= 8) {
+		xin_id -= 8;
+		reg_off = VBIF_OUT_AXI_AMEMTYPE_CONF1;
+	} else {
+		reg_off = VBIF_OUT_AXI_AMEMTYPE_CONF0;
+	}
+	bit_off = (xin_id & 0x7) * 4;
+	reg_val = DPU_REG_READ(c, reg_off);
+	reg_val &= ~(0x7 << bit_off);
+	reg_val |= (value & 0x7) << bit_off;
+	DPU_REG_WRITE(c, reg_off, reg_val);
+}
+
+static void dpu_hw_set_limit_conf(struct dpu_hw_vbif *vbif,
+		u32 xin_id, bool rd, u32 limit)
+{
+	struct dpu_hw_blk_reg_map *c = &vbif->hw;
+	u32 reg_val;
+	u32 reg_off;
+	u32 bit_off;
+
+	if (rd)
+		reg_off = VBIF_IN_RD_LIM_CONF0;
+	else
+		reg_off = VBIF_IN_WR_LIM_CONF0;
+
+	reg_off += (xin_id / 4) * 4;
+	bit_off = (xin_id % 4) * 8;
+	reg_val = DPU_REG_READ(c, reg_off);
+	reg_val &= ~(0xFF << bit_off);
+	reg_val |= (limit) << bit_off;
+	DPU_REG_WRITE(c, reg_off, reg_val);
+}
+
+static u32 dpu_hw_get_limit_conf(struct dpu_hw_vbif *vbif,
+		u32 xin_id, bool rd)
+{
+	struct dpu_hw_blk_reg_map *c = &vbif->hw;
+	u32 reg_val;
+	u32 reg_off;
+	u32 bit_off;
+	u32 limit;
+
+	if (rd)
+		reg_off = VBIF_IN_RD_LIM_CONF0;
+	else
+		reg_off = VBIF_IN_WR_LIM_CONF0;
+
+	reg_off += (xin_id / 4) * 4;
+	bit_off = (xin_id % 4) * 8;
+	reg_val = DPU_REG_READ(c, reg_off);
+	limit = (reg_val >> bit_off) & 0xFF;
+
+	return limit;
+}
+
+static void dpu_hw_set_halt_ctrl(struct dpu_hw_vbif *vbif,
+		u32 xin_id, bool enable)
+{
+	struct dpu_hw_blk_reg_map *c = &vbif->hw;
+	u32 reg_val;
+
+	reg_val = DPU_REG_READ(c, VBIF_XIN_HALT_CTRL0);
+
+	if (enable)
+		reg_val |= BIT(xin_id);
+	else
+		reg_val &= ~BIT(xin_id);
+
+	DPU_REG_WRITE(c, VBIF_XIN_HALT_CTRL0, reg_val);
+}
+
+static bool dpu_hw_get_halt_ctrl(struct dpu_hw_vbif *vbif,
+		u32 xin_id)
+{
+	struct dpu_hw_blk_reg_map *c = &vbif->hw;
+	u32 reg_val;
+
+	reg_val = DPU_REG_READ(c, VBIF_XIN_HALT_CTRL1);
+
+	return (reg_val & BIT(xin_id)) ? true : false;
+}
+
+static void dpu_hw_set_qos_remap(struct dpu_hw_vbif *vbif,
+		u32 xin_id, u32 level, u32 remap_level)
+{
+	struct dpu_hw_blk_reg_map *c;
+	u32 reg_val, reg_val_lvl, mask, reg_high, reg_shift;
+
+	if (!vbif)
+		return;
+
+	c = &vbif->hw;
+
+	reg_high = ((xin_id & 0x8) >> 3) * 4 + (level * 8);
+	reg_shift = (xin_id & 0x7) * 4;
+
+	reg_val = DPU_REG_READ(c, VBIF_XINL_QOS_RP_REMAP_000 + reg_high);
+	reg_val_lvl = DPU_REG_READ(c, VBIF_XINL_QOS_LVL_REMAP_000 + reg_high);
+
+	mask = 0x7 << reg_shift;
+
+	reg_val &= ~mask;
+	reg_val |= (remap_level << reg_shift) & mask;
+
+	reg_val_lvl &= ~mask;
+	reg_val_lvl |= (remap_level << reg_shift) & mask;
+
+	DPU_REG_WRITE(c, VBIF_XINL_QOS_RP_REMAP_000 + reg_high, reg_val);
+	DPU_REG_WRITE(c, VBIF_XINL_QOS_LVL_REMAP_000 + reg_high, reg_val_lvl);
+}
+
+static void dpu_hw_set_write_gather_en(struct dpu_hw_vbif *vbif, u32 xin_id)
+{
+	struct dpu_hw_blk_reg_map *c;
+	u32 reg_val;
+
+	if (!vbif || xin_id >= MAX_XIN_COUNT)
+		return;
+
+	c = &vbif->hw;
+
+	reg_val = DPU_REG_READ(c, VBIF_WRITE_GATHER_EN);
+	reg_val |= BIT(xin_id);
+	DPU_REG_WRITE(c, VBIF_WRITE_GATHER_EN, reg_val);
+}
+
+static void _setup_vbif_ops(struct dpu_hw_vbif_ops *ops,
+		unsigned long cap)
+{
+	ops->set_limit_conf = dpu_hw_set_limit_conf;
+	ops->get_limit_conf = dpu_hw_get_limit_conf;
+	ops->set_halt_ctrl = dpu_hw_set_halt_ctrl;
+	ops->get_halt_ctrl = dpu_hw_get_halt_ctrl;
+	if (test_bit(DPU_VBIF_QOS_REMAP, &cap))
+		ops->set_qos_remap = dpu_hw_set_qos_remap;
+	ops->set_mem_type = dpu_hw_set_mem_type;
+	ops->clear_errors = dpu_hw_clear_errors;
+	ops->set_write_gather_en = dpu_hw_set_write_gather_en;
+}
+
+static const struct dpu_vbif_cfg *_top_offset(enum dpu_vbif vbif,
+		const struct dpu_mdss_cfg *m,
+		void __iomem *addr,
+		struct dpu_hw_blk_reg_map *b)
+{
+	int i;
+
+	for (i = 0; i < m->vbif_count; i++) {
+		if (vbif == m->vbif[i].id) {
+			b->base_off = addr;
+			b->blk_off = m->vbif[i].base;
+			b->length = m->vbif[i].len;
+			b->hwversion = m->hwversion;
+			b->log_mask = DPU_DBG_MASK_VBIF;
+			return &m->vbif[i];
+		}
+	}
+
+	return ERR_PTR(-EINVAL);
+}
+
+struct dpu_hw_vbif *dpu_hw_vbif_init(enum dpu_vbif idx,
+		void __iomem *addr,
+		const struct dpu_mdss_cfg *m)
+{
+	struct dpu_hw_vbif *c;
+	const struct dpu_vbif_cfg *cfg;
+
+	c = kzalloc(sizeof(*c), GFP_KERNEL);
+	if (!c)
+		return ERR_PTR(-ENOMEM);
+
+	cfg = _top_offset(idx, m, addr, &c->hw);
+	if (IS_ERR_OR_NULL(cfg)) {
+		kfree(c);
+		return ERR_PTR(-EINVAL);
+	}
+
+	/*
+	 * Assign ops
+	 */
+	c->idx = idx;
+	c->cap = cfg;
+	_setup_vbif_ops(&c->ops, c->cap->features);
+
+	/* no need to register sub-range in dpu dbg, dump entire vbif io base */
+
+	return c;
+}
+
+void dpu_hw_vbif_destroy(struct dpu_hw_vbif *vbif)
+{
+	kfree(vbif);
+}
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_vbif.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_vbif.h
new file mode 100644
index 000000000000..471ff673c045
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_vbif.h
@@ -0,0 +1,128 @@
+/* Copyright (c) 2015-2018, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef _DPU_HW_VBIF_H
+#define _DPU_HW_VBIF_H
+
+#include "dpu_hw_catalog.h"
+#include "dpu_hw_mdss.h"
+#include "dpu_hw_util.h"
+
+struct dpu_hw_vbif;
+
+/**
+ * struct dpu_hw_vbif_ops : Interface to the VBIF hardware driver functions
+ *  Assumption is these functions will be called after clocks are enabled
+ */
+struct dpu_hw_vbif_ops {
+	/**
+	 * set_limit_conf - set transaction limit config
+	 * @vbif: vbif context driver
+	 * @xin_id: client interface identifier
+	 * @rd: true for read limit; false for write limit
+	 * @limit: outstanding transaction limit
+	 */
+	void (*set_limit_conf)(struct dpu_hw_vbif *vbif,
+			u32 xin_id, bool rd, u32 limit);
+
+	/**
+	 * get_limit_conf - get transaction limit config
+	 * @vbif: vbif context driver
+	 * @xin_id: client interface identifier
+	 * @rd: true for read limit; false for write limit
+	 * @return: outstanding transaction limit
+	 */
+	u32 (*get_limit_conf)(struct dpu_hw_vbif *vbif,
+			u32 xin_id, bool rd);
+
+	/**
+	 * set_halt_ctrl - set halt control
+	 * @vbif: vbif context driver
+	 * @xin_id: client interface identifier
+	 * @enable: halt control enable
+	 */
+	void (*set_halt_ctrl)(struct dpu_hw_vbif *vbif,
+			u32 xin_id, bool enable);
+
+	/**
+	 * get_halt_ctrl - get halt control
+	 * @vbif: vbif context driver
+	 * @xin_id: client interface identifier
+	 * @return: halt control enable
+	 */
+	bool (*get_halt_ctrl)(struct dpu_hw_vbif *vbif,
+			u32 xin_id);
+
+	/**
+	 * set_qos_remap - set QoS priority remap
+	 * @vbif: vbif context driver
+	 * @xin_id: client interface identifier
+	 * @level: priority level
+	 * @remap_level: remapped level
+	 */
+	void (*set_qos_remap)(struct dpu_hw_vbif *vbif,
+			u32 xin_id, u32 level, u32 remap_level);
+
+	/**
+	 * set_mem_type - set memory type
+	 * @vbif: vbif context driver
+	 * @xin_id: client interface identifier
+	 * @value: memory type value
+	 */
+	void (*set_mem_type)(struct dpu_hw_vbif *vbif,
+			u32 xin_id, u32 value);
+
+	/**
+	 * clear_errors - clear any vbif errors
+	 *	This function clears any detected pending/source errors
+	 *	on the VBIF interface, and optionally returns the detected
+	 *	error mask(s).
+	 * @vbif: vbif context driver
+	 * @pnd_errors: pointer to pending error reporting variable
+	 * @src_errors: pointer to source error reporting variable
+	 */
+	void (*clear_errors)(struct dpu_hw_vbif *vbif,
+		u32 *pnd_errors, u32 *src_errors);
+
+	/**
+	 * set_write_gather_en - set write_gather enable
+	 * @vbif: vbif context driver
+	 * @xin_id: client interface identifier
+	 */
+	void (*set_write_gather_en)(struct dpu_hw_vbif *vbif, u32 xin_id);
+};
+
+struct dpu_hw_vbif {
+	/* base */
+	struct dpu_hw_blk_reg_map hw;
+
+	/* vbif */
+	enum dpu_vbif idx;
+	const struct dpu_vbif_cfg *cap;
+
+	/* ops */
+	struct dpu_hw_vbif_ops ops;
+};
+
+/**
+ * dpu_hw_vbif_init - initializes the vbif driver for the passed interface idx
+ * @idx:  Interface index for which driver object is required
+ * @addr: Mapped register io address of MDSS
+ * @m:    Pointer to mdss catalog data
+ */
+struct dpu_hw_vbif *dpu_hw_vbif_init(enum dpu_vbif idx,
+		void __iomem *addr,
+		const struct dpu_mdss_cfg *m);
+
+void dpu_hw_vbif_destroy(struct dpu_hw_vbif *vbif);
+
+#endif /*_DPU_HW_VBIF_H */
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hwio.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hwio.h
new file mode 100644
index 000000000000..5b2bc9b65b15
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hwio.h
@@ -0,0 +1,56 @@
+/* Copyright (c) 2015-2018, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef _DPU_HWIO_H
+#define _DPU_HWIO_H
+
+#include "dpu_hw_util.h"
+
+/**
+ * MDP TOP block Register and bit fields and defines
+ */
+#define DISP_INTF_SEL                   0x004
+#define INTR_EN                         0x010
+#define INTR_STATUS                     0x014
+#define INTR_CLEAR                      0x018
+#define INTR2_EN                        0x008
+#define INTR2_STATUS                    0x00c
+#define INTR2_CLEAR                     0x02c
+#define HIST_INTR_EN                    0x01c
+#define HIST_INTR_STATUS                0x020
+#define HIST_INTR_CLEAR                 0x024
+#define INTF_INTR_EN                    0x1C0
+#define INTF_INTR_STATUS                0x1C4
+#define INTF_INTR_CLEAR                 0x1C8
+#define SPLIT_DISPLAY_EN                0x2F4
+#define SPLIT_DISPLAY_UPPER_PIPE_CTRL   0x2F8
+#define DSPP_IGC_COLOR0_RAM_LUTN        0x300
+#define DSPP_IGC_COLOR1_RAM_LUTN        0x304
+#define DSPP_IGC_COLOR2_RAM_LUTN        0x308
+#define HW_EVENTS_CTL                   0x37C
+#define CLK_CTRL3                       0x3A8
+#define CLK_STATUS3                     0x3AC
+#define CLK_CTRL4                       0x3B0
+#define CLK_STATUS4                     0x3B4
+#define CLK_CTRL5                       0x3B8
+#define CLK_STATUS5                     0x3BC
+#define CLK_CTRL7                       0x3D0
+#define CLK_STATUS7                     0x3D4
+#define SPLIT_DISPLAY_LOWER_PIPE_CTRL   0x3F0
+#define SPLIT_DISPLAY_TE_LINE_INTERVAL  0x3F4
+#define INTF_SW_RESET_MASK              0x3FC
+#define HDMI_DP_CORE_SELECT             0x408
+#define MDP_OUT_CTL_0                   0x410
+#define MDP_VSYNC_SEL                   0x414
+#define DCE_SEL                         0x450
+
+#endif /*_DPU_HWIO_H */
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_io_util.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_io_util.c
new file mode 100644
index 000000000000..790d39f816dc
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_io_util.c
@@ -0,0 +1,203 @@
+/* Copyright (c) 2012-2015, 2017-2018, The Linux Foundation.
+ * All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/clk.h>
+#include <linux/clk/clk-conf.h>
+#include <linux/err.h>
+#include <linux/delay.h>
+
+#include "dpu_io_util.h"
+
+void msm_dss_put_clk(struct dss_clk *clk_arry, int num_clk)
+{
+	int i;
+
+	for (i = num_clk - 1; i >= 0; i--) {
+		if (clk_arry[i].clk)
+			clk_put(clk_arry[i].clk);
+		clk_arry[i].clk = NULL;
+	}
+}
+
+int msm_dss_get_clk(struct device *dev, struct dss_clk *clk_arry, int num_clk)
+{
+	int i, rc = 0;
+
+	for (i = 0; i < num_clk; i++) {
+		clk_arry[i].clk = clk_get(dev, clk_arry[i].clk_name);
+		rc = PTR_ERR_OR_ZERO(clk_arry[i].clk);
+		if (rc) {
+			DEV_ERR("%pS->%s: '%s' get failed. rc=%d\n",
+				__builtin_return_address(0), __func__,
+				clk_arry[i].clk_name, rc);
+			goto error;
+		}
+	}
+
+	return rc;
+
+error:
+	for (i--; i >= 0; i--) {
+		if (clk_arry[i].clk)
+			clk_put(clk_arry[i].clk);
+		clk_arry[i].clk = NULL;
+	}
+
+	return rc;
+}
+
+int msm_dss_clk_set_rate(struct dss_clk *clk_arry, int num_clk)
+{
+	int i, rc = 0;
+
+	for (i = 0; i < num_clk; i++) {
+		if (clk_arry[i].clk) {
+			if (clk_arry[i].type != DSS_CLK_AHB) {
+				DEV_DBG("%pS->%s: '%s' rate %ld\n",
+					__builtin_return_address(0), __func__,
+					clk_arry[i].clk_name,
+					clk_arry[i].rate);
+				rc = clk_set_rate(clk_arry[i].clk,
+					clk_arry[i].rate);
+				if (rc) {
+					DEV_ERR("%pS->%s: %s failed. rc=%d\n",
+						__builtin_return_address(0),
+						__func__,
+						clk_arry[i].clk_name, rc);
+					break;
+				}
+			}
+		} else {
+			DEV_ERR("%pS->%s: '%s' is not available\n",
+				__builtin_return_address(0), __func__,
+				clk_arry[i].clk_name);
+			rc = -EPERM;
+			break;
+		}
+	}
+
+	return rc;
+}
+
+int msm_dss_enable_clk(struct dss_clk *clk_arry, int num_clk, int enable)
+{
+	int i, rc = 0;
+
+	if (enable) {
+		for (i = 0; i < num_clk; i++) {
+			DEV_DBG("%pS->%s: enable '%s'\n",
+				__builtin_return_address(0), __func__,
+				clk_arry[i].clk_name);
+			if (clk_arry[i].clk) {
+				rc = clk_prepare_enable(clk_arry[i].clk);
+				if (rc)
+					DEV_ERR("%pS->%s: %s en fail. rc=%d\n",
+						__builtin_return_address(0),
+						__func__,
+						clk_arry[i].clk_name, rc);
+			} else {
+				DEV_ERR("%pS->%s: '%s' is not available\n",
+					__builtin_return_address(0), __func__,
+					clk_arry[i].clk_name);
+				rc = -EPERM;
+			}
+
+			if (rc) {
+				msm_dss_enable_clk(&clk_arry[i],
+					i, false);
+				break;
+			}
+		}
+	} else {
+		for (i = num_clk - 1; i >= 0; i--) {
+			DEV_DBG("%pS->%s: disable '%s'\n",
+				__builtin_return_address(0), __func__,
+				clk_arry[i].clk_name);
+
+			if (clk_arry[i].clk)
+				clk_disable_unprepare(clk_arry[i].clk);
+			else
+				DEV_ERR("%pS->%s: '%s' is not available\n",
+					__builtin_return_address(0), __func__,
+					clk_arry[i].clk_name);
+		}
+	}
+
+	return rc;
+}
+
+int msm_dss_parse_clock(struct platform_device *pdev,
+			struct dss_module_power *mp)
+{
+	u32 i, rc = 0;
+	const char *clock_name;
+	int num_clk = 0;
+
+	if (!pdev || !mp)
+		return -EINVAL;
+
+	mp->num_clk = 0;
+	num_clk = of_property_count_strings(pdev->dev.of_node, "clock-names");
+	if (num_clk <= 0) {
+		pr_debug("clocks are not defined\n");
+		return 0;
+	}
+
+	mp->clk_config = devm_kzalloc(&pdev->dev,
+				      sizeof(struct dss_clk) * num_clk,
+				      GFP_KERNEL);
+	if (!mp->clk_config)
+		return -ENOMEM;
+
+	for (i = 0; i < num_clk; i++) {
+		rc = of_property_read_string_index(pdev->dev.of_node,
+						   "clock-names", i,
+						   &clock_name);
+		if (rc) {
+			dev_err(&pdev->dev, "Failed to get clock name for %d\n",
+				i);
+			break;
+		}
+		strlcpy(mp->clk_config[i].clk_name, clock_name,
+			sizeof(mp->clk_config[i].clk_name));
+
+		mp->clk_config[i].type = DSS_CLK_AHB;
+	}
+
+	rc = msm_dss_get_clk(&pdev->dev, mp->clk_config, num_clk);
+	if (rc) {
+		dev_err(&pdev->dev, "Failed to get clock refs %d\n", rc);
+		goto err;
+	}
+
+	rc = of_clk_set_defaults(pdev->dev.of_node, false);
+	if (rc) {
+		dev_err(&pdev->dev, "Failed to set clock defaults %d\n", rc);
+		goto err;
+	}
+
+	for (i = 0; i < num_clk; i++) {
+		u32 rate = clk_get_rate(mp->clk_config[i].clk);
+		if (!rate)
+			continue;
+		mp->clk_config[i].rate = rate;
+		mp->clk_config[i].type = DSS_CLK_PCLK;
+	}
+
+	mp->num_clk = num_clk;
+	return 0;
+
+err:
+	msm_dss_put_clk(mp->clk_config, num_clk);
+	return rc;
+}
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_io_util.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_io_util.h
new file mode 100644
index 000000000000..bc07381d7429
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_io_util.h
@@ -0,0 +1,57 @@
+/* Copyright (c) 2012, 2017-2018, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef __DPU_IO_UTIL_H__
+#define __DPU_IO_UTIL_H__
+
+#include <linux/gpio.h>
+#include <linux/platform_device.h>
+#include <linux/types.h>
+
+#define DEV_DBG(fmt, args...)   pr_debug(fmt, ##args)
+#define DEV_INFO(fmt, args...)  pr_info(fmt, ##args)
+#define DEV_WARN(fmt, args...)  pr_warn(fmt, ##args)
+#define DEV_ERR(fmt, args...)   pr_err(fmt, ##args)
+
+struct dss_gpio {
+	unsigned int gpio;
+	unsigned int value;
+	char gpio_name[32];
+};
+
+enum dss_clk_type {
+	DSS_CLK_AHB, /* no set rate. rate controlled through rpm */
+	DSS_CLK_PCLK,
+};
+
+struct dss_clk {
+	struct clk *clk; /* clk handle */
+	char clk_name[32];
+	enum dss_clk_type type;
+	unsigned long rate;
+	unsigned long max_rate;
+};
+
+struct dss_module_power {
+	unsigned int num_gpio;
+	struct dss_gpio *gpio_config;
+	unsigned int num_clk;
+	struct dss_clk *clk_config;
+};
+
+int msm_dss_get_clk(struct device *dev, struct dss_clk *clk_arry, int num_clk);
+void msm_dss_put_clk(struct dss_clk *clk_arry, int num_clk);
+int msm_dss_clk_set_rate(struct dss_clk *clk_arry, int num_clk);
+int msm_dss_enable_clk(struct dss_clk *clk_arry, int num_clk, int enable);
+int msm_dss_parse_clock(struct platform_device *pdev,
+		struct dss_module_power *mp);
+#endif /* __DPU_IO_UTIL_H__ */
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_irq.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_irq.c
new file mode 100644
index 000000000000..d5e6ce0140cf
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_irq.c
@@ -0,0 +1,66 @@
+/* Copyright (c) 2015-2018, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#define pr_fmt(fmt)	"[drm:%s:%d] " fmt, __func__, __LINE__
+
+#include <linux/irqdomain.h>
+#include <linux/irq.h>
+#include <linux/kthread.h>
+
+#include "dpu_irq.h"
+#include "dpu_core_irq.h"
+
+irqreturn_t dpu_irq(struct msm_kms *kms)
+{
+	struct dpu_kms *dpu_kms = to_dpu_kms(kms);
+
+	return dpu_core_irq(dpu_kms);
+}
+
+void dpu_irq_preinstall(struct msm_kms *kms)
+{
+	struct dpu_kms *dpu_kms = to_dpu_kms(kms);
+
+	if (!dpu_kms->dev || !dpu_kms->dev->dev) {
+		pr_err("invalid device handles\n");
+		return;
+	}
+
+	dpu_core_irq_preinstall(dpu_kms);
+}
+
+int dpu_irq_postinstall(struct msm_kms *kms)
+{
+	struct dpu_kms *dpu_kms = to_dpu_kms(kms);
+	int rc;
+
+	if (!kms) {
+		DPU_ERROR("invalid parameters\n");
+		return -EINVAL;
+	}
+
+	rc = dpu_core_irq_postinstall(dpu_kms);
+
+	return rc;
+}
+
+void dpu_irq_uninstall(struct msm_kms *kms)
+{
+	struct dpu_kms *dpu_kms = to_dpu_kms(kms);
+
+	if (!kms) {
+		DPU_ERROR("invalid parameters\n");
+		return;
+	}
+
+	dpu_core_irq_uninstall(dpu_kms);
+}
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_irq.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_irq.h
new file mode 100644
index 000000000000..3e147f7176e2
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_irq.h
@@ -0,0 +1,59 @@
+/* Copyright (c) 2015-2018, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef __DPU_IRQ_H__
+#define __DPU_IRQ_H__
+
+#include <linux/kernel.h>
+#include <linux/irqdomain.h>
+
+#include "msm_kms.h"
+
+/**
+ * dpu_irq_controller - define MDSS level interrupt controller context
+ * @enabled_mask:	enable status of MDSS level interrupt
+ * @domain:		interrupt domain of this controller
+ */
+struct dpu_irq_controller {
+	unsigned long enabled_mask;
+	struct irq_domain *domain;
+};
+
+/**
+ * dpu_irq_preinstall - perform pre-installation of MDSS IRQ handler
+ * @kms:		pointer to kms context
+ * @return:		none
+ */
+void dpu_irq_preinstall(struct msm_kms *kms);
+
+/**
+ * dpu_irq_postinstall - perform post-installation of MDSS IRQ handler
+ * @kms:		pointer to kms context
+ * @return:		0 if success; error code otherwise
+ */
+int dpu_irq_postinstall(struct msm_kms *kms);
+
+/**
+ * dpu_irq_uninstall - uninstall MDSS IRQ handler
+ * @drm_dev:		pointer to kms context
+ * @return:		none
+ */
+void dpu_irq_uninstall(struct msm_kms *kms);
+
+/**
+ * dpu_irq - MDSS level IRQ handler
+ * @kms:		pointer to kms context
+ * @return:		interrupt handling status
+ */
+irqreturn_t dpu_irq(struct msm_kms *kms);
+
+#endif /* __DPU_IRQ_H__ */
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c
new file mode 100644
index 000000000000..7dd6bd2d6d37
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c
@@ -0,0 +1,1345 @@
+/*
+ * Copyright (c) 2014-2018, The Linux Foundation. All rights reserved.
+ * Copyright (C) 2013 Red Hat
+ * Author: Rob Clark <robdclark@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define pr_fmt(fmt)	"[drm:%s:%d] " fmt, __func__, __LINE__
+
+#include <drm/drm_crtc.h>
+#include <linux/debugfs.h>
+#include <linux/of_irq.h>
+#include <linux/dma-buf.h>
+
+#include "msm_drv.h"
+#include "msm_mmu.h"
+#include "msm_gem.h"
+
+#include "dpu_kms.h"
+#include "dpu_core_irq.h"
+#include "dpu_formats.h"
+#include "dpu_hw_vbif.h"
+#include "dpu_vbif.h"
+#include "dpu_encoder.h"
+#include "dpu_plane.h"
+#include "dpu_crtc.h"
+
+#define CREATE_TRACE_POINTS
+#include "dpu_trace.h"
+
+static const char * const iommu_ports[] = {
+		"mdp_0",
+};
+
+/*
+ * To enable overall DRM driver logging
+ * # echo 0x2 > /sys/module/drm/parameters/debug
+ *
+ * To enable DRM driver h/w logging
+ * # echo <mask> > /sys/kernel/debug/dri/0/debug/hw_log_mask
+ *
+ * See dpu_hw_mdss.h for h/w logging mask definitions (search for DPU_DBG_MASK_)
+ */
+#define DPU_DEBUGFS_DIR "msm_dpu"
+#define DPU_DEBUGFS_HWMASKNAME "hw_log_mask"
+
+static int dpu_kms_hw_init(struct msm_kms *kms);
+static int _dpu_kms_mmu_destroy(struct dpu_kms *dpu_kms);
+
+static unsigned long dpu_iomap_size(struct platform_device *pdev,
+				    const char *name)
+{
+	struct resource *res;
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, name);
+	if (!res) {
+		DRM_ERROR("failed to get memory resource: %s\n", name);
+		return 0;
+	}
+
+	return resource_size(res);
+}
+
+#ifdef CONFIG_DEBUG_FS
+static int _dpu_danger_signal_status(struct seq_file *s,
+		bool danger_status)
+{
+	struct dpu_kms *kms = (struct dpu_kms *)s->private;
+	struct msm_drm_private *priv;
+	struct dpu_danger_safe_status status;
+	int i;
+
+	if (!kms || !kms->dev || !kms->dev->dev_private || !kms->hw_mdp) {
+		DPU_ERROR("invalid arg(s)\n");
+		return 0;
+	}
+
+	priv = kms->dev->dev_private;
+	memset(&status, 0, sizeof(struct dpu_danger_safe_status));
+
+	pm_runtime_get_sync(&kms->pdev->dev);
+	if (danger_status) {
+		seq_puts(s, "\nDanger signal status:\n");
+		if (kms->hw_mdp->ops.get_danger_status)
+			kms->hw_mdp->ops.get_danger_status(kms->hw_mdp,
+					&status);
+	} else {
+		seq_puts(s, "\nSafe signal status:\n");
+		if (kms->hw_mdp->ops.get_danger_status)
+			kms->hw_mdp->ops.get_danger_status(kms->hw_mdp,
+					&status);
+	}
+	pm_runtime_put_sync(&kms->pdev->dev);
+
+	seq_printf(s, "MDP     :  0x%x\n", status.mdp);
+
+	for (i = SSPP_VIG0; i < SSPP_MAX; i++)
+		seq_printf(s, "SSPP%d   :  0x%x  \t", i - SSPP_VIG0,
+				status.sspp[i]);
+	seq_puts(s, "\n");
+
+	return 0;
+}
+
+#define DEFINE_DPU_DEBUGFS_SEQ_FOPS(__prefix)				\
+static int __prefix ## _open(struct inode *inode, struct file *file)	\
+{									\
+	return single_open(file, __prefix ## _show, inode->i_private);	\
+}									\
+static const struct file_operations __prefix ## _fops = {		\
+	.owner = THIS_MODULE,						\
+	.open = __prefix ## _open,					\
+	.release = single_release,					\
+	.read = seq_read,						\
+	.llseek = seq_lseek,						\
+}
+
+static int dpu_debugfs_danger_stats_show(struct seq_file *s, void *v)
+{
+	return _dpu_danger_signal_status(s, true);
+}
+DEFINE_DPU_DEBUGFS_SEQ_FOPS(dpu_debugfs_danger_stats);
+
+static int dpu_debugfs_safe_stats_show(struct seq_file *s, void *v)
+{
+	return _dpu_danger_signal_status(s, false);
+}
+DEFINE_DPU_DEBUGFS_SEQ_FOPS(dpu_debugfs_safe_stats);
+
+static void dpu_debugfs_danger_destroy(struct dpu_kms *dpu_kms)
+{
+	debugfs_remove_recursive(dpu_kms->debugfs_danger);
+	dpu_kms->debugfs_danger = NULL;
+}
+
+static int dpu_debugfs_danger_init(struct dpu_kms *dpu_kms,
+		struct dentry *parent)
+{
+	dpu_kms->debugfs_danger = debugfs_create_dir("danger",
+			parent);
+	if (!dpu_kms->debugfs_danger) {
+		DPU_ERROR("failed to create danger debugfs\n");
+		return -EINVAL;
+	}
+
+	debugfs_create_file("danger_status", 0600, dpu_kms->debugfs_danger,
+			dpu_kms, &dpu_debugfs_danger_stats_fops);
+	debugfs_create_file("safe_status", 0600, dpu_kms->debugfs_danger,
+			dpu_kms, &dpu_debugfs_safe_stats_fops);
+
+	return 0;
+}
+
+static int _dpu_debugfs_show_regset32(struct seq_file *s, void *data)
+{
+	struct dpu_debugfs_regset32 *regset;
+	struct dpu_kms *dpu_kms;
+	struct drm_device *dev;
+	struct msm_drm_private *priv;
+	void __iomem *base;
+	uint32_t i, addr;
+
+	if (!s || !s->private)
+		return 0;
+
+	regset = s->private;
+
+	dpu_kms = regset->dpu_kms;
+	if (!dpu_kms || !dpu_kms->mmio)
+		return 0;
+
+	dev = dpu_kms->dev;
+	if (!dev)
+		return 0;
+
+	priv = dev->dev_private;
+	if (!priv)
+		return 0;
+
+	base = dpu_kms->mmio + regset->offset;
+
+	/* insert padding spaces, if needed */
+	if (regset->offset & 0xF) {
+		seq_printf(s, "[%x]", regset->offset & ~0xF);
+		for (i = 0; i < (regset->offset & 0xF); i += 4)
+			seq_puts(s, "         ");
+	}
+
+	pm_runtime_get_sync(&dpu_kms->pdev->dev);
+
+	/* main register output */
+	for (i = 0; i < regset->blk_len; i += 4) {
+		addr = regset->offset + i;
+		if ((addr & 0xF) == 0x0)
+			seq_printf(s, i ? "\n[%x]" : "[%x]", addr);
+		seq_printf(s, " %08x", readl_relaxed(base + i));
+	}
+	seq_puts(s, "\n");
+	pm_runtime_put_sync(&dpu_kms->pdev->dev);
+
+	return 0;
+}
+
+static int dpu_debugfs_open_regset32(struct inode *inode,
+		struct file *file)
+{
+	return single_open(file, _dpu_debugfs_show_regset32, inode->i_private);
+}
+
+static const struct file_operations dpu_fops_regset32 = {
+	.open =		dpu_debugfs_open_regset32,
+	.read =		seq_read,
+	.llseek =	seq_lseek,
+	.release =	single_release,
+};
+
+void dpu_debugfs_setup_regset32(struct dpu_debugfs_regset32 *regset,
+		uint32_t offset, uint32_t length, struct dpu_kms *dpu_kms)
+{
+	if (regset) {
+		regset->offset = offset;
+		regset->blk_len = length;
+		regset->dpu_kms = dpu_kms;
+	}
+}
+
+void *dpu_debugfs_create_regset32(const char *name, umode_t mode,
+		void *parent, struct dpu_debugfs_regset32 *regset)
+{
+	if (!name || !regset || !regset->dpu_kms || !regset->blk_len)
+		return NULL;
+
+	/* make sure offset is a multiple of 4 */
+	regset->offset = round_down(regset->offset, 4);
+
+	return debugfs_create_file(name, mode, parent,
+			regset, &dpu_fops_regset32);
+}
+
+static int _dpu_debugfs_init(struct dpu_kms *dpu_kms)
+{
+	void *p;
+	int rc;
+
+	p = dpu_hw_util_get_log_mask_ptr();
+
+	if (!dpu_kms || !p)
+		return -EINVAL;
+
+	dpu_kms->debugfs_root = debugfs_create_dir("debug",
+					   dpu_kms->dev->primary->debugfs_root);
+	if (IS_ERR_OR_NULL(dpu_kms->debugfs_root)) {
+		DRM_ERROR("debugfs create_dir failed %ld\n",
+			  PTR_ERR(dpu_kms->debugfs_root));
+		return PTR_ERR(dpu_kms->debugfs_root);
+	}
+
+	rc = dpu_dbg_debugfs_register(dpu_kms->debugfs_root);
+	if (rc) {
+		DRM_ERROR("failed to reg dpu dbg debugfs: %d\n", rc);
+		return rc;
+	}
+
+	/* allow root to be NULL */
+	debugfs_create_x32(DPU_DEBUGFS_HWMASKNAME, 0600, dpu_kms->debugfs_root, p);
+
+	(void) dpu_debugfs_danger_init(dpu_kms, dpu_kms->debugfs_root);
+	(void) dpu_debugfs_vbif_init(dpu_kms, dpu_kms->debugfs_root);
+	(void) dpu_debugfs_core_irq_init(dpu_kms, dpu_kms->debugfs_root);
+
+	rc = dpu_core_perf_debugfs_init(&dpu_kms->perf, dpu_kms->debugfs_root);
+	if (rc) {
+		DPU_ERROR("failed to init perf %d\n", rc);
+		return rc;
+	}
+
+	return 0;
+}
+
+static void _dpu_debugfs_destroy(struct dpu_kms *dpu_kms)
+{
+	/* don't need to NULL check debugfs_root */
+	if (dpu_kms) {
+		dpu_debugfs_vbif_destroy(dpu_kms);
+		dpu_debugfs_danger_destroy(dpu_kms);
+		dpu_debugfs_core_irq_destroy(dpu_kms);
+		debugfs_remove_recursive(dpu_kms->debugfs_root);
+	}
+}
+#else
+static void _dpu_debugfs_destroy(struct dpu_kms *dpu_kms)
+{
+}
+#endif
+
+static int dpu_kms_enable_vblank(struct msm_kms *kms, struct drm_crtc *crtc)
+{
+	return dpu_crtc_vblank(crtc, true);
+}
+
+static void dpu_kms_disable_vblank(struct msm_kms *kms, struct drm_crtc *crtc)
+{
+	dpu_crtc_vblank(crtc, false);
+}
+
+static void dpu_kms_prepare_commit(struct msm_kms *kms,
+		struct drm_atomic_state *state)
+{
+	struct dpu_kms *dpu_kms;
+	struct msm_drm_private *priv;
+	struct drm_device *dev;
+	struct drm_encoder *encoder;
+
+	if (!kms)
+		return;
+	dpu_kms = to_dpu_kms(kms);
+	dev = dpu_kms->dev;
+
+	if (!dev || !dev->dev_private)
+		return;
+	priv = dev->dev_private;
+	pm_runtime_get_sync(&dpu_kms->pdev->dev);
+
+	list_for_each_entry(encoder, &dev->mode_config.encoder_list, head)
+		if (encoder->crtc != NULL)
+			dpu_encoder_prepare_commit(encoder);
+}
+
+/*
+ * Override the encoder enable since we need to setup the inline rotator and do
+ * some crtc magic before enabling any bridge that might be present.
+ */
+void dpu_kms_encoder_enable(struct drm_encoder *encoder)
+{
+	const struct drm_encoder_helper_funcs *funcs = encoder->helper_private;
+	struct drm_crtc *crtc = encoder->crtc;
+
+	/* Forward this enable call to the commit hook */
+	if (funcs && funcs->commit)
+		funcs->commit(encoder);
+
+	if (crtc && crtc->state->active) {
+		trace_dpu_kms_enc_enable(DRMID(crtc));
+		dpu_crtc_commit_kickoff(crtc);
+	}
+}
+
+static void dpu_kms_commit(struct msm_kms *kms, struct drm_atomic_state *state)
+{
+	struct drm_crtc *crtc;
+	struct drm_crtc_state *crtc_state;
+	int i;
+
+	for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
+		/* If modeset is required, kickoff is run in encoder_enable */
+		if (drm_atomic_crtc_needs_modeset(crtc_state))
+			continue;
+
+		if (crtc->state->active) {
+			trace_dpu_kms_commit(DRMID(crtc));
+			dpu_crtc_commit_kickoff(crtc);
+		}
+	}
+}
+
+static void dpu_kms_complete_commit(struct msm_kms *kms,
+		struct drm_atomic_state *old_state)
+{
+	struct dpu_kms *dpu_kms;
+	struct msm_drm_private *priv;
+	struct drm_crtc *crtc;
+	struct drm_crtc_state *old_crtc_state;
+	int i;
+
+	if (!kms || !old_state)
+		return;
+	dpu_kms = to_dpu_kms(kms);
+
+	if (!dpu_kms->dev || !dpu_kms->dev->dev_private)
+		return;
+	priv = dpu_kms->dev->dev_private;
+
+	DPU_ATRACE_BEGIN("kms_complete_commit");
+
+	for_each_old_crtc_in_state(old_state, crtc, old_crtc_state, i)
+		dpu_crtc_complete_commit(crtc, old_crtc_state);
+
+	pm_runtime_put_sync(&dpu_kms->pdev->dev);
+
+	DPU_ATRACE_END("kms_complete_commit");
+}
+
+static void dpu_kms_wait_for_commit_done(struct msm_kms *kms,
+		struct drm_crtc *crtc)
+{
+	struct drm_encoder *encoder;
+	struct drm_device *dev;
+	int ret;
+
+	if (!kms || !crtc || !crtc->state) {
+		DPU_ERROR("invalid params\n");
+		return;
+	}
+
+	dev = crtc->dev;
+
+	if (!crtc->state->enable) {
+		DPU_DEBUG("[crtc:%d] not enable\n", crtc->base.id);
+		return;
+	}
+
+	if (!crtc->state->active) {
+		DPU_DEBUG("[crtc:%d] not active\n", crtc->base.id);
+		return;
+	}
+
+	list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
+		if (encoder->crtc != crtc)
+			continue;
+		/*
+		 * Wait for post-flush if necessary to delay before
+		 * plane_cleanup. For example, wait for vsync in case of video
+		 * mode panels. This may be a no-op for command mode panels.
+		 */
+		trace_dpu_kms_wait_for_commit_done(DRMID(crtc));
+		ret = dpu_encoder_wait_for_event(encoder, MSM_ENC_COMMIT_DONE);
+		if (ret && ret != -EWOULDBLOCK) {
+			DPU_ERROR("wait for commit done returned %d\n", ret);
+			break;
+		}
+	}
+}
+
+static void _dpu_kms_initialize_dsi(struct drm_device *dev,
+				    struct msm_drm_private *priv,
+				    struct dpu_kms *dpu_kms)
+{
+	struct drm_encoder *encoder = NULL;
+	int i, rc;
+
+	/*TODO: Support two independent DSI connectors */
+	encoder = dpu_encoder_init(dev, DRM_MODE_CONNECTOR_DSI);
+	if (IS_ERR_OR_NULL(encoder)) {
+		DPU_ERROR("encoder init failed for dsi display\n");
+		return;
+	}
+
+	priv->encoders[priv->num_encoders++] = encoder;
+
+	for (i = 0; i < ARRAY_SIZE(priv->dsi); i++) {
+		if (!priv->dsi[i]) {
+			DPU_DEBUG("invalid msm_dsi for ctrl %d\n", i);
+			return;
+		}
+
+		rc = msm_dsi_modeset_init(priv->dsi[i], dev, encoder);
+		if (rc) {
+			DPU_ERROR("modeset_init failed for dsi[%d], rc = %d\n",
+				i, rc);
+			continue;
+		}
+	}
+}
+
+/**
+ * _dpu_kms_setup_displays - create encoders, bridges and connectors
+ *                           for underlying displays
+ * @dev:        Pointer to drm device structure
+ * @priv:       Pointer to private drm device data
+ * @dpu_kms:    Pointer to dpu kms structure
+ * Returns:     Zero on success
+ */
+static void _dpu_kms_setup_displays(struct drm_device *dev,
+				    struct msm_drm_private *priv,
+				    struct dpu_kms *dpu_kms)
+{
+	_dpu_kms_initialize_dsi(dev, priv, dpu_kms);
+
+	/**
+	 * Extend this function to initialize other
+	 * types of displays
+	 */
+}
+
+static void _dpu_kms_drm_obj_destroy(struct dpu_kms *dpu_kms)
+{
+	struct msm_drm_private *priv;
+	int i;
+
+	if (!dpu_kms) {
+		DPU_ERROR("invalid dpu_kms\n");
+		return;
+	} else if (!dpu_kms->dev) {
+		DPU_ERROR("invalid dev\n");
+		return;
+	} else if (!dpu_kms->dev->dev_private) {
+		DPU_ERROR("invalid dev_private\n");
+		return;
+	}
+	priv = dpu_kms->dev->dev_private;
+
+	for (i = 0; i < priv->num_crtcs; i++)
+		priv->crtcs[i]->funcs->destroy(priv->crtcs[i]);
+	priv->num_crtcs = 0;
+
+	for (i = 0; i < priv->num_planes; i++)
+		priv->planes[i]->funcs->destroy(priv->planes[i]);
+	priv->num_planes = 0;
+
+	for (i = 0; i < priv->num_connectors; i++)
+		priv->connectors[i]->funcs->destroy(priv->connectors[i]);
+	priv->num_connectors = 0;
+
+	for (i = 0; i < priv->num_encoders; i++)
+		priv->encoders[i]->funcs->destroy(priv->encoders[i]);
+	priv->num_encoders = 0;
+}
+
+static int _dpu_kms_drm_obj_init(struct dpu_kms *dpu_kms)
+{
+	struct drm_device *dev;
+	struct drm_plane *primary_planes[MAX_PLANES], *plane;
+	struct drm_crtc *crtc;
+
+	struct msm_drm_private *priv;
+	struct dpu_mdss_cfg *catalog;
+
+	int primary_planes_idx = 0, i, ret;
+	int max_crtc_count;
+
+	if (!dpu_kms || !dpu_kms->dev || !dpu_kms->dev->dev) {
+		DPU_ERROR("invalid dpu_kms\n");
+		return -EINVAL;
+	}
+
+	dev = dpu_kms->dev;
+	priv = dev->dev_private;
+	catalog = dpu_kms->catalog;
+
+	/*
+	 * Create encoder and query display drivers to create
+	 * bridges and connectors
+	 */
+	_dpu_kms_setup_displays(dev, priv, dpu_kms);
+
+	max_crtc_count = min(catalog->mixer_count, priv->num_encoders);
+
+	/* Create the planes */
+	for (i = 0; i < catalog->sspp_count; i++) {
+		bool primary = true;
+
+		if (catalog->sspp[i].features & BIT(DPU_SSPP_CURSOR)
+			|| primary_planes_idx >= max_crtc_count)
+			primary = false;
+
+		plane = dpu_plane_init(dev, catalog->sspp[i].id, primary,
+				(1UL << max_crtc_count) - 1, 0);
+		if (IS_ERR(plane)) {
+			DPU_ERROR("dpu_plane_init failed\n");
+			ret = PTR_ERR(plane);
+			goto fail;
+		}
+		priv->planes[priv->num_planes++] = plane;
+
+		if (primary)
+			primary_planes[primary_planes_idx++] = plane;
+	}
+
+	max_crtc_count = min(max_crtc_count, primary_planes_idx);
+
+	/* Create one CRTC per encoder */
+	for (i = 0; i < max_crtc_count; i++) {
+		crtc = dpu_crtc_init(dev, primary_planes[i]);
+		if (IS_ERR(crtc)) {
+			ret = PTR_ERR(crtc);
+			goto fail;
+		}
+		priv->crtcs[priv->num_crtcs++] = crtc;
+	}
+
+	/* All CRTCs are compatible with all encoders */
+	for (i = 0; i < priv->num_encoders; i++)
+		priv->encoders[i]->possible_crtcs = (1 << priv->num_crtcs) - 1;
+
+	return 0;
+fail:
+	_dpu_kms_drm_obj_destroy(dpu_kms);
+	return ret;
+}
+
+#ifdef CONFIG_DEBUG_FS
+static int dpu_kms_debugfs_init(struct msm_kms *kms, struct drm_minor *minor)
+{
+	struct dpu_kms *dpu_kms = to_dpu_kms(kms);
+	struct drm_device *dev;
+	int rc;
+
+	if (!dpu_kms || !dpu_kms->dev || !dpu_kms->dev->dev) {
+		DPU_ERROR("invalid dpu_kms\n");
+		return -EINVAL;
+	}
+
+	dev = dpu_kms->dev;
+
+	rc = _dpu_debugfs_init(dpu_kms);
+	if (rc)
+		DPU_ERROR("dpu_debugfs init failed: %d\n", rc);
+
+	return rc;
+}
+#endif
+
+static long dpu_kms_round_pixclk(struct msm_kms *kms, unsigned long rate,
+		struct drm_encoder *encoder)
+{
+	return rate;
+}
+
+static void _dpu_kms_hw_destroy(struct dpu_kms *dpu_kms)
+{
+	struct drm_device *dev;
+	int i;
+
+	dev = dpu_kms->dev;
+	if (!dev)
+		return;
+
+	if (dpu_kms->hw_intr)
+		dpu_hw_intr_destroy(dpu_kms->hw_intr);
+	dpu_kms->hw_intr = NULL;
+
+	if (dpu_kms->power_event)
+		dpu_power_handle_unregister_event(
+				&dpu_kms->phandle, dpu_kms->power_event);
+
+	/* safe to call these more than once during shutdown */
+	_dpu_debugfs_destroy(dpu_kms);
+	_dpu_kms_mmu_destroy(dpu_kms);
+
+	if (dpu_kms->catalog) {
+		for (i = 0; i < dpu_kms->catalog->vbif_count; i++) {
+			u32 vbif_idx = dpu_kms->catalog->vbif[i].id;
+
+			if ((vbif_idx < VBIF_MAX) && dpu_kms->hw_vbif[vbif_idx])
+				dpu_hw_vbif_destroy(dpu_kms->hw_vbif[vbif_idx]);
+		}
+	}
+
+	if (dpu_kms->rm_init)
+		dpu_rm_destroy(&dpu_kms->rm);
+	dpu_kms->rm_init = false;
+
+	if (dpu_kms->catalog)
+		dpu_hw_catalog_deinit(dpu_kms->catalog);
+	dpu_kms->catalog = NULL;
+
+	if (dpu_kms->core_client)
+		dpu_power_client_destroy(&dpu_kms->phandle,
+			dpu_kms->core_client);
+	dpu_kms->core_client = NULL;
+
+	if (dpu_kms->vbif[VBIF_NRT])
+		devm_iounmap(&dpu_kms->pdev->dev, dpu_kms->vbif[VBIF_NRT]);
+	dpu_kms->vbif[VBIF_NRT] = NULL;
+
+	if (dpu_kms->vbif[VBIF_RT])
+		devm_iounmap(&dpu_kms->pdev->dev, dpu_kms->vbif[VBIF_RT]);
+	dpu_kms->vbif[VBIF_RT] = NULL;
+
+	if (dpu_kms->mmio)
+		devm_iounmap(&dpu_kms->pdev->dev, dpu_kms->mmio);
+	dpu_kms->mmio = NULL;
+}
+
+static void dpu_kms_destroy(struct msm_kms *kms)
+{
+	struct dpu_kms *dpu_kms;
+
+	if (!kms) {
+		DPU_ERROR("invalid kms\n");
+		return;
+	}
+
+	dpu_kms = to_dpu_kms(kms);
+
+	dpu_dbg_destroy();
+	_dpu_kms_hw_destroy(dpu_kms);
+}
+
+static int dpu_kms_pm_suspend(struct device *dev)
+{
+	struct drm_device *ddev;
+	struct drm_modeset_acquire_ctx ctx;
+	struct drm_atomic_state *state;
+	struct dpu_kms *dpu_kms;
+	int ret = 0, num_crtcs = 0;
+
+	if (!dev)
+		return -EINVAL;
+
+	ddev = dev_get_drvdata(dev);
+	if (!ddev || !ddev_to_msm_kms(ddev))
+		return -EINVAL;
+
+	dpu_kms = to_dpu_kms(ddev_to_msm_kms(ddev));
+
+	/* disable hot-plug polling */
+	drm_kms_helper_poll_disable(ddev);
+
+	/* acquire modeset lock(s) */
+	drm_modeset_acquire_init(&ctx, 0);
+
+retry:
+	DPU_ATRACE_BEGIN("kms_pm_suspend");
+
+	ret = drm_modeset_lock_all_ctx(ddev, &ctx);
+	if (ret)
+		goto unlock;
+
+	/* save current state for resume */
+	if (dpu_kms->suspend_state)
+		drm_atomic_state_put(dpu_kms->suspend_state);
+	dpu_kms->suspend_state = drm_atomic_helper_duplicate_state(ddev, &ctx);
+	if (IS_ERR_OR_NULL(dpu_kms->suspend_state)) {
+		DRM_ERROR("failed to back up suspend state\n");
+		dpu_kms->suspend_state = NULL;
+		goto unlock;
+	}
+
+	/* create atomic state to disable all CRTCs */
+	state = drm_atomic_state_alloc(ddev);
+	if (IS_ERR_OR_NULL(state)) {
+		DRM_ERROR("failed to allocate crtc disable state\n");
+		goto unlock;
+	}
+
+	state->acquire_ctx = &ctx;
+
+	/* check for nothing to do */
+	if (num_crtcs == 0) {
+		DRM_DEBUG("all crtcs are already in the off state\n");
+		drm_atomic_state_put(state);
+		goto suspended;
+	}
+
+	/* commit the "disable all" state */
+	ret = drm_atomic_commit(state);
+	if (ret < 0) {
+		DRM_ERROR("failed to disable crtcs, %d\n", ret);
+		drm_atomic_state_put(state);
+		goto unlock;
+	}
+
+suspended:
+	dpu_kms->suspend_block = true;
+
+unlock:
+	if (ret == -EDEADLK) {
+		drm_modeset_backoff(&ctx);
+		goto retry;
+	}
+	drm_modeset_drop_locks(&ctx);
+	drm_modeset_acquire_fini(&ctx);
+
+	DPU_ATRACE_END("kms_pm_suspend");
+	return 0;
+}
+
+static int dpu_kms_pm_resume(struct device *dev)
+{
+	struct drm_device *ddev;
+	struct dpu_kms *dpu_kms;
+	int ret;
+
+	if (!dev)
+		return -EINVAL;
+
+	ddev = dev_get_drvdata(dev);
+	if (!ddev || !ddev_to_msm_kms(ddev))
+		return -EINVAL;
+
+	dpu_kms = to_dpu_kms(ddev_to_msm_kms(ddev));
+
+	DPU_ATRACE_BEGIN("kms_pm_resume");
+
+	drm_mode_config_reset(ddev);
+
+	drm_modeset_lock_all(ddev);
+
+	dpu_kms->suspend_block = false;
+
+	if (dpu_kms->suspend_state) {
+		dpu_kms->suspend_state->acquire_ctx =
+			ddev->mode_config.acquire_ctx;
+		ret = drm_atomic_commit(dpu_kms->suspend_state);
+		if (ret < 0) {
+			DRM_ERROR("failed to restore state, %d\n", ret);
+			drm_atomic_state_put(dpu_kms->suspend_state);
+		}
+		dpu_kms->suspend_state = NULL;
+	}
+	drm_modeset_unlock_all(ddev);
+
+	/* enable hot-plug polling */
+	drm_kms_helper_poll_enable(ddev);
+
+	DPU_ATRACE_END("kms_pm_resume");
+	return 0;
+}
+
+static void _dpu_kms_set_encoder_mode(struct msm_kms *kms,
+				 struct drm_encoder *encoder,
+				 bool cmd_mode)
+{
+	struct msm_display_info info;
+	struct msm_drm_private *priv = encoder->dev->dev_private;
+	int i, rc = 0;
+
+	memset(&info, 0, sizeof(info));
+
+	info.intf_type = encoder->encoder_type;
+	info.capabilities = cmd_mode ? MSM_DISPLAY_CAP_CMD_MODE :
+			MSM_DISPLAY_CAP_VID_MODE;
+
+	/* TODO: No support for DSI swap */
+	for (i = 0; i < ARRAY_SIZE(priv->dsi); i++) {
+		if (priv->dsi[i]) {
+			info.h_tile_instance[info.num_of_h_tiles] = i;
+			info.num_of_h_tiles++;
+		}
+	}
+
+	rc = dpu_encoder_setup(encoder->dev, encoder, &info);
+	if (rc)
+		DPU_ERROR("failed to setup DPU encoder %d: rc:%d\n",
+			encoder->base.id, rc);
+}
+
+static const struct msm_kms_funcs kms_funcs = {
+	.hw_init         = dpu_kms_hw_init,
+	.irq_preinstall  = dpu_irq_preinstall,
+	.irq_postinstall = dpu_irq_postinstall,
+	.irq_uninstall   = dpu_irq_uninstall,
+	.irq             = dpu_irq,
+	.prepare_commit  = dpu_kms_prepare_commit,
+	.commit          = dpu_kms_commit,
+	.complete_commit = dpu_kms_complete_commit,
+	.wait_for_crtc_commit_done = dpu_kms_wait_for_commit_done,
+	.enable_vblank   = dpu_kms_enable_vblank,
+	.disable_vblank  = dpu_kms_disable_vblank,
+	.check_modified_format = dpu_format_check_modified_format,
+	.get_format      = dpu_get_msm_format,
+	.round_pixclk    = dpu_kms_round_pixclk,
+	.pm_suspend      = dpu_kms_pm_suspend,
+	.pm_resume       = dpu_kms_pm_resume,
+	.destroy         = dpu_kms_destroy,
+	.set_encoder_mode = _dpu_kms_set_encoder_mode,
+#ifdef CONFIG_DEBUG_FS
+	.debugfs_init    = dpu_kms_debugfs_init,
+#endif
+};
+
+/* the caller api needs to turn on clock before calling it */
+static inline void _dpu_kms_core_hw_rev_init(struct dpu_kms *dpu_kms)
+{
+	dpu_kms->core_rev = readl_relaxed(dpu_kms->mmio + 0x0);
+}
+
+static int _dpu_kms_mmu_destroy(struct dpu_kms *dpu_kms)
+{
+	struct msm_mmu *mmu;
+
+	mmu = dpu_kms->base.aspace->mmu;
+
+	mmu->funcs->detach(mmu, (const char **)iommu_ports,
+			ARRAY_SIZE(iommu_ports));
+	msm_gem_address_space_put(dpu_kms->base.aspace);
+
+	return 0;
+}
+
+static int _dpu_kms_mmu_init(struct dpu_kms *dpu_kms)
+{
+	struct iommu_domain *domain;
+	struct msm_gem_address_space *aspace;
+	int ret;
+
+	domain = iommu_domain_alloc(&platform_bus_type);
+	if (!domain)
+		return 0;
+
+	aspace = msm_gem_address_space_create(dpu_kms->dev->dev,
+			domain, "dpu1");
+	if (IS_ERR(aspace)) {
+		ret = PTR_ERR(aspace);
+		goto fail;
+	}
+
+	dpu_kms->base.aspace = aspace;
+
+	ret = aspace->mmu->funcs->attach(aspace->mmu, iommu_ports,
+			ARRAY_SIZE(iommu_ports));
+	if (ret) {
+		DPU_ERROR("failed to attach iommu %d\n", ret);
+		msm_gem_address_space_put(aspace);
+		goto fail;
+	}
+
+	return 0;
+fail:
+	_dpu_kms_mmu_destroy(dpu_kms);
+
+	return ret;
+}
+
+static struct dss_clk *_dpu_kms_get_clk(struct dpu_kms *dpu_kms,
+		char *clock_name)
+{
+	struct dss_module_power *mp = &dpu_kms->mp;
+	int i;
+
+	for (i = 0; i < mp->num_clk; i++) {
+		if (!strcmp(mp->clk_config[i].clk_name, clock_name))
+			return &mp->clk_config[i];
+	}
+
+	return NULL;
+}
+
+u64 dpu_kms_get_clk_rate(struct dpu_kms *dpu_kms, char *clock_name)
+{
+	struct dss_clk *clk;
+
+	clk = _dpu_kms_get_clk(dpu_kms, clock_name);
+	if (!clk)
+		return -EINVAL;
+
+	return clk_get_rate(clk->clk);
+}
+
+static void dpu_kms_handle_power_event(u32 event_type, void *usr)
+{
+	struct dpu_kms *dpu_kms = usr;
+
+	if (!dpu_kms)
+		return;
+
+	if (event_type == DPU_POWER_EVENT_POST_ENABLE)
+		dpu_vbif_init_memtypes(dpu_kms);
+}
+
+static int dpu_kms_hw_init(struct msm_kms *kms)
+{
+	struct dpu_kms *dpu_kms;
+	struct drm_device *dev;
+	struct msm_drm_private *priv;
+	int i, rc = -EINVAL;
+
+	if (!kms) {
+		DPU_ERROR("invalid kms\n");
+		goto end;
+	}
+
+	dpu_kms = to_dpu_kms(kms);
+	dev = dpu_kms->dev;
+	if (!dev) {
+		DPU_ERROR("invalid device\n");
+		goto end;
+	}
+
+	rc = dpu_dbg_init(&dpu_kms->pdev->dev);
+	if (rc) {
+		DRM_ERROR("failed to init dpu dbg: %d\n", rc);
+		goto end;
+	}
+
+	priv = dev->dev_private;
+	if (!priv) {
+		DPU_ERROR("invalid private data\n");
+		goto dbg_destroy;
+	}
+
+	dpu_kms->mmio = msm_ioremap(dpu_kms->pdev, "mdp", "mdp");
+	if (IS_ERR(dpu_kms->mmio)) {
+		rc = PTR_ERR(dpu_kms->mmio);
+		DPU_ERROR("mdp register memory map failed: %d\n", rc);
+		dpu_kms->mmio = NULL;
+		goto error;
+	}
+	DRM_DEBUG("mapped dpu address space @%pK\n", dpu_kms->mmio);
+	dpu_kms->mmio_len = dpu_iomap_size(dpu_kms->pdev, "mdp");
+
+	dpu_kms->vbif[VBIF_RT] = msm_ioremap(dpu_kms->pdev, "vbif", "vbif");
+	if (IS_ERR(dpu_kms->vbif[VBIF_RT])) {
+		rc = PTR_ERR(dpu_kms->vbif[VBIF_RT]);
+		DPU_ERROR("vbif register memory map failed: %d\n", rc);
+		dpu_kms->vbif[VBIF_RT] = NULL;
+		goto error;
+	}
+	dpu_kms->vbif_len[VBIF_RT] = dpu_iomap_size(dpu_kms->pdev, "vbif");
+	dpu_kms->vbif[VBIF_NRT] = msm_ioremap(dpu_kms->pdev, "vbif_nrt", "vbif_nrt");
+	if (IS_ERR(dpu_kms->vbif[VBIF_NRT])) {
+		dpu_kms->vbif[VBIF_NRT] = NULL;
+		DPU_DEBUG("VBIF NRT is not defined");
+	} else {
+		dpu_kms->vbif_len[VBIF_NRT] = dpu_iomap_size(dpu_kms->pdev,
+							     "vbif_nrt");
+	}
+
+	dpu_kms->reg_dma = msm_ioremap(dpu_kms->pdev, "regdma", "regdma");
+	if (IS_ERR(dpu_kms->reg_dma)) {
+		dpu_kms->reg_dma = NULL;
+		DPU_DEBUG("REG_DMA is not defined");
+	} else {
+		dpu_kms->reg_dma_len = dpu_iomap_size(dpu_kms->pdev, "regdma");
+	}
+
+	dpu_kms->core_client = dpu_power_client_create(&dpu_kms->phandle,
+					"core");
+	if (IS_ERR_OR_NULL(dpu_kms->core_client)) {
+		rc = PTR_ERR(dpu_kms->core_client);
+		if (!dpu_kms->core_client)
+			rc = -EINVAL;
+		DPU_ERROR("dpu power client create failed: %d\n", rc);
+		dpu_kms->core_client = NULL;
+		goto error;
+	}
+
+	pm_runtime_get_sync(&dpu_kms->pdev->dev);
+
+	_dpu_kms_core_hw_rev_init(dpu_kms);
+
+	pr_info("dpu hardware revision:0x%x\n", dpu_kms->core_rev);
+
+	dpu_kms->catalog = dpu_hw_catalog_init(dpu_kms->core_rev);
+	if (IS_ERR_OR_NULL(dpu_kms->catalog)) {
+		rc = PTR_ERR(dpu_kms->catalog);
+		if (!dpu_kms->catalog)
+			rc = -EINVAL;
+		DPU_ERROR("catalog init failed: %d\n", rc);
+		dpu_kms->catalog = NULL;
+		goto power_error;
+	}
+
+	dpu_dbg_init_dbg_buses(dpu_kms->core_rev);
+
+	/*
+	 * Now we need to read the HW catalog and initialize resources such as
+	 * clocks, regulators, GDSC/MMAGIC, ioremap the register ranges etc
+	 */
+	rc = _dpu_kms_mmu_init(dpu_kms);
+	if (rc) {
+		DPU_ERROR("dpu_kms_mmu_init failed: %d\n", rc);
+		goto power_error;
+	}
+
+	rc = dpu_rm_init(&dpu_kms->rm, dpu_kms->catalog, dpu_kms->mmio,
+			dpu_kms->dev);
+	if (rc) {
+		DPU_ERROR("rm init failed: %d\n", rc);
+		goto power_error;
+	}
+
+	dpu_kms->rm_init = true;
+
+	dpu_kms->hw_mdp = dpu_rm_get_mdp(&dpu_kms->rm);
+	if (IS_ERR_OR_NULL(dpu_kms->hw_mdp)) {
+		rc = PTR_ERR(dpu_kms->hw_mdp);
+		if (!dpu_kms->hw_mdp)
+			rc = -EINVAL;
+		DPU_ERROR("failed to get hw_mdp: %d\n", rc);
+		dpu_kms->hw_mdp = NULL;
+		goto power_error;
+	}
+
+	for (i = 0; i < dpu_kms->catalog->vbif_count; i++) {
+		u32 vbif_idx = dpu_kms->catalog->vbif[i].id;
+
+		dpu_kms->hw_vbif[i] = dpu_hw_vbif_init(vbif_idx,
+				dpu_kms->vbif[vbif_idx], dpu_kms->catalog);
+		if (IS_ERR_OR_NULL(dpu_kms->hw_vbif[vbif_idx])) {
+			rc = PTR_ERR(dpu_kms->hw_vbif[vbif_idx]);
+			if (!dpu_kms->hw_vbif[vbif_idx])
+				rc = -EINVAL;
+			DPU_ERROR("failed to init vbif %d: %d\n", vbif_idx, rc);
+			dpu_kms->hw_vbif[vbif_idx] = NULL;
+			goto power_error;
+		}
+	}
+
+	rc = dpu_core_perf_init(&dpu_kms->perf, dev, dpu_kms->catalog,
+			&dpu_kms->phandle,
+			_dpu_kms_get_clk(dpu_kms, "core"));
+	if (rc) {
+		DPU_ERROR("failed to init perf %d\n", rc);
+		goto perf_err;
+	}
+
+	dpu_kms->hw_intr = dpu_hw_intr_init(dpu_kms->mmio, dpu_kms->catalog);
+	if (IS_ERR_OR_NULL(dpu_kms->hw_intr)) {
+		rc = PTR_ERR(dpu_kms->hw_intr);
+		DPU_ERROR("hw_intr init failed: %d\n", rc);
+		dpu_kms->hw_intr = NULL;
+		goto hw_intr_init_err;
+	}
+
+	/*
+	 * _dpu_kms_drm_obj_init should create the DRM related objects
+	 * i.e. CRTCs, planes, encoders, connectors and so forth
+	 */
+	rc = _dpu_kms_drm_obj_init(dpu_kms);
+	if (rc) {
+		DPU_ERROR("modeset init failed: %d\n", rc);
+		goto drm_obj_init_err;
+	}
+
+	dev->mode_config.min_width = 0;
+	dev->mode_config.min_height = 0;
+
+	/*
+	 * max crtc width is equal to the max mixer width * 2 and max height is
+	 * is 4K
+	 */
+	dev->mode_config.max_width =
+			dpu_kms->catalog->caps->max_mixer_width * 2;
+	dev->mode_config.max_height = 4096;
+
+	/*
+	 * Support format modifiers for compression etc.
+	 */
+	dev->mode_config.allow_fb_modifiers = true;
+
+	/*
+	 * Handle (re)initializations during power enable
+	 */
+	dpu_kms_handle_power_event(DPU_POWER_EVENT_POST_ENABLE, dpu_kms);
+	dpu_kms->power_event = dpu_power_handle_register_event(
+			&dpu_kms->phandle,
+			DPU_POWER_EVENT_POST_ENABLE,
+			dpu_kms_handle_power_event, dpu_kms, "kms");
+
+	pm_runtime_put_sync(&dpu_kms->pdev->dev);
+
+	return 0;
+
+drm_obj_init_err:
+	dpu_core_perf_destroy(&dpu_kms->perf);
+hw_intr_init_err:
+perf_err:
+power_error:
+	pm_runtime_put_sync(&dpu_kms->pdev->dev);
+error:
+	_dpu_kms_hw_destroy(dpu_kms);
+dbg_destroy:
+	dpu_dbg_destroy();
+end:
+	return rc;
+}
+
+struct msm_kms *dpu_kms_init(struct drm_device *dev)
+{
+	struct msm_drm_private *priv;
+	struct dpu_kms *dpu_kms;
+	int irq;
+
+	if (!dev || !dev->dev_private) {
+		DPU_ERROR("drm device node invalid\n");
+		return ERR_PTR(-EINVAL);
+	}
+
+	priv = dev->dev_private;
+	dpu_kms = to_dpu_kms(priv->kms);
+
+	irq = irq_of_parse_and_map(dpu_kms->pdev->dev.of_node, 0);
+	if (irq < 0) {
+		DPU_ERROR("failed to get irq: %d\n", irq);
+		return ERR_PTR(irq);
+	}
+	dpu_kms->base.irq = irq;
+
+	return &dpu_kms->base;
+}
+
+static int dpu_bind(struct device *dev, struct device *master, void *data)
+{
+	struct drm_device *ddev = dev_get_drvdata(master);
+	struct platform_device *pdev = to_platform_device(dev);
+	struct msm_drm_private *priv = ddev->dev_private;
+	struct dpu_kms *dpu_kms;
+	struct dss_module_power *mp;
+	int ret = 0;
+
+	dpu_kms = devm_kzalloc(&pdev->dev, sizeof(*dpu_kms), GFP_KERNEL);
+	if (!dpu_kms)
+		return -ENOMEM;
+
+	mp = &dpu_kms->mp;
+	ret = msm_dss_parse_clock(pdev, mp);
+	if (ret) {
+		DPU_ERROR("failed to parse clocks, ret=%d\n", ret);
+		return ret;
+	}
+
+	dpu_power_resource_init(pdev, &dpu_kms->phandle);
+
+	platform_set_drvdata(pdev, dpu_kms);
+
+	msm_kms_init(&dpu_kms->base, &kms_funcs);
+	dpu_kms->dev = ddev;
+	dpu_kms->pdev = pdev;
+
+	pm_runtime_enable(&pdev->dev);
+	dpu_kms->rpm_enabled = true;
+
+	priv->kms = &dpu_kms->base;
+	return ret;
+}
+
+static void dpu_unbind(struct device *dev, struct device *master, void *data)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct dpu_kms *dpu_kms = platform_get_drvdata(pdev);
+	struct dss_module_power *mp = &dpu_kms->mp;
+
+	dpu_power_resource_deinit(pdev, &dpu_kms->phandle);
+	msm_dss_put_clk(mp->clk_config, mp->num_clk);
+	devm_kfree(&pdev->dev, mp->clk_config);
+	mp->num_clk = 0;
+
+	if (dpu_kms->rpm_enabled)
+		pm_runtime_disable(&pdev->dev);
+}
+
+static const struct component_ops dpu_ops = {
+	.bind   = dpu_bind,
+	.unbind = dpu_unbind,
+};
+
+static int dpu_dev_probe(struct platform_device *pdev)
+{
+	return component_add(&pdev->dev, &dpu_ops);
+}
+
+static int dpu_dev_remove(struct platform_device *pdev)
+{
+	component_del(&pdev->dev, &dpu_ops);
+	return 0;
+}
+
+static int __maybe_unused dpu_runtime_suspend(struct device *dev)
+{
+	int rc = -1;
+	struct platform_device *pdev = to_platform_device(dev);
+	struct dpu_kms *dpu_kms = platform_get_drvdata(pdev);
+	struct drm_device *ddev;
+	struct dss_module_power *mp = &dpu_kms->mp;
+
+	ddev = dpu_kms->dev;
+	if (!ddev) {
+		DPU_ERROR("invalid drm_device\n");
+		goto exit;
+	}
+
+	rc = dpu_power_resource_enable(&dpu_kms->phandle,
+			dpu_kms->core_client, false);
+	if (rc)
+		DPU_ERROR("resource disable failed: %d\n", rc);
+
+	rc = msm_dss_enable_clk(mp->clk_config, mp->num_clk, false);
+	if (rc)
+		DPU_ERROR("clock disable failed rc:%d\n", rc);
+
+exit:
+	return rc;
+}
+
+static int __maybe_unused dpu_runtime_resume(struct device *dev)
+{
+	int rc = -1;
+	struct platform_device *pdev = to_platform_device(dev);
+	struct dpu_kms *dpu_kms = platform_get_drvdata(pdev);
+	struct drm_device *ddev;
+	struct dss_module_power *mp = &dpu_kms->mp;
+
+	ddev = dpu_kms->dev;
+	if (!ddev) {
+		DPU_ERROR("invalid drm_device\n");
+		goto exit;
+	}
+
+	rc = msm_dss_enable_clk(mp->clk_config, mp->num_clk, true);
+	if (rc) {
+		DPU_ERROR("clock enable failed rc:%d\n", rc);
+		goto exit;
+	}
+
+	rc = dpu_power_resource_enable(&dpu_kms->phandle,
+			dpu_kms->core_client, true);
+	if (rc)
+		DPU_ERROR("resource enable failed: %d\n", rc);
+
+exit:
+	return rc;
+}
+
+static const struct dev_pm_ops dpu_pm_ops = {
+	SET_RUNTIME_PM_OPS(dpu_runtime_suspend, dpu_runtime_resume, NULL)
+};
+
+static const struct of_device_id dpu_dt_match[] = {
+	{ .compatible = "qcom,sdm845-dpu", },
+	{}
+};
+MODULE_DEVICE_TABLE(of, dpu_dt_match);
+
+static struct platform_driver dpu_driver = {
+	.probe = dpu_dev_probe,
+	.remove = dpu_dev_remove,
+	.driver = {
+		.name = "msm_dpu",
+		.of_match_table = dpu_dt_match,
+		.pm = &dpu_pm_ops,
+	},
+};
+
+void __init msm_dpu_register(void)
+{
+	platform_driver_register(&dpu_driver);
+}
+
+void __exit msm_dpu_unregister(void)
+{
+	platform_driver_unregister(&dpu_driver);
+}
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.h
new file mode 100644
index 000000000000..66d466628e2b
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.h
@@ -0,0 +1,290 @@
+/*
+ * Copyright (c) 2015-2018, The Linux Foundation. All rights reserved.
+ * Copyright (C) 2013 Red Hat
+ * Author: Rob Clark <robdclark@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __DPU_KMS_H__
+#define __DPU_KMS_H__
+
+#include "msm_drv.h"
+#include "msm_kms.h"
+#include "msm_mmu.h"
+#include "msm_gem.h"
+#include "dpu_dbg.h"
+#include "dpu_hw_catalog.h"
+#include "dpu_hw_ctl.h"
+#include "dpu_hw_lm.h"
+#include "dpu_hw_interrupts.h"
+#include "dpu_hw_top.h"
+#include "dpu_rm.h"
+#include "dpu_power_handle.h"
+#include "dpu_irq.h"
+#include "dpu_core_perf.h"
+
+#define DRMID(x) ((x) ? (x)->base.id : -1)
+
+/**
+ * DPU_DEBUG - macro for kms/plane/crtc/encoder/connector logs
+ * @fmt: Pointer to format string
+ */
+#define DPU_DEBUG(fmt, ...)                                                \
+	do {                                                               \
+		if (unlikely(drm_debug & DRM_UT_KMS))                      \
+			DRM_DEBUG(fmt, ##__VA_ARGS__); \
+		else                                                       \
+			pr_debug(fmt, ##__VA_ARGS__);                      \
+	} while (0)
+
+/**
+ * DPU_DEBUG_DRIVER - macro for hardware driver logging
+ * @fmt: Pointer to format string
+ */
+#define DPU_DEBUG_DRIVER(fmt, ...)                                         \
+	do {                                                               \
+		if (unlikely(drm_debug & DRM_UT_DRIVER))                   \
+			DRM_ERROR(fmt, ##__VA_ARGS__); \
+		else                                                       \
+			pr_debug(fmt, ##__VA_ARGS__);                      \
+	} while (0)
+
+#define DPU_ERROR(fmt, ...) pr_err("[dpu error]" fmt, ##__VA_ARGS__)
+
+/**
+ * ktime_compare_safe - compare two ktime structures
+ *	This macro is similar to the standard ktime_compare() function, but
+ *	attempts to also handle ktime overflows.
+ * @A: First ktime value
+ * @B: Second ktime value
+ * Returns: -1 if A < B, 0 if A == B, 1 if A > B
+ */
+#define ktime_compare_safe(A, B) \
+	ktime_compare(ktime_sub((A), (B)), ktime_set(0, 0))
+
+#define DPU_NAME_SIZE  12
+
+/* timeout in frames waiting for frame done */
+#define DPU_FRAME_DONE_TIMEOUT	60
+
+/*
+ * struct dpu_irq_callback - IRQ callback handlers
+ * @list: list to callback
+ * @func: intr handler
+ * @arg: argument for the handler
+ */
+struct dpu_irq_callback {
+	struct list_head list;
+	void (*func)(void *arg, int irq_idx);
+	void *arg;
+};
+
+/**
+ * struct dpu_irq: IRQ structure contains callback registration info
+ * @total_irq:    total number of irq_idx obtained from HW interrupts mapping
+ * @irq_cb_tbl:   array of IRQ callbacks setting
+ * @enable_counts array of IRQ enable counts
+ * @cb_lock:      callback lock
+ * @debugfs_file: debugfs file for irq statistics
+ */
+struct dpu_irq {
+	u32 total_irqs;
+	struct list_head *irq_cb_tbl;
+	atomic_t *enable_counts;
+	atomic_t *irq_counts;
+	spinlock_t cb_lock;
+	struct dentry *debugfs_file;
+};
+
+struct dpu_kms {
+	struct msm_kms base;
+	struct drm_device *dev;
+	int core_rev;
+	struct dpu_mdss_cfg *catalog;
+
+	struct dpu_power_handle phandle;
+	struct dpu_power_client *core_client;
+	struct dpu_power_event *power_event;
+
+	/* directory entry for debugfs */
+	struct dentry *debugfs_root;
+	struct dentry *debugfs_danger;
+	struct dentry *debugfs_vbif;
+
+	/* io/register spaces: */
+	void __iomem *mmio, *vbif[VBIF_MAX], *reg_dma;
+	unsigned long mmio_len, vbif_len[VBIF_MAX], reg_dma_len;
+
+	struct regulator *vdd;
+	struct regulator *mmagic;
+	struct regulator *venus;
+
+	struct dpu_hw_intr *hw_intr;
+	struct dpu_irq irq_obj;
+
+	struct dpu_core_perf perf;
+
+	/* saved atomic state during system suspend */
+	struct drm_atomic_state *suspend_state;
+	bool suspend_block;
+
+	struct dpu_rm rm;
+	bool rm_init;
+
+	struct dpu_hw_vbif *hw_vbif[VBIF_MAX];
+	struct dpu_hw_mdp *hw_mdp;
+
+	bool has_danger_ctrl;
+
+	struct platform_device *pdev;
+	bool rpm_enabled;
+	struct dss_module_power mp;
+};
+
+struct vsync_info {
+	u32 frame_count;
+	u32 line_count;
+};
+
+#define to_dpu_kms(x) container_of(x, struct dpu_kms, base)
+
+/* get struct msm_kms * from drm_device * */
+#define ddev_to_msm_kms(D) ((D) && (D)->dev_private ? \
+		((struct msm_drm_private *)((D)->dev_private))->kms : NULL)
+
+/**
+ * dpu_kms_is_suspend_state - whether or not the system is pm suspended
+ * @dev: Pointer to drm device
+ * Return: Suspend status
+ */
+static inline bool dpu_kms_is_suspend_state(struct drm_device *dev)
+{
+	if (!ddev_to_msm_kms(dev))
+		return false;
+
+	return to_dpu_kms(ddev_to_msm_kms(dev))->suspend_state != NULL;
+}
+
+/**
+ * dpu_kms_is_suspend_blocked - whether or not commits are blocked due to pm
+ *				suspend status
+ * @dev: Pointer to drm device
+ * Return: True if commits should be rejected due to pm suspend
+ */
+static inline bool dpu_kms_is_suspend_blocked(struct drm_device *dev)
+{
+	if (!dpu_kms_is_suspend_state(dev))
+		return false;
+
+	return to_dpu_kms(ddev_to_msm_kms(dev))->suspend_block;
+}
+
+/**
+ * Debugfs functions - extra helper functions for debugfs support
+ *
+ * Main debugfs documentation is located at,
+ *
+ * Documentation/filesystems/debugfs.txt
+ *
+ * @dpu_debugfs_setup_regset32: Initialize data for dpu_debugfs_create_regset32
+ * @dpu_debugfs_create_regset32: Create 32-bit register dump file
+ * @dpu_debugfs_get_root: Get root dentry for DPU_KMS's debugfs node
+ */
+
+/**
+ * Companion structure for dpu_debugfs_create_regset32. Do not initialize the
+ * members of this structure explicitly; use dpu_debugfs_setup_regset32 instead.
+ */
+struct dpu_debugfs_regset32 {
+	uint32_t offset;
+	uint32_t blk_len;
+	struct dpu_kms *dpu_kms;
+};
+
+/**
+ * dpu_debugfs_setup_regset32 - Initialize register block definition for debugfs
+ * This function is meant to initialize dpu_debugfs_regset32 structures for use
+ * with dpu_debugfs_create_regset32.
+ * @regset: opaque register definition structure
+ * @offset: sub-block offset
+ * @length: sub-block length, in bytes
+ * @dpu_kms: pointer to dpu kms structure
+ */
+void dpu_debugfs_setup_regset32(struct dpu_debugfs_regset32 *regset,
+		uint32_t offset, uint32_t length, struct dpu_kms *dpu_kms);
+
+/**
+ * dpu_debugfs_create_regset32 - Create register read back file for debugfs
+ *
+ * This function is almost identical to the standard debugfs_create_regset32()
+ * function, with the main difference being that a list of register
+ * names/offsets do not need to be provided. The 'read' function simply outputs
+ * sequential register values over a specified range.
+ *
+ * Similar to the related debugfs_create_regset32 API, the structure pointed to
+ * by regset needs to persist for the lifetime of the created file. The calling
+ * code is responsible for initialization/management of this structure.
+ *
+ * The structure pointed to by regset is meant to be opaque. Please use
+ * dpu_debugfs_setup_regset32 to initialize it.
+ *
+ * @name:   File name within debugfs
+ * @mode:   File mode within debugfs
+ * @parent: Parent directory entry within debugfs, can be NULL
+ * @regset: Pointer to persistent register block definition
+ *
+ * Return: dentry pointer for newly created file, use either debugfs_remove()
+ *         or debugfs_remove_recursive() (on a parent directory) to remove the
+ *         file
+ */
+void *dpu_debugfs_create_regset32(const char *name, umode_t mode,
+		void *parent, struct dpu_debugfs_regset32 *regset);
+
+/**
+ * dpu_debugfs_get_root - Return root directory entry for KMS's debugfs
+ *
+ * The return value should be passed as the 'parent' argument to subsequent
+ * debugfs create calls.
+ *
+ * @dpu_kms: Pointer to DPU's KMS structure
+ *
+ * Return: dentry pointer for DPU's debugfs location
+ */
+void *dpu_debugfs_get_root(struct dpu_kms *dpu_kms);
+
+/**
+ * DPU info management functions
+ * These functions/definitions allow for building up a 'dpu_info' structure
+ * containing one or more "key=value\n" entries.
+ */
+#define DPU_KMS_INFO_MAX_SIZE	4096
+
+/**
+ * Vblank enable/disable functions
+ */
+int dpu_enable_vblank(struct msm_kms *kms, struct drm_crtc *crtc);
+void dpu_disable_vblank(struct msm_kms *kms, struct drm_crtc *crtc);
+
+void dpu_kms_encoder_enable(struct drm_encoder *encoder);
+
+/**
+ * dpu_kms_get_clk_rate() - get the clock rate
+ * @dpu_kms:  poiner to dpu_kms structure
+ * @clock_name: clock name to get the rate
+ *
+ * Return: current clock rate
+ */
+u64 dpu_kms_get_clk_rate(struct dpu_kms *dpu_kms, char *clock_name);
+
+#endif /* __dpu_kms_H__ */
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_mdss.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_mdss.c
new file mode 100644
index 000000000000..9e533b86682c
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_mdss.c
@@ -0,0 +1,245 @@
+/*
+ * SPDX-License-Identifier: GPL-2.0
+ * Copyright (c) 2018, The Linux Foundation
+ */
+
+#include "dpu_kms.h"
+
+#define to_dpu_mdss(x) container_of(x, struct dpu_mdss, base)
+
+#define HW_INTR_STATUS			0x0010
+
+struct dpu_mdss {
+	struct msm_mdss base;
+	void __iomem *mmio;
+	unsigned long mmio_len;
+	u32 hwversion;
+	struct dss_module_power mp;
+	struct dpu_irq_controller irq_controller;
+};
+
+static irqreturn_t dpu_mdss_irq(int irq, void *arg)
+{
+	struct dpu_mdss *dpu_mdss = arg;
+	u32 interrupts;
+
+	interrupts = readl_relaxed(dpu_mdss->mmio + HW_INTR_STATUS);
+
+	while (interrupts) {
+		irq_hw_number_t hwirq = fls(interrupts) - 1;
+		unsigned int mapping;
+		int rc;
+
+		mapping = irq_find_mapping(dpu_mdss->irq_controller.domain,
+					   hwirq);
+		if (mapping == 0) {
+			DRM_ERROR("couldn't find irq mapping for %lu\n", hwirq);
+			return IRQ_NONE;
+		}
+
+		rc = generic_handle_irq(mapping);
+		if (rc < 0) {
+			DRM_ERROR("handle irq fail: irq=%lu mapping=%u rc=%d\n",
+				  hwirq, mapping, rc);
+			return IRQ_NONE;
+		}
+
+		interrupts &= ~(1 << hwirq);
+	}
+
+	return IRQ_HANDLED;
+}
+
+static void dpu_mdss_irq_mask(struct irq_data *irqd)
+{
+	struct dpu_mdss *dpu_mdss = irq_data_get_irq_chip_data(irqd);
+
+	/* memory barrier */
+	smp_mb__before_atomic();
+	clear_bit(irqd->hwirq, &dpu_mdss->irq_controller.enabled_mask);
+	/* memory barrier */
+	smp_mb__after_atomic();
+}
+
+static void dpu_mdss_irq_unmask(struct irq_data *irqd)
+{
+	struct dpu_mdss *dpu_mdss = irq_data_get_irq_chip_data(irqd);
+
+	/* memory barrier */
+	smp_mb__before_atomic();
+	set_bit(irqd->hwirq, &dpu_mdss->irq_controller.enabled_mask);
+	/* memory barrier */
+	smp_mb__after_atomic();
+}
+
+static struct irq_chip dpu_mdss_irq_chip = {
+	.name = "dpu_mdss",
+	.irq_mask = dpu_mdss_irq_mask,
+	.irq_unmask = dpu_mdss_irq_unmask,
+};
+
+static int dpu_mdss_irqdomain_map(struct irq_domain *domain,
+		unsigned int irq, irq_hw_number_t hwirq)
+{
+	struct dpu_mdss *dpu_mdss = domain->host_data;
+	int ret;
+
+	irq_set_chip_and_handler(irq, &dpu_mdss_irq_chip, handle_level_irq);
+	ret = irq_set_chip_data(irq, dpu_mdss);
+
+	return ret;
+}
+
+static const struct irq_domain_ops dpu_mdss_irqdomain_ops = {
+	.map = dpu_mdss_irqdomain_map,
+	.xlate = irq_domain_xlate_onecell,
+};
+
+static int _dpu_mdss_irq_domain_add(struct dpu_mdss *dpu_mdss)
+{
+	struct device *dev;
+	struct irq_domain *domain;
+
+	dev = dpu_mdss->base.dev->dev;
+
+	domain = irq_domain_add_linear(dev->of_node, 32,
+			&dpu_mdss_irqdomain_ops, dpu_mdss);
+	if (!domain) {
+		DPU_ERROR("failed to add irq_domain\n");
+		return -EINVAL;
+	}
+
+	dpu_mdss->irq_controller.enabled_mask = 0;
+	dpu_mdss->irq_controller.domain = domain;
+
+	return 0;
+}
+
+static int _dpu_mdss_irq_domain_fini(struct dpu_mdss *dpu_mdss)
+{
+	if (dpu_mdss->irq_controller.domain) {
+		irq_domain_remove(dpu_mdss->irq_controller.domain);
+		dpu_mdss->irq_controller.domain = NULL;
+	}
+	return 0;
+}
+static int dpu_mdss_enable(struct msm_mdss *mdss)
+{
+	struct dpu_mdss *dpu_mdss = to_dpu_mdss(mdss);
+	struct dss_module_power *mp = &dpu_mdss->mp;
+	int ret;
+
+	ret = msm_dss_enable_clk(mp->clk_config, mp->num_clk, true);
+	if (ret)
+		DPU_ERROR("clock enable failed, ret:%d\n", ret);
+
+	return ret;
+}
+
+static int dpu_mdss_disable(struct msm_mdss *mdss)
+{
+	struct dpu_mdss *dpu_mdss = to_dpu_mdss(mdss);
+	struct dss_module_power *mp = &dpu_mdss->mp;
+	int ret;
+
+	ret = msm_dss_enable_clk(mp->clk_config, mp->num_clk, false);
+	if (ret)
+		DPU_ERROR("clock disable failed, ret:%d\n", ret);
+
+	return ret;
+}
+
+static void dpu_mdss_destroy(struct drm_device *dev)
+{
+	struct platform_device *pdev = to_platform_device(dev->dev);
+	struct msm_drm_private *priv = dev->dev_private;
+	struct dpu_mdss *dpu_mdss = to_dpu_mdss(priv->mdss);
+	struct dss_module_power *mp = &dpu_mdss->mp;
+
+	_dpu_mdss_irq_domain_fini(dpu_mdss);
+
+	msm_dss_put_clk(mp->clk_config, mp->num_clk);
+	devm_kfree(&pdev->dev, mp->clk_config);
+
+	if (dpu_mdss->mmio)
+		devm_iounmap(&pdev->dev, dpu_mdss->mmio);
+	dpu_mdss->mmio = NULL;
+
+	pm_runtime_disable(dev->dev);
+	priv->mdss = NULL;
+}
+
+static const struct msm_mdss_funcs mdss_funcs = {
+	.enable	= dpu_mdss_enable,
+	.disable = dpu_mdss_disable,
+	.destroy = dpu_mdss_destroy,
+};
+
+int dpu_mdss_init(struct drm_device *dev)
+{
+	struct platform_device *pdev = to_platform_device(dev->dev);
+	struct msm_drm_private *priv = dev->dev_private;
+	struct resource *res;
+	struct dpu_mdss *dpu_mdss;
+	struct dss_module_power *mp;
+	int ret = 0;
+
+	dpu_mdss = devm_kzalloc(dev->dev, sizeof(*dpu_mdss), GFP_KERNEL);
+	if (!dpu_mdss)
+		return -ENOMEM;
+
+	dpu_mdss->mmio = msm_ioremap(pdev, "mdss", "mdss");
+	if (IS_ERR(dpu_mdss->mmio))
+		return PTR_ERR(dpu_mdss->mmio);
+
+	DRM_DEBUG("mapped mdss address space @%pK\n", dpu_mdss->mmio);
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "mdss");
+	if (!res) {
+		DRM_ERROR("failed to get memory resource for mdss\n");
+		return -ENOMEM;
+	}
+	dpu_mdss->mmio_len = resource_size(res);
+
+	mp = &dpu_mdss->mp;
+	ret = msm_dss_parse_clock(pdev, mp);
+	if (ret) {
+		DPU_ERROR("failed to parse clocks, ret=%d\n", ret);
+		goto clk_parse_err;
+	}
+
+	dpu_mdss->base.dev = dev;
+	dpu_mdss->base.funcs = &mdss_funcs;
+
+	ret = _dpu_mdss_irq_domain_add(dpu_mdss);
+	if (ret)
+		goto irq_domain_error;
+
+	ret = devm_request_irq(dev->dev, platform_get_irq(pdev, 0),
+			dpu_mdss_irq, 0, "dpu_mdss_isr", dpu_mdss);
+	if (ret) {
+		DPU_ERROR("failed to init irq: %d\n", ret);
+		goto irq_error;
+	}
+
+	pm_runtime_enable(dev->dev);
+
+	pm_runtime_get_sync(dev->dev);
+	dpu_mdss->hwversion = readl_relaxed(dpu_mdss->mmio);
+	pm_runtime_put_sync(dev->dev);
+
+	priv->mdss = &dpu_mdss->base;
+
+	return ret;
+
+irq_error:
+	_dpu_mdss_irq_domain_fini(dpu_mdss);
+irq_domain_error:
+	msm_dss_put_clk(mp->clk_config, mp->num_clk);
+clk_parse_err:
+	devm_kfree(&pdev->dev, mp->clk_config);
+	if (dpu_mdss->mmio)
+		devm_iounmap(&pdev->dev, dpu_mdss->mmio);
+	dpu_mdss->mmio = NULL;
+	return ret;
+}
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.c
new file mode 100644
index 000000000000..b640e39ebaca
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.c
@@ -0,0 +1,1963 @@
+/*
+ * Copyright (C) 2014-2018 The Linux Foundation. All rights reserved.
+ * Copyright (C) 2013 Red Hat
+ * Author: Rob Clark <robdclark@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define pr_fmt(fmt)	"[drm:%s:%d] " fmt, __func__, __LINE__
+
+#include <linux/debugfs.h>
+#include <linux/dma-buf.h>
+
+#include "msm_drv.h"
+#include "dpu_kms.h"
+#include "dpu_formats.h"
+#include "dpu_hw_sspp.h"
+#include "dpu_hw_catalog_format.h"
+#include "dpu_trace.h"
+#include "dpu_crtc.h"
+#include "dpu_vbif.h"
+#include "dpu_plane.h"
+
+#define DPU_DEBUG_PLANE(pl, fmt, ...) DPU_DEBUG("plane%d " fmt,\
+		(pl) ? (pl)->base.base.id : -1, ##__VA_ARGS__)
+
+#define DPU_ERROR_PLANE(pl, fmt, ...) DPU_ERROR("plane%d " fmt,\
+		(pl) ? (pl)->base.base.id : -1, ##__VA_ARGS__)
+
+#define DECIMATED_DIMENSION(dim, deci) (((dim) + ((1 << (deci)) - 1)) >> (deci))
+#define PHASE_STEP_SHIFT	21
+#define PHASE_STEP_UNIT_SCALE   ((int) (1 << PHASE_STEP_SHIFT))
+#define PHASE_RESIDUAL		15
+
+#define SHARP_STRENGTH_DEFAULT	32
+#define SHARP_EDGE_THR_DEFAULT	112
+#define SHARP_SMOOTH_THR_DEFAULT	8
+#define SHARP_NOISE_THR_DEFAULT	2
+
+#define DPU_NAME_SIZE  12
+
+#define DPU_PLANE_COLOR_FILL_FLAG	BIT(31)
+#define DPU_ZPOS_MAX 255
+
+/* multirect rect index */
+enum {
+	R0,
+	R1,
+	R_MAX
+};
+
+#define DPU_QSEED3_DEFAULT_PRELOAD_H 0x4
+#define DPU_QSEED3_DEFAULT_PRELOAD_V 0x3
+
+#define DEFAULT_REFRESH_RATE	60
+
+/**
+ * enum dpu_plane_qos - Different qos configurations for each pipe
+ *
+ * @DPU_PLANE_QOS_VBLANK_CTRL: Setup VBLANK qos for the pipe.
+ * @DPU_PLANE_QOS_VBLANK_AMORTIZE: Enables Amortization within pipe.
+ *	this configuration is mutually exclusive from VBLANK_CTRL.
+ * @DPU_PLANE_QOS_PANIC_CTRL: Setup panic for the pipe.
+ */
+enum dpu_plane_qos {
+	DPU_PLANE_QOS_VBLANK_CTRL = BIT(0),
+	DPU_PLANE_QOS_VBLANK_AMORTIZE = BIT(1),
+	DPU_PLANE_QOS_PANIC_CTRL = BIT(2),
+};
+
+/*
+ * struct dpu_plane - local dpu plane structure
+ * @aspace: address space pointer
+ * @csc_ptr: Points to dpu_csc_cfg structure to use for current
+ * @mplane_list: List of multirect planes of the same pipe
+ * @catalog: Points to dpu catalog structure
+ * @revalidate: force revalidation of all the plane properties
+ */
+struct dpu_plane {
+	struct drm_plane base;
+
+	struct mutex lock;
+
+	enum dpu_sspp pipe;
+	uint32_t features;      /* capabilities from catalog */
+	uint32_t nformats;
+	uint32_t formats[64];
+
+	struct dpu_hw_pipe *pipe_hw;
+	struct dpu_hw_pipe_cfg pipe_cfg;
+	struct dpu_hw_pipe_qos_cfg pipe_qos_cfg;
+	uint32_t color_fill;
+	bool is_error;
+	bool is_rt_pipe;
+	bool is_virtual;
+	struct list_head mplane_list;
+	struct dpu_mdss_cfg *catalog;
+
+	struct dpu_csc_cfg *csc_ptr;
+
+	const struct dpu_sspp_sub_blks *pipe_sblk;
+	char pipe_name[DPU_NAME_SIZE];
+
+	/* debugfs related stuff */
+	struct dentry *debugfs_root;
+	struct dpu_debugfs_regset32 debugfs_src;
+	struct dpu_debugfs_regset32 debugfs_scaler;
+	struct dpu_debugfs_regset32 debugfs_csc;
+	bool debugfs_default_scale;
+};
+
+#define to_dpu_plane(x) container_of(x, struct dpu_plane, base)
+
+static struct dpu_kms *_dpu_plane_get_kms(struct drm_plane *plane)
+{
+	struct msm_drm_private *priv;
+
+	if (!plane || !plane->dev)
+		return NULL;
+	priv = plane->dev->dev_private;
+	if (!priv)
+		return NULL;
+	return to_dpu_kms(priv->kms);
+}
+
+static bool dpu_plane_enabled(struct drm_plane_state *state)
+{
+	return state && state->fb && state->crtc;
+}
+
+static bool dpu_plane_sspp_enabled(struct drm_plane_state *state)
+{
+	return state && state->crtc;
+}
+
+/**
+ * _dpu_plane_calc_fill_level - calculate fill level of the given source format
+ * @plane:		Pointer to drm plane
+ * @fmt:		Pointer to source buffer format
+ * @src_wdith:		width of source buffer
+ * Return: fill level corresponding to the source buffer/format or 0 if error
+ */
+static inline int _dpu_plane_calc_fill_level(struct drm_plane *plane,
+		const struct dpu_format *fmt, u32 src_width)
+{
+	struct dpu_plane *pdpu, *tmp;
+	struct dpu_plane_state *pstate;
+	u32 fixed_buff_size;
+	u32 total_fl;
+
+	if (!plane || !fmt || !plane->state || !src_width || !fmt->bpp) {
+		DPU_ERROR("invalid arguments\n");
+		return 0;
+	}
+
+	pdpu = to_dpu_plane(plane);
+	pstate = to_dpu_plane_state(plane->state);
+	fixed_buff_size = pdpu->pipe_sblk->common->pixel_ram_size;
+
+	list_for_each_entry(tmp, &pdpu->mplane_list, mplane_list) {
+		if (!dpu_plane_enabled(tmp->base.state))
+			continue;
+		DPU_DEBUG("plane%d/%d src_width:%d/%d\n",
+				pdpu->base.base.id, tmp->base.base.id,
+				src_width,
+				drm_rect_width(&tmp->pipe_cfg.src_rect));
+		src_width = max_t(u32, src_width,
+				  drm_rect_width(&tmp->pipe_cfg.src_rect));
+	}
+
+	if (fmt->fetch_planes == DPU_PLANE_PSEUDO_PLANAR) {
+		if (fmt->chroma_sample == DPU_CHROMA_420) {
+			/* NV12 */
+			total_fl = (fixed_buff_size / 2) /
+				((src_width + 32) * fmt->bpp);
+		} else {
+			/* non NV12 */
+			total_fl = (fixed_buff_size / 2) * 2 /
+				((src_width + 32) * fmt->bpp);
+		}
+	} else {
+		if (pstate->multirect_mode == DPU_SSPP_MULTIRECT_PARALLEL) {
+			total_fl = (fixed_buff_size / 2) * 2 /
+				((src_width + 32) * fmt->bpp);
+		} else {
+			total_fl = (fixed_buff_size) * 2 /
+				((src_width + 32) * fmt->bpp);
+		}
+	}
+
+	DPU_DEBUG("plane%u: pnum:%d fmt: %4.4s w:%u fl:%u\n",
+			plane->base.id, pdpu->pipe - SSPP_VIG0,
+			(char *)&fmt->base.pixel_format,
+			src_width, total_fl);
+
+	return total_fl;
+}
+
+/**
+ * _dpu_plane_get_qos_lut - get LUT mapping based on fill level
+ * @tbl:		Pointer to LUT table
+ * @total_fl:		fill level
+ * Return: LUT setting corresponding to the fill level
+ */
+static u64 _dpu_plane_get_qos_lut(const struct dpu_qos_lut_tbl *tbl,
+		u32 total_fl)
+{
+	int i;
+
+	if (!tbl || !tbl->nentry || !tbl->entries)
+		return 0;
+
+	for (i = 0; i < tbl->nentry; i++)
+		if (total_fl <= tbl->entries[i].fl)
+			return tbl->entries[i].lut;
+
+	/* if last fl is zero, use as default */
+	if (!tbl->entries[i-1].fl)
+		return tbl->entries[i-1].lut;
+
+	return 0;
+}
+
+/**
+ * _dpu_plane_set_qos_lut - set QoS LUT of the given plane
+ * @plane:		Pointer to drm plane
+ * @fb:			Pointer to framebuffer associated with the given plane
+ */
+static void _dpu_plane_set_qos_lut(struct drm_plane *plane,
+		struct drm_framebuffer *fb)
+{
+	struct dpu_plane *pdpu;
+	const struct dpu_format *fmt = NULL;
+	u64 qos_lut;
+	u32 total_fl = 0, lut_usage;
+
+	if (!plane || !fb) {
+		DPU_ERROR("invalid arguments plane %d fb %d\n",
+				plane != 0, fb != 0);
+		return;
+	}
+
+	pdpu = to_dpu_plane(plane);
+
+	if (!pdpu->pipe_hw || !pdpu->pipe_sblk || !pdpu->catalog) {
+		DPU_ERROR("invalid arguments\n");
+		return;
+	} else if (!pdpu->pipe_hw->ops.setup_creq_lut) {
+		return;
+	}
+
+	if (!pdpu->is_rt_pipe) {
+		lut_usage = DPU_QOS_LUT_USAGE_NRT;
+	} else {
+		fmt = dpu_get_dpu_format_ext(
+				fb->format->format,
+				fb->modifier);
+		total_fl = _dpu_plane_calc_fill_level(plane, fmt,
+				drm_rect_width(&pdpu->pipe_cfg.src_rect));
+
+		if (fmt && DPU_FORMAT_IS_LINEAR(fmt))
+			lut_usage = DPU_QOS_LUT_USAGE_LINEAR;
+		else
+			lut_usage = DPU_QOS_LUT_USAGE_MACROTILE;
+	}
+
+	qos_lut = _dpu_plane_get_qos_lut(
+			&pdpu->catalog->perf.qos_lut_tbl[lut_usage], total_fl);
+
+	pdpu->pipe_qos_cfg.creq_lut = qos_lut;
+
+	trace_dpu_perf_set_qos_luts(pdpu->pipe - SSPP_VIG0,
+			(fmt) ? fmt->base.pixel_format : 0,
+			pdpu->is_rt_pipe, total_fl, qos_lut, lut_usage);
+
+	DPU_DEBUG("plane%u: pnum:%d fmt: %4.4s rt:%d fl:%u lut:0x%llx\n",
+			plane->base.id,
+			pdpu->pipe - SSPP_VIG0,
+			fmt ? (char *)&fmt->base.pixel_format : NULL,
+			pdpu->is_rt_pipe, total_fl, qos_lut);
+
+	pdpu->pipe_hw->ops.setup_creq_lut(pdpu->pipe_hw, &pdpu->pipe_qos_cfg);
+}
+
+/**
+ * _dpu_plane_set_panic_lut - set danger/safe LUT of the given plane
+ * @plane:		Pointer to drm plane
+ * @fb:			Pointer to framebuffer associated with the given plane
+ */
+static void _dpu_plane_set_danger_lut(struct drm_plane *plane,
+		struct drm_framebuffer *fb)
+{
+	struct dpu_plane *pdpu;
+	const struct dpu_format *fmt = NULL;
+	u32 danger_lut, safe_lut;
+
+	if (!plane || !fb) {
+		DPU_ERROR("invalid arguments\n");
+		return;
+	}
+
+	pdpu = to_dpu_plane(plane);
+
+	if (!pdpu->pipe_hw || !pdpu->pipe_sblk || !pdpu->catalog) {
+		DPU_ERROR("invalid arguments\n");
+		return;
+	} else if (!pdpu->pipe_hw->ops.setup_danger_safe_lut) {
+		return;
+	}
+
+	if (!pdpu->is_rt_pipe) {
+		danger_lut = pdpu->catalog->perf.danger_lut_tbl
+				[DPU_QOS_LUT_USAGE_NRT];
+		safe_lut = pdpu->catalog->perf.safe_lut_tbl
+				[DPU_QOS_LUT_USAGE_NRT];
+	} else {
+		fmt = dpu_get_dpu_format_ext(
+				fb->format->format,
+				fb->modifier);
+
+		if (fmt && DPU_FORMAT_IS_LINEAR(fmt)) {
+			danger_lut = pdpu->catalog->perf.danger_lut_tbl
+					[DPU_QOS_LUT_USAGE_LINEAR];
+			safe_lut = pdpu->catalog->perf.safe_lut_tbl
+					[DPU_QOS_LUT_USAGE_LINEAR];
+		} else {
+			danger_lut = pdpu->catalog->perf.danger_lut_tbl
+					[DPU_QOS_LUT_USAGE_MACROTILE];
+			safe_lut = pdpu->catalog->perf.safe_lut_tbl
+					[DPU_QOS_LUT_USAGE_MACROTILE];
+		}
+	}
+
+	pdpu->pipe_qos_cfg.danger_lut = danger_lut;
+	pdpu->pipe_qos_cfg.safe_lut = safe_lut;
+
+	trace_dpu_perf_set_danger_luts(pdpu->pipe - SSPP_VIG0,
+			(fmt) ? fmt->base.pixel_format : 0,
+			(fmt) ? fmt->fetch_mode : 0,
+			pdpu->pipe_qos_cfg.danger_lut,
+			pdpu->pipe_qos_cfg.safe_lut);
+
+	DPU_DEBUG("plane%u: pnum:%d fmt: %4.4s mode:%d luts[0x%x, 0x%x]\n",
+		plane->base.id,
+		pdpu->pipe - SSPP_VIG0,
+		fmt ? (char *)&fmt->base.pixel_format : NULL,
+		fmt ? fmt->fetch_mode : -1,
+		pdpu->pipe_qos_cfg.danger_lut,
+		pdpu->pipe_qos_cfg.safe_lut);
+
+	pdpu->pipe_hw->ops.setup_danger_safe_lut(pdpu->pipe_hw,
+			&pdpu->pipe_qos_cfg);
+}
+
+/**
+ * _dpu_plane_set_qos_ctrl - set QoS control of the given plane
+ * @plane:		Pointer to drm plane
+ * @enable:		true to enable QoS control
+ * @flags:		QoS control mode (enum dpu_plane_qos)
+ */
+static void _dpu_plane_set_qos_ctrl(struct drm_plane *plane,
+	bool enable, u32 flags)
+{
+	struct dpu_plane *pdpu;
+
+	if (!plane) {
+		DPU_ERROR("invalid arguments\n");
+		return;
+	}
+
+	pdpu = to_dpu_plane(plane);
+
+	if (!pdpu->pipe_hw || !pdpu->pipe_sblk) {
+		DPU_ERROR("invalid arguments\n");
+		return;
+	} else if (!pdpu->pipe_hw->ops.setup_qos_ctrl) {
+		return;
+	}
+
+	if (flags & DPU_PLANE_QOS_VBLANK_CTRL) {
+		pdpu->pipe_qos_cfg.creq_vblank = pdpu->pipe_sblk->creq_vblank;
+		pdpu->pipe_qos_cfg.danger_vblank =
+				pdpu->pipe_sblk->danger_vblank;
+		pdpu->pipe_qos_cfg.vblank_en = enable;
+	}
+
+	if (flags & DPU_PLANE_QOS_VBLANK_AMORTIZE) {
+		/* this feature overrules previous VBLANK_CTRL */
+		pdpu->pipe_qos_cfg.vblank_en = false;
+		pdpu->pipe_qos_cfg.creq_vblank = 0; /* clear vblank bits */
+	}
+
+	if (flags & DPU_PLANE_QOS_PANIC_CTRL)
+		pdpu->pipe_qos_cfg.danger_safe_en = enable;
+
+	if (!pdpu->is_rt_pipe) {
+		pdpu->pipe_qos_cfg.vblank_en = false;
+		pdpu->pipe_qos_cfg.danger_safe_en = false;
+	}
+
+	DPU_DEBUG("plane%u: pnum:%d ds:%d vb:%d pri[0x%x, 0x%x] is_rt:%d\n",
+		plane->base.id,
+		pdpu->pipe - SSPP_VIG0,
+		pdpu->pipe_qos_cfg.danger_safe_en,
+		pdpu->pipe_qos_cfg.vblank_en,
+		pdpu->pipe_qos_cfg.creq_vblank,
+		pdpu->pipe_qos_cfg.danger_vblank,
+		pdpu->is_rt_pipe);
+
+	pdpu->pipe_hw->ops.setup_qos_ctrl(pdpu->pipe_hw,
+			&pdpu->pipe_qos_cfg);
+}
+
+int dpu_plane_danger_signal_ctrl(struct drm_plane *plane, bool enable)
+{
+	struct dpu_plane *pdpu;
+	struct msm_drm_private *priv;
+	struct dpu_kms *dpu_kms;
+
+	if (!plane || !plane->dev) {
+		DPU_ERROR("invalid arguments\n");
+		return -EINVAL;
+	}
+
+	priv = plane->dev->dev_private;
+	if (!priv || !priv->kms) {
+		DPU_ERROR("invalid KMS reference\n");
+		return -EINVAL;
+	}
+
+	dpu_kms = to_dpu_kms(priv->kms);
+	pdpu = to_dpu_plane(plane);
+
+	if (!pdpu->is_rt_pipe)
+		goto end;
+
+	pm_runtime_get_sync(&dpu_kms->pdev->dev);
+	_dpu_plane_set_qos_ctrl(plane, enable, DPU_PLANE_QOS_PANIC_CTRL);
+	pm_runtime_put_sync(&dpu_kms->pdev->dev);
+
+end:
+	return 0;
+}
+
+/**
+ * _dpu_plane_set_ot_limit - set OT limit for the given plane
+ * @plane:		Pointer to drm plane
+ * @crtc:		Pointer to drm crtc
+ */
+static void _dpu_plane_set_ot_limit(struct drm_plane *plane,
+		struct drm_crtc *crtc)
+{
+	struct dpu_plane *pdpu;
+	struct dpu_vbif_set_ot_params ot_params;
+	struct msm_drm_private *priv;
+	struct dpu_kms *dpu_kms;
+
+	if (!plane || !plane->dev || !crtc) {
+		DPU_ERROR("invalid arguments plane %d crtc %d\n",
+				plane != 0, crtc != 0);
+		return;
+	}
+
+	priv = plane->dev->dev_private;
+	if (!priv || !priv->kms) {
+		DPU_ERROR("invalid KMS reference\n");
+		return;
+	}
+
+	dpu_kms = to_dpu_kms(priv->kms);
+	pdpu = to_dpu_plane(plane);
+	if (!pdpu->pipe_hw) {
+		DPU_ERROR("invalid pipe reference\n");
+		return;
+	}
+
+	memset(&ot_params, 0, sizeof(ot_params));
+	ot_params.xin_id = pdpu->pipe_hw->cap->xin_id;
+	ot_params.num = pdpu->pipe_hw->idx - SSPP_NONE;
+	ot_params.width = drm_rect_width(&pdpu->pipe_cfg.src_rect);
+	ot_params.height = drm_rect_height(&pdpu->pipe_cfg.src_rect);
+	ot_params.is_wfd = !pdpu->is_rt_pipe;
+	ot_params.frame_rate = crtc->mode.vrefresh;
+	ot_params.vbif_idx = VBIF_RT;
+	ot_params.clk_ctrl = pdpu->pipe_hw->cap->clk_ctrl;
+	ot_params.rd = true;
+
+	dpu_vbif_set_ot_limit(dpu_kms, &ot_params);
+}
+
+/**
+ * _dpu_plane_set_vbif_qos - set vbif QoS for the given plane
+ * @plane:		Pointer to drm plane
+ */
+static void _dpu_plane_set_qos_remap(struct drm_plane *plane)
+{
+	struct dpu_plane *pdpu;
+	struct dpu_vbif_set_qos_params qos_params;
+	struct msm_drm_private *priv;
+	struct dpu_kms *dpu_kms;
+
+	if (!plane || !plane->dev) {
+		DPU_ERROR("invalid arguments\n");
+		return;
+	}
+
+	priv = plane->dev->dev_private;
+	if (!priv || !priv->kms) {
+		DPU_ERROR("invalid KMS reference\n");
+		return;
+	}
+
+	dpu_kms = to_dpu_kms(priv->kms);
+	pdpu = to_dpu_plane(plane);
+	if (!pdpu->pipe_hw) {
+		DPU_ERROR("invalid pipe reference\n");
+		return;
+	}
+
+	memset(&qos_params, 0, sizeof(qos_params));
+	qos_params.vbif_idx = VBIF_RT;
+	qos_params.clk_ctrl = pdpu->pipe_hw->cap->clk_ctrl;
+	qos_params.xin_id = pdpu->pipe_hw->cap->xin_id;
+	qos_params.num = pdpu->pipe_hw->idx - SSPP_VIG0;
+	qos_params.is_rt = pdpu->is_rt_pipe;
+
+	DPU_DEBUG("plane%d pipe:%d vbif:%d xin:%d rt:%d, clk_ctrl:%d\n",
+			plane->base.id, qos_params.num,
+			qos_params.vbif_idx,
+			qos_params.xin_id, qos_params.is_rt,
+			qos_params.clk_ctrl);
+
+	dpu_vbif_set_qos_remap(dpu_kms, &qos_params);
+}
+
+/**
+ * _dpu_plane_get_aspace: gets the address space
+ */
+static int _dpu_plane_get_aspace(
+		struct dpu_plane *pdpu,
+		struct dpu_plane_state *pstate,
+		struct msm_gem_address_space **aspace)
+{
+	struct dpu_kms *kms;
+
+	if (!pdpu || !pstate || !aspace) {
+		DPU_ERROR("invalid parameters\n");
+		return -EINVAL;
+	}
+
+	kms = _dpu_plane_get_kms(&pdpu->base);
+	if (!kms) {
+		DPU_ERROR("invalid kms\n");
+		return -EINVAL;
+	}
+
+	*aspace = kms->base.aspace;
+
+	return 0;
+}
+
+static inline void _dpu_plane_set_scanout(struct drm_plane *plane,
+		struct dpu_plane_state *pstate,
+		struct dpu_hw_pipe_cfg *pipe_cfg,
+		struct drm_framebuffer *fb)
+{
+	struct dpu_plane *pdpu;
+	struct msm_gem_address_space *aspace = NULL;
+	int ret;
+
+	if (!plane || !pstate || !pipe_cfg || !fb) {
+		DPU_ERROR(
+			"invalid arg(s), plane %d state %d cfg %d fb %d\n",
+			plane != 0, pstate != 0, pipe_cfg != 0, fb != 0);
+		return;
+	}
+
+	pdpu = to_dpu_plane(plane);
+	if (!pdpu->pipe_hw) {
+		DPU_ERROR_PLANE(pdpu, "invalid pipe_hw\n");
+		return;
+	}
+
+	ret = _dpu_plane_get_aspace(pdpu, pstate, &aspace);
+	if (ret) {
+		DPU_ERROR_PLANE(pdpu, "Failed to get aspace %d\n", ret);
+		return;
+	}
+
+	ret = dpu_format_populate_layout(aspace, fb, &pipe_cfg->layout);
+	if (ret == -EAGAIN)
+		DPU_DEBUG_PLANE(pdpu, "not updating same src addrs\n");
+	else if (ret)
+		DPU_ERROR_PLANE(pdpu, "failed to get format layout, %d\n", ret);
+	else if (pdpu->pipe_hw->ops.setup_sourceaddress) {
+		trace_dpu_plane_set_scanout(pdpu->pipe_hw->idx,
+					    &pipe_cfg->layout,
+					    pstate->multirect_index);
+		pdpu->pipe_hw->ops.setup_sourceaddress(pdpu->pipe_hw, pipe_cfg,
+						pstate->multirect_index);
+	}
+}
+
+static void _dpu_plane_setup_scaler3(struct dpu_plane *pdpu,
+		struct dpu_plane_state *pstate,
+		uint32_t src_w, uint32_t src_h, uint32_t dst_w, uint32_t dst_h,
+		struct dpu_hw_scaler3_cfg *scale_cfg,
+		const struct dpu_format *fmt,
+		uint32_t chroma_subsmpl_h, uint32_t chroma_subsmpl_v)
+{
+	uint32_t i;
+
+	if (!pdpu || !pstate || !scale_cfg || !fmt || !chroma_subsmpl_h ||
+			!chroma_subsmpl_v) {
+		DPU_ERROR(
+			"pdpu %d pstate %d scale_cfg %d fmt %d smp_h %d smp_v %d\n",
+			!!pdpu, !!pstate, !!scale_cfg, !!fmt, chroma_subsmpl_h,
+			chroma_subsmpl_v);
+		return;
+	}
+
+	memset(scale_cfg, 0, sizeof(*scale_cfg));
+	memset(&pstate->pixel_ext, 0, sizeof(struct dpu_hw_pixel_ext));
+
+	scale_cfg->phase_step_x[DPU_SSPP_COMP_0] =
+		mult_frac((1 << PHASE_STEP_SHIFT), src_w, dst_w);
+	scale_cfg->phase_step_y[DPU_SSPP_COMP_0] =
+		mult_frac((1 << PHASE_STEP_SHIFT), src_h, dst_h);
+
+
+	scale_cfg->phase_step_y[DPU_SSPP_COMP_1_2] =
+		scale_cfg->phase_step_y[DPU_SSPP_COMP_0] / chroma_subsmpl_v;
+	scale_cfg->phase_step_x[DPU_SSPP_COMP_1_2] =
+		scale_cfg->phase_step_x[DPU_SSPP_COMP_0] / chroma_subsmpl_h;
+
+	scale_cfg->phase_step_x[DPU_SSPP_COMP_2] =
+		scale_cfg->phase_step_x[DPU_SSPP_COMP_1_2];
+	scale_cfg->phase_step_y[DPU_SSPP_COMP_2] =
+		scale_cfg->phase_step_y[DPU_SSPP_COMP_1_2];
+
+	scale_cfg->phase_step_x[DPU_SSPP_COMP_3] =
+		scale_cfg->phase_step_x[DPU_SSPP_COMP_0];
+	scale_cfg->phase_step_y[DPU_SSPP_COMP_3] =
+		scale_cfg->phase_step_y[DPU_SSPP_COMP_0];
+
+	for (i = 0; i < DPU_MAX_PLANES; i++) {
+		scale_cfg->src_width[i] = src_w;
+		scale_cfg->src_height[i] = src_h;
+		if (i == DPU_SSPP_COMP_1_2 || i == DPU_SSPP_COMP_2) {
+			scale_cfg->src_width[i] /= chroma_subsmpl_h;
+			scale_cfg->src_height[i] /= chroma_subsmpl_v;
+		}
+		scale_cfg->preload_x[i] = DPU_QSEED3_DEFAULT_PRELOAD_H;
+		scale_cfg->preload_y[i] = DPU_QSEED3_DEFAULT_PRELOAD_V;
+		pstate->pixel_ext.num_ext_pxls_top[i] =
+			scale_cfg->src_height[i];
+		pstate->pixel_ext.num_ext_pxls_left[i] =
+			scale_cfg->src_width[i];
+	}
+	if (!(DPU_FORMAT_IS_YUV(fmt)) && (src_h == dst_h)
+		&& (src_w == dst_w))
+		return;
+
+	scale_cfg->dst_width = dst_w;
+	scale_cfg->dst_height = dst_h;
+	scale_cfg->y_rgb_filter_cfg = DPU_SCALE_BIL;
+	scale_cfg->uv_filter_cfg = DPU_SCALE_BIL;
+	scale_cfg->alpha_filter_cfg = DPU_SCALE_ALPHA_BIL;
+	scale_cfg->lut_flag = 0;
+	scale_cfg->blend_cfg = 1;
+	scale_cfg->enable = 1;
+}
+
+static inline void _dpu_plane_setup_csc(struct dpu_plane *pdpu)
+{
+	static const struct dpu_csc_cfg dpu_csc_YUV2RGB_601L = {
+		{
+			/* S15.16 format */
+			0x00012A00, 0x00000000, 0x00019880,
+			0x00012A00, 0xFFFF9B80, 0xFFFF3000,
+			0x00012A00, 0x00020480, 0x00000000,
+		},
+		/* signed bias */
+		{ 0xfff0, 0xff80, 0xff80,},
+		{ 0x0, 0x0, 0x0,},
+		/* unsigned clamp */
+		{ 0x10, 0xeb, 0x10, 0xf0, 0x10, 0xf0,},
+		{ 0x00, 0xff, 0x00, 0xff, 0x00, 0xff,},
+	};
+	static const struct dpu_csc_cfg dpu_csc10_YUV2RGB_601L = {
+		{
+			/* S15.16 format */
+			0x00012A00, 0x00000000, 0x00019880,
+			0x00012A00, 0xFFFF9B80, 0xFFFF3000,
+			0x00012A00, 0x00020480, 0x00000000,
+			},
+		/* signed bias */
+		{ 0xffc0, 0xfe00, 0xfe00,},
+		{ 0x0, 0x0, 0x0,},
+		/* unsigned clamp */
+		{ 0x40, 0x3ac, 0x40, 0x3c0, 0x40, 0x3c0,},
+		{ 0x00, 0x3ff, 0x00, 0x3ff, 0x00, 0x3ff,},
+	};
+
+	if (!pdpu) {
+		DPU_ERROR("invalid plane\n");
+		return;
+	}
+
+	if (BIT(DPU_SSPP_CSC_10BIT) & pdpu->features)
+		pdpu->csc_ptr = (struct dpu_csc_cfg *)&dpu_csc10_YUV2RGB_601L;
+	else
+		pdpu->csc_ptr = (struct dpu_csc_cfg *)&dpu_csc_YUV2RGB_601L;
+
+	DPU_DEBUG_PLANE(pdpu, "using 0x%X 0x%X 0x%X...\n",
+			pdpu->csc_ptr->csc_mv[0],
+			pdpu->csc_ptr->csc_mv[1],
+			pdpu->csc_ptr->csc_mv[2]);
+}
+
+static void _dpu_plane_setup_scaler(struct dpu_plane *pdpu,
+		struct dpu_plane_state *pstate,
+		const struct dpu_format *fmt, bool color_fill)
+{
+	struct dpu_hw_pixel_ext *pe;
+	uint32_t chroma_subsmpl_h, chroma_subsmpl_v;
+
+	if (!pdpu || !fmt || !pstate) {
+		DPU_ERROR("invalid arg(s), plane %d fmt %d state %d\n",
+				pdpu != 0, fmt != 0, pstate != 0);
+		return;
+	}
+
+	pe = &pstate->pixel_ext;
+
+	/* don't chroma subsample if decimating */
+	chroma_subsmpl_h =
+		drm_format_horz_chroma_subsampling(fmt->base.pixel_format);
+	chroma_subsmpl_v =
+		drm_format_vert_chroma_subsampling(fmt->base.pixel_format);
+
+	/* update scaler. calculate default config for QSEED3 */
+	_dpu_plane_setup_scaler3(pdpu, pstate,
+			drm_rect_width(&pdpu->pipe_cfg.src_rect),
+			drm_rect_height(&pdpu->pipe_cfg.src_rect),
+			drm_rect_width(&pdpu->pipe_cfg.dst_rect),
+			drm_rect_height(&pdpu->pipe_cfg.dst_rect),
+			&pstate->scaler3_cfg, fmt,
+			chroma_subsmpl_h, chroma_subsmpl_v);
+}
+
+/**
+ * _dpu_plane_color_fill - enables color fill on plane
+ * @pdpu:   Pointer to DPU plane object
+ * @color:  RGB fill color value, [23..16] Blue, [15..8] Green, [7..0] Red
+ * @alpha:  8-bit fill alpha value, 255 selects 100% alpha
+ * Returns: 0 on success
+ */
+static int _dpu_plane_color_fill(struct dpu_plane *pdpu,
+		uint32_t color, uint32_t alpha)
+{
+	const struct dpu_format *fmt;
+	const struct drm_plane *plane;
+	struct dpu_plane_state *pstate;
+
+	if (!pdpu || !pdpu->base.state) {
+		DPU_ERROR("invalid plane\n");
+		return -EINVAL;
+	}
+
+	if (!pdpu->pipe_hw) {
+		DPU_ERROR_PLANE(pdpu, "invalid plane h/w pointer\n");
+		return -EINVAL;
+	}
+
+	plane = &pdpu->base;
+	pstate = to_dpu_plane_state(plane->state);
+
+	DPU_DEBUG_PLANE(pdpu, "\n");
+
+	/*
+	 * select fill format to match user property expectation,
+	 * h/w only supports RGB variants
+	 */
+	fmt = dpu_get_dpu_format(DRM_FORMAT_ABGR8888);
+
+	/* update sspp */
+	if (fmt && pdpu->pipe_hw->ops.setup_solidfill) {
+		pdpu->pipe_hw->ops.setup_solidfill(pdpu->pipe_hw,
+				(color & 0xFFFFFF) | ((alpha & 0xFF) << 24),
+				pstate->multirect_index);
+
+		/* override scaler/decimation if solid fill */
+		pdpu->pipe_cfg.src_rect.x1 = 0;
+		pdpu->pipe_cfg.src_rect.y1 = 0;
+		pdpu->pipe_cfg.src_rect.x2 =
+			drm_rect_width(&pdpu->pipe_cfg.dst_rect);
+		pdpu->pipe_cfg.src_rect.y2 =
+			drm_rect_height(&pdpu->pipe_cfg.dst_rect);
+		_dpu_plane_setup_scaler(pdpu, pstate, fmt, true);
+
+		if (pdpu->pipe_hw->ops.setup_format)
+			pdpu->pipe_hw->ops.setup_format(pdpu->pipe_hw,
+					fmt, DPU_SSPP_SOLID_FILL,
+					pstate->multirect_index);
+
+		if (pdpu->pipe_hw->ops.setup_rects)
+			pdpu->pipe_hw->ops.setup_rects(pdpu->pipe_hw,
+					&pdpu->pipe_cfg,
+					pstate->multirect_index);
+
+		if (pdpu->pipe_hw->ops.setup_pe)
+			pdpu->pipe_hw->ops.setup_pe(pdpu->pipe_hw,
+					&pstate->pixel_ext);
+
+		if (pdpu->pipe_hw->ops.setup_scaler &&
+				pstate->multirect_index != DPU_SSPP_RECT_1)
+			pdpu->pipe_hw->ops.setup_scaler(pdpu->pipe_hw,
+					&pdpu->pipe_cfg, &pstate->pixel_ext,
+					&pstate->scaler3_cfg);
+	}
+
+	return 0;
+}
+
+void dpu_plane_clear_multirect(const struct drm_plane_state *drm_state)
+{
+	struct dpu_plane_state *pstate;
+
+	if (!drm_state)
+		return;
+
+	pstate = to_dpu_plane_state(drm_state);
+
+	pstate->multirect_index = DPU_SSPP_RECT_SOLO;
+	pstate->multirect_mode = DPU_SSPP_MULTIRECT_NONE;
+}
+
+int dpu_plane_validate_multirect_v2(struct dpu_multirect_plane_states *plane)
+{
+	struct dpu_plane_state *pstate[R_MAX];
+	const struct drm_plane_state *drm_state[R_MAX];
+	struct drm_rect src[R_MAX], dst[R_MAX];
+	struct dpu_plane *dpu_plane[R_MAX];
+	const struct dpu_format *fmt[R_MAX];
+	int i, buffer_lines;
+	unsigned int max_tile_height = 1;
+	bool parallel_fetch_qualified = true;
+	bool has_tiled_rect = false;
+
+	for (i = 0; i < R_MAX; i++) {
+		const struct msm_format *msm_fmt;
+
+		drm_state[i] = i ? plane->r1 : plane->r0;
+		msm_fmt = msm_framebuffer_format(drm_state[i]->fb);
+		fmt[i] = to_dpu_format(msm_fmt);
+
+		if (DPU_FORMAT_IS_UBWC(fmt[i])) {
+			has_tiled_rect = true;
+			if (fmt[i]->tile_height > max_tile_height)
+				max_tile_height = fmt[i]->tile_height;
+		}
+	}
+
+	for (i = 0; i < R_MAX; i++) {
+		int width_threshold;
+
+		pstate[i] = to_dpu_plane_state(drm_state[i]);
+		dpu_plane[i] = to_dpu_plane(drm_state[i]->plane);
+
+		if (pstate[i] == NULL) {
+			DPU_ERROR("DPU plane state of plane id %d is NULL\n",
+				drm_state[i]->plane->base.id);
+			return -EINVAL;
+		}
+
+		src[i].x1 = drm_state[i]->src_x >> 16;
+		src[i].y1 = drm_state[i]->src_y >> 16;
+		src[i].x2 = src[i].x1 + (drm_state[i]->src_w >> 16);
+		src[i].y2 = src[i].y1 + (drm_state[i]->src_h >> 16);
+
+		dst[i] = drm_plane_state_dest(drm_state[i]);
+
+		if (drm_rect_calc_hscale(&src[i], &dst[i], 1, 1) != 1 ||
+		    drm_rect_calc_vscale(&src[i], &dst[i], 1, 1) != 1) {
+			DPU_ERROR_PLANE(dpu_plane[i],
+				"scaling is not supported in multirect mode\n");
+			return -EINVAL;
+		}
+
+		if (DPU_FORMAT_IS_YUV(fmt[i])) {
+			DPU_ERROR_PLANE(dpu_plane[i],
+				"Unsupported format for multirect mode\n");
+			return -EINVAL;
+		}
+
+		/**
+		 * SSPP PD_MEM is split half - one for each RECT.
+		 * Tiled formats need 5 lines of buffering while fetching
+		 * whereas linear formats need only 2 lines.
+		 * So we cannot support more than half of the supported SSPP
+		 * width for tiled formats.
+		 */
+		width_threshold = dpu_plane[i]->pipe_sblk->common->maxlinewidth;
+		if (has_tiled_rect)
+			width_threshold /= 2;
+
+		if (parallel_fetch_qualified &&
+		    drm_rect_width(&src[i]) > width_threshold)
+			parallel_fetch_qualified = false;
+
+	}
+
+	/* Validate RECT's and set the mode */
+
+	/* Prefer PARALLEL FETCH Mode over TIME_MX Mode */
+	if (parallel_fetch_qualified) {
+		pstate[R0]->multirect_mode = DPU_SSPP_MULTIRECT_PARALLEL;
+		pstate[R1]->multirect_mode = DPU_SSPP_MULTIRECT_PARALLEL;
+
+		goto done;
+	}
+
+	/* TIME_MX Mode */
+	buffer_lines = 2 * max_tile_height;
+
+	if (dst[R1].y1 >= dst[R0].y2 + buffer_lines ||
+	    dst[R0].y1 >= dst[R1].y2 + buffer_lines) {
+		pstate[R0]->multirect_mode = DPU_SSPP_MULTIRECT_TIME_MX;
+		pstate[R1]->multirect_mode = DPU_SSPP_MULTIRECT_TIME_MX;
+	} else {
+		DPU_ERROR(
+			"No multirect mode possible for the planes (%d - %d)\n",
+			drm_state[R0]->plane->base.id,
+			drm_state[R1]->plane->base.id);
+		return -EINVAL;
+	}
+
+done:
+	if (dpu_plane[R0]->is_virtual) {
+		pstate[R0]->multirect_index = DPU_SSPP_RECT_1;
+		pstate[R1]->multirect_index = DPU_SSPP_RECT_0;
+	} else {
+		pstate[R0]->multirect_index = DPU_SSPP_RECT_0;
+		pstate[R1]->multirect_index = DPU_SSPP_RECT_1;
+	};
+
+	DPU_DEBUG_PLANE(dpu_plane[R0], "R0: %d - %d\n",
+		pstate[R0]->multirect_mode, pstate[R0]->multirect_index);
+	DPU_DEBUG_PLANE(dpu_plane[R1], "R1: %d - %d\n",
+		pstate[R1]->multirect_mode, pstate[R1]->multirect_index);
+	return 0;
+}
+
+/**
+ * dpu_plane_get_ctl_flush - get control flush for the given plane
+ * @plane: Pointer to drm plane structure
+ * @ctl: Pointer to hardware control driver
+ * @flush_sspp: Pointer to sspp flush control word
+ */
+void dpu_plane_get_ctl_flush(struct drm_plane *plane, struct dpu_hw_ctl *ctl,
+		u32 *flush_sspp)
+{
+	struct dpu_plane_state *pstate;
+
+	if (!plane || !flush_sspp) {
+		DPU_ERROR("invalid parameters\n");
+		return;
+	}
+
+	pstate = to_dpu_plane_state(plane->state);
+
+	*flush_sspp = ctl->ops.get_bitmask_sspp(ctl, dpu_plane_pipe(plane));
+}
+
+static int dpu_plane_prepare_fb(struct drm_plane *plane,
+		struct drm_plane_state *new_state)
+{
+	struct drm_framebuffer *fb = new_state->fb;
+	struct dpu_plane *pdpu = to_dpu_plane(plane);
+	struct dpu_plane_state *pstate = to_dpu_plane_state(new_state);
+	struct dpu_hw_fmt_layout layout;
+	struct drm_gem_object *obj;
+	struct msm_gem_object *msm_obj;
+	struct dma_fence *fence;
+	struct msm_gem_address_space *aspace;
+	int ret;
+
+	if (!new_state->fb)
+		return 0;
+
+	DPU_DEBUG_PLANE(pdpu, "FB[%u]\n", fb->base.id);
+
+	ret = _dpu_plane_get_aspace(pdpu, pstate, &aspace);
+	if (ret) {
+		DPU_ERROR_PLANE(pdpu, "Failed to get aspace\n");
+		return ret;
+	}
+
+	/* cache aspace */
+	pstate->aspace = aspace;
+
+	/*
+	 * TODO: Need to sort out the msm_framebuffer_prepare() call below so
+	 *       we can use msm_atomic_prepare_fb() instead of doing the
+	 *       implicit fence and fb prepare by hand here.
+	 */
+	obj = msm_framebuffer_bo(new_state->fb, 0);
+	msm_obj = to_msm_bo(obj);
+	fence = reservation_object_get_excl_rcu(msm_obj->resv);
+	if (fence)
+		drm_atomic_set_fence_for_plane(new_state, fence);
+
+	if (pstate->aspace) {
+		ret = msm_framebuffer_prepare(new_state->fb,
+				pstate->aspace);
+		if (ret) {
+			DPU_ERROR("failed to prepare framebuffer\n");
+			return ret;
+		}
+	}
+
+	/* validate framebuffer layout before commit */
+	ret = dpu_format_populate_layout(pstate->aspace,
+			new_state->fb, &layout);
+	if (ret) {
+		DPU_ERROR_PLANE(pdpu, "failed to get format layout, %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static void dpu_plane_cleanup_fb(struct drm_plane *plane,
+		struct drm_plane_state *old_state)
+{
+	struct dpu_plane *pdpu = to_dpu_plane(plane);
+	struct dpu_plane_state *old_pstate;
+
+	if (!old_state || !old_state->fb)
+		return;
+
+	old_pstate = to_dpu_plane_state(old_state);
+
+	DPU_DEBUG_PLANE(pdpu, "FB[%u]\n", old_state->fb->base.id);
+
+	msm_framebuffer_cleanup(old_state->fb, old_pstate->aspace);
+}
+
+static bool dpu_plane_validate_src(struct drm_rect *src,
+				   struct drm_rect *fb_rect,
+				   uint32_t min_src_size)
+{
+	/* Ensure fb size is supported */
+	if (drm_rect_width(fb_rect) > MAX_IMG_WIDTH ||
+	    drm_rect_height(fb_rect) > MAX_IMG_HEIGHT)
+		return false;
+
+	/* Ensure src rect is above the minimum size */
+	if (drm_rect_width(src) < min_src_size ||
+	    drm_rect_height(src) < min_src_size)
+		return false;
+
+	/* Ensure src is fully encapsulated in fb */
+	return drm_rect_intersect(fb_rect, src) &&
+		drm_rect_equals(fb_rect, src);
+}
+
+static int dpu_plane_sspp_atomic_check(struct drm_plane *plane,
+		struct drm_plane_state *state)
+{
+	int ret = 0;
+	struct dpu_plane *pdpu;
+	struct dpu_plane_state *pstate;
+	const struct dpu_format *fmt;
+	struct drm_rect src, dst, fb_rect = { 0 };
+	uint32_t max_upscale = 1, max_downscale = 1;
+	uint32_t min_src_size, max_linewidth;
+	int hscale = 1, vscale = 1;
+
+	if (!plane || !state) {
+		DPU_ERROR("invalid arg(s), plane %d state %d\n",
+				plane != 0, state != 0);
+		ret = -EINVAL;
+		goto exit;
+	}
+
+	pdpu = to_dpu_plane(plane);
+	pstate = to_dpu_plane_state(state);
+
+	if (!pdpu->pipe_sblk) {
+		DPU_ERROR_PLANE(pdpu, "invalid catalog\n");
+		ret = -EINVAL;
+		goto exit;
+	}
+
+	src.x1 = state->src_x >> 16;
+	src.y1 = state->src_y >> 16;
+	src.x2 = src.x1 + (state->src_w >> 16);
+	src.y2 = src.y1 + (state->src_h >> 16);
+
+	dst = drm_plane_state_dest(state);
+
+	fb_rect.x2 = state->fb->width;
+	fb_rect.y2 = state->fb->height;
+
+	max_linewidth = pdpu->pipe_sblk->common->maxlinewidth;
+
+	if (pdpu->features & DPU_SSPP_SCALER) {
+		max_downscale = pdpu->pipe_sblk->maxdwnscale;
+		max_upscale = pdpu->pipe_sblk->maxupscale;
+	}
+	if (drm_rect_width(&src) < drm_rect_width(&dst))
+		hscale = drm_rect_calc_hscale(&src, &dst, 1, max_upscale);
+	else
+		hscale = drm_rect_calc_hscale(&dst, &src, 1, max_downscale);
+	if (drm_rect_height(&src) < drm_rect_height(&dst))
+		vscale = drm_rect_calc_vscale(&src, &dst, 1, max_upscale);
+	else
+		vscale = drm_rect_calc_vscale(&dst, &src, 1, max_downscale);
+
+	DPU_DEBUG_PLANE(pdpu, "check %d -> %d\n",
+		dpu_plane_enabled(plane->state), dpu_plane_enabled(state));
+
+	if (!dpu_plane_enabled(state))
+		goto exit;
+
+	fmt = to_dpu_format(msm_framebuffer_format(state->fb));
+
+	min_src_size = DPU_FORMAT_IS_YUV(fmt) ? 2 : 1;
+
+	if (DPU_FORMAT_IS_YUV(fmt) &&
+		(!(pdpu->features & DPU_SSPP_SCALER) ||
+		 !(pdpu->features & (BIT(DPU_SSPP_CSC)
+		 | BIT(DPU_SSPP_CSC_10BIT))))) {
+		DPU_ERROR_PLANE(pdpu,
+				"plane doesn't have scaler/csc for yuv\n");
+		ret = -EINVAL;
+
+	/* check src bounds */
+	} else if (!dpu_plane_validate_src(&src, &fb_rect, min_src_size)) {
+		DPU_ERROR_PLANE(pdpu, "invalid source " DRM_RECT_FMT "\n",
+				DRM_RECT_ARG(&src));
+		ret = -E2BIG;
+
+	/* valid yuv image */
+	} else if (DPU_FORMAT_IS_YUV(fmt) &&
+		   (src.x1 & 0x1 || src.y1 & 0x1 ||
+		    drm_rect_width(&src) & 0x1 ||
+		    drm_rect_height(&src) & 0x1)) {
+		DPU_ERROR_PLANE(pdpu, "invalid yuv source " DRM_RECT_FMT "\n",
+				DRM_RECT_ARG(&src));
+		ret = -EINVAL;
+
+	/* min dst support */
+	} else if (drm_rect_width(&dst) < 0x1 || drm_rect_height(&dst) < 0x1) {
+		DPU_ERROR_PLANE(pdpu, "invalid dest rect " DRM_RECT_FMT "\n",
+				DRM_RECT_ARG(&dst));
+		ret = -EINVAL;
+
+	/* check decimated source width */
+	} else if (drm_rect_width(&src) > max_linewidth) {
+		DPU_ERROR_PLANE(pdpu, "invalid src " DRM_RECT_FMT " line:%u\n",
+				DRM_RECT_ARG(&src), max_linewidth);
+		ret = -E2BIG;
+
+	/* check scaler capability */
+	} else if (hscale < 0 || vscale < 0) {
+		DPU_ERROR_PLANE(pdpu, "invalid scaling requested src="
+				DRM_RECT_FMT " dst=" DRM_RECT_FMT "\n",
+				DRM_RECT_ARG(&src), DRM_RECT_ARG(&dst));
+		ret = -E2BIG;
+	}
+
+exit:
+	return ret;
+}
+
+static int dpu_plane_atomic_check(struct drm_plane *plane,
+		struct drm_plane_state *state)
+{
+	if (!state->fb)
+		return 0;
+
+	DPU_DEBUG_PLANE(to_dpu_plane(plane), "\n");
+
+	return dpu_plane_sspp_atomic_check(plane, state);
+}
+
+void dpu_plane_flush(struct drm_plane *plane)
+{
+	struct dpu_plane *pdpu;
+	struct dpu_plane_state *pstate;
+
+	if (!plane || !plane->state) {
+		DPU_ERROR("invalid plane\n");
+		return;
+	}
+
+	pdpu = to_dpu_plane(plane);
+	pstate = to_dpu_plane_state(plane->state);
+
+	/*
+	 * These updates have to be done immediately before the plane flush
+	 * timing, and may not be moved to the atomic_update/mode_set functions.
+	 */
+	if (pdpu->is_error)
+		/* force white frame with 100% alpha pipe output on error */
+		_dpu_plane_color_fill(pdpu, 0xFFFFFF, 0xFF);
+	else if (pdpu->color_fill & DPU_PLANE_COLOR_FILL_FLAG)
+		/* force 100% alpha */
+		_dpu_plane_color_fill(pdpu, pdpu->color_fill, 0xFF);
+	else if (pdpu->pipe_hw && pdpu->csc_ptr && pdpu->pipe_hw->ops.setup_csc)
+		pdpu->pipe_hw->ops.setup_csc(pdpu->pipe_hw, pdpu->csc_ptr);
+
+	/* flag h/w flush complete */
+	if (plane->state)
+		pstate->pending = false;
+}
+
+/**
+ * dpu_plane_set_error: enable/disable error condition
+ * @plane: pointer to drm_plane structure
+ */
+void dpu_plane_set_error(struct drm_plane *plane, bool error)
+{
+	struct dpu_plane *pdpu;
+
+	if (!plane)
+		return;
+
+	pdpu = to_dpu_plane(plane);
+	pdpu->is_error = error;
+}
+
+static int dpu_plane_sspp_atomic_update(struct drm_plane *plane,
+				struct drm_plane_state *old_state)
+{
+	uint32_t nplanes, src_flags;
+	struct dpu_plane *pdpu;
+	struct drm_plane_state *state;
+	struct dpu_plane_state *pstate;
+	struct dpu_plane_state *old_pstate;
+	const struct dpu_format *fmt;
+	struct drm_crtc *crtc;
+	struct drm_framebuffer *fb;
+	struct drm_rect src, dst;
+
+	if (!plane) {
+		DPU_ERROR("invalid plane\n");
+		return -EINVAL;
+	} else if (!plane->state) {
+		DPU_ERROR("invalid plane state\n");
+		return -EINVAL;
+	} else if (!old_state) {
+		DPU_ERROR("invalid old state\n");
+		return -EINVAL;
+	}
+
+	pdpu = to_dpu_plane(plane);
+	state = plane->state;
+
+	pstate = to_dpu_plane_state(state);
+
+	old_pstate = to_dpu_plane_state(old_state);
+
+	crtc = state->crtc;
+	fb = state->fb;
+	if (!crtc || !fb) {
+		DPU_ERROR_PLANE(pdpu, "invalid crtc %d or fb %d\n",
+				crtc != 0, fb != 0);
+		return -EINVAL;
+	}
+	fmt = to_dpu_format(msm_framebuffer_format(fb));
+	nplanes = fmt->num_planes;
+
+	memset(&(pdpu->pipe_cfg), 0, sizeof(struct dpu_hw_pipe_cfg));
+
+	_dpu_plane_set_scanout(plane, pstate, &pdpu->pipe_cfg, fb);
+
+	pstate->pending = true;
+
+	pdpu->is_rt_pipe = (dpu_crtc_get_client_type(crtc) != NRT_CLIENT);
+	_dpu_plane_set_qos_ctrl(plane, false, DPU_PLANE_QOS_PANIC_CTRL);
+
+	src.x1 = state->src_x >> 16;
+	src.y1 = state->src_y >> 16;
+	src.x2 = src.x1 + (state->src_w >> 16);
+	src.y2 = src.y1 + (state->src_h >> 16);
+
+	dst = drm_plane_state_dest(state);
+
+	DPU_DEBUG_PLANE(pdpu, "FB[%u] " DRM_RECT_FMT "->crtc%u " DRM_RECT_FMT
+			", %4.4s ubwc %d\n", fb->base.id, DRM_RECT_ARG(&src),
+			crtc->base.id, DRM_RECT_ARG(&dst),
+			(char *)&fmt->base.pixel_format,
+			DPU_FORMAT_IS_UBWC(fmt));
+
+	pdpu->pipe_cfg.src_rect = src;
+	pdpu->pipe_cfg.dst_rect = dst;
+
+	_dpu_plane_setup_scaler(pdpu, pstate, fmt, false);
+
+	/* override for color fill */
+	if (pdpu->color_fill & DPU_PLANE_COLOR_FILL_FLAG) {
+		/* skip remaining processing on color fill */
+		return 0;
+	}
+
+	if (pdpu->pipe_hw->ops.setup_rects) {
+		pdpu->pipe_hw->ops.setup_rects(pdpu->pipe_hw,
+				&pdpu->pipe_cfg,
+				pstate->multirect_index);
+	}
+
+	if (pdpu->pipe_hw->ops.setup_pe &&
+			(pstate->multirect_index != DPU_SSPP_RECT_1))
+		pdpu->pipe_hw->ops.setup_pe(pdpu->pipe_hw,
+				&pstate->pixel_ext);
+
+	/**
+	 * when programmed in multirect mode, scalar block will be
+	 * bypassed. Still we need to update alpha and bitwidth
+	 * ONLY for RECT0
+	 */
+	if (pdpu->pipe_hw->ops.setup_scaler &&
+			pstate->multirect_index != DPU_SSPP_RECT_1)
+		pdpu->pipe_hw->ops.setup_scaler(pdpu->pipe_hw,
+				&pdpu->pipe_cfg, &pstate->pixel_ext,
+				&pstate->scaler3_cfg);
+
+	if (pdpu->pipe_hw->ops.setup_multirect)
+		pdpu->pipe_hw->ops.setup_multirect(
+				pdpu->pipe_hw,
+				pstate->multirect_index,
+				pstate->multirect_mode);
+
+	if (pdpu->pipe_hw->ops.setup_format) {
+		src_flags = 0x0;
+
+		/* update format */
+		pdpu->pipe_hw->ops.setup_format(pdpu->pipe_hw, fmt, src_flags,
+				pstate->multirect_index);
+
+		if (pdpu->pipe_hw->ops.setup_cdp) {
+			struct dpu_hw_pipe_cdp_cfg *cdp_cfg = &pstate->cdp_cfg;
+
+			memset(cdp_cfg, 0, sizeof(struct dpu_hw_pipe_cdp_cfg));
+
+			cdp_cfg->enable = pdpu->catalog->perf.cdp_cfg
+					[DPU_PERF_CDP_USAGE_RT].rd_enable;
+			cdp_cfg->ubwc_meta_enable =
+					DPU_FORMAT_IS_UBWC(fmt);
+			cdp_cfg->tile_amortize_enable =
+					DPU_FORMAT_IS_UBWC(fmt) ||
+					DPU_FORMAT_IS_TILE(fmt);
+			cdp_cfg->preload_ahead = DPU_SSPP_CDP_PRELOAD_AHEAD_64;
+
+			pdpu->pipe_hw->ops.setup_cdp(pdpu->pipe_hw, cdp_cfg);
+		}
+
+		/* update csc */
+		if (DPU_FORMAT_IS_YUV(fmt))
+			_dpu_plane_setup_csc(pdpu);
+		else
+			pdpu->csc_ptr = 0;
+	}
+
+	_dpu_plane_set_qos_lut(plane, fb);
+	_dpu_plane_set_danger_lut(plane, fb);
+
+	if (plane->type != DRM_PLANE_TYPE_CURSOR) {
+		_dpu_plane_set_qos_ctrl(plane, true, DPU_PLANE_QOS_PANIC_CTRL);
+		_dpu_plane_set_ot_limit(plane, crtc);
+	}
+
+	_dpu_plane_set_qos_remap(plane);
+	return 0;
+}
+
+static void _dpu_plane_atomic_disable(struct drm_plane *plane,
+				struct drm_plane_state *old_state)
+{
+	struct dpu_plane *pdpu;
+	struct drm_plane_state *state;
+	struct dpu_plane_state *pstate;
+
+	if (!plane) {
+		DPU_ERROR("invalid plane\n");
+		return;
+	} else if (!plane->state) {
+		DPU_ERROR("invalid plane state\n");
+		return;
+	} else if (!old_state) {
+		DPU_ERROR("invalid old state\n");
+		return;
+	}
+
+	pdpu = to_dpu_plane(plane);
+	state = plane->state;
+	pstate = to_dpu_plane_state(state);
+
+	trace_dpu_plane_disable(DRMID(plane), is_dpu_plane_virtual(plane),
+				pstate->multirect_mode);
+
+	pstate->pending = true;
+
+	if (is_dpu_plane_virtual(plane) &&
+			pdpu->pipe_hw && pdpu->pipe_hw->ops.setup_multirect)
+		pdpu->pipe_hw->ops.setup_multirect(pdpu->pipe_hw,
+				DPU_SSPP_RECT_SOLO, DPU_SSPP_MULTIRECT_NONE);
+}
+
+static void dpu_plane_atomic_update(struct drm_plane *plane,
+				struct drm_plane_state *old_state)
+{
+	struct dpu_plane *pdpu;
+	struct drm_plane_state *state;
+
+	if (!plane) {
+		DPU_ERROR("invalid plane\n");
+		return;
+	} else if (!plane->state) {
+		DPU_ERROR("invalid plane state\n");
+		return;
+	}
+
+	pdpu = to_dpu_plane(plane);
+	pdpu->is_error = false;
+	state = plane->state;
+
+	DPU_DEBUG_PLANE(pdpu, "\n");
+
+	if (!dpu_plane_sspp_enabled(state)) {
+		_dpu_plane_atomic_disable(plane, old_state);
+	} else {
+		int ret;
+
+		ret = dpu_plane_sspp_atomic_update(plane, old_state);
+		/* atomic_check should have ensured that this doesn't fail */
+		WARN_ON(ret < 0);
+	}
+}
+
+void dpu_plane_restore(struct drm_plane *plane)
+{
+	struct dpu_plane *pdpu;
+
+	if (!plane || !plane->state) {
+		DPU_ERROR("invalid plane\n");
+		return;
+	}
+
+	pdpu = to_dpu_plane(plane);
+
+	DPU_DEBUG_PLANE(pdpu, "\n");
+
+	/* last plane state is same as current state */
+	dpu_plane_atomic_update(plane, plane->state);
+}
+
+static void dpu_plane_destroy(struct drm_plane *plane)
+{
+	struct dpu_plane *pdpu = plane ? to_dpu_plane(plane) : NULL;
+
+	DPU_DEBUG_PLANE(pdpu, "\n");
+
+	if (pdpu) {
+		_dpu_plane_set_qos_ctrl(plane, false, DPU_PLANE_QOS_PANIC_CTRL);
+
+		mutex_destroy(&pdpu->lock);
+
+		drm_plane_helper_disable(plane, NULL);
+
+		/* this will destroy the states as well */
+		drm_plane_cleanup(plane);
+
+		if (pdpu->pipe_hw)
+			dpu_hw_sspp_destroy(pdpu->pipe_hw);
+
+		kfree(pdpu);
+	}
+}
+
+static void dpu_plane_destroy_state(struct drm_plane *plane,
+		struct drm_plane_state *state)
+{
+	struct dpu_plane_state *pstate;
+
+	if (!plane || !state) {
+		DPU_ERROR("invalid arg(s), plane %d state %d\n",
+				plane != 0, state != 0);
+		return;
+	}
+
+	pstate = to_dpu_plane_state(state);
+
+	/* remove ref count for frame buffers */
+	if (state->fb)
+		drm_framebuffer_put(state->fb);
+
+	kfree(pstate);
+}
+
+static struct drm_plane_state *
+dpu_plane_duplicate_state(struct drm_plane *plane)
+{
+	struct dpu_plane *pdpu;
+	struct dpu_plane_state *pstate;
+	struct dpu_plane_state *old_state;
+
+	if (!plane) {
+		DPU_ERROR("invalid plane\n");
+		return NULL;
+	} else if (!plane->state) {
+		DPU_ERROR("invalid plane state\n");
+		return NULL;
+	}
+
+	old_state = to_dpu_plane_state(plane->state);
+	pdpu = to_dpu_plane(plane);
+	pstate = kmemdup(old_state, sizeof(*old_state), GFP_KERNEL);
+	if (!pstate) {
+		DPU_ERROR_PLANE(pdpu, "failed to allocate state\n");
+		return NULL;
+	}
+
+	DPU_DEBUG_PLANE(pdpu, "\n");
+
+	pstate->pending = false;
+
+	__drm_atomic_helper_plane_duplicate_state(plane, &pstate->base);
+
+	return &pstate->base;
+}
+
+static void dpu_plane_reset(struct drm_plane *plane)
+{
+	struct dpu_plane *pdpu;
+	struct dpu_plane_state *pstate;
+
+	if (!plane) {
+		DPU_ERROR("invalid plane\n");
+		return;
+	}
+
+	pdpu = to_dpu_plane(plane);
+	DPU_DEBUG_PLANE(pdpu, "\n");
+
+	/* remove previous state, if present */
+	if (plane->state) {
+		dpu_plane_destroy_state(plane, plane->state);
+		plane->state = 0;
+	}
+
+	pstate = kzalloc(sizeof(*pstate), GFP_KERNEL);
+	if (!pstate) {
+		DPU_ERROR_PLANE(pdpu, "failed to allocate state\n");
+		return;
+	}
+
+	pstate->base.plane = plane;
+
+	plane->state = &pstate->base;
+}
+
+#ifdef CONFIG_DEBUG_FS
+static ssize_t _dpu_plane_danger_read(struct file *file,
+			char __user *buff, size_t count, loff_t *ppos)
+{
+	struct dpu_kms *kms = file->private_data;
+	struct dpu_mdss_cfg *cfg = kms->catalog;
+	int len = 0;
+	char buf[40] = {'\0'};
+
+	if (!cfg)
+		return -ENODEV;
+
+	if (*ppos)
+		return 0; /* the end */
+
+	len = snprintf(buf, sizeof(buf), "%d\n", !kms->has_danger_ctrl);
+	if (len < 0 || len >= sizeof(buf))
+		return 0;
+
+	if ((count < sizeof(buf)) || copy_to_user(buff, buf, len))
+		return -EFAULT;
+
+	*ppos += len;   /* increase offset */
+
+	return len;
+}
+
+static void _dpu_plane_set_danger_state(struct dpu_kms *kms, bool enable)
+{
+	struct drm_plane *plane;
+
+	drm_for_each_plane(plane, kms->dev) {
+		if (plane->fb && plane->state) {
+			dpu_plane_danger_signal_ctrl(plane, enable);
+			DPU_DEBUG("plane:%d img:%dx%d ",
+				plane->base.id, plane->fb->width,
+				plane->fb->height);
+			DPU_DEBUG("src[%d,%d,%d,%d] dst[%d,%d,%d,%d]\n",
+				plane->state->src_x >> 16,
+				plane->state->src_y >> 16,
+				plane->state->src_w >> 16,
+				plane->state->src_h >> 16,
+				plane->state->crtc_x, plane->state->crtc_y,
+				plane->state->crtc_w, plane->state->crtc_h);
+		} else {
+			DPU_DEBUG("Inactive plane:%d\n", plane->base.id);
+		}
+	}
+}
+
+static ssize_t _dpu_plane_danger_write(struct file *file,
+		    const char __user *user_buf, size_t count, loff_t *ppos)
+{
+	struct dpu_kms *kms = file->private_data;
+	struct dpu_mdss_cfg *cfg = kms->catalog;
+	int disable_panic;
+	char buf[10];
+
+	if (!cfg)
+		return -EFAULT;
+
+	if (count >= sizeof(buf))
+		return -EFAULT;
+
+	if (copy_from_user(buf, user_buf, count))
+		return -EFAULT;
+
+	buf[count] = 0;	/* end of string */
+
+	if (kstrtoint(buf, 0, &disable_panic))
+		return -EFAULT;
+
+	if (disable_panic) {
+		/* Disable panic signal for all active pipes */
+		DPU_DEBUG("Disabling danger:\n");
+		_dpu_plane_set_danger_state(kms, false);
+		kms->has_danger_ctrl = false;
+	} else {
+		/* Enable panic signal for all active pipes */
+		DPU_DEBUG("Enabling danger:\n");
+		kms->has_danger_ctrl = true;
+		_dpu_plane_set_danger_state(kms, true);
+	}
+
+	return count;
+}
+
+static const struct file_operations dpu_plane_danger_enable = {
+	.open = simple_open,
+	.read = _dpu_plane_danger_read,
+	.write = _dpu_plane_danger_write,
+};
+
+static int _dpu_plane_init_debugfs(struct drm_plane *plane)
+{
+	struct dpu_plane *pdpu;
+	struct dpu_kms *kms;
+	struct msm_drm_private *priv;
+	const struct dpu_sspp_sub_blks *sblk = 0;
+	const struct dpu_sspp_cfg *cfg = 0;
+
+	if (!plane || !plane->dev) {
+		DPU_ERROR("invalid arguments\n");
+		return -EINVAL;
+	}
+
+	priv = plane->dev->dev_private;
+	if (!priv || !priv->kms) {
+		DPU_ERROR("invalid KMS reference\n");
+		return -EINVAL;
+	}
+
+	kms = to_dpu_kms(priv->kms);
+	pdpu = to_dpu_plane(plane);
+
+	if (pdpu && pdpu->pipe_hw)
+		cfg = pdpu->pipe_hw->cap;
+	if (cfg)
+		sblk = cfg->sblk;
+
+	if (!sblk)
+		return 0;
+
+	/* create overall sub-directory for the pipe */
+	pdpu->debugfs_root =
+		debugfs_create_dir(pdpu->pipe_name,
+				plane->dev->primary->debugfs_root);
+
+	if (!pdpu->debugfs_root)
+		return -ENOMEM;
+
+	/* don't error check these */
+	debugfs_create_x32("features", 0600,
+			pdpu->debugfs_root, &pdpu->features);
+
+	/* add register dump support */
+	dpu_debugfs_setup_regset32(&pdpu->debugfs_src,
+			sblk->src_blk.base + cfg->base,
+			sblk->src_blk.len,
+			kms);
+	dpu_debugfs_create_regset32("src_blk", 0400,
+			pdpu->debugfs_root, &pdpu->debugfs_src);
+
+	if (cfg->features & BIT(DPU_SSPP_SCALER_QSEED3) ||
+			cfg->features & BIT(DPU_SSPP_SCALER_QSEED2)) {
+		dpu_debugfs_setup_regset32(&pdpu->debugfs_scaler,
+				sblk->scaler_blk.base + cfg->base,
+				sblk->scaler_blk.len,
+				kms);
+		dpu_debugfs_create_regset32("scaler_blk", 0400,
+				pdpu->debugfs_root,
+				&pdpu->debugfs_scaler);
+		debugfs_create_bool("default_scaling",
+				0600,
+				pdpu->debugfs_root,
+				&pdpu->debugfs_default_scale);
+	}
+
+	if (cfg->features & BIT(DPU_SSPP_CSC) ||
+			cfg->features & BIT(DPU_SSPP_CSC_10BIT)) {
+		dpu_debugfs_setup_regset32(&pdpu->debugfs_csc,
+				sblk->csc_blk.base + cfg->base,
+				sblk->csc_blk.len,
+				kms);
+		dpu_debugfs_create_regset32("csc_blk", 0400,
+				pdpu->debugfs_root, &pdpu->debugfs_csc);
+	}
+
+	debugfs_create_u32("xin_id",
+			0400,
+			pdpu->debugfs_root,
+			(u32 *) &cfg->xin_id);
+	debugfs_create_u32("clk_ctrl",
+			0400,
+			pdpu->debugfs_root,
+			(u32 *) &cfg->clk_ctrl);
+	debugfs_create_x32("creq_vblank",
+			0600,
+			pdpu->debugfs_root,
+			(u32 *) &sblk->creq_vblank);
+	debugfs_create_x32("danger_vblank",
+			0600,
+			pdpu->debugfs_root,
+			(u32 *) &sblk->danger_vblank);
+
+	debugfs_create_file("disable_danger",
+			0600,
+			pdpu->debugfs_root,
+			kms, &dpu_plane_danger_enable);
+
+	return 0;
+}
+
+static void _dpu_plane_destroy_debugfs(struct drm_plane *plane)
+{
+	struct dpu_plane *pdpu;
+
+	if (!plane)
+		return;
+	pdpu = to_dpu_plane(plane);
+
+	debugfs_remove_recursive(pdpu->debugfs_root);
+}
+#else
+static int _dpu_plane_init_debugfs(struct drm_plane *plane)
+{
+	return 0;
+}
+static void _dpu_plane_destroy_debugfs(struct drm_plane *plane)
+{
+}
+#endif
+
+static int dpu_plane_late_register(struct drm_plane *plane)
+{
+	return _dpu_plane_init_debugfs(plane);
+}
+
+static void dpu_plane_early_unregister(struct drm_plane *plane)
+{
+	_dpu_plane_destroy_debugfs(plane);
+}
+
+static const struct drm_plane_funcs dpu_plane_funcs = {
+		.update_plane = drm_atomic_helper_update_plane,
+		.disable_plane = drm_atomic_helper_disable_plane,
+		.destroy = dpu_plane_destroy,
+		.reset = dpu_plane_reset,
+		.atomic_duplicate_state = dpu_plane_duplicate_state,
+		.atomic_destroy_state = dpu_plane_destroy_state,
+		.late_register = dpu_plane_late_register,
+		.early_unregister = dpu_plane_early_unregister,
+};
+
+static const struct drm_plane_helper_funcs dpu_plane_helper_funcs = {
+		.prepare_fb = dpu_plane_prepare_fb,
+		.cleanup_fb = dpu_plane_cleanup_fb,
+		.atomic_check = dpu_plane_atomic_check,
+		.atomic_update = dpu_plane_atomic_update,
+};
+
+enum dpu_sspp dpu_plane_pipe(struct drm_plane *plane)
+{
+	return plane ? to_dpu_plane(plane)->pipe : SSPP_NONE;
+}
+
+bool is_dpu_plane_virtual(struct drm_plane *plane)
+{
+	return plane ? to_dpu_plane(plane)->is_virtual : false;
+}
+
+/* initialize plane */
+struct drm_plane *dpu_plane_init(struct drm_device *dev,
+		uint32_t pipe, bool primary_plane,
+		unsigned long possible_crtcs, u32 master_plane_id)
+{
+	struct drm_plane *plane = NULL, *master_plane = NULL;
+	const struct dpu_format_extended *format_list;
+	struct dpu_plane *pdpu;
+	struct msm_drm_private *priv;
+	struct dpu_kms *kms;
+	enum drm_plane_type type;
+	int zpos_max = DPU_ZPOS_MAX;
+	int ret = -EINVAL;
+
+	if (!dev) {
+		DPU_ERROR("[%u]device is NULL\n", pipe);
+		goto exit;
+	}
+
+	priv = dev->dev_private;
+	if (!priv) {
+		DPU_ERROR("[%u]private data is NULL\n", pipe);
+		goto exit;
+	}
+
+	if (!priv->kms) {
+		DPU_ERROR("[%u]invalid KMS reference\n", pipe);
+		goto exit;
+	}
+	kms = to_dpu_kms(priv->kms);
+
+	if (!kms->catalog) {
+		DPU_ERROR("[%u]invalid catalog reference\n", pipe);
+		goto exit;
+	}
+
+	/* create and zero local structure */
+	pdpu = kzalloc(sizeof(*pdpu), GFP_KERNEL);
+	if (!pdpu) {
+		DPU_ERROR("[%u]failed to allocate local plane struct\n", pipe);
+		ret = -ENOMEM;
+		goto exit;
+	}
+
+	/* cache local stuff for later */
+	plane = &pdpu->base;
+	pdpu->pipe = pipe;
+	pdpu->is_virtual = (master_plane_id != 0);
+	INIT_LIST_HEAD(&pdpu->mplane_list);
+	master_plane = drm_plane_find(dev, NULL, master_plane_id);
+	if (master_plane) {
+		struct dpu_plane *mpdpu = to_dpu_plane(master_plane);
+
+		list_add_tail(&pdpu->mplane_list, &mpdpu->mplane_list);
+	}
+
+	/* initialize underlying h/w driver */
+	pdpu->pipe_hw = dpu_hw_sspp_init(pipe, kms->mmio, kms->catalog,
+							master_plane_id != 0);
+	if (IS_ERR(pdpu->pipe_hw)) {
+		DPU_ERROR("[%u]SSPP init failed\n", pipe);
+		ret = PTR_ERR(pdpu->pipe_hw);
+		goto clean_plane;
+	} else if (!pdpu->pipe_hw->cap || !pdpu->pipe_hw->cap->sblk) {
+		DPU_ERROR("[%u]SSPP init returned invalid cfg\n", pipe);
+		goto clean_sspp;
+	}
+
+	/* cache features mask for later */
+	pdpu->features = pdpu->pipe_hw->cap->features;
+	pdpu->pipe_sblk = pdpu->pipe_hw->cap->sblk;
+	if (!pdpu->pipe_sblk) {
+		DPU_ERROR("[%u]invalid sblk\n", pipe);
+		goto clean_sspp;
+	}
+
+	if (!master_plane_id)
+		format_list = pdpu->pipe_sblk->format_list;
+	else
+		format_list = pdpu->pipe_sblk->virt_format_list;
+
+	pdpu->nformats = dpu_populate_formats(format_list,
+				pdpu->formats,
+				0,
+				ARRAY_SIZE(pdpu->formats));
+
+	if (!pdpu->nformats) {
+		DPU_ERROR("[%u]no valid formats for plane\n", pipe);
+		goto clean_sspp;
+	}
+
+	if (pdpu->features & BIT(DPU_SSPP_CURSOR))
+		type = DRM_PLANE_TYPE_CURSOR;
+	else if (primary_plane)
+		type = DRM_PLANE_TYPE_PRIMARY;
+	else
+		type = DRM_PLANE_TYPE_OVERLAY;
+	ret = drm_universal_plane_init(dev, plane, 0xff, &dpu_plane_funcs,
+				pdpu->formats, pdpu->nformats,
+				NULL, type, NULL);
+	if (ret)
+		goto clean_sspp;
+
+	pdpu->catalog = kms->catalog;
+
+	if (kms->catalog->mixer_count &&
+		kms->catalog->mixer[0].sblk->maxblendstages) {
+		zpos_max = kms->catalog->mixer[0].sblk->maxblendstages - 1;
+		if (zpos_max > DPU_STAGE_MAX - DPU_STAGE_0 - 1)
+			zpos_max = DPU_STAGE_MAX - DPU_STAGE_0 - 1;
+	}
+
+	ret = drm_plane_create_zpos_property(plane, 0, 0, zpos_max);
+	if (ret)
+		DPU_ERROR("failed to install zpos property, rc = %d\n", ret);
+
+	/* success! finalize initialization */
+	drm_plane_helper_add(plane, &dpu_plane_helper_funcs);
+
+	/* save user friendly pipe name for later */
+	snprintf(pdpu->pipe_name, DPU_NAME_SIZE, "plane%u", plane->base.id);
+
+	mutex_init(&pdpu->lock);
+
+	DPU_DEBUG("%s created for pipe:%u id:%u virtual:%u\n", pdpu->pipe_name,
+					pipe, plane->base.id, master_plane_id);
+	return plane;
+
+clean_sspp:
+	if (pdpu && pdpu->pipe_hw)
+		dpu_hw_sspp_destroy(pdpu->pipe_hw);
+clean_plane:
+	kfree(pdpu);
+exit:
+	return ERR_PTR(ret);
+}
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.h
new file mode 100644
index 000000000000..f6fe6ddc7a3a
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.h
@@ -0,0 +1,175 @@
+/*
+ * Copyright (c) 2015-2018, The Linux Foundation. All rights reserved.
+ * Copyright (C) 2013 Red Hat
+ * Author: Rob Clark <robdclark@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _DPU_PLANE_H_
+#define _DPU_PLANE_H_
+
+#include <drm/drm_crtc.h>
+
+#include "dpu_kms.h"
+#include "dpu_hw_mdss.h"
+#include "dpu_hw_sspp.h"
+
+/**
+ * struct dpu_plane_state: Define dpu extension of drm plane state object
+ * @base:	base drm plane state object
+ * @property_state: Local storage for msm_prop properties
+ * @property_values:	cached plane property values
+ * @aspace:	pointer to address space for input/output buffers
+ * @input_fence:	dereferenced input fence pointer
+ * @stage:	assigned by crtc blender
+ * @multirect_index: index of the rectangle of SSPP
+ * @multirect_mode: parallel or time multiplex multirect mode
+ * @pending:	whether the current update is still pending
+ * @scaler3_cfg: configuration data for scaler3
+ * @pixel_ext: configuration data for pixel extensions
+ * @scaler_check_state: indicates status of user provided pixel extension data
+ * @cdp_cfg:	CDP configuration
+ */
+struct dpu_plane_state {
+	struct drm_plane_state base;
+	struct msm_gem_address_space *aspace;
+	void *input_fence;
+	enum dpu_stage stage;
+	uint32_t multirect_index;
+	uint32_t multirect_mode;
+	bool pending;
+
+	/* scaler configuration */
+	struct dpu_hw_scaler3_cfg scaler3_cfg;
+	struct dpu_hw_pixel_ext pixel_ext;
+
+	struct dpu_hw_pipe_cdp_cfg cdp_cfg;
+};
+
+/**
+ * struct dpu_multirect_plane_states: Defines multirect pair of drm plane states
+ * @r0: drm plane configured on rect 0
+ * @r1: drm plane configured on rect 1
+ */
+struct dpu_multirect_plane_states {
+	const struct drm_plane_state *r0;
+	const struct drm_plane_state *r1;
+};
+
+#define to_dpu_plane_state(x) \
+	container_of(x, struct dpu_plane_state, base)
+
+/**
+ * dpu_plane_pipe - return sspp identifier for the given plane
+ * @plane:   Pointer to DRM plane object
+ * Returns: sspp identifier of the given plane
+ */
+enum dpu_sspp dpu_plane_pipe(struct drm_plane *plane);
+
+/**
+ * is_dpu_plane_virtual - check for virtual plane
+ * @plane: Pointer to DRM plane object
+ * returns: true - if the plane is virtual
+ *          false - if the plane is primary
+ */
+bool is_dpu_plane_virtual(struct drm_plane *plane);
+
+/**
+ * dpu_plane_get_ctl_flush - get control flush mask
+ * @plane:   Pointer to DRM plane object
+ * @ctl: Pointer to control hardware
+ * @flush_sspp: Pointer to sspp flush control word
+ */
+void dpu_plane_get_ctl_flush(struct drm_plane *plane, struct dpu_hw_ctl *ctl,
+		u32 *flush_sspp);
+
+/**
+ * dpu_plane_restore - restore hw state if previously power collapsed
+ * @plane: Pointer to drm plane structure
+ */
+void dpu_plane_restore(struct drm_plane *plane);
+
+/**
+ * dpu_plane_flush - final plane operations before commit flush
+ * @plane: Pointer to drm plane structure
+ */
+void dpu_plane_flush(struct drm_plane *plane);
+
+/**
+ * dpu_plane_kickoff - final plane operations before commit kickoff
+ * @plane: Pointer to drm plane structure
+ */
+void dpu_plane_kickoff(struct drm_plane *plane);
+
+/**
+ * dpu_plane_set_error: enable/disable error condition
+ * @plane: pointer to drm_plane structure
+ */
+void dpu_plane_set_error(struct drm_plane *plane, bool error);
+
+/**
+ * dpu_plane_init - create new dpu plane for the given pipe
+ * @dev:   Pointer to DRM device
+ * @pipe:  dpu hardware pipe identifier
+ * @primary_plane: true if this pipe is primary plane for crtc
+ * @possible_crtcs: bitmask of crtc that can be attached to the given pipe
+ * @master_plane_id: primary plane id of a multirect pipe. 0 value passed for
+ *                   a regular plane initialization. A non-zero primary plane
+ *                   id will be passed for a virtual pipe initialization.
+ *
+ */
+struct drm_plane *dpu_plane_init(struct drm_device *dev,
+		uint32_t pipe, bool primary_plane,
+		unsigned long possible_crtcs, u32 master_plane_id);
+
+/**
+ * dpu_plane_validate_multirecti_v2 - validate the multirect planes
+ *				      against hw limitations
+ * @plane: drm plate states of the multirect pair
+ */
+int dpu_plane_validate_multirect_v2(struct dpu_multirect_plane_states *plane);
+
+/**
+ * dpu_plane_clear_multirect - clear multirect bits for the given pipe
+ * @drm_state: Pointer to DRM plane state
+ */
+void dpu_plane_clear_multirect(const struct drm_plane_state *drm_state);
+
+/**
+ * dpu_plane_wait_input_fence - wait for input fence object
+ * @plane:   Pointer to DRM plane object
+ * @wait_ms: Wait timeout value
+ * Returns: Zero on success
+ */
+int dpu_plane_wait_input_fence(struct drm_plane *plane, uint32_t wait_ms);
+
+/**
+ * dpu_plane_color_fill - enables color fill on plane
+ * @plane:  Pointer to DRM plane object
+ * @color:  RGB fill color value, [23..16] Blue, [15..8] Green, [7..0] Red
+ * @alpha:  8-bit fill alpha value, 255 selects 100% alpha
+ * Returns: 0 on success
+ */
+int dpu_plane_color_fill(struct drm_plane *plane,
+		uint32_t color, uint32_t alpha);
+
+/**
+ * dpu_plane_set_revalidate - sets revalidate flag which forces a full
+ *	validation of the plane properties in the next atomic check
+ * @plane: Pointer to DRM plane object
+ * @enable: Boolean to set/unset the flag
+ */
+void dpu_plane_set_revalidate(struct drm_plane *plane, bool enable);
+
+#endif /* _DPU_PLANE_H_ */
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_power_handle.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_power_handle.c
new file mode 100644
index 000000000000..a75eebca2f37
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_power_handle.c
@@ -0,0 +1,249 @@
+/* Copyright (c) 2014-2018, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#define pr_fmt(fmt)	"[drm:%s:%d]: " fmt, __func__, __LINE__
+
+#include <linux/kernel.h>
+#include <linux/of.h>
+#include <linux/string.h>
+#include <linux/of_address.h>
+#include <linux/slab.h>
+#include <linux/mutex.h>
+#include <linux/of_platform.h>
+
+#include "dpu_power_handle.h"
+#include "dpu_trace.h"
+
+static const char *data_bus_name[DPU_POWER_HANDLE_DBUS_ID_MAX] = {
+	[DPU_POWER_HANDLE_DBUS_ID_MNOC] = "qcom,dpu-data-bus",
+	[DPU_POWER_HANDLE_DBUS_ID_LLCC] = "qcom,dpu-llcc-bus",
+	[DPU_POWER_HANDLE_DBUS_ID_EBI] = "qcom,dpu-ebi-bus",
+};
+
+const char *dpu_power_handle_get_dbus_name(u32 bus_id)
+{
+	if (bus_id < DPU_POWER_HANDLE_DBUS_ID_MAX)
+		return data_bus_name[bus_id];
+
+	return NULL;
+}
+
+static void dpu_power_event_trigger_locked(struct dpu_power_handle *phandle,
+		u32 event_type)
+{
+	struct dpu_power_event *event;
+
+	list_for_each_entry(event, &phandle->event_list, list) {
+		if (event->event_type & event_type)
+			event->cb_fnc(event_type, event->usr);
+	}
+}
+
+struct dpu_power_client *dpu_power_client_create(
+	struct dpu_power_handle *phandle, char *client_name)
+{
+	struct dpu_power_client *client;
+	static u32 id;
+
+	if (!client_name || !phandle) {
+		pr_err("client name is null or invalid power data\n");
+		return ERR_PTR(-EINVAL);
+	}
+
+	client = kzalloc(sizeof(struct dpu_power_client), GFP_KERNEL);
+	if (!client)
+		return ERR_PTR(-ENOMEM);
+
+	mutex_lock(&phandle->phandle_lock);
+	strlcpy(client->name, client_name, MAX_CLIENT_NAME_LEN);
+	client->usecase_ndx = VOTE_INDEX_DISABLE;
+	client->id = id;
+	client->active = true;
+	pr_debug("client %s created:%pK id :%d\n", client_name,
+		client, id);
+	id++;
+	list_add(&client->list, &phandle->power_client_clist);
+	mutex_unlock(&phandle->phandle_lock);
+
+	return client;
+}
+
+void dpu_power_client_destroy(struct dpu_power_handle *phandle,
+	struct dpu_power_client *client)
+{
+	if (!client  || !phandle) {
+		pr_err("reg bus vote: invalid client handle\n");
+	} else if (!client->active) {
+		pr_err("dpu power deinit already done\n");
+		kfree(client);
+	} else {
+		pr_debug("bus vote client %s destroyed:%pK id:%u\n",
+			client->name, client, client->id);
+		mutex_lock(&phandle->phandle_lock);
+		list_del_init(&client->list);
+		mutex_unlock(&phandle->phandle_lock);
+		kfree(client);
+	}
+}
+
+void dpu_power_resource_init(struct platform_device *pdev,
+	struct dpu_power_handle *phandle)
+{
+	phandle->dev = &pdev->dev;
+
+	INIT_LIST_HEAD(&phandle->power_client_clist);
+	INIT_LIST_HEAD(&phandle->event_list);
+
+	mutex_init(&phandle->phandle_lock);
+}
+
+void dpu_power_resource_deinit(struct platform_device *pdev,
+	struct dpu_power_handle *phandle)
+{
+	struct dpu_power_client *curr_client, *next_client;
+	struct dpu_power_event *curr_event, *next_event;
+
+	if (!phandle || !pdev) {
+		pr_err("invalid input param\n");
+		return;
+	}
+
+	mutex_lock(&phandle->phandle_lock);
+	list_for_each_entry_safe(curr_client, next_client,
+			&phandle->power_client_clist, list) {
+		pr_err("client:%s-%d still registered with refcount:%d\n",
+				curr_client->name, curr_client->id,
+				curr_client->refcount);
+		curr_client->active = false;
+		list_del(&curr_client->list);
+	}
+
+	list_for_each_entry_safe(curr_event, next_event,
+			&phandle->event_list, list) {
+		pr_err("event:%d, client:%s still registered\n",
+				curr_event->event_type,
+				curr_event->client_name);
+		curr_event->active = false;
+		list_del(&curr_event->list);
+	}
+	mutex_unlock(&phandle->phandle_lock);
+}
+
+int dpu_power_resource_enable(struct dpu_power_handle *phandle,
+	struct dpu_power_client *pclient, bool enable)
+{
+	bool changed = false;
+	u32 max_usecase_ndx = VOTE_INDEX_DISABLE, prev_usecase_ndx;
+	struct dpu_power_client *client;
+
+	if (!phandle || !pclient) {
+		pr_err("invalid input argument\n");
+		return -EINVAL;
+	}
+
+	mutex_lock(&phandle->phandle_lock);
+	if (enable)
+		pclient->refcount++;
+	else if (pclient->refcount)
+		pclient->refcount--;
+
+	if (pclient->refcount)
+		pclient->usecase_ndx = VOTE_INDEX_LOW;
+	else
+		pclient->usecase_ndx = VOTE_INDEX_DISABLE;
+
+	list_for_each_entry(client, &phandle->power_client_clist, list) {
+		if (client->usecase_ndx < VOTE_INDEX_MAX &&
+		    client->usecase_ndx > max_usecase_ndx)
+			max_usecase_ndx = client->usecase_ndx;
+	}
+
+	if (phandle->current_usecase_ndx != max_usecase_ndx) {
+		changed = true;
+		prev_usecase_ndx = phandle->current_usecase_ndx;
+		phandle->current_usecase_ndx = max_usecase_ndx;
+	}
+
+	pr_debug("%pS: changed=%d current idx=%d request client %s id:%u enable:%d refcount:%d\n",
+		__builtin_return_address(0), changed, max_usecase_ndx,
+		pclient->name, pclient->id, enable, pclient->refcount);
+
+	if (!changed)
+		goto end;
+
+	if (enable) {
+		dpu_power_event_trigger_locked(phandle,
+				DPU_POWER_EVENT_PRE_ENABLE);
+		dpu_power_event_trigger_locked(phandle,
+				DPU_POWER_EVENT_POST_ENABLE);
+
+	} else {
+		dpu_power_event_trigger_locked(phandle,
+				DPU_POWER_EVENT_PRE_DISABLE);
+		dpu_power_event_trigger_locked(phandle,
+				DPU_POWER_EVENT_POST_DISABLE);
+	}
+
+end:
+	mutex_unlock(&phandle->phandle_lock);
+	return 0;
+}
+
+struct dpu_power_event *dpu_power_handle_register_event(
+		struct dpu_power_handle *phandle,
+		u32 event_type, void (*cb_fnc)(u32 event_type, void *usr),
+		void *usr, char *client_name)
+{
+	struct dpu_power_event *event;
+
+	if (!phandle) {
+		pr_err("invalid power handle\n");
+		return ERR_PTR(-EINVAL);
+	} else if (!cb_fnc || !event_type) {
+		pr_err("no callback fnc or event type\n");
+		return ERR_PTR(-EINVAL);
+	}
+
+	event = kzalloc(sizeof(struct dpu_power_event), GFP_KERNEL);
+	if (!event)
+		return ERR_PTR(-ENOMEM);
+
+	event->event_type = event_type;
+	event->cb_fnc = cb_fnc;
+	event->usr = usr;
+	strlcpy(event->client_name, client_name, MAX_CLIENT_NAME_LEN);
+	event->active = true;
+
+	mutex_lock(&phandle->phandle_lock);
+	list_add(&event->list, &phandle->event_list);
+	mutex_unlock(&phandle->phandle_lock);
+
+	return event;
+}
+
+void dpu_power_handle_unregister_event(
+		struct dpu_power_handle *phandle,
+		struct dpu_power_event *event)
+{
+	if (!phandle || !event) {
+		pr_err("invalid phandle or event\n");
+	} else if (!event->active) {
+		pr_err("power handle deinit already done\n");
+		kfree(event);
+	} else {
+		mutex_lock(&phandle->phandle_lock);
+		list_del_init(&event->list);
+		mutex_unlock(&phandle->phandle_lock);
+		kfree(event);
+	}
+}
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_power_handle.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_power_handle.h
new file mode 100644
index 000000000000..344f74464eca
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_power_handle.h
@@ -0,0 +1,225 @@
+/* Copyright (c) 2016-2018, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#ifndef _DPU_POWER_HANDLE_H_
+#define _DPU_POWER_HANDLE_H_
+
+#define MAX_CLIENT_NAME_LEN 128
+
+#define DPU_POWER_HANDLE_ENABLE_BUS_AB_QUOTA	0
+#define DPU_POWER_HANDLE_DISABLE_BUS_AB_QUOTA	0
+#define DPU_POWER_HANDLE_ENABLE_BUS_IB_QUOTA	1600000000
+#define DPU_POWER_HANDLE_DISABLE_BUS_IB_QUOTA	0
+
+#include "dpu_io_util.h"
+
+/* event will be triggered before power handler disable */
+#define DPU_POWER_EVENT_PRE_DISABLE	0x1
+
+/* event will be triggered after power handler disable */
+#define DPU_POWER_EVENT_POST_DISABLE	0x2
+
+/* event will be triggered before power handler enable */
+#define DPU_POWER_EVENT_PRE_ENABLE	0x4
+
+/* event will be triggered after power handler enable */
+#define DPU_POWER_EVENT_POST_ENABLE	0x8
+
+/**
+ * mdss_bus_vote_type: register bus vote type
+ * VOTE_INDEX_DISABLE: removes the client vote
+ * VOTE_INDEX_LOW: keeps the lowest vote for register bus
+ * VOTE_INDEX_MAX: invalid
+ */
+enum mdss_bus_vote_type {
+	VOTE_INDEX_DISABLE,
+	VOTE_INDEX_LOW,
+	VOTE_INDEX_MAX,
+};
+
+/**
+ * enum dpu_power_handle_data_bus_client - type of axi bus clients
+ * @DPU_POWER_HANDLE_DATA_BUS_CLIENT_RT: core real-time bus client
+ * @DPU_POWER_HANDLE_DATA_BUS_CLIENT_NRT: core non-real-time bus client
+ * @DPU_POWER_HANDLE_DATA_BUS_CLIENT_MAX: maximum number of bus client type
+ */
+enum dpu_power_handle_data_bus_client {
+	DPU_POWER_HANDLE_DATA_BUS_CLIENT_RT,
+	DPU_POWER_HANDLE_DATA_BUS_CLIENT_NRT,
+	DPU_POWER_HANDLE_DATA_BUS_CLIENT_MAX
+};
+
+/**
+ * enum DPU_POWER_HANDLE_DBUS_ID - data bus identifier
+ * @DPU_POWER_HANDLE_DBUS_ID_MNOC: DPU/MNOC data bus
+ * @DPU_POWER_HANDLE_DBUS_ID_LLCC: MNOC/LLCC data bus
+ * @DPU_POWER_HANDLE_DBUS_ID_EBI: LLCC/EBI data bus
+ */
+enum DPU_POWER_HANDLE_DBUS_ID {
+	DPU_POWER_HANDLE_DBUS_ID_MNOC,
+	DPU_POWER_HANDLE_DBUS_ID_LLCC,
+	DPU_POWER_HANDLE_DBUS_ID_EBI,
+	DPU_POWER_HANDLE_DBUS_ID_MAX,
+};
+
+/**
+ * struct dpu_power_client: stores the power client for dpu driver
+ * @name:	name of the client
+ * @usecase_ndx: current regs bus vote type
+ * @refcount:	current refcount if multiple modules are using same
+ *              same client for enable/disable. Power module will
+ *              aggregate the refcount and vote accordingly for this
+ *              client.
+ * @id:		assigned during create. helps for debugging.
+ * @list:	list to attach power handle master list
+ * @ab:         arbitrated bandwidth for each bus client
+ * @ib:         instantaneous bandwidth for each bus client
+ * @active:	inidcates the state of dpu power handle
+ */
+struct dpu_power_client {
+	char name[MAX_CLIENT_NAME_LEN];
+	short usecase_ndx;
+	short refcount;
+	u32 id;
+	struct list_head list;
+	u64 ab[DPU_POWER_HANDLE_DATA_BUS_CLIENT_MAX];
+	u64 ib[DPU_POWER_HANDLE_DATA_BUS_CLIENT_MAX];
+	bool active;
+};
+
+/*
+ * struct dpu_power_event - local event registration structure
+ * @client_name: name of the client registering
+ * @cb_fnc: pointer to desired callback function
+ * @usr: user pointer to pass to callback event trigger
+ * @event: refer to DPU_POWER_HANDLE_EVENT_*
+ * @list: list to attach event master list
+ * @active: indicates the state of dpu power handle
+ */
+struct dpu_power_event {
+	char client_name[MAX_CLIENT_NAME_LEN];
+	void (*cb_fnc)(u32 event_type, void *usr);
+	void *usr;
+	u32 event_type;
+	struct list_head list;
+	bool active;
+};
+
+/**
+ * struct dpu_power_handle: power handle main struct
+ * @client_clist: master list to store all clients
+ * @phandle_lock: lock to synchronize the enable/disable
+ * @dev: pointer to device structure
+ * @usecase_ndx: current usecase index
+ * @event_list: current power handle event list
+ */
+struct dpu_power_handle {
+	struct list_head power_client_clist;
+	struct mutex phandle_lock;
+	struct device *dev;
+	u32 current_usecase_ndx;
+	struct list_head event_list;
+};
+
+/**
+ * dpu_power_resource_init() - initializes the dpu power handle
+ * @pdev:   platform device to search the power resources
+ * @pdata:  power handle to store the power resources
+ */
+void dpu_power_resource_init(struct platform_device *pdev,
+	struct dpu_power_handle *pdata);
+
+/**
+ * dpu_power_resource_deinit() - release the dpu power handle
+ * @pdev:   platform device for power resources
+ * @pdata:  power handle containing the resources
+ *
+ * Return: error code.
+ */
+void dpu_power_resource_deinit(struct platform_device *pdev,
+	struct dpu_power_handle *pdata);
+
+/**
+ * dpu_power_client_create() - create the client on power handle
+ * @pdata:  power handle containing the resources
+ * @client_name: new client name for registration
+ *
+ * Return: error code.
+ */
+struct dpu_power_client *dpu_power_client_create(struct dpu_power_handle *pdata,
+	char *client_name);
+
+/**
+ * dpu_power_client_destroy() - destroy the client on power handle
+ * @pdata:  power handle containing the resources
+ * @client_name: new client name for registration
+ *
+ * Return: none
+ */
+void dpu_power_client_destroy(struct dpu_power_handle *phandle,
+	struct dpu_power_client *client);
+
+/**
+ * dpu_power_resource_enable() - enable/disable the power resources
+ * @pdata:  power handle containing the resources
+ * @client: client information to enable/disable its vote
+ * @enable: boolean request for enable/disable
+ *
+ * Return: error code.
+ */
+int dpu_power_resource_enable(struct dpu_power_handle *pdata,
+	struct dpu_power_client *pclient, bool enable);
+
+/**
+ * dpu_power_data_bus_bandwidth_ctrl() - control data bus bandwidth enable
+ * @phandle:  power handle containing the resources
+ * @client: client information to bandwidth control
+ * @enable: true to enable bandwidth for data base
+ *
+ * Return: none
+ */
+void dpu_power_data_bus_bandwidth_ctrl(struct dpu_power_handle *phandle,
+		struct dpu_power_client *pclient, int enable);
+
+/**
+ * dpu_power_handle_register_event - register a callback function for an event.
+ *	Clients can register for multiple events with a single register.
+ *	Any block with access to phandle can register for the event
+ *	notification.
+ * @phandle:	power handle containing the resources
+ * @event_type:	event type to register; refer DPU_POWER_HANDLE_EVENT_*
+ * @cb_fnc:	pointer to desired callback function
+ * @usr:	user pointer to pass to callback on event trigger
+ *
+ * Return:	event pointer if success, or error code otherwise
+ */
+struct dpu_power_event *dpu_power_handle_register_event(
+		struct dpu_power_handle *phandle,
+		u32 event_type, void (*cb_fnc)(u32 event_type, void *usr),
+		void *usr, char *client_name);
+/**
+ * dpu_power_handle_unregister_event - unregister callback for event(s)
+ * @phandle:	power handle containing the resources
+ * @event:	event pointer returned after power handle register
+ */
+void dpu_power_handle_unregister_event(struct dpu_power_handle *phandle,
+		struct dpu_power_event *event);
+
+/**
+ * dpu_power_handle_get_dbus_name - get name of given data bus identifier
+ * @bus_id:	data bus identifier
+ * Return:	Pointer to name string if success; NULL otherwise
+ */
+const char *dpu_power_handle_get_dbus_name(u32 bus_id);
+
+#endif /* _DPU_POWER_HANDLE_H_ */
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_rm.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_rm.c
new file mode 100644
index 000000000000..13c0a36d4ef9
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_rm.c
@@ -0,0 +1,1079 @@
+/*
+ * Copyright (c) 2016-2018, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#define pr_fmt(fmt)	"[drm:%s] " fmt, __func__
+#include "dpu_kms.h"
+#include "dpu_hw_lm.h"
+#include "dpu_hw_ctl.h"
+#include "dpu_hw_cdm.h"
+#include "dpu_hw_pingpong.h"
+#include "dpu_hw_intf.h"
+#include "dpu_encoder.h"
+#include "dpu_trace.h"
+
+#define RESERVED_BY_OTHER(h, r) \
+	((h)->rsvp && ((h)->rsvp->enc_id != (r)->enc_id))
+
+#define RM_RQ_LOCK(r) ((r)->top_ctrl & BIT(DPU_RM_TOPCTL_RESERVE_LOCK))
+#define RM_RQ_CLEAR(r) ((r)->top_ctrl & BIT(DPU_RM_TOPCTL_RESERVE_CLEAR))
+#define RM_RQ_DS(r) ((r)->top_ctrl & BIT(DPU_RM_TOPCTL_DS))
+#define RM_IS_TOPOLOGY_MATCH(t, r) ((t).num_lm == (r).num_lm && \
+				(t).num_comp_enc == (r).num_enc && \
+				(t).num_intf == (r).num_intf)
+
+struct dpu_rm_topology_def {
+	enum dpu_rm_topology_name top_name;
+	int num_lm;
+	int num_comp_enc;
+	int num_intf;
+	int num_ctl;
+	int needs_split_display;
+};
+
+static const struct dpu_rm_topology_def g_top_table[] = {
+	{   DPU_RM_TOPOLOGY_NONE,                 0, 0, 0, 0, false },
+	{   DPU_RM_TOPOLOGY_SINGLEPIPE,           1, 0, 1, 1, false },
+	{   DPU_RM_TOPOLOGY_DUALPIPE,             2, 0, 2, 2, true  },
+	{   DPU_RM_TOPOLOGY_DUALPIPE_3DMERGE,     2, 0, 1, 1, false },
+};
+
+/**
+ * struct dpu_rm_requirements - Reservation requirements parameter bundle
+ * @top_ctrl:  topology control preference from kernel client
+ * @top:       selected topology for the display
+ * @hw_res:	   Hardware resources required as reported by the encoders
+ */
+struct dpu_rm_requirements {
+	uint64_t top_ctrl;
+	const struct dpu_rm_topology_def *topology;
+	struct dpu_encoder_hw_resources hw_res;
+};
+
+/**
+ * struct dpu_rm_rsvp - Use Case Reservation tagging structure
+ *	Used to tag HW blocks as reserved by a CRTC->Encoder->Connector chain
+ *	By using as a tag, rather than lists of pointers to HW blocks used
+ *	we can avoid some list management since we don't know how many blocks
+ *	of each type a given use case may require.
+ * @list:	List head for list of all reservations
+ * @seq:	Global RSVP sequence number for debugging, especially for
+ *		differentiating differenct allocations for same encoder.
+ * @enc_id:	Reservations are tracked by Encoder DRM object ID.
+ *		CRTCs may be connected to multiple Encoders.
+ *		An encoder or connector id identifies the display path.
+ * @topology	DRM<->HW topology use case
+ */
+struct dpu_rm_rsvp {
+	struct list_head list;
+	uint32_t seq;
+	uint32_t enc_id;
+	enum dpu_rm_topology_name topology;
+};
+
+/**
+ * struct dpu_rm_hw_blk - hardware block tracking list member
+ * @list:	List head for list of all hardware blocks tracking items
+ * @rsvp:	Pointer to use case reservation if reserved by a client
+ * @rsvp_nxt:	Temporary pointer used during reservation to the incoming
+ *		request. Will be swapped into rsvp if proposal is accepted
+ * @type:	Type of hardware block this structure tracks
+ * @id:		Hardware ID number, within it's own space, ie. LM_X
+ * @catalog:	Pointer to the hardware catalog entry for this block
+ * @hw:		Pointer to the hardware register access object for this block
+ */
+struct dpu_rm_hw_blk {
+	struct list_head list;
+	struct dpu_rm_rsvp *rsvp;
+	struct dpu_rm_rsvp *rsvp_nxt;
+	enum dpu_hw_blk_type type;
+	uint32_t id;
+	struct dpu_hw_blk *hw;
+};
+
+/**
+ * dpu_rm_dbg_rsvp_stage - enum of steps in making reservation for event logging
+ */
+enum dpu_rm_dbg_rsvp_stage {
+	DPU_RM_STAGE_BEGIN,
+	DPU_RM_STAGE_AFTER_CLEAR,
+	DPU_RM_STAGE_AFTER_RSVPNEXT,
+	DPU_RM_STAGE_FINAL
+};
+
+static void _dpu_rm_print_rsvps(
+		struct dpu_rm *rm,
+		enum dpu_rm_dbg_rsvp_stage stage)
+{
+	struct dpu_rm_rsvp *rsvp;
+	struct dpu_rm_hw_blk *blk;
+	enum dpu_hw_blk_type type;
+
+	DPU_DEBUG("%d\n", stage);
+
+	list_for_each_entry(rsvp, &rm->rsvps, list) {
+		DRM_DEBUG_KMS("%d rsvp[s%ue%u] topology %d\n", stage, rsvp->seq,
+			      rsvp->enc_id, rsvp->topology);
+	}
+
+	for (type = 0; type < DPU_HW_BLK_MAX; type++) {
+		list_for_each_entry(blk, &rm->hw_blks[type], list) {
+			if (!blk->rsvp && !blk->rsvp_nxt)
+				continue;
+
+			DRM_DEBUG_KMS("%d rsvp[s%ue%u->s%ue%u] %d %d\n", stage,
+				(blk->rsvp) ? blk->rsvp->seq : 0,
+				(blk->rsvp) ? blk->rsvp->enc_id : 0,
+				(blk->rsvp_nxt) ? blk->rsvp_nxt->seq : 0,
+				(blk->rsvp_nxt) ? blk->rsvp_nxt->enc_id : 0,
+				blk->type, blk->id);
+		}
+	}
+}
+
+struct dpu_hw_mdp *dpu_rm_get_mdp(struct dpu_rm *rm)
+{
+	return rm->hw_mdp;
+}
+
+enum dpu_rm_topology_name
+dpu_rm_get_topology_name(struct msm_display_topology topology)
+{
+	int i;
+
+	for (i = 0; i < DPU_RM_TOPOLOGY_MAX; i++)
+		if (RM_IS_TOPOLOGY_MATCH(g_top_table[i], topology))
+			return g_top_table[i].top_name;
+
+	return DPU_RM_TOPOLOGY_NONE;
+}
+
+void dpu_rm_init_hw_iter(
+		struct dpu_rm_hw_iter *iter,
+		uint32_t enc_id,
+		enum dpu_hw_blk_type type)
+{
+	memset(iter, 0, sizeof(*iter));
+	iter->enc_id = enc_id;
+	iter->type = type;
+}
+
+static bool _dpu_rm_get_hw_locked(struct dpu_rm *rm, struct dpu_rm_hw_iter *i)
+{
+	struct list_head *blk_list;
+
+	if (!rm || !i || i->type >= DPU_HW_BLK_MAX) {
+		DPU_ERROR("invalid rm\n");
+		return false;
+	}
+
+	i->hw = NULL;
+	blk_list = &rm->hw_blks[i->type];
+
+	if (i->blk && (&i->blk->list == blk_list)) {
+		DPU_DEBUG("attempt resume iteration past last\n");
+		return false;
+	}
+
+	i->blk = list_prepare_entry(i->blk, blk_list, list);
+
+	list_for_each_entry_continue(i->blk, blk_list, list) {
+		struct dpu_rm_rsvp *rsvp = i->blk->rsvp;
+
+		if (i->blk->type != i->type) {
+			DPU_ERROR("found incorrect block type %d on %d list\n",
+					i->blk->type, i->type);
+			return false;
+		}
+
+		if ((i->enc_id == 0) || (rsvp && rsvp->enc_id == i->enc_id)) {
+			i->hw = i->blk->hw;
+			DPU_DEBUG("found type %d id %d for enc %d\n",
+					i->type, i->blk->id, i->enc_id);
+			return true;
+		}
+	}
+
+	DPU_DEBUG("no match, type %d for enc %d\n", i->type, i->enc_id);
+
+	return false;
+}
+
+bool dpu_rm_get_hw(struct dpu_rm *rm, struct dpu_rm_hw_iter *i)
+{
+	bool ret;
+
+	mutex_lock(&rm->rm_lock);
+	ret = _dpu_rm_get_hw_locked(rm, i);
+	mutex_unlock(&rm->rm_lock);
+
+	return ret;
+}
+
+static void _dpu_rm_hw_destroy(enum dpu_hw_blk_type type, void *hw)
+{
+	switch (type) {
+	case DPU_HW_BLK_LM:
+		dpu_hw_lm_destroy(hw);
+		break;
+	case DPU_HW_BLK_CTL:
+		dpu_hw_ctl_destroy(hw);
+		break;
+	case DPU_HW_BLK_CDM:
+		dpu_hw_cdm_destroy(hw);
+		break;
+	case DPU_HW_BLK_PINGPONG:
+		dpu_hw_pingpong_destroy(hw);
+		break;
+	case DPU_HW_BLK_INTF:
+		dpu_hw_intf_destroy(hw);
+		break;
+	case DPU_HW_BLK_SSPP:
+		/* SSPPs are not managed by the resource manager */
+	case DPU_HW_BLK_TOP:
+		/* Top is a singleton, not managed in hw_blks list */
+	case DPU_HW_BLK_MAX:
+	default:
+		DPU_ERROR("unsupported block type %d\n", type);
+		break;
+	}
+}
+
+int dpu_rm_destroy(struct dpu_rm *rm)
+{
+
+	struct dpu_rm_rsvp *rsvp_cur, *rsvp_nxt;
+	struct dpu_rm_hw_blk *hw_cur, *hw_nxt;
+	enum dpu_hw_blk_type type;
+
+	if (!rm) {
+		DPU_ERROR("invalid rm\n");
+		return -EINVAL;
+	}
+
+	list_for_each_entry_safe(rsvp_cur, rsvp_nxt, &rm->rsvps, list) {
+		list_del(&rsvp_cur->list);
+		kfree(rsvp_cur);
+	}
+
+
+	for (type = 0; type < DPU_HW_BLK_MAX; type++) {
+		list_for_each_entry_safe(hw_cur, hw_nxt, &rm->hw_blks[type],
+				list) {
+			list_del(&hw_cur->list);
+			_dpu_rm_hw_destroy(hw_cur->type, hw_cur->hw);
+			kfree(hw_cur);
+		}
+	}
+
+	dpu_hw_mdp_destroy(rm->hw_mdp);
+	rm->hw_mdp = NULL;
+
+	mutex_destroy(&rm->rm_lock);
+
+	return 0;
+}
+
+static int _dpu_rm_hw_blk_create(
+		struct dpu_rm *rm,
+		struct dpu_mdss_cfg *cat,
+		void __iomem *mmio,
+		enum dpu_hw_blk_type type,
+		uint32_t id,
+		void *hw_catalog_info)
+{
+	struct dpu_rm_hw_blk *blk;
+	struct dpu_hw_mdp *hw_mdp;
+	void *hw;
+
+	hw_mdp = rm->hw_mdp;
+
+	switch (type) {
+	case DPU_HW_BLK_LM:
+		hw = dpu_hw_lm_init(id, mmio, cat);
+		break;
+	case DPU_HW_BLK_CTL:
+		hw = dpu_hw_ctl_init(id, mmio, cat);
+		break;
+	case DPU_HW_BLK_CDM:
+		hw = dpu_hw_cdm_init(id, mmio, cat, hw_mdp);
+		break;
+	case DPU_HW_BLK_PINGPONG:
+		hw = dpu_hw_pingpong_init(id, mmio, cat);
+		break;
+	case DPU_HW_BLK_INTF:
+		hw = dpu_hw_intf_init(id, mmio, cat);
+		break;
+	case DPU_HW_BLK_SSPP:
+		/* SSPPs are not managed by the resource manager */
+	case DPU_HW_BLK_TOP:
+		/* Top is a singleton, not managed in hw_blks list */
+	case DPU_HW_BLK_MAX:
+	default:
+		DPU_ERROR("unsupported block type %d\n", type);
+		return -EINVAL;
+	}
+
+	if (IS_ERR_OR_NULL(hw)) {
+		DPU_ERROR("failed hw object creation: type %d, err %ld\n",
+				type, PTR_ERR(hw));
+		return -EFAULT;
+	}
+
+	blk = kzalloc(sizeof(*blk), GFP_KERNEL);
+	if (!blk) {
+		_dpu_rm_hw_destroy(type, hw);
+		return -ENOMEM;
+	}
+
+	blk->type = type;
+	blk->id = id;
+	blk->hw = hw;
+	list_add_tail(&blk->list, &rm->hw_blks[type]);
+
+	return 0;
+}
+
+int dpu_rm_init(struct dpu_rm *rm,
+		struct dpu_mdss_cfg *cat,
+		void __iomem *mmio,
+		struct drm_device *dev)
+{
+	int rc, i;
+	enum dpu_hw_blk_type type;
+
+	if (!rm || !cat || !mmio || !dev) {
+		DPU_ERROR("invalid kms\n");
+		return -EINVAL;
+	}
+
+	/* Clear, setup lists */
+	memset(rm, 0, sizeof(*rm));
+
+	mutex_init(&rm->rm_lock);
+
+	INIT_LIST_HEAD(&rm->rsvps);
+	for (type = 0; type < DPU_HW_BLK_MAX; type++)
+		INIT_LIST_HEAD(&rm->hw_blks[type]);
+
+	rm->dev = dev;
+
+	/* Some of the sub-blocks require an mdptop to be created */
+	rm->hw_mdp = dpu_hw_mdptop_init(MDP_TOP, mmio, cat);
+	if (IS_ERR_OR_NULL(rm->hw_mdp)) {
+		rc = PTR_ERR(rm->hw_mdp);
+		rm->hw_mdp = NULL;
+		DPU_ERROR("failed: mdp hw not available\n");
+		goto fail;
+	}
+
+	/* Interrogate HW catalog and create tracking items for hw blocks */
+	for (i = 0; i < cat->mixer_count; i++) {
+		struct dpu_lm_cfg *lm = &cat->mixer[i];
+
+		if (lm->pingpong == PINGPONG_MAX) {
+			DPU_DEBUG("skip mixer %d without pingpong\n", lm->id);
+			continue;
+		}
+
+		rc = _dpu_rm_hw_blk_create(rm, cat, mmio, DPU_HW_BLK_LM,
+				cat->mixer[i].id, &cat->mixer[i]);
+		if (rc) {
+			DPU_ERROR("failed: lm hw not available\n");
+			goto fail;
+		}
+
+		if (!rm->lm_max_width) {
+			rm->lm_max_width = lm->sblk->maxwidth;
+		} else if (rm->lm_max_width != lm->sblk->maxwidth) {
+			/*
+			 * Don't expect to have hw where lm max widths differ.
+			 * If found, take the min.
+			 */
+			DPU_ERROR("unsupported: lm maxwidth differs\n");
+			if (rm->lm_max_width > lm->sblk->maxwidth)
+				rm->lm_max_width = lm->sblk->maxwidth;
+		}
+	}
+
+	for (i = 0; i < cat->pingpong_count; i++) {
+		rc = _dpu_rm_hw_blk_create(rm, cat, mmio, DPU_HW_BLK_PINGPONG,
+				cat->pingpong[i].id, &cat->pingpong[i]);
+		if (rc) {
+			DPU_ERROR("failed: pp hw not available\n");
+			goto fail;
+		}
+	}
+
+	for (i = 0; i < cat->intf_count; i++) {
+		if (cat->intf[i].type == INTF_NONE) {
+			DPU_DEBUG("skip intf %d with type none\n", i);
+			continue;
+		}
+
+		rc = _dpu_rm_hw_blk_create(rm, cat, mmio, DPU_HW_BLK_INTF,
+				cat->intf[i].id, &cat->intf[i]);
+		if (rc) {
+			DPU_ERROR("failed: intf hw not available\n");
+			goto fail;
+		}
+	}
+
+	for (i = 0; i < cat->ctl_count; i++) {
+		rc = _dpu_rm_hw_blk_create(rm, cat, mmio, DPU_HW_BLK_CTL,
+				cat->ctl[i].id, &cat->ctl[i]);
+		if (rc) {
+			DPU_ERROR("failed: ctl hw not available\n");
+			goto fail;
+		}
+	}
+
+	for (i = 0; i < cat->cdm_count; i++) {
+		rc = _dpu_rm_hw_blk_create(rm, cat, mmio, DPU_HW_BLK_CDM,
+				cat->cdm[i].id, &cat->cdm[i]);
+		if (rc) {
+			DPU_ERROR("failed: cdm hw not available\n");
+			goto fail;
+		}
+	}
+
+	return 0;
+
+fail:
+	dpu_rm_destroy(rm);
+
+	return rc;
+}
+
+/**
+ * _dpu_rm_check_lm_and_get_connected_blks - check if proposed layer mixer meets
+ *	proposed use case requirements, incl. hardwired dependent blocks like
+ *	pingpong
+ * @rm: dpu resource manager handle
+ * @rsvp: reservation currently being created
+ * @reqs: proposed use case requirements
+ * @lm: proposed layer mixer, function checks if lm, and all other hardwired
+ *      blocks connected to the lm (pp) is available and appropriate
+ * @pp: output parameter, pingpong block attached to the layer mixer.
+ *      NULL if pp was not available, or not matching requirements.
+ * @primary_lm: if non-null, this function check if lm is compatible primary_lm
+ *              as well as satisfying all other requirements
+ * @Return: true if lm matches all requirements, false otherwise
+ */
+static bool _dpu_rm_check_lm_and_get_connected_blks(
+		struct dpu_rm *rm,
+		struct dpu_rm_rsvp *rsvp,
+		struct dpu_rm_requirements *reqs,
+		struct dpu_rm_hw_blk *lm,
+		struct dpu_rm_hw_blk **pp,
+		struct dpu_rm_hw_blk *primary_lm)
+{
+	const struct dpu_lm_cfg *lm_cfg = to_dpu_hw_mixer(lm->hw)->cap;
+	struct dpu_rm_hw_iter iter;
+
+	*pp = NULL;
+
+	DPU_DEBUG("check lm %d pp %d\n",
+			   lm_cfg->id, lm_cfg->pingpong);
+
+	/* Check if this layer mixer is a peer of the proposed primary LM */
+	if (primary_lm) {
+		const struct dpu_lm_cfg *prim_lm_cfg =
+				to_dpu_hw_mixer(primary_lm->hw)->cap;
+
+		if (!test_bit(lm_cfg->id, &prim_lm_cfg->lm_pair_mask)) {
+			DPU_DEBUG("lm %d not peer of lm %d\n", lm_cfg->id,
+					prim_lm_cfg->id);
+			return false;
+		}
+	}
+
+	/* Already reserved? */
+	if (RESERVED_BY_OTHER(lm, rsvp)) {
+		DPU_DEBUG("lm %d already reserved\n", lm_cfg->id);
+		return false;
+	}
+
+	dpu_rm_init_hw_iter(&iter, 0, DPU_HW_BLK_PINGPONG);
+	while (_dpu_rm_get_hw_locked(rm, &iter)) {
+		if (iter.blk->id == lm_cfg->pingpong) {
+			*pp = iter.blk;
+			break;
+		}
+	}
+
+	if (!*pp) {
+		DPU_ERROR("failed to get pp on lm %d\n", lm_cfg->pingpong);
+		return false;
+	}
+
+	if (RESERVED_BY_OTHER(*pp, rsvp)) {
+		DPU_DEBUG("lm %d pp %d already reserved\n", lm->id,
+				(*pp)->id);
+		return false;
+	}
+
+	return true;
+}
+
+static int _dpu_rm_reserve_lms(
+		struct dpu_rm *rm,
+		struct dpu_rm_rsvp *rsvp,
+		struct dpu_rm_requirements *reqs)
+
+{
+	struct dpu_rm_hw_blk *lm[MAX_BLOCKS];
+	struct dpu_rm_hw_blk *pp[MAX_BLOCKS];
+	struct dpu_rm_hw_iter iter_i, iter_j;
+	int lm_count = 0;
+	int i, rc = 0;
+
+	if (!reqs->topology->num_lm) {
+		DPU_ERROR("invalid number of lm: %d\n", reqs->topology->num_lm);
+		return -EINVAL;
+	}
+
+	/* Find a primary mixer */
+	dpu_rm_init_hw_iter(&iter_i, 0, DPU_HW_BLK_LM);
+	while (lm_count != reqs->topology->num_lm &&
+			_dpu_rm_get_hw_locked(rm, &iter_i)) {
+		memset(&lm, 0, sizeof(lm));
+		memset(&pp, 0, sizeof(pp));
+
+		lm_count = 0;
+		lm[lm_count] = iter_i.blk;
+
+		if (!_dpu_rm_check_lm_and_get_connected_blks(
+				rm, rsvp, reqs, lm[lm_count],
+				&pp[lm_count], NULL))
+			continue;
+
+		++lm_count;
+
+		/* Valid primary mixer found, find matching peers */
+		dpu_rm_init_hw_iter(&iter_j, 0, DPU_HW_BLK_LM);
+
+		while (lm_count != reqs->topology->num_lm &&
+				_dpu_rm_get_hw_locked(rm, &iter_j)) {
+			if (iter_i.blk == iter_j.blk)
+				continue;
+
+			if (!_dpu_rm_check_lm_and_get_connected_blks(
+					rm, rsvp, reqs, iter_j.blk,
+					&pp[lm_count], iter_i.blk))
+				continue;
+
+			lm[lm_count] = iter_j.blk;
+			++lm_count;
+		}
+	}
+
+	if (lm_count != reqs->topology->num_lm) {
+		DPU_DEBUG("unable to find appropriate mixers\n");
+		return -ENAVAIL;
+	}
+
+	for (i = 0; i < ARRAY_SIZE(lm); i++) {
+		if (!lm[i])
+			break;
+
+		lm[i]->rsvp_nxt = rsvp;
+		pp[i]->rsvp_nxt = rsvp;
+
+		trace_dpu_rm_reserve_lms(lm[i]->id, lm[i]->type, rsvp->enc_id,
+					 pp[i]->id);
+	}
+
+	return rc;
+}
+
+static int _dpu_rm_reserve_ctls(
+		struct dpu_rm *rm,
+		struct dpu_rm_rsvp *rsvp,
+		const struct dpu_rm_topology_def *top)
+{
+	struct dpu_rm_hw_blk *ctls[MAX_BLOCKS];
+	struct dpu_rm_hw_iter iter;
+	int i = 0;
+
+	memset(&ctls, 0, sizeof(ctls));
+
+	dpu_rm_init_hw_iter(&iter, 0, DPU_HW_BLK_CTL);
+	while (_dpu_rm_get_hw_locked(rm, &iter)) {
+		const struct dpu_hw_ctl *ctl = to_dpu_hw_ctl(iter.blk->hw);
+		unsigned long features = ctl->caps->features;
+		bool has_split_display;
+
+		if (RESERVED_BY_OTHER(iter.blk, rsvp))
+			continue;
+
+		has_split_display = BIT(DPU_CTL_SPLIT_DISPLAY) & features;
+
+		DPU_DEBUG("ctl %d caps 0x%lX\n", iter.blk->id, features);
+
+		if (top->needs_split_display != has_split_display)
+			continue;
+
+		ctls[i] = iter.blk;
+		DPU_DEBUG("ctl %d match\n", iter.blk->id);
+
+		if (++i == top->num_ctl)
+			break;
+	}
+
+	if (i != top->num_ctl)
+		return -ENAVAIL;
+
+	for (i = 0; i < ARRAY_SIZE(ctls) && i < top->num_ctl; i++) {
+		ctls[i]->rsvp_nxt = rsvp;
+		trace_dpu_rm_reserve_ctls(ctls[i]->id, ctls[i]->type,
+					  rsvp->enc_id);
+	}
+
+	return 0;
+}
+
+static int _dpu_rm_reserve_cdm(
+		struct dpu_rm *rm,
+		struct dpu_rm_rsvp *rsvp,
+		uint32_t id,
+		enum dpu_hw_blk_type type)
+{
+	struct dpu_rm_hw_iter iter;
+
+	DRM_DEBUG_KMS("type %d id %d\n", type, id);
+
+	dpu_rm_init_hw_iter(&iter, 0, DPU_HW_BLK_CDM);
+	while (_dpu_rm_get_hw_locked(rm, &iter)) {
+		const struct dpu_hw_cdm *cdm = to_dpu_hw_cdm(iter.blk->hw);
+		const struct dpu_cdm_cfg *caps = cdm->caps;
+		bool match = false;
+
+		if (RESERVED_BY_OTHER(iter.blk, rsvp))
+			continue;
+
+		if (type == DPU_HW_BLK_INTF && id != INTF_MAX)
+			match = test_bit(id, &caps->intf_connect);
+
+		DRM_DEBUG_KMS("iter: type:%d id:%d enc:%d cdm:%lu match:%d\n",
+			      iter.blk->type, iter.blk->id, rsvp->enc_id,
+			      caps->intf_connect, match);
+
+		if (!match)
+			continue;
+
+		trace_dpu_rm_reserve_cdm(iter.blk->id, iter.blk->type,
+					 rsvp->enc_id);
+		iter.blk->rsvp_nxt = rsvp;
+		break;
+	}
+
+	if (!iter.hw) {
+		DPU_ERROR("couldn't reserve cdm for type %d id %d\n", type, id);
+		return -ENAVAIL;
+	}
+
+	return 0;
+}
+
+static int _dpu_rm_reserve_intf(
+		struct dpu_rm *rm,
+		struct dpu_rm_rsvp *rsvp,
+		uint32_t id,
+		enum dpu_hw_blk_type type,
+		bool needs_cdm)
+{
+	struct dpu_rm_hw_iter iter;
+	int ret = 0;
+
+	/* Find the block entry in the rm, and note the reservation */
+	dpu_rm_init_hw_iter(&iter, 0, type);
+	while (_dpu_rm_get_hw_locked(rm, &iter)) {
+		if (iter.blk->id != id)
+			continue;
+
+		if (RESERVED_BY_OTHER(iter.blk, rsvp)) {
+			DPU_ERROR("type %d id %d already reserved\n", type, id);
+			return -ENAVAIL;
+		}
+
+		iter.blk->rsvp_nxt = rsvp;
+		trace_dpu_rm_reserve_intf(iter.blk->id, iter.blk->type,
+					  rsvp->enc_id);
+		break;
+	}
+
+	/* Shouldn't happen since intfs are fixed at probe */
+	if (!iter.hw) {
+		DPU_ERROR("couldn't find type %d id %d\n", type, id);
+		return -EINVAL;
+	}
+
+	if (needs_cdm)
+		ret = _dpu_rm_reserve_cdm(rm, rsvp, id, type);
+
+	return ret;
+}
+
+static int _dpu_rm_reserve_intf_related_hw(
+		struct dpu_rm *rm,
+		struct dpu_rm_rsvp *rsvp,
+		struct dpu_encoder_hw_resources *hw_res)
+{
+	int i, ret = 0;
+	u32 id;
+
+	for (i = 0; i < ARRAY_SIZE(hw_res->intfs); i++) {
+		if (hw_res->intfs[i] == INTF_MODE_NONE)
+			continue;
+		id = i + INTF_0;
+		ret = _dpu_rm_reserve_intf(rm, rsvp, id,
+				DPU_HW_BLK_INTF, hw_res->needs_cdm);
+		if (ret)
+			return ret;
+	}
+
+	return ret;
+}
+
+static int _dpu_rm_make_next_rsvp(
+		struct dpu_rm *rm,
+		struct drm_encoder *enc,
+		struct drm_crtc_state *crtc_state,
+		struct drm_connector_state *conn_state,
+		struct dpu_rm_rsvp *rsvp,
+		struct dpu_rm_requirements *reqs)
+{
+	int ret;
+	struct dpu_rm_topology_def topology;
+
+	/* Create reservation info, tag reserved blocks with it as we go */
+	rsvp->seq = ++rm->rsvp_next_seq;
+	rsvp->enc_id = enc->base.id;
+	rsvp->topology = reqs->topology->top_name;
+	list_add_tail(&rsvp->list, &rm->rsvps);
+
+	ret = _dpu_rm_reserve_lms(rm, rsvp, reqs);
+	if (ret) {
+		DPU_ERROR("unable to find appropriate mixers\n");
+		return ret;
+	}
+
+	/*
+	 * Do assignment preferring to give away low-resource CTLs first:
+	 * - Check mixers without Split Display
+	 * - Only then allow to grab from CTLs with split display capability
+	 */
+	_dpu_rm_reserve_ctls(rm, rsvp, reqs->topology);
+	if (ret && !reqs->topology->needs_split_display) {
+		memcpy(&topology, reqs->topology, sizeof(topology));
+		topology.needs_split_display = true;
+		_dpu_rm_reserve_ctls(rm, rsvp, &topology);
+	}
+	if (ret) {
+		DPU_ERROR("unable to find appropriate CTL\n");
+		return ret;
+	}
+
+	/* Assign INTFs and blks whose usage is tied to them: CTL & CDM */
+	ret = _dpu_rm_reserve_intf_related_hw(rm, rsvp, &reqs->hw_res);
+	if (ret)
+		return ret;
+
+	return ret;
+}
+
+static int _dpu_rm_populate_requirements(
+		struct dpu_rm *rm,
+		struct drm_encoder *enc,
+		struct drm_crtc_state *crtc_state,
+		struct drm_connector_state *conn_state,
+		struct dpu_rm_requirements *reqs,
+		struct msm_display_topology req_topology)
+{
+	int i;
+
+	memset(reqs, 0, sizeof(*reqs));
+
+	dpu_encoder_get_hw_resources(enc, &reqs->hw_res, conn_state);
+
+	for (i = 0; i < DPU_RM_TOPOLOGY_MAX; i++) {
+		if (RM_IS_TOPOLOGY_MATCH(g_top_table[i],
+					req_topology)) {
+			reqs->topology = &g_top_table[i];
+			break;
+		}
+	}
+
+	if (!reqs->topology) {
+		DPU_ERROR("invalid topology for the display\n");
+		return -EINVAL;
+	}
+
+	/**
+	 * Set the requirement based on caps if not set from user space
+	 * This will ensure to select LM tied with DS blocks
+	 * Currently, DS blocks are tied with LM 0 and LM 1 (primary display)
+	 */
+	if (!RM_RQ_DS(reqs) && rm->hw_mdp->caps->has_dest_scaler &&
+		conn_state->connector->connector_type == DRM_MODE_CONNECTOR_DSI)
+		reqs->top_ctrl |= BIT(DPU_RM_TOPCTL_DS);
+
+	DRM_DEBUG_KMS("top_ctrl: 0x%llX num_h_tiles: %d\n", reqs->top_ctrl,
+		      reqs->hw_res.display_num_of_h_tiles);
+	DRM_DEBUG_KMS("num_lm: %d num_ctl: %d topology: %d split_display: %d\n",
+		      reqs->topology->num_lm, reqs->topology->num_ctl,
+		      reqs->topology->top_name,
+		      reqs->topology->needs_split_display);
+
+	return 0;
+}
+
+static struct dpu_rm_rsvp *_dpu_rm_get_rsvp(
+		struct dpu_rm *rm,
+		struct drm_encoder *enc)
+{
+	struct dpu_rm_rsvp *i;
+
+	if (!rm || !enc) {
+		DPU_ERROR("invalid params\n");
+		return NULL;
+	}
+
+	if (list_empty(&rm->rsvps))
+		return NULL;
+
+	list_for_each_entry(i, &rm->rsvps, list)
+		if (i->enc_id == enc->base.id)
+			return i;
+
+	return NULL;
+}
+
+static struct drm_connector *_dpu_rm_get_connector(
+		struct drm_encoder *enc)
+{
+	struct drm_connector *conn = NULL;
+	struct list_head *connector_list =
+			&enc->dev->mode_config.connector_list;
+
+	list_for_each_entry(conn, connector_list, head)
+		if (conn->encoder == enc)
+			return conn;
+
+	return NULL;
+}
+
+/**
+ * _dpu_rm_release_rsvp - release resources and release a reservation
+ * @rm:	KMS handle
+ * @rsvp:	RSVP pointer to release and release resources for
+ */
+static void _dpu_rm_release_rsvp(
+		struct dpu_rm *rm,
+		struct dpu_rm_rsvp *rsvp,
+		struct drm_connector *conn)
+{
+	struct dpu_rm_rsvp *rsvp_c, *rsvp_n;
+	struct dpu_rm_hw_blk *blk;
+	enum dpu_hw_blk_type type;
+
+	if (!rsvp)
+		return;
+
+	DPU_DEBUG("rel rsvp %d enc %d\n", rsvp->seq, rsvp->enc_id);
+
+	list_for_each_entry_safe(rsvp_c, rsvp_n, &rm->rsvps, list) {
+		if (rsvp == rsvp_c) {
+			list_del(&rsvp_c->list);
+			break;
+		}
+	}
+
+	for (type = 0; type < DPU_HW_BLK_MAX; type++) {
+		list_for_each_entry(blk, &rm->hw_blks[type], list) {
+			if (blk->rsvp == rsvp) {
+				blk->rsvp = NULL;
+				DPU_DEBUG("rel rsvp %d enc %d %d %d\n",
+						rsvp->seq, rsvp->enc_id,
+						blk->type, blk->id);
+			}
+			if (blk->rsvp_nxt == rsvp) {
+				blk->rsvp_nxt = NULL;
+				DPU_DEBUG("rel rsvp_nxt %d enc %d %d %d\n",
+						rsvp->seq, rsvp->enc_id,
+						blk->type, blk->id);
+			}
+		}
+	}
+
+	kfree(rsvp);
+}
+
+void dpu_rm_release(struct dpu_rm *rm, struct drm_encoder *enc)
+{
+	struct dpu_rm_rsvp *rsvp;
+	struct drm_connector *conn;
+
+	if (!rm || !enc) {
+		DPU_ERROR("invalid params\n");
+		return;
+	}
+
+	mutex_lock(&rm->rm_lock);
+
+	rsvp = _dpu_rm_get_rsvp(rm, enc);
+	if (!rsvp) {
+		DPU_ERROR("failed to find rsvp for enc %d\n", enc->base.id);
+		goto end;
+	}
+
+	conn = _dpu_rm_get_connector(enc);
+	if (!conn) {
+		DPU_ERROR("failed to get connector for enc %d\n", enc->base.id);
+		goto end;
+	}
+
+	_dpu_rm_release_rsvp(rm, rsvp, conn);
+end:
+	mutex_unlock(&rm->rm_lock);
+}
+
+static int _dpu_rm_commit_rsvp(
+		struct dpu_rm *rm,
+		struct dpu_rm_rsvp *rsvp,
+		struct drm_connector_state *conn_state)
+{
+	struct dpu_rm_hw_blk *blk;
+	enum dpu_hw_blk_type type;
+	int ret = 0;
+
+	/* Swap next rsvp to be the active */
+	for (type = 0; type < DPU_HW_BLK_MAX; type++) {
+		list_for_each_entry(blk, &rm->hw_blks[type], list) {
+			if (blk->rsvp_nxt) {
+				blk->rsvp = blk->rsvp_nxt;
+				blk->rsvp_nxt = NULL;
+			}
+		}
+	}
+
+	if (!ret)
+		DRM_DEBUG_KMS("rsrv enc %d topology %d\n", rsvp->enc_id,
+			      rsvp->topology);
+
+	return ret;
+}
+
+int dpu_rm_reserve(
+		struct dpu_rm *rm,
+		struct drm_encoder *enc,
+		struct drm_crtc_state *crtc_state,
+		struct drm_connector_state *conn_state,
+		struct msm_display_topology topology,
+		bool test_only)
+{
+	struct dpu_rm_rsvp *rsvp_cur, *rsvp_nxt;
+	struct dpu_rm_requirements reqs;
+	int ret;
+
+	if (!rm || !enc || !crtc_state || !conn_state) {
+		DPU_ERROR("invalid arguments\n");
+		return -EINVAL;
+	}
+
+	/* Check if this is just a page-flip */
+	if (!drm_atomic_crtc_needs_modeset(crtc_state))
+		return 0;
+
+	DRM_DEBUG_KMS("reserving hw for conn %d enc %d crtc %d test_only %d\n",
+		      conn_state->connector->base.id, enc->base.id,
+		      crtc_state->crtc->base.id, test_only);
+
+	mutex_lock(&rm->rm_lock);
+
+	_dpu_rm_print_rsvps(rm, DPU_RM_STAGE_BEGIN);
+
+	ret = _dpu_rm_populate_requirements(rm, enc, crtc_state,
+			conn_state, &reqs, topology);
+	if (ret) {
+		DPU_ERROR("failed to populate hw requirements\n");
+		goto end;
+	}
+
+	/*
+	 * We only support one active reservation per-hw-block. But to implement
+	 * transactional semantics for test-only, and for allowing failure while
+	 * modifying your existing reservation, over the course of this
+	 * function we can have two reservations:
+	 * Current: Existing reservation
+	 * Next: Proposed reservation. The proposed reservation may fail, or may
+	 *       be discarded if in test-only mode.
+	 * If reservation is successful, and we're not in test-only, then we
+	 * replace the current with the next.
+	 */
+	rsvp_nxt = kzalloc(sizeof(*rsvp_nxt), GFP_KERNEL);
+	if (!rsvp_nxt) {
+		ret = -ENOMEM;
+		goto end;
+	}
+
+	rsvp_cur = _dpu_rm_get_rsvp(rm, enc);
+
+	/*
+	 * User can request that we clear out any reservation during the
+	 * atomic_check phase by using this CLEAR bit
+	 */
+	if (rsvp_cur && test_only && RM_RQ_CLEAR(&reqs)) {
+		DPU_DEBUG("test_only & CLEAR: clear rsvp[s%de%d]\n",
+				rsvp_cur->seq, rsvp_cur->enc_id);
+		_dpu_rm_release_rsvp(rm, rsvp_cur, conn_state->connector);
+		rsvp_cur = NULL;
+		_dpu_rm_print_rsvps(rm, DPU_RM_STAGE_AFTER_CLEAR);
+	}
+
+	/* Check the proposed reservation, store it in hw's "next" field */
+	ret = _dpu_rm_make_next_rsvp(rm, enc, crtc_state, conn_state,
+			rsvp_nxt, &reqs);
+
+	_dpu_rm_print_rsvps(rm, DPU_RM_STAGE_AFTER_RSVPNEXT);
+
+	if (ret) {
+		DPU_ERROR("failed to reserve hw resources: %d\n", ret);
+		_dpu_rm_release_rsvp(rm, rsvp_nxt, conn_state->connector);
+	} else if (test_only && !RM_RQ_LOCK(&reqs)) {
+		/*
+		 * Normally, if test_only, test the reservation and then undo
+		 * However, if the user requests LOCK, then keep the reservation
+		 * made during the atomic_check phase.
+		 */
+		DPU_DEBUG("test_only: discard test rsvp[s%de%d]\n",
+				rsvp_nxt->seq, rsvp_nxt->enc_id);
+		_dpu_rm_release_rsvp(rm, rsvp_nxt, conn_state->connector);
+	} else {
+		if (test_only && RM_RQ_LOCK(&reqs))
+			DPU_DEBUG("test_only & LOCK: lock rsvp[s%de%d]\n",
+					rsvp_nxt->seq, rsvp_nxt->enc_id);
+
+		_dpu_rm_release_rsvp(rm, rsvp_cur, conn_state->connector);
+
+		ret = _dpu_rm_commit_rsvp(rm, rsvp_nxt, conn_state);
+	}
+
+	_dpu_rm_print_rsvps(rm, DPU_RM_STAGE_FINAL);
+
+end:
+	mutex_unlock(&rm->rm_lock);
+
+	return ret;
+}
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_rm.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_rm.h
new file mode 100644
index 000000000000..ffd1841a6067
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_rm.h
@@ -0,0 +1,199 @@
+/*
+ * Copyright (c) 2016-2018, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#ifndef __DPU_RM_H__
+#define __DPU_RM_H__
+
+#include <linux/list.h>
+
+#include "msm_kms.h"
+#include "dpu_hw_top.h"
+
+/**
+ * enum dpu_rm_topology_name - HW resource use case in use by connector
+ * @DPU_RM_TOPOLOGY_NONE:                 No topology in use currently
+ * @DPU_RM_TOPOLOGY_SINGLEPIPE:           1 LM, 1 PP, 1 INTF/WB
+ * @DPU_RM_TOPOLOGY_DUALPIPE:             2 LM, 2 PP, 2 INTF/WB
+ * @DPU_RM_TOPOLOGY_DUALPIPE_3DMERGE:     2 LM, 2 PP, 3DMux, 1 INTF/WB
+ */
+enum dpu_rm_topology_name {
+	DPU_RM_TOPOLOGY_NONE = 0,
+	DPU_RM_TOPOLOGY_SINGLEPIPE,
+	DPU_RM_TOPOLOGY_DUALPIPE,
+	DPU_RM_TOPOLOGY_DUALPIPE_3DMERGE,
+	DPU_RM_TOPOLOGY_MAX,
+};
+
+/**
+ * enum dpu_rm_topology_control - HW resource use case in use by connector
+ * @DPU_RM_TOPCTL_RESERVE_LOCK: If set, in AtomicTest phase, after a successful
+ *                              test, reserve the resources for this display.
+ *                              Normal behavior would not impact the reservation
+ *                              list during the AtomicTest phase.
+ * @DPU_RM_TOPCTL_RESERVE_CLEAR: If set, in AtomicTest phase, before testing,
+ *                               release any reservation held by this display.
+ *                               Normal behavior would not impact the
+ *                               reservation list during the AtomicTest phase.
+ * @DPU_RM_TOPCTL_DS  : Require layer mixers with DS capabilities
+ */
+enum dpu_rm_topology_control {
+	DPU_RM_TOPCTL_RESERVE_LOCK,
+	DPU_RM_TOPCTL_RESERVE_CLEAR,
+	DPU_RM_TOPCTL_DS,
+};
+
+/**
+ * struct dpu_rm - DPU dynamic hardware resource manager
+ * @dev: device handle for event logging purposes
+ * @rsvps: list of hardware reservations by each crtc->encoder->connector
+ * @hw_blks: array of lists of hardware resources present in the system, one
+ *	list per type of hardware block
+ * @hw_mdp: hardware object for mdp_top
+ * @lm_max_width: cached layer mixer maximum width
+ * @rsvp_next_seq: sequence number for next reservation for debugging purposes
+ * @rm_lock: resource manager mutex
+ */
+struct dpu_rm {
+	struct drm_device *dev;
+	struct list_head rsvps;
+	struct list_head hw_blks[DPU_HW_BLK_MAX];
+	struct dpu_hw_mdp *hw_mdp;
+	uint32_t lm_max_width;
+	uint32_t rsvp_next_seq;
+	struct mutex rm_lock;
+};
+
+/**
+ *  struct dpu_rm_hw_blk - resource manager internal structure
+ *	forward declaration for single iterator definition without void pointer
+ */
+struct dpu_rm_hw_blk;
+
+/**
+ * struct dpu_rm_hw_iter - iterator for use with dpu_rm
+ * @hw: dpu_hw object requested, or NULL on failure
+ * @blk: dpu_rm internal block representation. Clients ignore. Used as iterator.
+ * @enc_id: DRM ID of Encoder client wishes to search for, or 0 for Any Encoder
+ * @type: Hardware Block Type client wishes to search for.
+ */
+struct dpu_rm_hw_iter {
+	void *hw;
+	struct dpu_rm_hw_blk *blk;
+	uint32_t enc_id;
+	enum dpu_hw_blk_type type;
+};
+
+/**
+ * dpu_rm_init - Read hardware catalog and create reservation tracking objects
+ *	for all HW blocks.
+ * @rm: DPU Resource Manager handle
+ * @cat: Pointer to hardware catalog
+ * @mmio: mapped register io address of MDP
+ * @dev: device handle for event logging purposes
+ * @Return: 0 on Success otherwise -ERROR
+ */
+int dpu_rm_init(struct dpu_rm *rm,
+		struct dpu_mdss_cfg *cat,
+		void __iomem *mmio,
+		struct drm_device *dev);
+
+/**
+ * dpu_rm_destroy - Free all memory allocated by dpu_rm_init
+ * @rm: DPU Resource Manager handle
+ * @Return: 0 on Success otherwise -ERROR
+ */
+int dpu_rm_destroy(struct dpu_rm *rm);
+
+/**
+ * dpu_rm_reserve - Given a CRTC->Encoder->Connector display chain, analyze
+ *	the use connections and user requirements, specified through related
+ *	topology control properties, and reserve hardware blocks to that
+ *	display chain.
+ *	HW blocks can then be accessed through dpu_rm_get_* functions.
+ *	HW Reservations should be released via dpu_rm_release_hw.
+ * @rm: DPU Resource Manager handle
+ * @drm_enc: DRM Encoder handle
+ * @crtc_state: Proposed Atomic DRM CRTC State handle
+ * @conn_state: Proposed Atomic DRM Connector State handle
+ * @topology: Pointer to topology info for the display
+ * @test_only: Atomic-Test phase, discard results (unless property overrides)
+ * @Return: 0 on Success otherwise -ERROR
+ */
+int dpu_rm_reserve(struct dpu_rm *rm,
+		struct drm_encoder *drm_enc,
+		struct drm_crtc_state *crtc_state,
+		struct drm_connector_state *conn_state,
+		struct msm_display_topology topology,
+		bool test_only);
+
+/**
+ * dpu_rm_reserve - Given the encoder for the display chain, release any
+ *	HW blocks previously reserved for that use case.
+ * @rm: DPU Resource Manager handle
+ * @enc: DRM Encoder handle
+ * @Return: 0 on Success otherwise -ERROR
+ */
+void dpu_rm_release(struct dpu_rm *rm, struct drm_encoder *enc);
+
+/**
+ * dpu_rm_get_mdp - Retrieve HW block for MDP TOP.
+ *	This is never reserved, and is usable by any display.
+ * @rm: DPU Resource Manager handle
+ * @Return: Pointer to hw block or NULL
+ */
+struct dpu_hw_mdp *dpu_rm_get_mdp(struct dpu_rm *rm);
+
+/**
+ * dpu_rm_init_hw_iter - setup given iterator for new iteration over hw list
+ *	using dpu_rm_get_hw
+ * @iter: iter object to initialize
+ * @enc_id: DRM ID of Encoder client wishes to search for, or 0 for Any Encoder
+ * @type: Hardware Block Type client wishes to search for.
+ */
+void dpu_rm_init_hw_iter(
+		struct dpu_rm_hw_iter *iter,
+		uint32_t enc_id,
+		enum dpu_hw_blk_type type);
+/**
+ * dpu_rm_get_hw - retrieve reserved hw object given encoder and hw type
+ *	Meant to do a single pass through the hardware list to iteratively
+ *	retrieve hardware blocks of a given type for a given encoder.
+ *	Initialize an iterator object.
+ *	Set hw block type of interest. Set encoder id of interest, 0 for any.
+ *	Function returns first hw of type for that encoder.
+ *	Subsequent calls will return the next reserved hw of that type in-order.
+ *	Iterator HW pointer will be null on failure to find hw.
+ * @rm: DPU Resource Manager handle
+ * @iter: iterator object
+ * @Return: true on match found, false on no match found
+ */
+bool dpu_rm_get_hw(struct dpu_rm *rm, struct dpu_rm_hw_iter *iter);
+
+/**
+ * dpu_rm_check_property_topctl - validate property bitmask before it is set
+ * @val: user's proposed topology control bitmask
+ * @Return: 0 on success or error
+ */
+int dpu_rm_check_property_topctl(uint64_t val);
+
+/**
+ * dpu_rm_get_topology_name - returns the name of the the given topology
+ *                            definition
+ * @topology: topology definition
+ * @Return: name of the topology
+ */
+enum dpu_rm_topology_name
+dpu_rm_get_topology_name(struct msm_display_topology topology);
+
+#endif /* __DPU_RM_H__ */
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_trace.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_trace.h
new file mode 100644
index 000000000000..ae0ca5076238
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_trace.h
@@ -0,0 +1,1007 @@
+/* Copyright (c) 2014-2018, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#if !defined(_DPU_TRACE_H_) || defined(TRACE_HEADER_MULTI_READ)
+#define _DPU_TRACE_H_
+
+#include <linux/stringify.h>
+#include <linux/types.h>
+#include <linux/tracepoint.h>
+
+#include <drm/drm_rect.h>
+#include "dpu_crtc.h"
+#include "dpu_encoder_phys.h"
+#include "dpu_hw_mdss.h"
+#include "dpu_hw_vbif.h"
+#include "dpu_plane.h"
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM dpu
+#undef TRACE_INCLUDE_FILE
+#define TRACE_INCLUDE_FILE dpu_trace
+
+TRACE_EVENT(dpu_perf_set_qos_luts,
+	TP_PROTO(u32 pnum, u32 fmt, bool rt, u32 fl,
+		u32 lut, u32 lut_usage),
+	TP_ARGS(pnum, fmt, rt, fl, lut, lut_usage),
+	TP_STRUCT__entry(
+			__field(u32, pnum)
+			__field(u32, fmt)
+			__field(bool, rt)
+			__field(u32, fl)
+			__field(u64, lut)
+			__field(u32, lut_usage)
+	),
+	TP_fast_assign(
+			__entry->pnum = pnum;
+			__entry->fmt = fmt;
+			__entry->rt = rt;
+			__entry->fl = fl;
+			__entry->lut = lut;
+			__entry->lut_usage = lut_usage;
+	),
+	TP_printk("pnum=%d fmt=%x rt=%d fl=%d lut=0x%llx lut_usage=%d",
+			__entry->pnum, __entry->fmt,
+			__entry->rt, __entry->fl,
+			__entry->lut, __entry->lut_usage)
+);
+
+TRACE_EVENT(dpu_perf_set_danger_luts,
+	TP_PROTO(u32 pnum, u32 fmt, u32 mode, u32 danger_lut,
+		u32 safe_lut),
+	TP_ARGS(pnum, fmt, mode, danger_lut, safe_lut),
+	TP_STRUCT__entry(
+			__field(u32, pnum)
+			__field(u32, fmt)
+			__field(u32, mode)
+			__field(u32, danger_lut)
+			__field(u32, safe_lut)
+	),
+	TP_fast_assign(
+			__entry->pnum = pnum;
+			__entry->fmt = fmt;
+			__entry->mode = mode;
+			__entry->danger_lut = danger_lut;
+			__entry->safe_lut = safe_lut;
+	),
+	TP_printk("pnum=%d fmt=%x mode=%d luts[0x%x, 0x%x]",
+			__entry->pnum, __entry->fmt,
+			__entry->mode, __entry->danger_lut,
+			__entry->safe_lut)
+);
+
+TRACE_EVENT(dpu_perf_set_ot,
+	TP_PROTO(u32 pnum, u32 xin_id, u32 rd_lim, u32 vbif_idx),
+	TP_ARGS(pnum, xin_id, rd_lim, vbif_idx),
+	TP_STRUCT__entry(
+			__field(u32, pnum)
+			__field(u32, xin_id)
+			__field(u32, rd_lim)
+			__field(u32, vbif_idx)
+	),
+	TP_fast_assign(
+			__entry->pnum = pnum;
+			__entry->xin_id = xin_id;
+			__entry->rd_lim = rd_lim;
+			__entry->vbif_idx = vbif_idx;
+	),
+	TP_printk("pnum:%d xin_id:%d ot:%d vbif:%d",
+			__entry->pnum, __entry->xin_id, __entry->rd_lim,
+			__entry->vbif_idx)
+)
+
+TRACE_EVENT(dpu_perf_update_bus,
+	TP_PROTO(int client, unsigned long long ab_quota,
+	unsigned long long ib_quota),
+	TP_ARGS(client, ab_quota, ib_quota),
+	TP_STRUCT__entry(
+			__field(int, client)
+			__field(u64, ab_quota)
+			__field(u64, ib_quota)
+	),
+	TP_fast_assign(
+			__entry->client = client;
+			__entry->ab_quota = ab_quota;
+			__entry->ib_quota = ib_quota;
+	),
+	TP_printk("Request client:%d ab=%llu ib=%llu",
+			__entry->client,
+			__entry->ab_quota,
+			__entry->ib_quota)
+)
+
+
+TRACE_EVENT(dpu_cmd_release_bw,
+	TP_PROTO(u32 crtc_id),
+	TP_ARGS(crtc_id),
+	TP_STRUCT__entry(
+			__field(u32, crtc_id)
+	),
+	TP_fast_assign(
+			__entry->crtc_id = crtc_id;
+	),
+	TP_printk("crtc:%d", __entry->crtc_id)
+);
+
+TRACE_EVENT(tracing_mark_write,
+	TP_PROTO(int pid, const char *name, bool trace_begin),
+	TP_ARGS(pid, name, trace_begin),
+	TP_STRUCT__entry(
+			__field(int, pid)
+			__string(trace_name, name)
+			__field(bool, trace_begin)
+	),
+	TP_fast_assign(
+			__entry->pid = pid;
+			__assign_str(trace_name, name);
+			__entry->trace_begin = trace_begin;
+	),
+	TP_printk("%s|%d|%s", __entry->trace_begin ? "B" : "E",
+		__entry->pid, __get_str(trace_name))
+)
+
+TRACE_EVENT(dpu_trace_counter,
+	TP_PROTO(int pid, char *name, int value),
+	TP_ARGS(pid, name, value),
+	TP_STRUCT__entry(
+			__field(int, pid)
+			__string(counter_name, name)
+			__field(int, value)
+	),
+	TP_fast_assign(
+			__entry->pid = current->tgid;
+			__assign_str(counter_name, name);
+			__entry->value = value;
+	),
+	TP_printk("%d|%s|%d", __entry->pid,
+			__get_str(counter_name), __entry->value)
+)
+
+TRACE_EVENT(dpu_perf_crtc_update,
+	TP_PROTO(u32 crtc, u64 bw_ctl_mnoc, u64 bw_ctl_llcc,
+			u64 bw_ctl_ebi, u32 core_clk_rate,
+			bool stop_req, u32 update_bus, u32 update_clk),
+	TP_ARGS(crtc, bw_ctl_mnoc, bw_ctl_llcc, bw_ctl_ebi, core_clk_rate,
+		stop_req, update_bus, update_clk),
+	TP_STRUCT__entry(
+			__field(u32, crtc)
+			__field(u64, bw_ctl_mnoc)
+			__field(u64, bw_ctl_llcc)
+			__field(u64, bw_ctl_ebi)
+			__field(u32, core_clk_rate)
+			__field(bool, stop_req)
+			__field(u32, update_bus)
+			__field(u32, update_clk)
+	),
+	TP_fast_assign(
+			__entry->crtc = crtc;
+			__entry->bw_ctl_mnoc = bw_ctl_mnoc;
+			__entry->bw_ctl_llcc = bw_ctl_llcc;
+			__entry->bw_ctl_ebi = bw_ctl_ebi;
+			__entry->core_clk_rate = core_clk_rate;
+			__entry->stop_req = stop_req;
+			__entry->update_bus = update_bus;
+			__entry->update_clk = update_clk;
+	),
+	 TP_printk(
+		"crtc=%d bw_mnoc=%llu bw_llcc=%llu bw_ebi=%llu clk_rate=%u stop_req=%d u_bus=%d u_clk=%d",
+			__entry->crtc,
+			__entry->bw_ctl_mnoc,
+			__entry->bw_ctl_llcc,
+			__entry->bw_ctl_ebi,
+			__entry->core_clk_rate,
+			__entry->stop_req,
+			__entry->update_bus,
+			__entry->update_clk)
+);
+
+DECLARE_EVENT_CLASS(dpu_enc_irq_template,
+	TP_PROTO(uint32_t drm_id, enum dpu_intr_idx intr_idx, int hw_idx,
+		 int irq_idx),
+	TP_ARGS(drm_id, intr_idx, hw_idx, irq_idx),
+	TP_STRUCT__entry(
+		__field(	uint32_t,		drm_id		)
+		__field(	enum dpu_intr_idx,	intr_idx	)
+		__field(	int,			hw_idx		)
+		__field(	int,			irq_idx		)
+	),
+	TP_fast_assign(
+		__entry->drm_id = drm_id;
+		__entry->intr_idx = intr_idx;
+		__entry->hw_idx = hw_idx;
+		__entry->irq_idx = irq_idx;
+	),
+	TP_printk("id=%u, intr=%d, hw=%d, irq=%d",
+		  __entry->drm_id, __entry->intr_idx, __entry->hw_idx,
+		  __entry->irq_idx)
+);
+DEFINE_EVENT(dpu_enc_irq_template, dpu_enc_irq_register_success,
+	TP_PROTO(uint32_t drm_id, enum dpu_intr_idx intr_idx, int hw_idx,
+		 int irq_idx),
+	TP_ARGS(drm_id, intr_idx, hw_idx, irq_idx)
+);
+DEFINE_EVENT(dpu_enc_irq_template, dpu_enc_irq_unregister_success,
+	TP_PROTO(uint32_t drm_id, enum dpu_intr_idx intr_idx, int hw_idx,
+		 int irq_idx),
+	TP_ARGS(drm_id, intr_idx, hw_idx, irq_idx)
+);
+
+TRACE_EVENT(dpu_enc_irq_wait_success,
+	TP_PROTO(uint32_t drm_id, enum dpu_intr_idx intr_idx, int hw_idx,
+		 int irq_idx, enum dpu_pingpong pp_idx, int atomic_cnt),
+	TP_ARGS(drm_id, intr_idx, hw_idx, irq_idx, pp_idx, atomic_cnt),
+	TP_STRUCT__entry(
+		__field(	uint32_t,		drm_id		)
+		__field(	enum dpu_intr_idx,	intr_idx	)
+		__field(	int,			hw_idx		)
+		__field(	int,			irq_idx		)
+		__field(	enum dpu_pingpong,	pp_idx		)
+		__field(	int,			atomic_cnt	)
+	),
+	TP_fast_assign(
+		__entry->drm_id = drm_id;
+		__entry->intr_idx = intr_idx;
+		__entry->hw_idx = hw_idx;
+		__entry->irq_idx = irq_idx;
+		__entry->pp_idx = pp_idx;
+		__entry->atomic_cnt = atomic_cnt;
+	),
+	TP_printk("id=%u, intr=%d, hw=%d, irq=%d, pp=%d, atomic_cnt=%d",
+		  __entry->drm_id, __entry->intr_idx, __entry->hw_idx,
+		  __entry->irq_idx, __entry->pp_idx, __entry->atomic_cnt)
+);
+
+DECLARE_EVENT_CLASS(dpu_drm_obj_template,
+	TP_PROTO(uint32_t drm_id),
+	TP_ARGS(drm_id),
+	TP_STRUCT__entry(
+		__field(	uint32_t,		drm_id		)
+	),
+	TP_fast_assign(
+		__entry->drm_id = drm_id;
+	),
+	TP_printk("id=%u", __entry->drm_id)
+);
+DEFINE_EVENT(dpu_drm_obj_template, dpu_enc_atomic_check,
+	TP_PROTO(uint32_t drm_id),
+	TP_ARGS(drm_id)
+);
+DEFINE_EVENT(dpu_drm_obj_template, dpu_enc_mode_set,
+	TP_PROTO(uint32_t drm_id),
+	TP_ARGS(drm_id)
+);
+DEFINE_EVENT(dpu_drm_obj_template, dpu_enc_disable,
+	TP_PROTO(uint32_t drm_id),
+	TP_ARGS(drm_id)
+);
+DEFINE_EVENT(dpu_drm_obj_template, dpu_enc_kickoff,
+	TP_PROTO(uint32_t drm_id),
+	TP_ARGS(drm_id)
+);
+DEFINE_EVENT(dpu_drm_obj_template, dpu_enc_prepare_kickoff,
+	TP_PROTO(uint32_t drm_id),
+	TP_ARGS(drm_id)
+);
+DEFINE_EVENT(dpu_drm_obj_template, dpu_enc_prepare_kickoff_reset,
+	TP_PROTO(uint32_t drm_id),
+	TP_ARGS(drm_id)
+);
+DEFINE_EVENT(dpu_drm_obj_template, dpu_crtc_complete_flip,
+	TP_PROTO(uint32_t drm_id),
+	TP_ARGS(drm_id)
+);
+DEFINE_EVENT(dpu_drm_obj_template, dpu_crtc_vblank_cb,
+	TP_PROTO(uint32_t drm_id),
+	TP_ARGS(drm_id)
+);
+DEFINE_EVENT(dpu_drm_obj_template, dpu_crtc_complete_commit,
+	TP_PROTO(uint32_t drm_id),
+	TP_ARGS(drm_id)
+);
+DEFINE_EVENT(dpu_drm_obj_template, dpu_kms_enc_enable,
+	TP_PROTO(uint32_t drm_id),
+	TP_ARGS(drm_id)
+);
+DEFINE_EVENT(dpu_drm_obj_template, dpu_kms_commit,
+	TP_PROTO(uint32_t drm_id),
+	TP_ARGS(drm_id)
+);
+DEFINE_EVENT(dpu_drm_obj_template, dpu_kms_wait_for_commit_done,
+	TP_PROTO(uint32_t drm_id),
+	TP_ARGS(drm_id)
+);
+
+TRACE_EVENT(dpu_enc_enable,
+	TP_PROTO(uint32_t drm_id, int hdisplay, int vdisplay),
+	TP_ARGS(drm_id, hdisplay, vdisplay),
+	TP_STRUCT__entry(
+		__field(	uint32_t,		drm_id		)
+		__field(	int,			hdisplay	)
+		__field(	int,			vdisplay	)
+	),
+	TP_fast_assign(
+		__entry->drm_id = drm_id;
+		__entry->hdisplay = hdisplay;
+		__entry->vdisplay = vdisplay;
+	),
+	TP_printk("id=%u, mode=%dx%d",
+		  __entry->drm_id, __entry->hdisplay, __entry->vdisplay)
+);
+
+DECLARE_EVENT_CLASS(dpu_enc_keyval_template,
+	TP_PROTO(uint32_t drm_id, int val),
+	TP_ARGS(drm_id, val),
+	TP_STRUCT__entry(
+		__field(	uint32_t,	drm_id	)
+		__field(	int,		val	)
+	),
+	TP_fast_assign(
+		__entry->drm_id = drm_id;
+		__entry->val = val;
+	),
+	TP_printk("id=%u, val=%d", __entry->drm_id, __entry->val)
+);
+DEFINE_EVENT(dpu_enc_keyval_template, dpu_enc_underrun_cb,
+	TP_PROTO(uint32_t drm_id, int count),
+	TP_ARGS(drm_id, count)
+);
+DEFINE_EVENT(dpu_enc_keyval_template, dpu_enc_trigger_start,
+	TP_PROTO(uint32_t drm_id, int ctl_idx),
+	TP_ARGS(drm_id, ctl_idx)
+);
+
+TRACE_EVENT(dpu_enc_atomic_check_flags,
+	TP_PROTO(uint32_t drm_id, unsigned int flags, int private_flags),
+	TP_ARGS(drm_id, flags, private_flags),
+	TP_STRUCT__entry(
+		__field(	uint32_t,		drm_id		)
+		__field(	unsigned int,		flags		)
+		__field(	int,			private_flags	)
+	),
+	TP_fast_assign(
+		__entry->drm_id = drm_id;
+		__entry->flags = flags;
+		__entry->private_flags = private_flags;
+	),
+	TP_printk("id=%u, flags=%u, private_flags=%d",
+		  __entry->drm_id, __entry->flags, __entry->private_flags)
+);
+
+DECLARE_EVENT_CLASS(dpu_enc_id_enable_template,
+	TP_PROTO(uint32_t drm_id, bool enable),
+	TP_ARGS(drm_id, enable),
+	TP_STRUCT__entry(
+		__field(	uint32_t,		drm_id		)
+		__field(	bool,			enable		)
+	),
+	TP_fast_assign(
+		__entry->drm_id = drm_id;
+		__entry->enable = enable;
+	),
+	TP_printk("id=%u, enable=%s",
+		  __entry->drm_id, __entry->enable ? "true" : "false")
+);
+DEFINE_EVENT(dpu_enc_id_enable_template, dpu_enc_rc_helper,
+	TP_PROTO(uint32_t drm_id, bool enable),
+	TP_ARGS(drm_id, enable)
+);
+DEFINE_EVENT(dpu_enc_id_enable_template, dpu_enc_vblank_cb,
+	TP_PROTO(uint32_t drm_id, bool enable),
+	TP_ARGS(drm_id, enable)
+);
+DEFINE_EVENT(dpu_enc_id_enable_template, dpu_enc_frame_event_cb,
+	TP_PROTO(uint32_t drm_id, bool enable),
+	TP_ARGS(drm_id, enable)
+);
+DEFINE_EVENT(dpu_enc_id_enable_template, dpu_enc_phys_cmd_connect_te,
+	TP_PROTO(uint32_t drm_id, bool enable),
+	TP_ARGS(drm_id, enable)
+);
+
+TRACE_EVENT(dpu_enc_rc,
+	TP_PROTO(uint32_t drm_id, u32 sw_event, bool idle_pc_supported,
+		 int rc_state, const char *stage),
+	TP_ARGS(drm_id, sw_event, idle_pc_supported, rc_state, stage),
+	TP_STRUCT__entry(
+		__field(	uint32_t,	drm_id			)
+		__field(	u32,		sw_event		)
+		__field(	bool,		idle_pc_supported	)
+		__field(	int,		rc_state		)
+		__string(	stage_str,	stage			)
+	),
+	TP_fast_assign(
+		__entry->drm_id = drm_id;
+		__entry->sw_event = sw_event;
+		__entry->idle_pc_supported = idle_pc_supported;
+		__entry->rc_state = rc_state;
+		__assign_str(stage_str, stage);
+	),
+	TP_printk("%s: id:%u, sw_event:%d, idle_pc_supported:%s, rc_state:%d\n",
+		  __get_str(stage_str), __entry->drm_id, __entry->sw_event,
+		  __entry->idle_pc_supported ? "true" : "false",
+		  __entry->rc_state)
+);
+
+TRACE_EVENT(dpu_enc_frame_done_cb_not_busy,
+	TP_PROTO(uint32_t drm_id, u32 event, enum dpu_intf intf_idx),
+	TP_ARGS(drm_id, event, intf_idx),
+	TP_STRUCT__entry(
+		__field(	uint32_t,	drm_id		)
+		__field(	u32,		event		)
+		__field(	enum dpu_intf,	intf_idx	)
+	),
+	TP_fast_assign(
+		__entry->drm_id = drm_id;
+		__entry->event = event;
+		__entry->intf_idx = intf_idx;
+	),
+	TP_printk("id=%u, event=%u, intf=%d", __entry->drm_id, __entry->event,
+		  __entry->intf_idx)
+);
+
+TRACE_EVENT(dpu_enc_frame_done_cb,
+	TP_PROTO(uint32_t drm_id, unsigned int idx,
+		 unsigned long frame_busy_mask),
+	TP_ARGS(drm_id, idx, frame_busy_mask),
+	TP_STRUCT__entry(
+		__field(	uint32_t,		drm_id		)
+		__field(	unsigned int,		idx		)
+		__field(	unsigned long,		frame_busy_mask	)
+	),
+	TP_fast_assign(
+		__entry->drm_id = drm_id;
+		__entry->idx = idx;
+		__entry->frame_busy_mask = frame_busy_mask;
+	),
+	TP_printk("id=%u, idx=%u, frame_busy_mask=%lx", __entry->drm_id,
+		  __entry->idx, __entry->frame_busy_mask)
+);
+
+TRACE_EVENT(dpu_enc_trigger_flush,
+	TP_PROTO(uint32_t drm_id, enum dpu_intf intf_idx,
+		 int pending_kickoff_cnt, int ctl_idx, u32 pending_flush_ret),
+	TP_ARGS(drm_id, intf_idx, pending_kickoff_cnt, ctl_idx,
+		pending_flush_ret),
+	TP_STRUCT__entry(
+		__field(	uint32_t,	drm_id			)
+		__field(	enum dpu_intf,	intf_idx		)
+		__field(	int,		pending_kickoff_cnt	)
+		__field(	int,		ctl_idx			)
+		__field(	u32,		pending_flush_ret	)
+	),
+	TP_fast_assign(
+		__entry->drm_id = drm_id;
+		__entry->intf_idx = intf_idx;
+		__entry->pending_kickoff_cnt = pending_kickoff_cnt;
+		__entry->ctl_idx = ctl_idx;
+		__entry->pending_flush_ret = pending_flush_ret;
+	),
+	TP_printk("id=%u, intf_idx=%d, pending_kickoff_cnt=%d ctl_idx=%d "
+		  "pending_flush_ret=%u", __entry->drm_id,
+		  __entry->intf_idx, __entry->pending_kickoff_cnt,
+		  __entry->ctl_idx, __entry->pending_flush_ret)
+);
+
+DECLARE_EVENT_CLASS(dpu_enc_ktime_template,
+	TP_PROTO(uint32_t drm_id, ktime_t time),
+	TP_ARGS(drm_id, time),
+	TP_STRUCT__entry(
+		__field(	uint32_t,	drm_id	)
+		__field(	ktime_t,	time	)
+	),
+	TP_fast_assign(
+		__entry->drm_id = drm_id;
+		__entry->time = time;
+	),
+	TP_printk("id=%u, time=%lld", __entry->drm_id,
+		  ktime_to_ms(__entry->time))
+);
+DEFINE_EVENT(dpu_enc_ktime_template, dpu_enc_vsync_event_work,
+	TP_PROTO(uint32_t drm_id, ktime_t time),
+	TP_ARGS(drm_id, time)
+);
+DEFINE_EVENT(dpu_enc_ktime_template, dpu_enc_early_kickoff,
+	TP_PROTO(uint32_t drm_id, ktime_t time),
+	TP_ARGS(drm_id, time)
+);
+
+DECLARE_EVENT_CLASS(dpu_id_event_template,
+	TP_PROTO(uint32_t drm_id, u32 event),
+	TP_ARGS(drm_id, event),
+	TP_STRUCT__entry(
+		__field(	uint32_t,	drm_id	)
+		__field(	u32,		event	)
+	),
+	TP_fast_assign(
+		__entry->drm_id = drm_id;
+		__entry->event = event;
+	),
+	TP_printk("id=%u, event=%u", __entry->drm_id, __entry->event)
+);
+DEFINE_EVENT(dpu_id_event_template, dpu_enc_frame_done_timeout,
+	TP_PROTO(uint32_t drm_id, u32 event),
+	TP_ARGS(drm_id, event)
+);
+DEFINE_EVENT(dpu_id_event_template, dpu_crtc_frame_event_cb,
+	TP_PROTO(uint32_t drm_id, u32 event),
+	TP_ARGS(drm_id, event)
+);
+DEFINE_EVENT(dpu_id_event_template, dpu_crtc_handle_power_event,
+	TP_PROTO(uint32_t drm_id, u32 event),
+	TP_ARGS(drm_id, event)
+);
+DEFINE_EVENT(dpu_id_event_template, dpu_crtc_frame_event_done,
+	TP_PROTO(uint32_t drm_id, u32 event),
+	TP_ARGS(drm_id, event)
+);
+DEFINE_EVENT(dpu_id_event_template, dpu_crtc_frame_event_more_pending,
+	TP_PROTO(uint32_t drm_id, u32 event),
+	TP_ARGS(drm_id, event)
+);
+
+TRACE_EVENT(dpu_enc_wait_event_timeout,
+	TP_PROTO(uint32_t drm_id, int32_t hw_id, int rc, s64 time,
+		 s64 expected_time, int atomic_cnt),
+	TP_ARGS(drm_id, hw_id, rc, time, expected_time, atomic_cnt),
+	TP_STRUCT__entry(
+		__field(	uint32_t,	drm_id		)
+		__field(	int32_t,	hw_id		)
+		__field(	int,		rc		)
+		__field(	s64,		time		)
+		__field(	s64,		expected_time	)
+		__field(	int,		atomic_cnt	)
+	),
+	TP_fast_assign(
+		__entry->drm_id = drm_id;
+		__entry->hw_id = hw_id;
+		__entry->rc = rc;
+		__entry->time = time;
+		__entry->expected_time = expected_time;
+		__entry->atomic_cnt = atomic_cnt;
+	),
+	TP_printk("id=%u, hw_id=%d, rc=%d, time=%lld, expected=%lld cnt=%d",
+		  __entry->drm_id, __entry->hw_id, __entry->rc, __entry->time,
+		  __entry->expected_time, __entry->atomic_cnt)
+);
+
+TRACE_EVENT(dpu_enc_phys_cmd_irq_ctrl,
+	TP_PROTO(uint32_t drm_id, enum dpu_pingpong pp, bool enable,
+		 int refcnt),
+	TP_ARGS(drm_id, pp, enable, refcnt),
+	TP_STRUCT__entry(
+		__field(	uint32_t,		drm_id	)
+		__field(	enum dpu_pingpong,	pp	)
+		__field(	bool,			enable	)
+		__field(	int,			refcnt	)
+	),
+	TP_fast_assign(
+		__entry->drm_id = drm_id;
+		__entry->pp = pp;
+		__entry->enable = enable;
+		__entry->refcnt = refcnt;
+	),
+	TP_printk("id=%u, pp=%d, enable=%s, refcnt=%d", __entry->drm_id,
+		  __entry->pp, __entry->enable ? "true" : "false",
+		  __entry->refcnt)
+);
+
+TRACE_EVENT(dpu_enc_phys_cmd_pp_tx_done,
+	TP_PROTO(uint32_t drm_id, enum dpu_pingpong pp, int new_count,
+		 u32 event),
+	TP_ARGS(drm_id, pp, new_count, event),
+	TP_STRUCT__entry(
+		__field(	uint32_t,		drm_id		)
+		__field(	enum dpu_pingpong,	pp		)
+		__field(	int,			new_count	)
+		__field(	u32,			event		)
+	),
+	TP_fast_assign(
+		__entry->drm_id = drm_id;
+		__entry->pp = pp;
+		__entry->new_count = new_count;
+		__entry->event = event;
+	),
+	TP_printk("id=%u, pp=%d, new_count=%d, event=%u", __entry->drm_id,
+		  __entry->pp, __entry->new_count, __entry->event)
+);
+
+TRACE_EVENT(dpu_enc_phys_cmd_pdone_timeout,
+	TP_PROTO(uint32_t drm_id, enum dpu_pingpong pp, int timeout_count,
+		 int kickoff_count, u32 event),
+	TP_ARGS(drm_id, pp, timeout_count, kickoff_count, event),
+	TP_STRUCT__entry(
+		__field(	uint32_t,		drm_id		)
+		__field(	enum dpu_pingpong,	pp		)
+		__field(	int,			timeout_count	)
+		__field(	int,			kickoff_count	)
+		__field(	u32,			event		)
+	),
+	TP_fast_assign(
+		__entry->drm_id = drm_id;
+		__entry->pp = pp;
+		__entry->timeout_count = timeout_count;
+		__entry->kickoff_count = kickoff_count;
+		__entry->event = event;
+	),
+	TP_printk("id=%u, pp=%d, timeout_count=%d, kickoff_count=%d, event=%u",
+		  __entry->drm_id, __entry->pp, __entry->timeout_count,
+		  __entry->kickoff_count, __entry->event)
+);
+
+TRACE_EVENT(dpu_enc_phys_vid_post_kickoff,
+	TP_PROTO(uint32_t drm_id, enum dpu_intf intf_idx),
+	TP_ARGS(drm_id, intf_idx),
+	TP_STRUCT__entry(
+		__field(	uint32_t,	drm_id			)
+		__field(	enum dpu_intf,	intf_idx		)
+	),
+	TP_fast_assign(
+		__entry->drm_id = drm_id;
+		__entry->intf_idx = intf_idx;
+	),
+	TP_printk("id=%u, intf_idx=%d", __entry->drm_id, __entry->intf_idx)
+);
+
+TRACE_EVENT(dpu_enc_phys_vid_irq_ctrl,
+	TP_PROTO(uint32_t drm_id, enum dpu_intf intf_idx, bool enable,
+		 int refcnt),
+	TP_ARGS(drm_id, intf_idx, enable, refcnt),
+	TP_STRUCT__entry(
+		__field(	uint32_t,	drm_id		)
+		__field(	enum dpu_intf,	intf_idx	)
+		__field(	bool,		enable		)
+		__field(	int,		refcnt		)
+	),
+	TP_fast_assign(
+		__entry->drm_id = drm_id;
+		__entry->intf_idx = intf_idx;
+		__entry->enable = enable;
+		__entry->refcnt = refcnt;
+	),
+	TP_printk("id=%u, intf_idx=%d enable=%s refcnt=%d", __entry->drm_id,
+		  __entry->intf_idx, __entry->enable ? "true" : "false",
+		  __entry->drm_id)
+);
+
+TRACE_EVENT(dpu_crtc_setup_mixer,
+	TP_PROTO(uint32_t crtc_id, uint32_t plane_id,
+		 struct drm_plane_state *state, struct dpu_plane_state *pstate,
+		 uint32_t stage_idx, enum dpu_sspp sspp, uint32_t pixel_format,
+		 uint64_t modifier),
+	TP_ARGS(crtc_id, plane_id, state, pstate, stage_idx, sspp,
+		pixel_format, modifier),
+	TP_STRUCT__entry(
+		__field(	uint32_t,		crtc_id		)
+		__field(	uint32_t,		plane_id	)
+		__field(	struct drm_plane_state*,state		)
+		__field(	struct dpu_plane_state*,pstate		)
+		__field(	uint32_t,		stage_idx	)
+		__field(	enum dpu_sspp,		sspp		)
+		__field(	uint32_t,		pixel_format	)
+		__field(	uint64_t,		modifier	)
+	),
+	TP_fast_assign(
+		__entry->crtc_id = crtc_id;
+		__entry->plane_id = plane_id;
+		__entry->state = state;
+		__entry->pstate = pstate;
+		__entry->stage_idx = stage_idx;
+		__entry->sspp = sspp;
+		__entry->pixel_format = pixel_format;
+		__entry->modifier = modifier;
+	),
+	TP_printk("crtc_id:%u plane_id:%u fb_id:%u src:{%ux%u+%ux%u} "
+		  "dst:{%ux%u+%ux%u} stage_idx:%u stage:%d, sspp:%d "
+		  "multirect_index:%d multirect_mode:%u pix_format:%u "
+		  "modifier:%llu",
+		  __entry->crtc_id, __entry->plane_id,
+		  __entry->state->fb ? __entry->state->fb->base.id : -1,
+		  __entry->state->src_w >> 16,  __entry->state->src_h >> 16,
+		  __entry->state->src_x >> 16,  __entry->state->src_y >> 16,
+		  __entry->state->crtc_w,  __entry->state->crtc_h,
+		  __entry->state->crtc_x,  __entry->state->crtc_y,
+		  __entry->stage_idx, __entry->pstate->stage, __entry->sspp,
+		  __entry->pstate->multirect_index,
+		  __entry->pstate->multirect_mode, __entry->pixel_format,
+		  __entry->modifier)
+);
+
+TRACE_EVENT(dpu_crtc_setup_lm_bounds,
+	TP_PROTO(uint32_t drm_id, int mixer, struct drm_rect *bounds),
+	TP_ARGS(drm_id, mixer, bounds),
+	TP_STRUCT__entry(
+		__field(	uint32_t,		drm_id	)
+		__field(	int,			mixer	)
+		__field(	struct drm_rect *,	bounds	)
+	),
+	TP_fast_assign(
+		__entry->drm_id = drm_id;
+		__entry->mixer = mixer;
+		__entry->bounds = bounds;
+	),
+	TP_printk("id:%u mixer:%d bounds:" DRM_RECT_FMT, __entry->drm_id,
+		  __entry->mixer, DRM_RECT_ARG(__entry->bounds))
+);
+
+TRACE_EVENT(dpu_crtc_vblank_enable,
+	TP_PROTO(uint32_t drm_id, uint32_t enc_id, bool enable,
+		 struct dpu_crtc *crtc),
+	TP_ARGS(drm_id, enc_id, enable, crtc),
+	TP_STRUCT__entry(
+		__field(	uint32_t,		drm_id	)
+		__field(	uint32_t,		enc_id	)
+		__field(	bool,			enable	)
+		__field(	struct dpu_crtc *,	crtc	)
+	),
+	TP_fast_assign(
+		__entry->drm_id = drm_id;
+		__entry->enc_id = enc_id;
+		__entry->enable = enable;
+		__entry->crtc = crtc;
+	),
+	TP_printk("id:%u encoder:%u enable:%s state{enabled:%s suspend:%s "
+		  "vblank_req:%s}",
+		  __entry->drm_id, __entry->enc_id,
+		  __entry->enable ? "true" : "false",
+		  __entry->crtc->enabled ? "true" : "false",
+		  __entry->crtc->suspend ? "true" : "false",
+		  __entry->crtc->vblank_requested ? "true" : "false")
+);
+
+DECLARE_EVENT_CLASS(dpu_crtc_enable_template,
+	TP_PROTO(uint32_t drm_id, bool enable, struct dpu_crtc *crtc),
+	TP_ARGS(drm_id, enable, crtc),
+	TP_STRUCT__entry(
+		__field(	uint32_t,		drm_id	)
+		__field(	bool,			enable	)
+		__field(	struct dpu_crtc *,	crtc	)
+	),
+	TP_fast_assign(
+		__entry->drm_id = drm_id;
+		__entry->enable = enable;
+		__entry->crtc = crtc;
+	),
+	TP_printk("id:%u enable:%s state{enabled:%s suspend:%s vblank_req:%s}",
+		  __entry->drm_id, __entry->enable ? "true" : "false",
+		  __entry->crtc->enabled ? "true" : "false",
+		  __entry->crtc->suspend ? "true" : "false",
+		  __entry->crtc->vblank_requested ? "true" : "false")
+);
+DEFINE_EVENT(dpu_crtc_enable_template, dpu_crtc_set_suspend,
+	TP_PROTO(uint32_t drm_id, bool enable, struct dpu_crtc *crtc),
+	TP_ARGS(drm_id, enable, crtc)
+);
+DEFINE_EVENT(dpu_crtc_enable_template, dpu_crtc_enable,
+	TP_PROTO(uint32_t drm_id, bool enable, struct dpu_crtc *crtc),
+	TP_ARGS(drm_id, enable, crtc)
+);
+DEFINE_EVENT(dpu_crtc_enable_template, dpu_crtc_disable,
+	TP_PROTO(uint32_t drm_id, bool enable, struct dpu_crtc *crtc),
+	TP_ARGS(drm_id, enable, crtc)
+);
+DEFINE_EVENT(dpu_crtc_enable_template, dpu_crtc_vblank,
+	TP_PROTO(uint32_t drm_id, bool enable, struct dpu_crtc *crtc),
+	TP_ARGS(drm_id, enable, crtc)
+);
+
+TRACE_EVENT(dpu_crtc_disable_frame_pending,
+	TP_PROTO(uint32_t drm_id, int frame_pending),
+	TP_ARGS(drm_id, frame_pending),
+	TP_STRUCT__entry(
+		__field(	uint32_t,		drm_id		)
+		__field(	int,			frame_pending	)
+	),
+	TP_fast_assign(
+		__entry->drm_id = drm_id;
+		__entry->frame_pending = frame_pending;
+	),
+	TP_printk("id:%u frame_pending:%d", __entry->drm_id,
+		  __entry->frame_pending)
+);
+
+TRACE_EVENT(dpu_plane_set_scanout,
+	TP_PROTO(enum dpu_sspp index, struct dpu_hw_fmt_layout *layout,
+		 enum dpu_sspp_multirect_index multirect_index),
+	TP_ARGS(index, layout, multirect_index),
+	TP_STRUCT__entry(
+		__field(	enum dpu_sspp,			index	)
+		__field(	struct dpu_hw_fmt_layout*,	layout	)
+		__field(	enum dpu_sspp_multirect_index,	multirect_index)
+	),
+	TP_fast_assign(
+		__entry->index = index;
+		__entry->layout = layout;
+		__entry->multirect_index = multirect_index;
+	),
+	TP_printk("index:%d layout:{%ux%u @ [%u/%u, %u/%u, %u/%u, %u/%u]} "
+		  "multirect_index:%d", __entry->index, __entry->layout->width,
+		  __entry->layout->height, __entry->layout->plane_addr[0],
+		  __entry->layout->plane_size[0],
+		  __entry->layout->plane_addr[1],
+		  __entry->layout->plane_size[1],
+		  __entry->layout->plane_addr[2],
+		  __entry->layout->plane_size[2],
+		  __entry->layout->plane_addr[3],
+		  __entry->layout->plane_size[3], __entry->multirect_index)
+);
+
+TRACE_EVENT(dpu_plane_disable,
+	TP_PROTO(uint32_t drm_id, bool is_virtual, uint32_t multirect_mode),
+	TP_ARGS(drm_id, is_virtual, multirect_mode),
+	TP_STRUCT__entry(
+		__field(	uint32_t,		drm_id		)
+		__field(	bool,			is_virtual	)
+		__field(	uint32_t,		multirect_mode	)
+	),
+	TP_fast_assign(
+		__entry->drm_id = drm_id;
+		__entry->is_virtual = is_virtual;
+		__entry->multirect_mode = multirect_mode;
+	),
+	TP_printk("id:%u is_virtual:%s multirect_mode:%u", __entry->drm_id,
+		  __entry->is_virtual ? "true" : "false",
+		  __entry->multirect_mode)
+);
+
+DECLARE_EVENT_CLASS(dpu_rm_iter_template,
+	TP_PROTO(uint32_t id, enum dpu_hw_blk_type type, uint32_t enc_id),
+	TP_ARGS(id, type, enc_id),
+	TP_STRUCT__entry(
+		__field(	uint32_t,		id	)
+		__field(	enum dpu_hw_blk_type,	type	)
+		__field(	uint32_t,		enc_id	)
+	),
+	TP_fast_assign(
+		__entry->id = id;
+		__entry->type = type;
+		__entry->enc_id = enc_id;
+	),
+	TP_printk("id:%d type:%d enc_id:%u", __entry->id, __entry->type,
+		  __entry->enc_id)
+);
+DEFINE_EVENT(dpu_rm_iter_template, dpu_rm_reserve_cdm,
+	TP_PROTO(uint32_t id, enum dpu_hw_blk_type type, uint32_t enc_id),
+	TP_ARGS(id, type, enc_id)
+);
+DEFINE_EVENT(dpu_rm_iter_template, dpu_rm_reserve_intf,
+	TP_PROTO(uint32_t id, enum dpu_hw_blk_type type, uint32_t enc_id),
+	TP_ARGS(id, type, enc_id)
+);
+DEFINE_EVENT(dpu_rm_iter_template, dpu_rm_reserve_ctls,
+	TP_PROTO(uint32_t id, enum dpu_hw_blk_type type, uint32_t enc_id),
+	TP_ARGS(id, type, enc_id)
+);
+
+TRACE_EVENT(dpu_rm_reserve_lms,
+	TP_PROTO(uint32_t id, enum dpu_hw_blk_type type, uint32_t enc_id,
+		 uint32_t pp_id),
+	TP_ARGS(id, type, enc_id, pp_id),
+	TP_STRUCT__entry(
+		__field(	uint32_t,		id	)
+		__field(	enum dpu_hw_blk_type,	type	)
+		__field(	uint32_t,		enc_id	)
+		__field(	uint32_t,		pp_id	)
+	),
+	TP_fast_assign(
+		__entry->id = id;
+		__entry->type = type;
+		__entry->enc_id = enc_id;
+		__entry->pp_id = pp_id;
+	),
+	TP_printk("id:%d type:%d enc_id:%u pp_id:%u", __entry->id,
+		  __entry->type, __entry->enc_id, __entry->pp_id)
+);
+
+TRACE_EVENT(dpu_vbif_wait_xin_halt_fail,
+	TP_PROTO(enum dpu_vbif index, u32 xin_id),
+	TP_ARGS(index, xin_id),
+	TP_STRUCT__entry(
+		__field(	enum dpu_vbif,	index	)
+		__field(	u32,		xin_id	)
+	),
+	TP_fast_assign(
+		__entry->index = index;
+		__entry->xin_id = xin_id;
+	),
+	TP_printk("index:%d xin_id:%u", __entry->index, __entry->xin_id)
+);
+
+TRACE_EVENT(dpu_pp_connect_ext_te,
+	TP_PROTO(enum dpu_pingpong pp, u32 cfg),
+	TP_ARGS(pp, cfg),
+	TP_STRUCT__entry(
+		__field(	enum dpu_pingpong,	pp	)
+		__field(	u32,			cfg	)
+	),
+	TP_fast_assign(
+		__entry->pp = pp;
+		__entry->cfg = cfg;
+	),
+	TP_printk("pp:%d cfg:%u", __entry->pp, __entry->cfg)
+);
+
+DECLARE_EVENT_CLASS(dpu_core_irq_idx_cnt_template,
+	TP_PROTO(int irq_idx, int enable_count),
+	TP_ARGS(irq_idx, enable_count),
+	TP_STRUCT__entry(
+		__field(	int,	irq_idx		)
+		__field(	int,	enable_count	)
+	),
+	TP_fast_assign(
+		__entry->irq_idx = irq_idx;
+		__entry->enable_count = enable_count;
+	),
+	TP_printk("irq_idx:%d enable_count:%u", __entry->irq_idx,
+		  __entry->enable_count)
+);
+DEFINE_EVENT(dpu_core_irq_idx_cnt_template, dpu_core_irq_enable_idx,
+	TP_PROTO(int irq_idx, int enable_count),
+	TP_ARGS(irq_idx, enable_count)
+);
+DEFINE_EVENT(dpu_core_irq_idx_cnt_template, dpu_core_irq_disable_idx,
+	TP_PROTO(int irq_idx, int enable_count),
+	TP_ARGS(irq_idx, enable_count)
+);
+
+DECLARE_EVENT_CLASS(dpu_core_irq_callback_template,
+	TP_PROTO(int irq_idx, struct dpu_irq_callback *callback),
+	TP_ARGS(irq_idx, callback),
+	TP_STRUCT__entry(
+		__field(	int,				irq_idx	)
+		__field(	struct dpu_irq_callback *,	callback)
+	),
+	TP_fast_assign(
+		__entry->irq_idx = irq_idx;
+		__entry->callback = callback;
+	),
+	TP_printk("irq_idx:%d callback:%pK", __entry->irq_idx,
+		  __entry->callback)
+);
+DEFINE_EVENT(dpu_core_irq_callback_template, dpu_core_irq_register_callback,
+	TP_PROTO(int irq_idx, struct dpu_irq_callback *callback),
+	TP_ARGS(irq_idx, callback)
+);
+DEFINE_EVENT(dpu_core_irq_callback_template, dpu_core_irq_unregister_callback,
+	TP_PROTO(int irq_idx, struct dpu_irq_callback *callback),
+	TP_ARGS(irq_idx, callback)
+);
+
+TRACE_EVENT(dpu_core_perf_update_clk,
+	TP_PROTO(struct drm_device *dev, bool stop_req, u64 clk_rate),
+	TP_ARGS(dev, stop_req, clk_rate),
+	TP_STRUCT__entry(
+		__field(	struct drm_device *,	dev		)
+		__field(	bool,			stop_req	)
+		__field(	u64,			clk_rate	)
+	),
+	TP_fast_assign(
+		__entry->dev = dev;
+		__entry->stop_req = stop_req;
+		__entry->clk_rate = clk_rate;
+	),
+	TP_printk("dev:%s stop_req:%s clk_rate:%llu", __entry->dev->unique,
+		  __entry->stop_req ? "true" : "false", __entry->clk_rate)
+);
+
+#define DPU_ATRACE_END(name) trace_tracing_mark_write(current->tgid, name, 0)
+#define DPU_ATRACE_BEGIN(name) trace_tracing_mark_write(current->tgid, name, 1)
+#define DPU_ATRACE_FUNC() DPU_ATRACE_BEGIN(__func__)
+
+#define DPU_ATRACE_INT(name, value) \
+	trace_dpu_trace_counter(current->tgid, name, value)
+
+#endif /* _DPU_TRACE_H_ */
+
+/* This part must be outside protection */
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH .
+#include <trace/define_trace.h>
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_vbif.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_vbif.c
new file mode 100644
index 000000000000..295528292296
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_vbif.c
@@ -0,0 +1,384 @@
+/* Copyright (c) 2015-2018, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#define pr_fmt(fmt)	"[drm:%s:%d] " fmt, __func__, __LINE__
+
+#include <linux/debugfs.h>
+
+#include "dpu_vbif.h"
+#include "dpu_hw_vbif.h"
+#include "dpu_trace.h"
+
+/**
+ * _dpu_vbif_wait_for_xin_halt - wait for the xin to halt
+ * @vbif:	Pointer to hardware vbif driver
+ * @xin_id:	Client interface identifier
+ * @return:	0 if success; error code otherwise
+ */
+static int _dpu_vbif_wait_for_xin_halt(struct dpu_hw_vbif *vbif, u32 xin_id)
+{
+	ktime_t timeout;
+	bool status;
+	int rc;
+
+	if (!vbif || !vbif->cap || !vbif->ops.get_halt_ctrl) {
+		DPU_ERROR("invalid arguments vbif %d\n", vbif != 0);
+		return -EINVAL;
+	}
+
+	timeout = ktime_add_us(ktime_get(), vbif->cap->xin_halt_timeout);
+	for (;;) {
+		status = vbif->ops.get_halt_ctrl(vbif, xin_id);
+		if (status)
+			break;
+		if (ktime_compare_safe(ktime_get(), timeout) > 0) {
+			status = vbif->ops.get_halt_ctrl(vbif, xin_id);
+			break;
+		}
+		usleep_range(501, 1000);
+	}
+
+	if (!status) {
+		rc = -ETIMEDOUT;
+		DPU_ERROR("VBIF %d client %d not halting. TIMEDOUT.\n",
+				vbif->idx - VBIF_0, xin_id);
+	} else {
+		rc = 0;
+		DPU_DEBUG("VBIF %d client %d is halted\n",
+				vbif->idx - VBIF_0, xin_id);
+	}
+
+	return rc;
+}
+
+/**
+ * _dpu_vbif_apply_dynamic_ot_limit - determine OT based on usecase parameters
+ * @vbif:	Pointer to hardware vbif driver
+ * @ot_lim:	Pointer to OT limit to be modified
+ * @params:	Pointer to usecase parameters
+ */
+static void _dpu_vbif_apply_dynamic_ot_limit(struct dpu_hw_vbif *vbif,
+		u32 *ot_lim, struct dpu_vbif_set_ot_params *params)
+{
+	u64 pps;
+	const struct dpu_vbif_dynamic_ot_tbl *tbl;
+	u32 i;
+
+	if (!vbif || !(vbif->cap->features & BIT(DPU_VBIF_QOS_OTLIM)))
+		return;
+
+	/* Dynamic OT setting done only for WFD */
+	if (!params->is_wfd)
+		return;
+
+	pps = params->frame_rate;
+	pps *= params->width;
+	pps *= params->height;
+
+	tbl = params->rd ? &vbif->cap->dynamic_ot_rd_tbl :
+			&vbif->cap->dynamic_ot_wr_tbl;
+
+	for (i = 0; i < tbl->count; i++) {
+		if (pps <= tbl->cfg[i].pps) {
+			*ot_lim = tbl->cfg[i].ot_limit;
+			break;
+		}
+	}
+
+	DPU_DEBUG("vbif:%d xin:%d w:%d h:%d fps:%d pps:%llu ot:%u\n",
+			vbif->idx - VBIF_0, params->xin_id,
+			params->width, params->height, params->frame_rate,
+			pps, *ot_lim);
+}
+
+/**
+ * _dpu_vbif_get_ot_limit - get OT based on usecase & configuration parameters
+ * @vbif:	Pointer to hardware vbif driver
+ * @params:	Pointer to usecase parameters
+ * @return:	OT limit
+ */
+static u32 _dpu_vbif_get_ot_limit(struct dpu_hw_vbif *vbif,
+	struct dpu_vbif_set_ot_params *params)
+{
+	u32 ot_lim = 0;
+	u32 val;
+
+	if (!vbif || !vbif->cap) {
+		DPU_ERROR("invalid arguments vbif %d\n", vbif != 0);
+		return -EINVAL;
+	}
+
+	if (vbif->cap->default_ot_wr_limit && !params->rd)
+		ot_lim = vbif->cap->default_ot_wr_limit;
+	else if (vbif->cap->default_ot_rd_limit && params->rd)
+		ot_lim = vbif->cap->default_ot_rd_limit;
+
+	/*
+	 * If default ot is not set from dt/catalog,
+	 * then do not configure it.
+	 */
+	if (ot_lim == 0)
+		goto exit;
+
+	/* Modify the limits if the target and the use case requires it */
+	_dpu_vbif_apply_dynamic_ot_limit(vbif, &ot_lim, params);
+
+	if (vbif && vbif->ops.get_limit_conf) {
+		val = vbif->ops.get_limit_conf(vbif,
+				params->xin_id, params->rd);
+		if (val == ot_lim)
+			ot_lim = 0;
+	}
+
+exit:
+	DPU_DEBUG("vbif:%d xin:%d ot_lim:%d\n",
+			vbif->idx - VBIF_0, params->xin_id, ot_lim);
+	return ot_lim;
+}
+
+/**
+ * dpu_vbif_set_ot_limit - set OT based on usecase & configuration parameters
+ * @vbif:	Pointer to hardware vbif driver
+ * @params:	Pointer to usecase parameters
+ *
+ * Note this function would block waiting for bus halt.
+ */
+void dpu_vbif_set_ot_limit(struct dpu_kms *dpu_kms,
+		struct dpu_vbif_set_ot_params *params)
+{
+	struct dpu_hw_vbif *vbif = NULL;
+	struct dpu_hw_mdp *mdp;
+	bool forced_on = false;
+	u32 ot_lim;
+	int ret, i;
+
+	if (!dpu_kms) {
+		DPU_ERROR("invalid arguments\n");
+		return;
+	}
+	mdp = dpu_kms->hw_mdp;
+
+	for (i = 0; i < ARRAY_SIZE(dpu_kms->hw_vbif); i++) {
+		if (dpu_kms->hw_vbif[i] &&
+				dpu_kms->hw_vbif[i]->idx == params->vbif_idx)
+			vbif = dpu_kms->hw_vbif[i];
+	}
+
+	if (!vbif || !mdp) {
+		DPU_DEBUG("invalid arguments vbif %d mdp %d\n",
+				vbif != 0, mdp != 0);
+		return;
+	}
+
+	if (!mdp->ops.setup_clk_force_ctrl ||
+			!vbif->ops.set_limit_conf ||
+			!vbif->ops.set_halt_ctrl)
+		return;
+
+	/* set write_gather_en for all write clients */
+	if (vbif->ops.set_write_gather_en && !params->rd)
+		vbif->ops.set_write_gather_en(vbif, params->xin_id);
+
+	ot_lim = _dpu_vbif_get_ot_limit(vbif, params) & 0xFF;
+
+	if (ot_lim == 0)
+		goto exit;
+
+	trace_dpu_perf_set_ot(params->num, params->xin_id, ot_lim,
+		params->vbif_idx);
+
+	forced_on = mdp->ops.setup_clk_force_ctrl(mdp, params->clk_ctrl, true);
+
+	vbif->ops.set_limit_conf(vbif, params->xin_id, params->rd, ot_lim);
+
+	vbif->ops.set_halt_ctrl(vbif, params->xin_id, true);
+
+	ret = _dpu_vbif_wait_for_xin_halt(vbif, params->xin_id);
+	if (ret)
+		trace_dpu_vbif_wait_xin_halt_fail(vbif->idx, params->xin_id);
+
+	vbif->ops.set_halt_ctrl(vbif, params->xin_id, false);
+
+	if (forced_on)
+		mdp->ops.setup_clk_force_ctrl(mdp, params->clk_ctrl, false);
+exit:
+	return;
+}
+
+void dpu_vbif_set_qos_remap(struct dpu_kms *dpu_kms,
+		struct dpu_vbif_set_qos_params *params)
+{
+	struct dpu_hw_vbif *vbif = NULL;
+	struct dpu_hw_mdp *mdp;
+	bool forced_on = false;
+	const struct dpu_vbif_qos_tbl *qos_tbl;
+	int i;
+
+	if (!dpu_kms || !params || !dpu_kms->hw_mdp) {
+		DPU_ERROR("invalid arguments\n");
+		return;
+	}
+	mdp = dpu_kms->hw_mdp;
+
+	for (i = 0; i < ARRAY_SIZE(dpu_kms->hw_vbif); i++) {
+		if (dpu_kms->hw_vbif[i] &&
+				dpu_kms->hw_vbif[i]->idx == params->vbif_idx) {
+			vbif = dpu_kms->hw_vbif[i];
+			break;
+		}
+	}
+
+	if (!vbif || !vbif->cap) {
+		DPU_ERROR("invalid vbif %d\n", params->vbif_idx);
+		return;
+	}
+
+	if (!vbif->ops.set_qos_remap || !mdp->ops.setup_clk_force_ctrl) {
+		DPU_DEBUG("qos remap not supported\n");
+		return;
+	}
+
+	qos_tbl = params->is_rt ? &vbif->cap->qos_rt_tbl :
+			&vbif->cap->qos_nrt_tbl;
+
+	if (!qos_tbl->npriority_lvl || !qos_tbl->priority_lvl) {
+		DPU_DEBUG("qos tbl not defined\n");
+		return;
+	}
+
+	forced_on = mdp->ops.setup_clk_force_ctrl(mdp, params->clk_ctrl, true);
+
+	for (i = 0; i < qos_tbl->npriority_lvl; i++) {
+		DPU_DEBUG("vbif:%d xin:%d lvl:%d/%d\n",
+				params->vbif_idx, params->xin_id, i,
+				qos_tbl->priority_lvl[i]);
+		vbif->ops.set_qos_remap(vbif, params->xin_id, i,
+				qos_tbl->priority_lvl[i]);
+	}
+
+	if (forced_on)
+		mdp->ops.setup_clk_force_ctrl(mdp, params->clk_ctrl, false);
+}
+
+void dpu_vbif_clear_errors(struct dpu_kms *dpu_kms)
+{
+	struct dpu_hw_vbif *vbif;
+	u32 i, pnd, src;
+
+	if (!dpu_kms) {
+		DPU_ERROR("invalid argument\n");
+		return;
+	}
+
+	for (i = 0; i < ARRAY_SIZE(dpu_kms->hw_vbif); i++) {
+		vbif = dpu_kms->hw_vbif[i];
+		if (vbif && vbif->ops.clear_errors) {
+			vbif->ops.clear_errors(vbif, &pnd, &src);
+			if (pnd || src) {
+				DRM_DEBUG_KMS("VBIF %d: pnd 0x%X, src 0x%X\n",
+					      vbif->idx - VBIF_0, pnd, src);
+			}
+		}
+	}
+}
+
+void dpu_vbif_init_memtypes(struct dpu_kms *dpu_kms)
+{
+	struct dpu_hw_vbif *vbif;
+	int i, j;
+
+	if (!dpu_kms) {
+		DPU_ERROR("invalid argument\n");
+		return;
+	}
+
+	for (i = 0; i < ARRAY_SIZE(dpu_kms->hw_vbif); i++) {
+		vbif = dpu_kms->hw_vbif[i];
+		if (vbif && vbif->cap && vbif->ops.set_mem_type) {
+			for (j = 0; j < vbif->cap->memtype_count; j++)
+				vbif->ops.set_mem_type(
+						vbif, j, vbif->cap->memtype[j]);
+		}
+	}
+}
+
+#ifdef CONFIG_DEBUG_FS
+void dpu_debugfs_vbif_destroy(struct dpu_kms *dpu_kms)
+{
+	debugfs_remove_recursive(dpu_kms->debugfs_vbif);
+	dpu_kms->debugfs_vbif = NULL;
+}
+
+int dpu_debugfs_vbif_init(struct dpu_kms *dpu_kms, struct dentry *debugfs_root)
+{
+	char vbif_name[32];
+	struct dentry *debugfs_vbif;
+	int i, j;
+
+	dpu_kms->debugfs_vbif = debugfs_create_dir("vbif", debugfs_root);
+	if (!dpu_kms->debugfs_vbif) {
+		DPU_ERROR("failed to create vbif debugfs\n");
+		return -EINVAL;
+	}
+
+	for (i = 0; i < dpu_kms->catalog->vbif_count; i++) {
+		struct dpu_vbif_cfg *vbif = &dpu_kms->catalog->vbif[i];
+
+		snprintf(vbif_name, sizeof(vbif_name), "%d", vbif->id);
+
+		debugfs_vbif = debugfs_create_dir(vbif_name,
+				dpu_kms->debugfs_vbif);
+
+		debugfs_create_u32("features", 0600, debugfs_vbif,
+			(u32 *)&vbif->features);
+
+		debugfs_create_u32("xin_halt_timeout", 0400, debugfs_vbif,
+			(u32 *)&vbif->xin_halt_timeout);
+
+		debugfs_create_u32("default_rd_ot_limit", 0400, debugfs_vbif,
+			(u32 *)&vbif->default_ot_rd_limit);
+
+		debugfs_create_u32("default_wr_ot_limit", 0400, debugfs_vbif,
+			(u32 *)&vbif->default_ot_wr_limit);
+
+		for (j = 0; j < vbif->dynamic_ot_rd_tbl.count; j++) {
+			struct dpu_vbif_dynamic_ot_cfg *cfg =
+					&vbif->dynamic_ot_rd_tbl.cfg[j];
+
+			snprintf(vbif_name, sizeof(vbif_name),
+					"dynamic_ot_rd_%d_pps", j);
+			debugfs_create_u64(vbif_name, 0400, debugfs_vbif,
+					(u64 *)&cfg->pps);
+			snprintf(vbif_name, sizeof(vbif_name),
+					"dynamic_ot_rd_%d_ot_limit", j);
+			debugfs_create_u32(vbif_name, 0400, debugfs_vbif,
+					(u32 *)&cfg->ot_limit);
+		}
+
+		for (j = 0; j < vbif->dynamic_ot_wr_tbl.count; j++) {
+			struct dpu_vbif_dynamic_ot_cfg *cfg =
+					&vbif->dynamic_ot_wr_tbl.cfg[j];
+
+			snprintf(vbif_name, sizeof(vbif_name),
+					"dynamic_ot_wr_%d_pps", j);
+			debugfs_create_u64(vbif_name, 0400, debugfs_vbif,
+					(u64 *)&cfg->pps);
+			snprintf(vbif_name, sizeof(vbif_name),
+					"dynamic_ot_wr_%d_ot_limit", j);
+			debugfs_create_u32(vbif_name, 0400, debugfs_vbif,
+					(u32 *)&cfg->ot_limit);
+		}
+	}
+
+	return 0;
+}
+#endif
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_vbif.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_vbif.h
new file mode 100644
index 000000000000..f17af52dbbd5
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_vbif.h
@@ -0,0 +1,94 @@
+/* Copyright (c) 2016-2018, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef __DPU_VBIF_H__
+#define __DPU_VBIF_H__
+
+#include "dpu_kms.h"
+
+struct dpu_vbif_set_ot_params {
+	u32 xin_id;
+	u32 num;
+	u32 width;
+	u32 height;
+	u32 frame_rate;
+	bool rd;
+	bool is_wfd;
+	u32 vbif_idx;
+	u32 clk_ctrl;
+};
+
+struct dpu_vbif_set_memtype_params {
+	u32 xin_id;
+	u32 vbif_idx;
+	u32 clk_ctrl;
+	bool is_cacheable;
+};
+
+/**
+ * struct dpu_vbif_set_qos_params - QoS remapper parameter
+ * @vbif_idx: vbif identifier
+ * @xin_id: client interface identifier
+ * @clk_ctrl: clock control identifier of the xin
+ * @num: pipe identifier (debug only)
+ * @is_rt: true if pipe is used in real-time use case
+ */
+struct dpu_vbif_set_qos_params {
+	u32 vbif_idx;
+	u32 xin_id;
+	u32 clk_ctrl;
+	u32 num;
+	bool is_rt;
+};
+
+/**
+ * dpu_vbif_set_ot_limit - set OT limit for vbif client
+ * @dpu_kms:	DPU handler
+ * @params:	Pointer to OT configuration parameters
+ */
+void dpu_vbif_set_ot_limit(struct dpu_kms *dpu_kms,
+		struct dpu_vbif_set_ot_params *params);
+
+/**
+ * dpu_vbif_set_qos_remap - set QoS priority level remap
+ * @dpu_kms:	DPU handler
+ * @params:	Pointer to QoS configuration parameters
+ */
+void dpu_vbif_set_qos_remap(struct dpu_kms *dpu_kms,
+		struct dpu_vbif_set_qos_params *params);
+
+/**
+ * dpu_vbif_clear_errors - clear any vbif errors
+ * @dpu_kms:	DPU handler
+ */
+void dpu_vbif_clear_errors(struct dpu_kms *dpu_kms);
+
+/**
+ * dpu_vbif_init_memtypes - initialize xin memory types for vbif
+ * @dpu_kms:	DPU handler
+ */
+void dpu_vbif_init_memtypes(struct dpu_kms *dpu_kms);
+
+#ifdef CONFIG_DEBUG_FS
+int dpu_debugfs_vbif_init(struct dpu_kms *dpu_kms, struct dentry *debugfs_root);
+void dpu_debugfs_vbif_destroy(struct dpu_kms *dpu_kms);
+#else
+static inline int dpu_debugfs_vbif_init(struct dpu_kms *dpu_kms,
+		struct dentry *debugfs_root)
+{
+	return 0;
+}
+static inline void dpu_debugfs_vbif_destroy(struct dpu_kms *dpu_kms)
+{
+}
+#endif
+#endif /* __DPU_VBIF_H__ */
diff --git a/drivers/gpu/drm/msm/disp/dpu1/msm_media_info.h b/drivers/gpu/drm/msm/disp/dpu1/msm_media_info.h
new file mode 100644
index 000000000000..4f12e5c534c8
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/msm_media_info.h
@@ -0,0 +1,1376 @@
+#ifndef __MEDIA_INFO_H__
+#define __MEDIA_INFO_H__
+
+#ifndef MSM_MEDIA_ALIGN
+#define MSM_MEDIA_ALIGN(__sz, __align) (((__align) & ((__align) - 1)) ?\
+	((((__sz) + (__align) - 1) / (__align)) * (__align)) :\
+	(((__sz) + (__align) - 1) & (~((__align) - 1))))
+#endif
+
+#ifndef MSM_MEDIA_ROUNDUP
+#define MSM_MEDIA_ROUNDUP(__sz, __r) (((__sz) + ((__r) - 1)) / (__r))
+#endif
+
+#ifndef MSM_MEDIA_MAX
+#define MSM_MEDIA_MAX(__a, __b) ((__a) > (__b)?(__a):(__b))
+#endif
+
+enum color_fmts {
+	/* Venus NV12:
+	 * YUV 4:2:0 image with a plane of 8 bit Y samples followed
+	 * by an interleaved U/V plane containing 8 bit 2x2 subsampled
+	 * colour difference samples.
+	 *
+	 * <-------- Y/UV_Stride -------->
+	 * <------- Width ------->
+	 * Y Y Y Y Y Y Y Y Y Y Y Y . . . .  ^           ^
+	 * Y Y Y Y Y Y Y Y Y Y Y Y . . . .  |           |
+	 * Y Y Y Y Y Y Y Y Y Y Y Y . . . .  Height      |
+	 * Y Y Y Y Y Y Y Y Y Y Y Y . . . .  |          Y_Scanlines
+	 * Y Y Y Y Y Y Y Y Y Y Y Y . . . .  |           |
+	 * Y Y Y Y Y Y Y Y Y Y Y Y . . . .  |           |
+	 * Y Y Y Y Y Y Y Y Y Y Y Y . . . .  |           |
+	 * Y Y Y Y Y Y Y Y Y Y Y Y . . . .  V           |
+	 * . . . . . . . . . . . . . . . .              |
+	 * . . . . . . . . . . . . . . . .              |
+	 * . . . . . . . . . . . . . . . .              |
+	 * . . . . . . . . . . . . . . . .              V
+	 * U V U V U V U V U V U V . . . .  ^
+	 * U V U V U V U V U V U V . . . .  |
+	 * U V U V U V U V U V U V . . . .  |
+	 * U V U V U V U V U V U V . . . .  UV_Scanlines
+	 * . . . . . . . . . . . . . . . .  |
+	 * . . . . . . . . . . . . . . . .  V
+	 * . . . . . . . . . . . . . . . .  --> Buffer size alignment
+	 *
+	 * Y_Stride : Width aligned to 128
+	 * UV_Stride : Width aligned to 128
+	 * Y_Scanlines: Height aligned to 32
+	 * UV_Scanlines: Height/2 aligned to 16
+	 * Extradata: Arbitrary (software-imposed) padding
+	 * Total size = align((Y_Stride * Y_Scanlines
+	 *          + UV_Stride * UV_Scanlines
+	 *          + max(Extradata, Y_Stride * 8), 4096)
+	 */
+	COLOR_FMT_NV12,
+
+	/* Venus NV21:
+	 * YUV 4:2:0 image with a plane of 8 bit Y samples followed
+	 * by an interleaved V/U plane containing 8 bit 2x2 subsampled
+	 * colour difference samples.
+	 *
+	 * <-------- Y/UV_Stride -------->
+	 * <------- Width ------->
+	 * Y Y Y Y Y Y Y Y Y Y Y Y . . . .  ^           ^
+	 * Y Y Y Y Y Y Y Y Y Y Y Y . . . .  |           |
+	 * Y Y Y Y Y Y Y Y Y Y Y Y . . . .  Height      |
+	 * Y Y Y Y Y Y Y Y Y Y Y Y . . . .  |          Y_Scanlines
+	 * Y Y Y Y Y Y Y Y Y Y Y Y . . . .  |           |
+	 * Y Y Y Y Y Y Y Y Y Y Y Y . . . .  |           |
+	 * Y Y Y Y Y Y Y Y Y Y Y Y . . . .  |           |
+	 * Y Y Y Y Y Y Y Y Y Y Y Y . . . .  V           |
+	 * . . . . . . . . . . . . . . . .              |
+	 * . . . . . . . . . . . . . . . .              |
+	 * . . . . . . . . . . . . . . . .              |
+	 * . . . . . . . . . . . . . . . .              V
+	 * V U V U V U V U V U V U . . . .  ^
+	 * V U V U V U V U V U V U . . . .  |
+	 * V U V U V U V U V U V U . . . .  |
+	 * V U V U V U V U V U V U . . . .  UV_Scanlines
+	 * . . . . . . . . . . . . . . . .  |
+	 * . . . . . . . . . . . . . . . .  V
+	 * . . . . . . . . . . . . . . . .  --> Padding & Buffer size alignment
+	 *
+	 * Y_Stride : Width aligned to 128
+	 * UV_Stride : Width aligned to 128
+	 * Y_Scanlines: Height aligned to 32
+	 * UV_Scanlines: Height/2 aligned to 16
+	 * Extradata: Arbitrary (software-imposed) padding
+	 * Total size = align((Y_Stride * Y_Scanlines
+	 *          + UV_Stride * UV_Scanlines
+	 *          + max(Extradata, Y_Stride * 8), 4096)
+	 */
+	COLOR_FMT_NV21,
+	/* Venus NV12_MVTB:
+	 * Two YUV 4:2:0 images/views one after the other
+	 * in a top-bottom layout, same as NV12
+	 * with a plane of 8 bit Y samples followed
+	 * by an interleaved U/V plane containing 8 bit 2x2 subsampled
+	 * colour difference samples.
+	 *
+	 *
+	 * <-------- Y/UV_Stride -------->
+	 * <------- Width ------->
+	 * Y Y Y Y Y Y Y Y Y Y Y Y . . . .  ^           ^               ^
+	 * Y Y Y Y Y Y Y Y Y Y Y Y . . . .  |           |               |
+	 * Y Y Y Y Y Y Y Y Y Y Y Y . . . .  Height      |               |
+	 * Y Y Y Y Y Y Y Y Y Y Y Y . . . .  |          Y_Scanlines      |
+	 * Y Y Y Y Y Y Y Y Y Y Y Y . . . .  |           |               |
+	 * Y Y Y Y Y Y Y Y Y Y Y Y . . . .  |           |               |
+	 * Y Y Y Y Y Y Y Y Y Y Y Y . . . .  |           |               |
+	 * Y Y Y Y Y Y Y Y Y Y Y Y . . . .  V           |               |
+	 * . . . . . . . . . . . . . . . .              |             View_1
+	 * . . . . . . . . . . . . . . . .              |               |
+	 * . . . . . . . . . . . . . . . .              |               |
+	 * . . . . . . . . . . . . . . . .              V               |
+	 * U V U V U V U V U V U V . . . .  ^                           |
+	 * U V U V U V U V U V U V . . . .  |                           |
+	 * U V U V U V U V U V U V . . . .  |                           |
+	 * U V U V U V U V U V U V . . . .  UV_Scanlines                |
+	 * . . . . . . . . . . . . . . . .  |                           |
+	 * . . . . . . . . . . . . . . . .  V                           V
+	 * Y Y Y Y Y Y Y Y Y Y Y Y . . . .  ^           ^               ^
+	 * Y Y Y Y Y Y Y Y Y Y Y Y . . . .  |           |               |
+	 * Y Y Y Y Y Y Y Y Y Y Y Y . . . .  Height      |               |
+	 * Y Y Y Y Y Y Y Y Y Y Y Y . . . .  |          Y_Scanlines      |
+	 * Y Y Y Y Y Y Y Y Y Y Y Y . . . .  |           |               |
+	 * Y Y Y Y Y Y Y Y Y Y Y Y . . . .  |           |               |
+	 * Y Y Y Y Y Y Y Y Y Y Y Y . . . .  |           |               |
+	 * Y Y Y Y Y Y Y Y Y Y Y Y . . . .  V           |               |
+	 * . . . . . . . . . . . . . . . .              |             View_2
+	 * . . . . . . . . . . . . . . . .              |               |
+	 * . . . . . . . . . . . . . . . .              |               |
+	 * . . . . . . . . . . . . . . . .              V               |
+	 * U V U V U V U V U V U V . . . .  ^                           |
+	 * U V U V U V U V U V U V . . . .  |                           |
+	 * U V U V U V U V U V U V . . . .  |                           |
+	 * U V U V U V U V U V U V . . . .  UV_Scanlines                |
+	 * . . . . . . . . . . . . . . . .  |                           |
+	 * . . . . . . . . . . . . . . . .  V                           V
+	 * . . . . . . . . . . . . . . . .  --> Buffer size alignment
+	 *
+	 * Y_Stride : Width aligned to 128
+	 * UV_Stride : Width aligned to 128
+	 * Y_Scanlines: Height aligned to 32
+	 * UV_Scanlines: Height/2 aligned to 16
+	 * View_1 begin at: 0 (zero)
+	 * View_2 begin at: Y_Stride * Y_Scanlines + UV_Stride * UV_Scanlines
+	 * Extradata: Arbitrary (software-imposed) padding
+	 * Total size = align((2*(Y_Stride * Y_Scanlines)
+	 *          + 2*(UV_Stride * UV_Scanlines) + Extradata), 4096)
+	 */
+	COLOR_FMT_NV12_MVTB,
+	/*
+	 * The buffer can be of 2 types:
+	 * (1) Venus NV12 UBWC Progressive
+	 * (2) Venus NV12 UBWC Interlaced
+	 *
+	 * (1) Venus NV12 UBWC Progressive Buffer Format:
+	 * Compressed Macro-tile format for NV12.
+	 * Contains 4 planes in the following order -
+	 * (A) Y_Meta_Plane
+	 * (B) Y_UBWC_Plane
+	 * (C) UV_Meta_Plane
+	 * (D) UV_UBWC_Plane
+	 *
+	 * Y_Meta_Plane consists of meta information to decode compressed
+	 * tile data in Y_UBWC_Plane.
+	 * Y_UBWC_Plane consists of Y data in compressed macro-tile format.
+	 * UBWC decoder block will use the Y_Meta_Plane data together with
+	 * Y_UBWC_Plane data to produce loss-less uncompressed 8 bit Y samples.
+	 *
+	 * UV_Meta_Plane consists of meta information to decode compressed
+	 * tile data in UV_UBWC_Plane.
+	 * UV_UBWC_Plane consists of UV data in compressed macro-tile format.
+	 * UBWC decoder block will use UV_Meta_Plane data together with
+	 * UV_UBWC_Plane data to produce loss-less uncompressed 8 bit 2x2
+	 * subsampled color difference samples.
+	 *
+	 * Each tile in Y_UBWC_Plane/UV_UBWC_Plane is independently decodable
+	 * and randomly accessible. There is no dependency between tiles.
+	 *
+	 * <----- Y_Meta_Stride ---->
+	 * <-------- Width ------>
+	 * M M M M M M M M M M M M . .      ^           ^
+	 * M M M M M M M M M M M M . .      |           |
+	 * M M M M M M M M M M M M . .      Height      |
+	 * M M M M M M M M M M M M . .      |         Meta_Y_Scanlines
+	 * M M M M M M M M M M M M . .      |           |
+	 * M M M M M M M M M M M M . .      |           |
+	 * M M M M M M M M M M M M . .      |           |
+	 * M M M M M M M M M M M M . .      V           |
+	 * . . . . . . . . . . . . . .                  |
+	 * . . . . . . . . . . . . . .                  |
+	 * . . . . . . . . . . . . . .      -------> Buffer size aligned to 4k
+	 * . . . . . . . . . . . . . .                  V
+	 * <--Compressed tile Y Stride--->
+	 * <------- Width ------->
+	 * Y* Y* Y* Y* Y* Y* Y* Y* . . . .  ^           ^
+	 * Y* Y* Y* Y* Y* Y* Y* Y* . . . .  |           |
+	 * Y* Y* Y* Y* Y* Y* Y* Y* . . . .  Height      |
+	 * Y* Y* Y* Y* Y* Y* Y* Y* . . . .  |        Macro_tile_Y_Scanlines
+	 * Y* Y* Y* Y* Y* Y* Y* Y* . . . .  |           |
+	 * Y* Y* Y* Y* Y* Y* Y* Y* . . . .  |           |
+	 * Y* Y* Y* Y* Y* Y* Y* Y* . . . .  |           |
+	 * Y* Y* Y* Y* Y* Y* Y* Y* . . . .  V           |
+	 * . . . . . . . . . . . . . . . .              |
+	 * . . . . . . . . . . . . . . . .              |
+	 * . . . . . . . . . . . . . . . .  -------> Buffer size aligned to 4k
+	 * . . . . . . . . . . . . . . . .              V
+	 * <----- UV_Meta_Stride ---->
+	 * M M M M M M M M M M M M . .      ^
+	 * M M M M M M M M M M M M . .      |
+	 * M M M M M M M M M M M M . .      |
+	 * M M M M M M M M M M M M . .      M_UV_Scanlines
+	 * . . . . . . . . . . . . . .      |
+	 * . . . . . . . . . . . . . .      V
+	 * . . . . . . . . . . . . . .      -------> Buffer size aligned to 4k
+	 * <--Compressed tile UV Stride--->
+	 * U* V* U* V* U* V* U* V* . . . .  ^
+	 * U* V* U* V* U* V* U* V* . . . .  |
+	 * U* V* U* V* U* V* U* V* . . . .  |
+	 * U* V* U* V* U* V* U* V* . . . .  UV_Scanlines
+	 * . . . . . . . . . . . . . . . .  |
+	 * . . . . . . . . . . . . . . . .  V
+	 * . . . . . . . . . . . . . . . .  -------> Buffer size aligned to 4k
+	 *
+	 * Y_Stride = align(Width, 128)
+	 * UV_Stride = align(Width, 128)
+	 * Y_Scanlines = align(Height, 32)
+	 * UV_Scanlines = align(Height/2, 16)
+	 * Y_UBWC_Plane_size = align(Y_Stride * Y_Scanlines, 4096)
+	 * UV_UBWC_Plane_size = align(UV_Stride * UV_Scanlines, 4096)
+	 * Y_Meta_Stride = align(roundup(Width, Y_TileWidth), 64)
+	 * Y_Meta_Scanlines = align(roundup(Height, Y_TileHeight), 16)
+	 * Y_Meta_Plane_size = align(Y_Meta_Stride * Y_Meta_Scanlines, 4096)
+	 * UV_Meta_Stride = align(roundup(Width, UV_TileWidth), 64)
+	 * UV_Meta_Scanlines = align(roundup(Height, UV_TileHeight), 16)
+	 * UV_Meta_Plane_size = align(UV_Meta_Stride * UV_Meta_Scanlines, 4096)
+	 * Extradata = 8k
+	 *
+	 * Total size = align( Y_UBWC_Plane_size + UV_UBWC_Plane_size +
+	 *           Y_Meta_Plane_size + UV_Meta_Plane_size
+	 *           + max(Extradata, Y_Stride * 48), 4096)
+	 *
+	 *
+	 * (2) Venus NV12 UBWC Interlaced Buffer Format:
+	 * Compressed Macro-tile format for NV12 interlaced.
+	 * Contains 8 planes in the following order -
+	 * (A) Y_Meta_Top_Field_Plane
+	 * (B) Y_UBWC_Top_Field_Plane
+	 * (C) UV_Meta_Top_Field_Plane
+	 * (D) UV_UBWC_Top_Field_Plane
+	 * (E) Y_Meta_Bottom_Field_Plane
+	 * (F) Y_UBWC_Bottom_Field_Plane
+	 * (G) UV_Meta_Bottom_Field_Plane
+	 * (H) UV_UBWC_Bottom_Field_Plane
+	 * Y_Meta_Top_Field_Plane consists of meta information to decode
+	 * compressed tile data for Y_UBWC_Top_Field_Plane.
+	 * Y_UBWC_Top_Field_Plane consists of Y data in compressed macro-tile
+	 * format for top field of an interlaced frame.
+	 * UBWC decoder block will use the Y_Meta_Top_Field_Plane data together
+	 * with Y_UBWC_Top_Field_Plane data to produce loss-less uncompressed
+	 * 8 bit Y samples for top field of an interlaced frame.
+	 *
+	 * UV_Meta_Top_Field_Plane consists of meta information to decode
+	 * compressed tile data in UV_UBWC_Top_Field_Plane.
+	 * UV_UBWC_Top_Field_Plane consists of UV data in compressed macro-tile
+	 * format for top field of an interlaced frame.
+	 * UBWC decoder block will use UV_Meta_Top_Field_Plane data together
+	 * with UV_UBWC_Top_Field_Plane data to produce loss-less uncompressed
+	 * 8 bit subsampled color difference samples for top field of an
+	 * interlaced frame.
+	 *
+	 * Each tile in Y_UBWC_Top_Field_Plane/UV_UBWC_Top_Field_Plane is
+	 * independently decodable and randomly accessible. There is no
+	 * dependency between tiles.
+	 *
+	 * Y_Meta_Bottom_Field_Plane consists of meta information to decode
+	 * compressed tile data for Y_UBWC_Bottom_Field_Plane.
+	 * Y_UBWC_Bottom_Field_Plane consists of Y data in compressed macro-tile
+	 * format for bottom field of an interlaced frame.
+	 * UBWC decoder block will use the Y_Meta_Bottom_Field_Plane data
+	 * together with Y_UBWC_Bottom_Field_Plane data to produce loss-less
+	 * uncompressed 8 bit Y samples for bottom field of an interlaced frame.
+	 *
+	 * UV_Meta_Bottom_Field_Plane consists of meta information to decode
+	 * compressed tile data in UV_UBWC_Bottom_Field_Plane.
+	 * UV_UBWC_Bottom_Field_Plane consists of UV data in compressed
+	 * macro-tile format for bottom field of an interlaced frame.
+	 * UBWC decoder block will use UV_Meta_Bottom_Field_Plane data together
+	 * with UV_UBWC_Bottom_Field_Plane data to produce loss-less
+	 * uncompressed 8 bit subsampled color difference samples for bottom
+	 * field of an interlaced frame.
+	 *
+	 * Each tile in Y_UBWC_Bottom_Field_Plane/UV_UBWC_Bottom_Field_Plane is
+	 * independently decodable and randomly accessible. There is no
+	 * dependency between tiles.
+	 *
+	 * <-----Y_TF_Meta_Stride---->
+	 * <-------- Width ------>
+	 * M M M M M M M M M M M M . .      ^           ^
+	 * M M M M M M M M M M M M . .      |           |
+	 * M M M M M M M M M M M M . . Half_height      |
+	 * M M M M M M M M M M M M . .      |         Meta_Y_TF_Scanlines
+	 * M M M M M M M M M M M M . .      |           |
+	 * M M M M M M M M M M M M . .      |           |
+	 * M M M M M M M M M M M M . .      |           |
+	 * M M M M M M M M M M M M . .      V           |
+	 * . . . . . . . . . . . . . .                  |
+	 * . . . . . . . . . . . . . .                  |
+	 * . . . . . . . . . . . . . .      -------> Buffer size aligned to 4k
+	 * . . . . . . . . . . . . . .                  V
+	 * <-Compressed tile Y_TF Stride->
+	 * <------- Width ------->
+	 * Y* Y* Y* Y* Y* Y* Y* Y* . . . .  ^           ^
+	 * Y* Y* Y* Y* Y* Y* Y* Y* . . . .  |           |
+	 * Y* Y* Y* Y* Y* Y* Y* Y* . . . . Half_height  |
+	 * Y* Y* Y* Y* Y* Y* Y* Y* . . . .  |        Macro_tile_Y_TF_Scanlines
+	 * Y* Y* Y* Y* Y* Y* Y* Y* . . . .  |           |
+	 * Y* Y* Y* Y* Y* Y* Y* Y* . . . .  |           |
+	 * Y* Y* Y* Y* Y* Y* Y* Y* . . . .  |           |
+	 * Y* Y* Y* Y* Y* Y* Y* Y* . . . .  V           |
+	 * . . . . . . . . . . . . . . . .              |
+	 * . . . . . . . . . . . . . . . .              |
+	 * . . . . . . . . . . . . . . . .  -------> Buffer size aligned to 4k
+	 * . . . . . . . . . . . . . . . .              V
+	 * <----UV_TF_Meta_Stride---->
+	 * M M M M M M M M M M M M . .      ^
+	 * M M M M M M M M M M M M . .      |
+	 * M M M M M M M M M M M M . .      |
+	 * M M M M M M M M M M M M . .      M_UV_TF_Scanlines
+	 * . . . . . . . . . . . . . .      |
+	 * . . . . . . . . . . . . . .      V
+	 * . . . . . . . . . . . . . .      -------> Buffer size aligned to 4k
+	 * <-Compressed tile UV_TF Stride->
+	 * U* V* U* V* U* V* U* V* . . . .  ^
+	 * U* V* U* V* U* V* U* V* . . . .  |
+	 * U* V* U* V* U* V* U* V* . . . .  |
+	 * U* V* U* V* U* V* U* V* . . . .  UV_TF_Scanlines
+	 * . . . . . . . . . . . . . . . .  |
+	 * . . . . . . . . . . . . . . . .  V
+	 * . . . . . . . . . . . . . . . .  -------> Buffer size aligned to 4k
+	 * <-----Y_BF_Meta_Stride---->
+	 * <-------- Width ------>
+	 * M M M M M M M M M M M M . .      ^           ^
+	 * M M M M M M M M M M M M . .      |           |
+	 * M M M M M M M M M M M M . . Half_height      |
+	 * M M M M M M M M M M M M . .      |         Meta_Y_BF_Scanlines
+	 * M M M M M M M M M M M M . .      |           |
+	 * M M M M M M M M M M M M . .      |           |
+	 * M M M M M M M M M M M M . .      |           |
+	 * M M M M M M M M M M M M . .      V           |
+	 * . . . . . . . . . . . . . .                  |
+	 * . . . . . . . . . . . . . .                  |
+	 * . . . . . . . . . . . . . .      -------> Buffer size aligned to 4k
+	 * . . . . . . . . . . . . . .                  V
+	 * <-Compressed tile Y_BF Stride->
+	 * <------- Width ------->
+	 * Y* Y* Y* Y* Y* Y* Y* Y* . . . .  ^           ^
+	 * Y* Y* Y* Y* Y* Y* Y* Y* . . . .  |           |
+	 * Y* Y* Y* Y* Y* Y* Y* Y* . . . . Half_height  |
+	 * Y* Y* Y* Y* Y* Y* Y* Y* . . . .  |        Macro_tile_Y_BF_Scanlines
+	 * Y* Y* Y* Y* Y* Y* Y* Y* . . . .  |           |
+	 * Y* Y* Y* Y* Y* Y* Y* Y* . . . .  |           |
+	 * Y* Y* Y* Y* Y* Y* Y* Y* . . . .  |           |
+	 * Y* Y* Y* Y* Y* Y* Y* Y* . . . .  V           |
+	 * . . . . . . . . . . . . . . . .              |
+	 * . . . . . . . . . . . . . . . .              |
+	 * . . . . . . . . . . . . . . . .  -------> Buffer size aligned to 4k
+	 * . . . . . . . . . . . . . . . .              V
+	 * <----UV_BF_Meta_Stride---->
+	 * M M M M M M M M M M M M . .      ^
+	 * M M M M M M M M M M M M . .      |
+	 * M M M M M M M M M M M M . .      |
+	 * M M M M M M M M M M M M . .      M_UV_BF_Scanlines
+	 * . . . . . . . . . . . . . .      |
+	 * . . . . . . . . . . . . . .      V
+	 * . . . . . . . . . . . . . .      -------> Buffer size aligned to 4k
+	 * <-Compressed tile UV_BF Stride->
+	 * U* V* U* V* U* V* U* V* . . . .  ^
+	 * U* V* U* V* U* V* U* V* . . . .  |
+	 * U* V* U* V* U* V* U* V* . . . .  |
+	 * U* V* U* V* U* V* U* V* . . . .  UV_BF_Scanlines
+	 * . . . . . . . . . . . . . . . .  |
+	 * . . . . . . . . . . . . . . . .  V
+	 * . . . . . . . . . . . . . . . .  -------> Buffer size aligned to 4k
+	 *
+	 * Half_height = (Height+1)>>1
+	 * Y_TF_Stride = align(Width, 128)
+	 * UV_TF_Stride = align(Width, 128)
+	 * Y_TF_Scanlines = align(Half_height, 32)
+	 * UV_TF_Scanlines = align((Half_height+1)/2, 32)
+	 * Y_UBWC_TF_Plane_size = align(Y_TF_Stride * Y_TF_Scanlines, 4096)
+	 * UV_UBWC_TF_Plane_size = align(UV_TF_Stride * UV_TF_Scanlines, 4096)
+	 * Y_TF_Meta_Stride = align(roundup(Width, Y_TileWidth), 64)
+	 * Y_TF_Meta_Scanlines = align(roundup(Half_height, Y_TileHeight), 16)
+	 * Y_TF_Meta_Plane_size =
+	 *     align(Y_TF_Meta_Stride * Y_TF_Meta_Scanlines, 4096)
+	 * UV_TF_Meta_Stride = align(roundup(Width, UV_TileWidth), 64)
+	 * UV_TF_Meta_Scanlines = align(roundup(Half_height, UV_TileHeight), 16)
+	 * UV_TF_Meta_Plane_size =
+	 *     align(UV_TF_Meta_Stride * UV_TF_Meta_Scanlines, 4096)
+	 * Y_BF_Stride = align(Width, 128)
+	 * UV_BF_Stride = align(Width, 128)
+	 * Y_BF_Scanlines = align(Half_height, 32)
+	 * UV_BF_Scanlines = align((Half_height+1)/2, 32)
+	 * Y_UBWC_BF_Plane_size = align(Y_BF_Stride * Y_BF_Scanlines, 4096)
+	 * UV_UBWC_BF_Plane_size = align(UV_BF_Stride * UV_BF_Scanlines, 4096)
+	 * Y_BF_Meta_Stride = align(roundup(Width, Y_TileWidth), 64)
+	 * Y_BF_Meta_Scanlines = align(roundup(Half_height, Y_TileHeight), 16)
+	 * Y_BF_Meta_Plane_size =
+	 *     align(Y_BF_Meta_Stride * Y_BF_Meta_Scanlines, 4096)
+	 * UV_BF_Meta_Stride = align(roundup(Width, UV_TileWidth), 64)
+	 * UV_BF_Meta_Scanlines = align(roundup(Half_height, UV_TileHeight), 16)
+	 * UV_BF_Meta_Plane_size =
+	 *     align(UV_BF_Meta_Stride * UV_BF_Meta_Scanlines, 4096)
+	 * Extradata = 8k
+	 *
+	 * Total size = align( Y_UBWC_TF_Plane_size + UV_UBWC_TF_Plane_size +
+	 *           Y_TF_Meta_Plane_size + UV_TF_Meta_Plane_size +
+	 *			 Y_UBWC_BF_Plane_size + UV_UBWC_BF_Plane_size +
+	 *           Y_BF_Meta_Plane_size + UV_BF_Meta_Plane_size +
+	 *           + max(Extradata, Y_TF_Stride * 48), 4096)
+	 */
+	COLOR_FMT_NV12_UBWC,
+	/* Venus NV12 10-bit UBWC:
+	 * Compressed Macro-tile format for NV12.
+	 * Contains 4 planes in the following order -
+	 * (A) Y_Meta_Plane
+	 * (B) Y_UBWC_Plane
+	 * (C) UV_Meta_Plane
+	 * (D) UV_UBWC_Plane
+	 *
+	 * Y_Meta_Plane consists of meta information to decode compressed
+	 * tile data in Y_UBWC_Plane.
+	 * Y_UBWC_Plane consists of Y data in compressed macro-tile format.
+	 * UBWC decoder block will use the Y_Meta_Plane data together with
+	 * Y_UBWC_Plane data to produce loss-less uncompressed 10 bit Y samples.
+	 *
+	 * UV_Meta_Plane consists of meta information to decode compressed
+	 * tile data in UV_UBWC_Plane.
+	 * UV_UBWC_Plane consists of UV data in compressed macro-tile format.
+	 * UBWC decoder block will use UV_Meta_Plane data together with
+	 * UV_UBWC_Plane data to produce loss-less uncompressed 10 bit 2x2
+	 * subsampled color difference samples.
+	 *
+	 * Each tile in Y_UBWC_Plane/UV_UBWC_Plane is independently decodable
+	 * and randomly accessible. There is no dependency between tiles.
+	 *
+	 * <----- Y_Meta_Stride ----->
+	 * <-------- Width ------>
+	 * M M M M M M M M M M M M . .      ^           ^
+	 * M M M M M M M M M M M M . .      |           |
+	 * M M M M M M M M M M M M . .      Height      |
+	 * M M M M M M M M M M M M . .      |         Meta_Y_Scanlines
+	 * M M M M M M M M M M M M . .      |           |
+	 * M M M M M M M M M M M M . .      |           |
+	 * M M M M M M M M M M M M . .      |           |
+	 * M M M M M M M M M M M M . .      V           |
+	 * . . . . . . . . . . . . . .                  |
+	 * . . . . . . . . . . . . . .                  |
+	 * . . . . . . . . . . . . . .      -------> Buffer size aligned to 4k
+	 * . . . . . . . . . . . . . .                  V
+	 * <--Compressed tile Y Stride--->
+	 * <------- Width ------->
+	 * Y* Y* Y* Y* Y* Y* Y* Y* . . . .  ^           ^
+	 * Y* Y* Y* Y* Y* Y* Y* Y* . . . .  |           |
+	 * Y* Y* Y* Y* Y* Y* Y* Y* . . . .  Height      |
+	 * Y* Y* Y* Y* Y* Y* Y* Y* . . . .  |        Macro_tile_Y_Scanlines
+	 * Y* Y* Y* Y* Y* Y* Y* Y* . . . .  |           |
+	 * Y* Y* Y* Y* Y* Y* Y* Y* . . . .  |           |
+	 * Y* Y* Y* Y* Y* Y* Y* Y* . . . .  |           |
+	 * Y* Y* Y* Y* Y* Y* Y* Y* . . . .  V           |
+	 * . . . . . . . . . . . . . . . .              |
+	 * . . . . . . . . . . . . . . . .              |
+	 * . . . . . . . . . . . . . . . .  -------> Buffer size aligned to 4k
+	 * . . . . . . . . . . . . . . . .              V
+	 * <----- UV_Meta_Stride ---->
+	 * M M M M M M M M M M M M . .      ^
+	 * M M M M M M M M M M M M . .      |
+	 * M M M M M M M M M M M M . .      |
+	 * M M M M M M M M M M M M . .      M_UV_Scanlines
+	 * . . . . . . . . . . . . . .      |
+	 * . . . . . . . . . . . . . .      V
+	 * . . . . . . . . . . . . . .      -------> Buffer size aligned to 4k
+	 * <--Compressed tile UV Stride--->
+	 * U* V* U* V* U* V* U* V* . . . .  ^
+	 * U* V* U* V* U* V* U* V* . . . .  |
+	 * U* V* U* V* U* V* U* V* . . . .  |
+	 * U* V* U* V* U* V* U* V* . . . .  UV_Scanlines
+	 * . . . . . . . . . . . . . . . .  |
+	 * . . . . . . . . . . . . . . . .  V
+	 * . . . . . . . . . . . . . . . .  -------> Buffer size aligned to 4k
+	 *
+	 *
+	 * Y_Stride = align(Width * 4/3, 128)
+	 * UV_Stride = align(Width * 4/3, 128)
+	 * Y_Scanlines = align(Height, 32)
+	 * UV_Scanlines = align(Height/2, 16)
+	 * Y_UBWC_Plane_Size = align(Y_Stride * Y_Scanlines, 4096)
+	 * UV_UBWC_Plane_Size = align(UV_Stride * UV_Scanlines, 4096)
+	 * Y_Meta_Stride = align(roundup(Width, Y_TileWidth), 64)
+	 * Y_Meta_Scanlines = align(roundup(Height, Y_TileHeight), 16)
+	 * Y_Meta_Plane_size = align(Y_Meta_Stride * Y_Meta_Scanlines, 4096)
+	 * UV_Meta_Stride = align(roundup(Width, UV_TileWidth), 64)
+	 * UV_Meta_Scanlines = align(roundup(Height, UV_TileHeight), 16)
+	 * UV_Meta_Plane_size = align(UV_Meta_Stride * UV_Meta_Scanlines, 4096)
+	 * Extradata = 8k
+	 *
+	 * Total size = align(Y_UBWC_Plane_size + UV_UBWC_Plane_size +
+	 *           Y_Meta_Plane_size + UV_Meta_Plane_size
+	 *           + max(Extradata, Y_Stride * 48), 4096)
+	 */
+	COLOR_FMT_NV12_BPP10_UBWC,
+	/* Venus RGBA8888 format:
+	 * Contains 1 plane in the following order -
+	 * (A) RGBA plane
+	 *
+	 * <-------- RGB_Stride -------->
+	 * <------- Width ------->
+	 * R R R R R R R R R R R R . . . .  ^           ^
+	 * R R R R R R R R R R R R . . . .  |           |
+	 * R R R R R R R R R R R R . . . .  Height      |
+	 * R R R R R R R R R R R R . . . .  |       RGB_Scanlines
+	 * R R R R R R R R R R R R . . . .  |           |
+	 * R R R R R R R R R R R R . . . .  |           |
+	 * R R R R R R R R R R R R . . . .  |           |
+	 * R R R R R R R R R R R R . . . .  V           |
+	 * . . . . . . . . . . . . . . . .              |
+	 * . . . . . . . . . . . . . . . .              |
+	 * . . . . . . . . . . . . . . . .              |
+	 * . . . . . . . . . . . . . . . .              V
+	 *
+	 * RGB_Stride = align(Width * 4, 128)
+	 * RGB_Scanlines = align(Height, 32)
+	 * RGB_Plane_size = align(RGB_Stride * RGB_Scanlines, 4096)
+	 * Extradata = 8k
+	 *
+	 * Total size = align(RGB_Plane_size + Extradata, 4096)
+	 */
+	COLOR_FMT_RGBA8888,
+	/* Venus RGBA8888 UBWC format:
+	 * Contains 2 planes in the following order -
+	 * (A) Meta plane
+	 * (B) RGBA plane
+	 *
+	 * <--- RGB_Meta_Stride ---->
+	 * <-------- Width ------>
+	 * M M M M M M M M M M M M . .      ^           ^
+	 * M M M M M M M M M M M M . .      |           |
+	 * M M M M M M M M M M M M . .      Height      |
+	 * M M M M M M M M M M M M . .      |       Meta_RGB_Scanlines
+	 * M M M M M M M M M M M M . .      |           |
+	 * M M M M M M M M M M M M . .      |           |
+	 * M M M M M M M M M M M M . .      |           |
+	 * M M M M M M M M M M M M . .      V           |
+	 * . . . . . . . . . . . . . .                  |
+	 * . . . . . . . . . . . . . .                  |
+	 * . . . . . . . . . . . . . .      -------> Buffer size aligned to 4k
+	 * . . . . . . . . . . . . . .                  V
+	 * <-------- RGB_Stride -------->
+	 * <------- Width ------->
+	 * R R R R R R R R R R R R . . . .  ^           ^
+	 * R R R R R R R R R R R R . . . .  |           |
+	 * R R R R R R R R R R R R . . . .  Height      |
+	 * R R R R R R R R R R R R . . . .  |       RGB_Scanlines
+	 * R R R R R R R R R R R R . . . .  |           |
+	 * R R R R R R R R R R R R . . . .  |           |
+	 * R R R R R R R R R R R R . . . .  |           |
+	 * R R R R R R R R R R R R . . . .  V           |
+	 * . . . . . . . . . . . . . . . .              |
+	 * . . . . . . . . . . . . . . . .              |
+	 * . . . . . . . . . . . . . . . .    -------> Buffer size aligned to 4k
+	 * . . . . . . . . . . . . . . . .              V
+	 *
+	 * RGB_Stride = align(Width * 4, 128)
+	 * RGB_Scanlines = align(Height, 32)
+	 * RGB_Plane_size = align(RGB_Stride * RGB_Scanlines, 4096)
+	 * RGB_Meta_Stride = align(roundup(Width, RGB_TileWidth), 64)
+	 * RGB_Meta_Scanline = align(roundup(Height, RGB_TileHeight), 16)
+	 * RGB_Meta_Plane_size = align(RGB_Meta_Stride *
+	 *		RGB_Meta_Scanlines, 4096)
+	 * Extradata = 8k
+	 *
+	 * Total size = align(RGB_Meta_Plane_size + RGB_Plane_size +
+	 *		Extradata, 4096)
+	 */
+	COLOR_FMT_RGBA8888_UBWC,
+	/* Venus RGBA1010102 UBWC format:
+	 * Contains 2 planes in the following order -
+	 * (A) Meta plane
+	 * (B) RGBA plane
+	 *
+	 * <--- RGB_Meta_Stride ---->
+	 * <-------- Width ------>
+	 * M M M M M M M M M M M M . .      ^           ^
+	 * M M M M M M M M M M M M . .      |           |
+	 * M M M M M M M M M M M M . .      Height      |
+	 * M M M M M M M M M M M M . .      |       Meta_RGB_Scanlines
+	 * M M M M M M M M M M M M . .      |           |
+	 * M M M M M M M M M M M M . .      |           |
+	 * M M M M M M M M M M M M . .      |           |
+	 * M M M M M M M M M M M M . .      V           |
+	 * . . . . . . . . . . . . . .                  |
+	 * . . . . . . . . . . . . . .                  |
+	 * . . . . . . . . . . . . . .      -------> Buffer size aligned to 4k
+	 * . . . . . . . . . . . . . .                  V
+	 * <-------- RGB_Stride -------->
+	 * <------- Width ------->
+	 * R R R R R R R R R R R R . . . .  ^           ^
+	 * R R R R R R R R R R R R . . . .  |           |
+	 * R R R R R R R R R R R R . . . .  Height      |
+	 * R R R R R R R R R R R R . . . .  |       RGB_Scanlines
+	 * R R R R R R R R R R R R . . . .  |           |
+	 * R R R R R R R R R R R R . . . .  |           |
+	 * R R R R R R R R R R R R . . . .  |           |
+	 * R R R R R R R R R R R R . . . .  V           |
+	 * . . . . . . . . . . . . . . . .              |
+	 * . . . . . . . . . . . . . . . .              |
+	 * . . . . . . . . . . . . . . . .    -------> Buffer size aligned to 4k
+	 * . . . . . . . . . . . . . . . .              V
+	 *
+	 * RGB_Stride = align(Width * 4, 256)
+	 * RGB_Scanlines = align(Height, 16)
+	 * RGB_Plane_size = align(RGB_Stride * RGB_Scanlines, 4096)
+	 * RGB_Meta_Stride = align(roundup(Width, RGB_TileWidth), 64)
+	 * RGB_Meta_Scanline = align(roundup(Height, RGB_TileHeight), 16)
+	 * RGB_Meta_Plane_size = align(RGB_Meta_Stride *
+	 *		RGB_Meta_Scanlines, 4096)
+	 * Extradata = 8k
+	 *
+	 * Total size = align(RGB_Meta_Plane_size + RGB_Plane_size +
+	 *		Extradata, 4096)
+	 */
+	COLOR_FMT_RGBA1010102_UBWC,
+	/* Venus RGB565 UBWC format:
+	 * Contains 2 planes in the following order -
+	 * (A) Meta plane
+	 * (B) RGB plane
+	 *
+	 * <--- RGB_Meta_Stride ---->
+	 * <-------- Width ------>
+	 * M M M M M M M M M M M M . .      ^           ^
+	 * M M M M M M M M M M M M . .      |           |
+	 * M M M M M M M M M M M M . .      Height      |
+	 * M M M M M M M M M M M M . .      |       Meta_RGB_Scanlines
+	 * M M M M M M M M M M M M . .      |           |
+	 * M M M M M M M M M M M M . .      |           |
+	 * M M M M M M M M M M M M . .      |           |
+	 * M M M M M M M M M M M M . .      V           |
+	 * . . . . . . . . . . . . . .                  |
+	 * . . . . . . . . . . . . . .                  |
+	 * . . . . . . . . . . . . . .      -------> Buffer size aligned to 4k
+	 * . . . . . . . . . . . . . .                  V
+	 * <-------- RGB_Stride -------->
+	 * <------- Width ------->
+	 * R R R R R R R R R R R R . . . .  ^           ^
+	 * R R R R R R R R R R R R . . . .  |           |
+	 * R R R R R R R R R R R R . . . .  Height      |
+	 * R R R R R R R R R R R R . . . .  |       RGB_Scanlines
+	 * R R R R R R R R R R R R . . . .  |           |
+	 * R R R R R R R R R R R R . . . .  |           |
+	 * R R R R R R R R R R R R . . . .  |           |
+	 * R R R R R R R R R R R R . . . .  V           |
+	 * . . . . . . . . . . . . . . . .              |
+	 * . . . . . . . . . . . . . . . .              |
+	 * . . . . . . . . . . . . . . . .    -------> Buffer size aligned to 4k
+	 * . . . . . . . . . . . . . . . .              V
+	 *
+	 * RGB_Stride = align(Width * 2, 128)
+	 * RGB_Scanlines = align(Height, 16)
+	 * RGB_Plane_size = align(RGB_Stride * RGB_Scanlines, 4096)
+	 * RGB_Meta_Stride = align(roundup(Width, RGB_TileWidth), 64)
+	 * RGB_Meta_Scanline = align(roundup(Height, RGB_TileHeight), 16)
+	 * RGB_Meta_Plane_size = align(RGB_Meta_Stride *
+	 *		RGB_Meta_Scanlines, 4096)
+	 * Extradata = 8k
+	 *
+	 * Total size = align(RGB_Meta_Plane_size + RGB_Plane_size +
+	 *		Extradata, 4096)
+	 */
+	COLOR_FMT_RGB565_UBWC,
+	/* P010 UBWC:
+	 * Compressed Macro-tile format for NV12.
+	 * Contains 4 planes in the following order -
+	 * (A) Y_Meta_Plane
+	 * (B) Y_UBWC_Plane
+	 * (C) UV_Meta_Plane
+	 * (D) UV_UBWC_Plane
+	 *
+	 * Y_Meta_Plane consists of meta information to decode compressed
+	 * tile data in Y_UBWC_Plane.
+	 * Y_UBWC_Plane consists of Y data in compressed macro-tile format.
+	 * UBWC decoder block will use the Y_Meta_Plane data together with
+	 * Y_UBWC_Plane data to produce loss-less uncompressed 10 bit Y samples.
+	 *
+	 * UV_Meta_Plane consists of meta information to decode compressed
+	 * tile data in UV_UBWC_Plane.
+	 * UV_UBWC_Plane consists of UV data in compressed macro-tile format.
+	 * UBWC decoder block will use UV_Meta_Plane data together with
+	 * UV_UBWC_Plane data to produce loss-less uncompressed 10 bit 2x2
+	 * subsampled color difference samples.
+	 *
+	 * Each tile in Y_UBWC_Plane/UV_UBWC_Plane is independently decodable
+	 * and randomly accessible. There is no dependency between tiles.
+	 *
+	 * <----- Y_Meta_Stride ----->
+	 * <-------- Width ------>
+	 * M M M M M M M M M M M M . .      ^           ^
+	 * M M M M M M M M M M M M . .      |           |
+	 * M M M M M M M M M M M M . .      Height      |
+	 * M M M M M M M M M M M M . .      |         Meta_Y_Scanlines
+	 * M M M M M M M M M M M M . .      |           |
+	 * M M M M M M M M M M M M . .      |           |
+	 * M M M M M M M M M M M M . .      |           |
+	 * M M M M M M M M M M M M . .      V           |
+	 * . . . . . . . . . . . . . .                  |
+	 * . . . . . . . . . . . . . .                  |
+	 * . . . . . . . . . . . . . .      -------> Buffer size aligned to 4k
+	 * . . . . . . . . . . . . . .                  V
+	 * <--Compressed tile Y Stride--->
+	 * <------- Width ------->
+	 * Y* Y* Y* Y* Y* Y* Y* Y* . . . .  ^           ^
+	 * Y* Y* Y* Y* Y* Y* Y* Y* . . . .  |           |
+	 * Y* Y* Y* Y* Y* Y* Y* Y* . . . .  Height      |
+	 * Y* Y* Y* Y* Y* Y* Y* Y* . . . .  |        Macro_tile_Y_Scanlines
+	 * Y* Y* Y* Y* Y* Y* Y* Y* . . . .  |           |
+	 * Y* Y* Y* Y* Y* Y* Y* Y* . . . .  |           |
+	 * Y* Y* Y* Y* Y* Y* Y* Y* . . . .  |           |
+	 * Y* Y* Y* Y* Y* Y* Y* Y* . . . .  V           |
+	 * . . . . . . . . . . . . . . . .              |
+	 * . . . . . . . . . . . . . . . .              |
+	 * . . . . . . . . . . . . . . . .  -------> Buffer size aligned to 4k
+	 * . . . . . . . . . . . . . . . .              V
+	 * <----- UV_Meta_Stride ---->
+	 * M M M M M M M M M M M M . .      ^
+	 * M M M M M M M M M M M M . .      |
+	 * M M M M M M M M M M M M . .      |
+	 * M M M M M M M M M M M M . .      M_UV_Scanlines
+	 * . . . . . . . . . . . . . .      |
+	 * . . . . . . . . . . . . . .      V
+	 * . . . . . . . . . . . . . .      -------> Buffer size aligned to 4k
+	 * <--Compressed tile UV Stride--->
+	 * U* V* U* V* U* V* U* V* . . . .  ^
+	 * U* V* U* V* U* V* U* V* . . . .  |
+	 * U* V* U* V* U* V* U* V* . . . .  |
+	 * U* V* U* V* U* V* U* V* . . . .  UV_Scanlines
+	 * . . . . . . . . . . . . . . . .  |
+	 * . . . . . . . . . . . . . . . .  V
+	 * . . . . . . . . . . . . . . . .  -------> Buffer size aligned to 4k
+	 *
+	 *
+	 * Y_Stride = align(Width * 2, 256)
+	 * UV_Stride = align(Width * 2, 256)
+	 * Y_Scanlines = align(Height, 16)
+	 * UV_Scanlines = align(Height/2, 16)
+	 * Y_UBWC_Plane_Size = align(Y_Stride * Y_Scanlines, 4096)
+	 * UV_UBWC_Plane_Size = align(UV_Stride * UV_Scanlines, 4096)
+	 * Y_Meta_Stride = align(roundup(Width, Y_TileWidth), 64)
+	 * Y_Meta_Scanlines = align(roundup(Height, Y_TileHeight), 16)
+	 * Y_Meta_Plane_size = align(Y_Meta_Stride * Y_Meta_Scanlines, 4096)
+	 * UV_Meta_Stride = align(roundup(Width, UV_TileWidth), 64)
+	 * UV_Meta_Scanlines = align(roundup(Height, UV_TileHeight), 16)
+	 * UV_Meta_Plane_size = align(UV_Meta_Stride * UV_Meta_Scanlines, 4096)
+	 * Extradata = 8k
+	 *
+	 * Total size = align(Y_UBWC_Plane_size + UV_UBWC_Plane_size +
+	 *           Y_Meta_Plane_size + UV_Meta_Plane_size
+	 *           + max(Extradata, Y_Stride * 48), 4096)
+	 */
+	COLOR_FMT_P010_UBWC,
+	/* Venus P010:
+	 * YUV 4:2:0 image with a plane of 10 bit Y samples followed
+	 * by an interleaved U/V plane containing 10 bit 2x2 subsampled
+	 * colour difference samples.
+	 *
+	 * <-------- Y/UV_Stride -------->
+	 * <------- Width ------->
+	 * Y Y Y Y Y Y Y Y Y Y Y Y . . . .  ^           ^
+	 * Y Y Y Y Y Y Y Y Y Y Y Y . . . .  |           |
+	 * Y Y Y Y Y Y Y Y Y Y Y Y . . . .  Height      |
+	 * Y Y Y Y Y Y Y Y Y Y Y Y . . . .  |          Y_Scanlines
+	 * Y Y Y Y Y Y Y Y Y Y Y Y . . . .  |           |
+	 * Y Y Y Y Y Y Y Y Y Y Y Y . . . .  |           |
+	 * Y Y Y Y Y Y Y Y Y Y Y Y . . . .  |           |
+	 * Y Y Y Y Y Y Y Y Y Y Y Y . . . .  V           |
+	 * . . . . . . . . . . . . . . . .              |
+	 * . . . . . . . . . . . . . . . .              |
+	 * . . . . . . . . . . . . . . . .              |
+	 * . . . . . . . . . . . . . . . .              V
+	 * U V U V U V U V U V U V . . . .  ^
+	 * U V U V U V U V U V U V . . . .  |
+	 * U V U V U V U V U V U V . . . .  |
+	 * U V U V U V U V U V U V . . . .  UV_Scanlines
+	 * . . . . . . . . . . . . . . . .  |
+	 * . . . . . . . . . . . . . . . .  V
+	 * . . . . . . . . . . . . . . . .  --> Buffer size alignment
+	 *
+	 * Y_Stride : Width * 2 aligned to 128
+	 * UV_Stride : Width * 2 aligned to 128
+	 * Y_Scanlines: Height aligned to 32
+	 * UV_Scanlines: Height/2 aligned to 16
+	 * Extradata: Arbitrary (software-imposed) padding
+	 * Total size = align((Y_Stride * Y_Scanlines
+	 *          + UV_Stride * UV_Scanlines
+	 *          + max(Extradata, Y_Stride * 8), 4096)
+	 */
+	COLOR_FMT_P010,
+};
+
+#define COLOR_FMT_RGBA1010102_UBWC	COLOR_FMT_RGBA1010102_UBWC
+#define COLOR_FMT_RGB565_UBWC		COLOR_FMT_RGB565_UBWC
+#define COLOR_FMT_P010_UBWC		COLOR_FMT_P010_UBWC
+#define COLOR_FMT_P010		COLOR_FMT_P010
+
+static inline unsigned int VENUS_EXTRADATA_SIZE(int width, int height)
+{
+	(void)height;
+	(void)width;
+
+	/*
+	 * In the future, calculate the size based on the w/h but just
+	 * hardcode it for now since 16K satisfies all current usecases.
+	 */
+	return 16 * 1024;
+}
+
+/*
+ * Function arguments:
+ * @color_fmt
+ * @width
+ * Progressive: width
+ * Interlaced: width
+ */
+static inline unsigned int VENUS_Y_STRIDE(int color_fmt, int width)
+{
+	unsigned int alignment, stride = 0;
+
+	if (!width)
+		goto invalid_input;
+
+	switch (color_fmt) {
+	case COLOR_FMT_NV21:
+	case COLOR_FMT_NV12:
+	case COLOR_FMT_NV12_MVTB:
+	case COLOR_FMT_NV12_UBWC:
+		alignment = 128;
+		stride = MSM_MEDIA_ALIGN(width, alignment);
+		break;
+	case COLOR_FMT_NV12_BPP10_UBWC:
+		alignment = 256;
+		stride = MSM_MEDIA_ALIGN(width, 192);
+		stride = MSM_MEDIA_ALIGN(stride * 4/3, alignment);
+		break;
+	case COLOR_FMT_P010_UBWC:
+		alignment = 256;
+		stride = MSM_MEDIA_ALIGN(width * 2, alignment);
+		break;
+	case COLOR_FMT_P010:
+		alignment = 128;
+		stride = MSM_MEDIA_ALIGN(width*2, alignment);
+		break;
+	default:
+		break;
+	}
+invalid_input:
+	return stride;
+}
+
+/*
+ * Function arguments:
+ * @color_fmt
+ * @width
+ * Progressive: width
+ * Interlaced: width
+ */
+static inline unsigned int VENUS_UV_STRIDE(int color_fmt, int width)
+{
+	unsigned int alignment, stride = 0;
+
+	if (!width)
+		goto invalid_input;
+
+	switch (color_fmt) {
+	case COLOR_FMT_NV21:
+	case COLOR_FMT_NV12:
+	case COLOR_FMT_NV12_MVTB:
+	case COLOR_FMT_NV12_UBWC:
+		alignment = 128;
+		stride = MSM_MEDIA_ALIGN(width, alignment);
+		break;
+	case COLOR_FMT_NV12_BPP10_UBWC:
+		alignment = 256;
+		stride = MSM_MEDIA_ALIGN(width, 192);
+		stride = MSM_MEDIA_ALIGN(stride * 4/3, alignment);
+		break;
+	case COLOR_FMT_P010_UBWC:
+		alignment = 256;
+		stride = MSM_MEDIA_ALIGN(width * 2, alignment);
+		break;
+	case COLOR_FMT_P010:
+		alignment = 128;
+		stride = MSM_MEDIA_ALIGN(width*2, alignment);
+		break;
+	default:
+		break;
+	}
+invalid_input:
+	return stride;
+}
+
+/*
+ * Function arguments:
+ * @color_fmt
+ * @height
+ * Progressive: height
+ * Interlaced: (height+1)>>1
+ */
+static inline unsigned int VENUS_Y_SCANLINES(int color_fmt, int height)
+{
+	unsigned int alignment, sclines = 0;
+
+	if (!height)
+		goto invalid_input;
+
+	switch (color_fmt) {
+	case COLOR_FMT_NV21:
+	case COLOR_FMT_NV12:
+	case COLOR_FMT_NV12_MVTB:
+	case COLOR_FMT_NV12_UBWC:
+	case COLOR_FMT_P010:
+		alignment = 32;
+		break;
+	case COLOR_FMT_NV12_BPP10_UBWC:
+	case COLOR_FMT_P010_UBWC:
+		alignment = 16;
+		break;
+	default:
+		return 0;
+	}
+	sclines = MSM_MEDIA_ALIGN(height, alignment);
+invalid_input:
+	return sclines;
+}
+
+/*
+ * Function arguments:
+ * @color_fmt
+ * @height
+ * Progressive: height
+ * Interlaced: (height+1)>>1
+ */
+static inline unsigned int VENUS_UV_SCANLINES(int color_fmt, int height)
+{
+	unsigned int alignment, sclines = 0;
+
+	if (!height)
+		goto invalid_input;
+
+	switch (color_fmt) {
+	case COLOR_FMT_NV21:
+	case COLOR_FMT_NV12:
+	case COLOR_FMT_NV12_MVTB:
+	case COLOR_FMT_NV12_BPP10_UBWC:
+	case COLOR_FMT_P010_UBWC:
+	case COLOR_FMT_P010:
+		alignment = 16;
+		break;
+	case COLOR_FMT_NV12_UBWC:
+		alignment = 32;
+		break;
+	default:
+		goto invalid_input;
+	}
+
+	sclines = MSM_MEDIA_ALIGN((height+1)>>1, alignment);
+
+invalid_input:
+	return sclines;
+}
+
+/*
+ * Function arguments:
+ * @color_fmt
+ * @width
+ * Progressive: width
+ * Interlaced: width
+ */
+static inline unsigned int VENUS_Y_META_STRIDE(int color_fmt, int width)
+{
+	int y_tile_width = 0, y_meta_stride = 0;
+
+	if (!width)
+		goto invalid_input;
+
+	switch (color_fmt) {
+	case COLOR_FMT_NV12_UBWC:
+	case COLOR_FMT_P010_UBWC:
+		y_tile_width = 32;
+		break;
+	case COLOR_FMT_NV12_BPP10_UBWC:
+		y_tile_width = 48;
+		break;
+	default:
+		goto invalid_input;
+	}
+
+	y_meta_stride = MSM_MEDIA_ROUNDUP(width, y_tile_width);
+	y_meta_stride = MSM_MEDIA_ALIGN(y_meta_stride, 64);
+
+invalid_input:
+	return y_meta_stride;
+}
+
+/*
+ * Function arguments:
+ * @color_fmt
+ * @height
+ * Progressive: height
+ * Interlaced: (height+1)>>1
+ */
+static inline unsigned int VENUS_Y_META_SCANLINES(int color_fmt, int height)
+{
+	int y_tile_height = 0, y_meta_scanlines = 0;
+
+	if (!height)
+		goto invalid_input;
+
+	switch (color_fmt) {
+	case COLOR_FMT_NV12_UBWC:
+		y_tile_height = 8;
+		break;
+	case COLOR_FMT_NV12_BPP10_UBWC:
+	case COLOR_FMT_P010_UBWC:
+		y_tile_height = 4;
+		break;
+	default:
+		goto invalid_input;
+	}
+
+	y_meta_scanlines = MSM_MEDIA_ROUNDUP(height, y_tile_height);
+	y_meta_scanlines = MSM_MEDIA_ALIGN(y_meta_scanlines, 16);
+
+invalid_input:
+	return y_meta_scanlines;
+}
+
+/*
+ * Function arguments:
+ * @color_fmt
+ * @width
+ * Progressive: width
+ * Interlaced: width
+ */
+static inline unsigned int VENUS_UV_META_STRIDE(int color_fmt, int width)
+{
+	int uv_tile_width = 0, uv_meta_stride = 0;
+
+	if (!width)
+		goto invalid_input;
+
+	switch (color_fmt) {
+	case COLOR_FMT_NV12_UBWC:
+	case COLOR_FMT_P010_UBWC:
+		uv_tile_width = 16;
+		break;
+	case COLOR_FMT_NV12_BPP10_UBWC:
+		uv_tile_width = 24;
+		break;
+	default:
+		goto invalid_input;
+	}
+
+	uv_meta_stride = MSM_MEDIA_ROUNDUP((width+1)>>1, uv_tile_width);
+	uv_meta_stride = MSM_MEDIA_ALIGN(uv_meta_stride, 64);
+
+invalid_input:
+	return uv_meta_stride;
+}
+
+/*
+ * Function arguments:
+ * @color_fmt
+ * @height
+ * Progressive: height
+ * Interlaced: (height+1)>>1
+ */
+static inline unsigned int VENUS_UV_META_SCANLINES(int color_fmt, int height)
+{
+	int uv_tile_height = 0, uv_meta_scanlines = 0;
+
+	if (!height)
+		goto invalid_input;
+
+	switch (color_fmt) {
+	case COLOR_FMT_NV12_UBWC:
+		uv_tile_height = 8;
+		break;
+	case COLOR_FMT_NV12_BPP10_UBWC:
+	case COLOR_FMT_P010_UBWC:
+		uv_tile_height = 4;
+		break;
+	default:
+		goto invalid_input;
+	}
+
+	uv_meta_scanlines = MSM_MEDIA_ROUNDUP((height+1)>>1, uv_tile_height);
+	uv_meta_scanlines = MSM_MEDIA_ALIGN(uv_meta_scanlines, 16);
+
+invalid_input:
+	return uv_meta_scanlines;
+}
+
+static inline unsigned int VENUS_RGB_STRIDE(int color_fmt, int width)
+{
+	unsigned int alignment = 0, stride = 0, bpp = 4;
+
+	if (!width)
+		goto invalid_input;
+
+	switch (color_fmt) {
+	case COLOR_FMT_RGBA8888:
+		alignment = 128;
+		break;
+	case COLOR_FMT_RGB565_UBWC:
+		alignment = 256;
+		bpp = 2;
+		break;
+	case COLOR_FMT_RGBA8888_UBWC:
+	case COLOR_FMT_RGBA1010102_UBWC:
+		alignment = 256;
+		break;
+	default:
+		goto invalid_input;
+	}
+
+	stride = MSM_MEDIA_ALIGN(width * bpp, alignment);
+
+invalid_input:
+	return stride;
+}
+
+static inline unsigned int VENUS_RGB_SCANLINES(int color_fmt, int height)
+{
+	unsigned int alignment = 0, scanlines = 0;
+
+	if (!height)
+		goto invalid_input;
+
+	switch (color_fmt) {
+	case COLOR_FMT_RGBA8888:
+		alignment = 32;
+		break;
+	case COLOR_FMT_RGBA8888_UBWC:
+	case COLOR_FMT_RGBA1010102_UBWC:
+	case COLOR_FMT_RGB565_UBWC:
+		alignment = 16;
+		break;
+	default:
+		goto invalid_input;
+	}
+
+	scanlines = MSM_MEDIA_ALIGN(height, alignment);
+
+invalid_input:
+	return scanlines;
+}
+
+static inline unsigned int VENUS_RGB_META_STRIDE(int color_fmt, int width)
+{
+	int rgb_tile_width = 0, rgb_meta_stride = 0;
+
+	if (!width)
+		goto invalid_input;
+
+	switch (color_fmt) {
+	case COLOR_FMT_RGBA8888_UBWC:
+	case COLOR_FMT_RGBA1010102_UBWC:
+	case COLOR_FMT_RGB565_UBWC:
+		rgb_tile_width = 16;
+		break;
+	default:
+		goto invalid_input;
+	}
+
+	rgb_meta_stride = MSM_MEDIA_ROUNDUP(width, rgb_tile_width);
+	rgb_meta_stride = MSM_MEDIA_ALIGN(rgb_meta_stride, 64);
+
+invalid_input:
+	return rgb_meta_stride;
+}
+
+static inline unsigned int VENUS_RGB_META_SCANLINES(int color_fmt, int height)
+{
+	int rgb_tile_height = 0, rgb_meta_scanlines = 0;
+
+	if (!height)
+		goto invalid_input;
+
+	switch (color_fmt) {
+	case COLOR_FMT_RGBA8888_UBWC:
+	case COLOR_FMT_RGBA1010102_UBWC:
+	case COLOR_FMT_RGB565_UBWC:
+		rgb_tile_height = 4;
+		break;
+	default:
+		goto invalid_input;
+	}
+
+	rgb_meta_scanlines = MSM_MEDIA_ROUNDUP(height, rgb_tile_height);
+	rgb_meta_scanlines = MSM_MEDIA_ALIGN(rgb_meta_scanlines, 16);
+
+invalid_input:
+	return rgb_meta_scanlines;
+}
+
+/*
+ * Function arguments:
+ * @color_fmt
+ * @width
+ * Progressive: width
+ * Interlaced: width
+ * @height
+ * Progressive: height
+ * Interlaced: height
+ */
+static inline unsigned int VENUS_BUFFER_SIZE(
+	int color_fmt, int width, int height)
+{
+	const unsigned int extra_size = VENUS_EXTRADATA_SIZE(width, height);
+	unsigned int uv_alignment = 0, size = 0;
+	unsigned int y_plane, uv_plane, y_stride,
+		uv_stride, y_sclines, uv_sclines;
+	unsigned int y_ubwc_plane = 0, uv_ubwc_plane = 0;
+	unsigned int y_meta_stride = 0, y_meta_scanlines = 0;
+	unsigned int uv_meta_stride = 0, uv_meta_scanlines = 0;
+	unsigned int y_meta_plane = 0, uv_meta_plane = 0;
+	unsigned int rgb_stride = 0, rgb_scanlines = 0;
+	unsigned int rgb_plane = 0, rgb_ubwc_plane = 0, rgb_meta_plane = 0;
+	unsigned int rgb_meta_stride = 0, rgb_meta_scanlines = 0;
+
+	if (!width || !height)
+		goto invalid_input;
+
+	y_stride = VENUS_Y_STRIDE(color_fmt, width);
+	uv_stride = VENUS_UV_STRIDE(color_fmt, width);
+	y_sclines = VENUS_Y_SCANLINES(color_fmt, height);
+	uv_sclines = VENUS_UV_SCANLINES(color_fmt, height);
+	rgb_stride = VENUS_RGB_STRIDE(color_fmt, width);
+	rgb_scanlines = VENUS_RGB_SCANLINES(color_fmt, height);
+
+	switch (color_fmt) {
+	case COLOR_FMT_NV21:
+	case COLOR_FMT_NV12:
+	case COLOR_FMT_P010:
+		uv_alignment = 4096;
+		y_plane = y_stride * y_sclines;
+		uv_plane = uv_stride * uv_sclines + uv_alignment;
+		size = y_plane + uv_plane +
+				MSM_MEDIA_MAX(extra_size, 8 * y_stride);
+		size = MSM_MEDIA_ALIGN(size, 4096);
+		break;
+	case COLOR_FMT_NV12_MVTB:
+		uv_alignment = 4096;
+		y_plane = y_stride * y_sclines;
+		uv_plane = uv_stride * uv_sclines + uv_alignment;
+		size = y_plane + uv_plane;
+		size = 2 * size + extra_size;
+		size = MSM_MEDIA_ALIGN(size, 4096);
+		break;
+	case COLOR_FMT_NV12_UBWC:
+		y_sclines = VENUS_Y_SCANLINES(color_fmt, (height+1)>>1);
+		y_ubwc_plane = MSM_MEDIA_ALIGN(y_stride * y_sclines, 4096);
+		uv_sclines = VENUS_UV_SCANLINES(color_fmt, (height+1)>>1);
+		uv_ubwc_plane = MSM_MEDIA_ALIGN(uv_stride * uv_sclines, 4096);
+		y_meta_stride = VENUS_Y_META_STRIDE(color_fmt, width);
+		y_meta_scanlines =
+			VENUS_Y_META_SCANLINES(color_fmt, (height+1)>>1);
+		y_meta_plane = MSM_MEDIA_ALIGN(
+			y_meta_stride * y_meta_scanlines, 4096);
+		uv_meta_stride = VENUS_UV_META_STRIDE(color_fmt, width);
+		uv_meta_scanlines =
+			VENUS_UV_META_SCANLINES(color_fmt, (height+1)>>1);
+		uv_meta_plane = MSM_MEDIA_ALIGN(uv_meta_stride *
+			uv_meta_scanlines, 4096);
+
+		size = (y_ubwc_plane + uv_ubwc_plane + y_meta_plane +
+			uv_meta_plane)*2 +
+			MSM_MEDIA_MAX(extra_size + 8192, 48 * y_stride);
+		size = MSM_MEDIA_ALIGN(size, 4096);
+		break;
+	case COLOR_FMT_NV12_BPP10_UBWC:
+		y_ubwc_plane = MSM_MEDIA_ALIGN(y_stride * y_sclines, 4096);
+		uv_ubwc_plane = MSM_MEDIA_ALIGN(uv_stride * uv_sclines, 4096);
+		y_meta_stride = VENUS_Y_META_STRIDE(color_fmt, width);
+		y_meta_scanlines = VENUS_Y_META_SCANLINES(color_fmt, height);
+		y_meta_plane = MSM_MEDIA_ALIGN(
+				y_meta_stride * y_meta_scanlines, 4096);
+		uv_meta_stride = VENUS_UV_META_STRIDE(color_fmt, width);
+		uv_meta_scanlines = VENUS_UV_META_SCANLINES(color_fmt, height);
+		uv_meta_plane = MSM_MEDIA_ALIGN(uv_meta_stride *
+					uv_meta_scanlines, 4096);
+
+		size = y_ubwc_plane + uv_ubwc_plane + y_meta_plane +
+			uv_meta_plane +
+			MSM_MEDIA_MAX(extra_size + 8192, 48 * y_stride);
+		size = MSM_MEDIA_ALIGN(size, 4096);
+		break;
+	case COLOR_FMT_P010_UBWC:
+		y_ubwc_plane = MSM_MEDIA_ALIGN(y_stride * y_sclines, 4096);
+		uv_ubwc_plane = MSM_MEDIA_ALIGN(uv_stride * uv_sclines, 4096);
+		y_meta_stride = VENUS_Y_META_STRIDE(color_fmt, width);
+		y_meta_scanlines = VENUS_Y_META_SCANLINES(color_fmt, height);
+		y_meta_plane = MSM_MEDIA_ALIGN(
+				y_meta_stride * y_meta_scanlines, 4096);
+		uv_meta_stride = VENUS_UV_META_STRIDE(color_fmt, width);
+		uv_meta_scanlines = VENUS_UV_META_SCANLINES(color_fmt, height);
+		uv_meta_plane = MSM_MEDIA_ALIGN(uv_meta_stride *
+					uv_meta_scanlines, 4096);
+
+		size = y_ubwc_plane + uv_ubwc_plane + y_meta_plane +
+			uv_meta_plane;
+		size = MSM_MEDIA_ALIGN(size, 4096);
+		break;
+	case COLOR_FMT_RGBA8888:
+		rgb_plane = MSM_MEDIA_ALIGN(rgb_stride  * rgb_scanlines, 4096);
+		size = rgb_plane;
+		size =  MSM_MEDIA_ALIGN(size, 4096);
+		break;
+	case COLOR_FMT_RGBA8888_UBWC:
+	case COLOR_FMT_RGBA1010102_UBWC:
+	case COLOR_FMT_RGB565_UBWC:
+		rgb_ubwc_plane = MSM_MEDIA_ALIGN(rgb_stride * rgb_scanlines,
+							4096);
+		rgb_meta_stride = VENUS_RGB_META_STRIDE(color_fmt, width);
+		rgb_meta_scanlines = VENUS_RGB_META_SCANLINES(color_fmt,
+					height);
+		rgb_meta_plane = MSM_MEDIA_ALIGN(rgb_meta_stride *
+					rgb_meta_scanlines, 4096);
+		size = rgb_ubwc_plane + rgb_meta_plane;
+		size = MSM_MEDIA_ALIGN(size, 4096);
+		break;
+	default:
+		break;
+	}
+invalid_input:
+	return size;
+}
+
+static inline unsigned int VENUS_VIEW2_OFFSET(
+	int color_fmt, int width, int height)
+{
+	unsigned int offset = 0;
+	unsigned int y_plane, uv_plane, y_stride,
+		uv_stride, y_sclines, uv_sclines;
+	if (!width || !height)
+		goto invalid_input;
+
+	y_stride = VENUS_Y_STRIDE(color_fmt, width);
+	uv_stride = VENUS_UV_STRIDE(color_fmt, width);
+	y_sclines = VENUS_Y_SCANLINES(color_fmt, height);
+	uv_sclines = VENUS_UV_SCANLINES(color_fmt, height);
+	switch (color_fmt) {
+	case COLOR_FMT_NV12_MVTB:
+		y_plane = y_stride * y_sclines;
+		uv_plane = uv_stride * uv_sclines;
+		offset = y_plane + uv_plane;
+		break;
+	default:
+		break;
+	}
+invalid_input:
+	return offset;
+}
+
+#endif
diff --git a/drivers/gpu/drm/msm/disp/mdp4/mdp4.xml.h b/drivers/gpu/drm/msm/disp/mdp4/mdp4.xml.h
index 576cea30d391..4b36b8954bae 100644
--- a/drivers/gpu/drm/msm/disp/mdp4/mdp4.xml.h
+++ b/drivers/gpu/drm/msm/disp/mdp4/mdp4.xml.h
@@ -8,19 +8,19 @@ http://github.com/freedreno/envytools/
 git clone https://github.com/freedreno/envytools.git
 
 The rules-ng-ng source files this header was generated from are:
-- /home/robclark/src/freedreno/envytools/rnndb/msm.xml                 (    676 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml (   1572 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp4.xml            (  20915 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp_common.xml      (   2849 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp5.xml            (  37411 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/dsi/dsi.xml             (  33004 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/dsi/sfpb.xml            (    602 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/dsi/mmss_cc.xml         (   1686 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/hdmi/qfprom.xml         (    600 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/hdmi/hdmi.xml           (  41799 bytes, from 2017-06-16 12:32:42)
-- /home/robclark/src/freedreno/envytools/rnndb/edp/edp.xml             (  10416 bytes, from 2017-05-17 13:21:27)
-
-Copyright (C) 2013-2017 by the following authors:
+- /home/robclark/src/envytools/rnndb/msm.xml                 (    676 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/freedreno_copyright.xml (   1572 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/mdp/mdp4.xml            (  20915 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/mdp/mdp_common.xml      (   2849 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/mdp/mdp5.xml            (  37411 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/dsi/dsi.xml             (  37239 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/dsi/sfpb.xml            (    602 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/dsi/mmss_cc.xml         (   1686 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/hdmi/qfprom.xml         (    600 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/hdmi/hdmi.xml           (  41799 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/edp/edp.xml             (  10416 bytes, from 2018-07-03 19:37:13)
+
+Copyright (C) 2013-2018 by the following authors:
 - Rob Clark <robdclark@gmail.com> (robclark)
 - Ilia Mirkin <imirkin@alum.mit.edu> (imirkin)
 
diff --git a/drivers/gpu/drm/msm/disp/mdp4/mdp4_kms.c b/drivers/gpu/drm/msm/disp/mdp4/mdp4_kms.c
index 4b646bf9c214..44d1cda56974 100644
--- a/drivers/gpu/drm/msm/disp/mdp4/mdp4_kms.c
+++ b/drivers/gpu/drm/msm/disp/mdp4/mdp4_kms.c
@@ -125,6 +125,8 @@ static void mdp4_complete_commit(struct msm_kms *kms, struct drm_atomic_state *s
 	struct drm_crtc *crtc;
 	struct drm_crtc_state *crtc_state;
 
+	drm_atomic_helper_wait_for_vblanks(mdp4_kms->dev, state);
+
 	/* see 119ecb7fd */
 	for_each_new_crtc_in_state(state, crtc, crtc_state, i)
 		drm_crtc_vblank_put(crtc);
diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5.xml.h b/drivers/gpu/drm/msm/disp/mdp5/mdp5.xml.h
index d9c10e02ee41..784d98989e3a 100644
--- a/drivers/gpu/drm/msm/disp/mdp5/mdp5.xml.h
+++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5.xml.h
@@ -8,19 +8,19 @@ http://github.com/freedreno/envytools/
 git clone https://github.com/freedreno/envytools.git
 
 The rules-ng-ng source files this header was generated from are:
-- /home/robclark/src/freedreno/envytools/rnndb/msm.xml                 (    676 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml (   1572 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp4.xml            (  20915 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp_common.xml      (   2849 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp5.xml            (  37411 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/dsi/dsi.xml             (  33004 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/dsi/sfpb.xml            (    602 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/dsi/mmss_cc.xml         (   1686 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/hdmi/qfprom.xml         (    600 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/hdmi/hdmi.xml           (  41799 bytes, from 2017-06-16 12:32:42)
-- /home/robclark/src/freedreno/envytools/rnndb/edp/edp.xml             (  10416 bytes, from 2017-05-17 13:21:27)
-
-Copyright (C) 2013-2017 by the following authors:
+- /home/robclark/src/envytools/rnndb/msm.xml                 (    676 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/freedreno_copyright.xml (   1572 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/mdp/mdp4.xml            (  20915 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/mdp/mdp_common.xml      (   2849 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/mdp/mdp5.xml            (  37411 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/dsi/dsi.xml             (  37239 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/dsi/sfpb.xml            (    602 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/dsi/mmss_cc.xml         (   1686 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/hdmi/qfprom.xml         (    600 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/hdmi/hdmi.xml           (  41799 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/edp/edp.xml             (  10416 bytes, from 2018-07-03 19:37:13)
+
+Copyright (C) 2013-2018 by the following authors:
 - Rob Clark <robdclark@gmail.com> (robclark)
 - Ilia Mirkin <imirkin@alum.mit.edu> (imirkin)
 
diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_crtc.c b/drivers/gpu/drm/msm/disp/mdp5/mdp5_crtc.c
index 24e00274844b..b1da9ce54379 100644
--- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_crtc.c
+++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_crtc.c
@@ -65,7 +65,7 @@ struct mdp5_crtc {
 		struct drm_gem_object *scanout_bo;
 		uint64_t iova;
 		uint32_t width, height;
-		uint32_t x, y;
+		int x, y;
 	} cursor;
 };
 #define to_mdp5_crtc(x) container_of(x, struct mdp5_crtc, base)
@@ -760,20 +760,31 @@ static void get_roi(struct drm_crtc *crtc, uint32_t *roi_w, uint32_t *roi_h)
 	 * Cursor Region Of Interest (ROI) is a plane read from cursor
 	 * buffer to render. The ROI region is determined by the visibility of
 	 * the cursor point. In the default Cursor image the cursor point will
-	 * be at the top left of the cursor image, unless it is specified
-	 * otherwise using hotspot feature.
+	 * be at the top left of the cursor image.
 	 *
+	 * Without rotation:
 	 * If the cursor point reaches the right (xres - x < cursor.width) or
 	 * bottom (yres - y < cursor.height) boundary of the screen, then ROI
 	 * width and ROI height need to be evaluated to crop the cursor image
 	 * accordingly.
 	 * (xres-x) will be new cursor width when x > (xres - cursor.width)
 	 * (yres-y) will be new cursor height when y > (yres - cursor.height)
+	 *
+	 * With rotation:
+	 * We get negative x and/or y coordinates.
+	 * (cursor.width - abs(x)) will be new cursor width when x < 0
+	 * (cursor.height - abs(y)) will be new cursor width when y < 0
 	 */
-	*roi_w = min(mdp5_crtc->cursor.width, xres -
+	if (mdp5_crtc->cursor.x >= 0)
+		*roi_w = min(mdp5_crtc->cursor.width, xres -
 			mdp5_crtc->cursor.x);
-	*roi_h = min(mdp5_crtc->cursor.height, yres -
+	else
+		*roi_w = mdp5_crtc->cursor.width - abs(mdp5_crtc->cursor.x);
+	if (mdp5_crtc->cursor.y >= 0)
+		*roi_h = min(mdp5_crtc->cursor.height, yres -
 			mdp5_crtc->cursor.y);
+	else
+		*roi_h = mdp5_crtc->cursor.height - abs(mdp5_crtc->cursor.y);
 }
 
 static void mdp5_crtc_restore_cursor(struct drm_crtc *crtc)
@@ -783,7 +794,7 @@ static void mdp5_crtc_restore_cursor(struct drm_crtc *crtc)
 	struct mdp5_kms *mdp5_kms = get_kms(crtc);
 	const enum mdp5_cursor_alpha cur_alpha = CURSOR_ALPHA_PER_PIXEL;
 	uint32_t blendcfg, stride;
-	uint32_t x, y, width, height;
+	uint32_t x, y, src_x, src_y, width, height;
 	uint32_t roi_w, roi_h;
 	int lm;
 
@@ -800,6 +811,26 @@ static void mdp5_crtc_restore_cursor(struct drm_crtc *crtc)
 
 	get_roi(crtc, &roi_w, &roi_h);
 
+	/* If cusror buffer overlaps due to rotation on the
+	 * upper or left screen border the pixel offset inside
+	 * the cursor buffer of the ROI is the positive overlap
+	 * distance.
+	 */
+	if (mdp5_crtc->cursor.x < 0) {
+		src_x = abs(mdp5_crtc->cursor.x);
+		x = 0;
+	} else {
+		src_x = 0;
+	}
+	if (mdp5_crtc->cursor.y < 0) {
+		src_y = abs(mdp5_crtc->cursor.y);
+		y = 0;
+	} else {
+		src_y = 0;
+	}
+	DBG("%s: x=%d, y=%d roi_w=%d roi_h=%d src_x=%d src_y=%d",
+		crtc->name, x, y, roi_w, roi_h, src_x, src_y);
+
 	mdp5_write(mdp5_kms, REG_MDP5_LM_CURSOR_STRIDE(lm), stride);
 	mdp5_write(mdp5_kms, REG_MDP5_LM_CURSOR_FORMAT(lm),
 			MDP5_LM_CURSOR_FORMAT_FORMAT(CURSOR_FMT_ARGB8888));
@@ -812,6 +843,9 @@ static void mdp5_crtc_restore_cursor(struct drm_crtc *crtc)
 	mdp5_write(mdp5_kms, REG_MDP5_LM_CURSOR_START_XY(lm),
 			MDP5_LM_CURSOR_START_XY_Y_START(y) |
 			MDP5_LM_CURSOR_START_XY_X_START(x));
+	mdp5_write(mdp5_kms, REG_MDP5_LM_CURSOR_XY(lm),
+			MDP5_LM_CURSOR_XY_SRC_Y(src_y) |
+			MDP5_LM_CURSOR_XY_SRC_X(src_x));
 	mdp5_write(mdp5_kms, REG_MDP5_LM_CURSOR_BASE_ADDR(lm),
 			mdp5_crtc->cursor.iova);
 
@@ -932,8 +966,9 @@ static int mdp5_crtc_cursor_move(struct drm_crtc *crtc, int x, int y)
 	if (unlikely(!crtc->state->enable))
 		return 0;
 
-	mdp5_crtc->cursor.x = x = max(x, 0);
-	mdp5_crtc->cursor.y = y = max(y, 0);
+	/* accept negative x/y coordinates up to maximum cursor overlap */
+	mdp5_crtc->cursor.x = x = max(x, -(int)mdp5_crtc->cursor.width);
+	mdp5_crtc->cursor.y = y = max(y, -(int)mdp5_crtc->cursor.height);
 
 	get_roi(crtc, &roi_w, &roi_h);
 
diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_encoder.c b/drivers/gpu/drm/msm/disp/mdp5/mdp5_encoder.c
index 9af94e35f678..fcd44d1d1068 100644
--- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_encoder.c
+++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_encoder.c
@@ -319,7 +319,17 @@ static int mdp5_encoder_atomic_check(struct drm_encoder *encoder,
 
 	mdp5_cstate->ctl = ctl;
 	mdp5_cstate->pipeline.intf = intf;
-	mdp5_cstate->defer_start = true;
+
+	/*
+	 * This is a bit awkward, but we want to flush the CTL and hit the
+	 * START bit at most once for an atomic update.  In the non-full-
+	 * modeset case, this is done from crtc->atomic_flush(), but that
+	 * is too early in the case of full modeset, in which case we
+	 * defer to encoder->enable().  But we need to *know* whether
+	 * encoder->enable() will be called to do this:
+	 */
+	if (drm_atomic_crtc_needs_modeset(crtc_state))
+		mdp5_cstate->defer_start = true;
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c b/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c
index 6e12e275deba..bddd625ab91b 100644
--- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c
+++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c
@@ -170,6 +170,8 @@ static void mdp5_complete_commit(struct msm_kms *kms, struct drm_atomic_state *s
 	struct device *dev = &mdp5_kms->pdev->dev;
 	struct mdp5_global_state *global_state;
 
+	drm_atomic_helper_wait_for_vblanks(mdp5_kms->dev, state);
+
 	global_state = mdp5_get_existing_global_state(mdp5_kms);
 
 	if (mdp5_kms->smp)
diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_mdss.c b/drivers/gpu/drm/msm/disp/mdp5/mdp5_mdss.c
index f2a0db7a8a03..1cc4e57f0226 100644
--- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_mdss.c
+++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_mdss.c
@@ -20,12 +20,10 @@
 #include "msm_drv.h"
 #include "mdp5_kms.h"
 
-/*
- * If needed, this can become more specific: something like struct mdp5_mdss,
- * which contains a 'struct msm_mdss base' member.
- */
-struct msm_mdss {
-	struct drm_device *dev;
+#define to_mdp5_mdss(x) container_of(x, struct mdp5_mdss, base)
+
+struct mdp5_mdss {
+	struct msm_mdss base;
 
 	void __iomem *mmio, *vbif;
 
@@ -41,22 +39,22 @@ struct msm_mdss {
 	} irqcontroller;
 };
 
-static inline void mdss_write(struct msm_mdss *mdss, u32 reg, u32 data)
+static inline void mdss_write(struct mdp5_mdss *mdp5_mdss, u32 reg, u32 data)
 {
-	msm_writel(data, mdss->mmio + reg);
+	msm_writel(data, mdp5_mdss->mmio + reg);
 }
 
-static inline u32 mdss_read(struct msm_mdss *mdss, u32 reg)
+static inline u32 mdss_read(struct mdp5_mdss *mdp5_mdss, u32 reg)
 {
-	return msm_readl(mdss->mmio + reg);
+	return msm_readl(mdp5_mdss->mmio + reg);
 }
 
 static irqreturn_t mdss_irq(int irq, void *arg)
 {
-	struct msm_mdss *mdss = arg;
+	struct mdp5_mdss *mdp5_mdss = arg;
 	u32 intr;
 
-	intr = mdss_read(mdss, REG_MDSS_HW_INTR_STATUS);
+	intr = mdss_read(mdp5_mdss, REG_MDSS_HW_INTR_STATUS);
 
 	VERB("intr=%08x", intr);
 
@@ -64,7 +62,7 @@ static irqreturn_t mdss_irq(int irq, void *arg)
 		irq_hw_number_t hwirq = fls(intr) - 1;
 
 		generic_handle_irq(irq_find_mapping(
-				mdss->irqcontroller.domain, hwirq));
+				mdp5_mdss->irqcontroller.domain, hwirq));
 		intr &= ~(1 << hwirq);
 	}
 
@@ -84,19 +82,19 @@ static irqreturn_t mdss_irq(int irq, void *arg)
 
 static void mdss_hw_mask_irq(struct irq_data *irqd)
 {
-	struct msm_mdss *mdss = irq_data_get_irq_chip_data(irqd);
+	struct mdp5_mdss *mdp5_mdss = irq_data_get_irq_chip_data(irqd);
 
 	smp_mb__before_atomic();
-	clear_bit(irqd->hwirq, &mdss->irqcontroller.enabled_mask);
+	clear_bit(irqd->hwirq, &mdp5_mdss->irqcontroller.enabled_mask);
 	smp_mb__after_atomic();
 }
 
 static void mdss_hw_unmask_irq(struct irq_data *irqd)
 {
-	struct msm_mdss *mdss = irq_data_get_irq_chip_data(irqd);
+	struct mdp5_mdss *mdp5_mdss = irq_data_get_irq_chip_data(irqd);
 
 	smp_mb__before_atomic();
-	set_bit(irqd->hwirq, &mdss->irqcontroller.enabled_mask);
+	set_bit(irqd->hwirq, &mdp5_mdss->irqcontroller.enabled_mask);
 	smp_mb__after_atomic();
 }
 
@@ -109,13 +107,13 @@ static struct irq_chip mdss_hw_irq_chip = {
 static int mdss_hw_irqdomain_map(struct irq_domain *d, unsigned int irq,
 				 irq_hw_number_t hwirq)
 {
-	struct msm_mdss *mdss = d->host_data;
+	struct mdp5_mdss *mdp5_mdss = d->host_data;
 
 	if (!(VALID_IRQS & (1 << hwirq)))
 		return -EPERM;
 
 	irq_set_chip_and_handler(irq, &mdss_hw_irq_chip, handle_level_irq);
-	irq_set_chip_data(irq, mdss);
+	irq_set_chip_data(irq, mdp5_mdss);
 
 	return 0;
 }
@@ -126,90 +124,99 @@ static const struct irq_domain_ops mdss_hw_irqdomain_ops = {
 };
 
 
-static int mdss_irq_domain_init(struct msm_mdss *mdss)
+static int mdss_irq_domain_init(struct mdp5_mdss *mdp5_mdss)
 {
-	struct device *dev = mdss->dev->dev;
+	struct device *dev = mdp5_mdss->base.dev->dev;
 	struct irq_domain *d;
 
 	d = irq_domain_add_linear(dev->of_node, 32, &mdss_hw_irqdomain_ops,
-				  mdss);
+				  mdp5_mdss);
 	if (!d) {
 		dev_err(dev, "mdss irq domain add failed\n");
 		return -ENXIO;
 	}
 
-	mdss->irqcontroller.enabled_mask = 0;
-	mdss->irqcontroller.domain = d;
+	mdp5_mdss->irqcontroller.enabled_mask = 0;
+	mdp5_mdss->irqcontroller.domain = d;
 
 	return 0;
 }
 
-int msm_mdss_enable(struct msm_mdss *mdss)
+static int mdp5_mdss_enable(struct msm_mdss *mdss)
 {
+	struct mdp5_mdss *mdp5_mdss = to_mdp5_mdss(mdss);
 	DBG("");
 
-	clk_prepare_enable(mdss->ahb_clk);
-	if (mdss->axi_clk)
-		clk_prepare_enable(mdss->axi_clk);
-	if (mdss->vsync_clk)
-		clk_prepare_enable(mdss->vsync_clk);
+	clk_prepare_enable(mdp5_mdss->ahb_clk);
+	if (mdp5_mdss->axi_clk)
+		clk_prepare_enable(mdp5_mdss->axi_clk);
+	if (mdp5_mdss->vsync_clk)
+		clk_prepare_enable(mdp5_mdss->vsync_clk);
 
 	return 0;
 }
 
-int msm_mdss_disable(struct msm_mdss *mdss)
+static int mdp5_mdss_disable(struct msm_mdss *mdss)
 {
+	struct mdp5_mdss *mdp5_mdss = to_mdp5_mdss(mdss);
 	DBG("");
 
-	if (mdss->vsync_clk)
-		clk_disable_unprepare(mdss->vsync_clk);
-	if (mdss->axi_clk)
-		clk_disable_unprepare(mdss->axi_clk);
-	clk_disable_unprepare(mdss->ahb_clk);
+	if (mdp5_mdss->vsync_clk)
+		clk_disable_unprepare(mdp5_mdss->vsync_clk);
+	if (mdp5_mdss->axi_clk)
+		clk_disable_unprepare(mdp5_mdss->axi_clk);
+	clk_disable_unprepare(mdp5_mdss->ahb_clk);
 
 	return 0;
 }
 
-static int msm_mdss_get_clocks(struct msm_mdss *mdss)
+static int msm_mdss_get_clocks(struct mdp5_mdss *mdp5_mdss)
 {
-	struct platform_device *pdev = to_platform_device(mdss->dev->dev);
+	struct platform_device *pdev =
+			to_platform_device(mdp5_mdss->base.dev->dev);
 
-	mdss->ahb_clk = msm_clk_get(pdev, "iface");
-	if (IS_ERR(mdss->ahb_clk))
-		mdss->ahb_clk = NULL;
+	mdp5_mdss->ahb_clk = msm_clk_get(pdev, "iface");
+	if (IS_ERR(mdp5_mdss->ahb_clk))
+		mdp5_mdss->ahb_clk = NULL;
 
-	mdss->axi_clk = msm_clk_get(pdev, "bus");
-	if (IS_ERR(mdss->axi_clk))
-		mdss->axi_clk = NULL;
+	mdp5_mdss->axi_clk = msm_clk_get(pdev, "bus");
+	if (IS_ERR(mdp5_mdss->axi_clk))
+		mdp5_mdss->axi_clk = NULL;
 
-	mdss->vsync_clk = msm_clk_get(pdev, "vsync");
-	if (IS_ERR(mdss->vsync_clk))
-		mdss->vsync_clk = NULL;
+	mdp5_mdss->vsync_clk = msm_clk_get(pdev, "vsync");
+	if (IS_ERR(mdp5_mdss->vsync_clk))
+		mdp5_mdss->vsync_clk = NULL;
 
 	return 0;
 }
 
-void msm_mdss_destroy(struct drm_device *dev)
+static void mdp5_mdss_destroy(struct drm_device *dev)
 {
 	struct msm_drm_private *priv = dev->dev_private;
-	struct msm_mdss *mdss = priv->mdss;
+	struct mdp5_mdss *mdp5_mdss = to_mdp5_mdss(priv->mdss);
 
-	if (!mdss)
+	if (!mdp5_mdss)
 		return;
 
-	irq_domain_remove(mdss->irqcontroller.domain);
-	mdss->irqcontroller.domain = NULL;
+	irq_domain_remove(mdp5_mdss->irqcontroller.domain);
+	mdp5_mdss->irqcontroller.domain = NULL;
 
-	regulator_disable(mdss->vdd);
+	regulator_disable(mdp5_mdss->vdd);
 
 	pm_runtime_disable(dev->dev);
 }
 
-int msm_mdss_init(struct drm_device *dev)
+static const struct msm_mdss_funcs mdss_funcs = {
+	.enable	= mdp5_mdss_enable,
+	.disable = mdp5_mdss_disable,
+	.destroy = mdp5_mdss_destroy,
+};
+
+int mdp5_mdss_init(struct drm_device *dev)
 {
 	struct platform_device *pdev = to_platform_device(dev->dev);
 	struct msm_drm_private *priv = dev->dev_private;
-	struct msm_mdss *mdss;
+	struct mdp5_mdss *mdp5_mdss;
 	int ret;
 
 	DBG("");
@@ -217,40 +224,40 @@ int msm_mdss_init(struct drm_device *dev)
 	if (!of_device_is_compatible(dev->dev->of_node, "qcom,mdss"))
 		return 0;
 
-	mdss = devm_kzalloc(dev->dev, sizeof(*mdss), GFP_KERNEL);
-	if (!mdss) {
+	mdp5_mdss = devm_kzalloc(dev->dev, sizeof(*mdp5_mdss), GFP_KERNEL);
+	if (!mdp5_mdss) {
 		ret = -ENOMEM;
 		goto fail;
 	}
 
-	mdss->dev = dev;
+	mdp5_mdss->base.dev = dev;
 
-	mdss->mmio = msm_ioremap(pdev, "mdss_phys", "MDSS");
-	if (IS_ERR(mdss->mmio)) {
-		ret = PTR_ERR(mdss->mmio);
+	mdp5_mdss->mmio = msm_ioremap(pdev, "mdss_phys", "MDSS");
+	if (IS_ERR(mdp5_mdss->mmio)) {
+		ret = PTR_ERR(mdp5_mdss->mmio);
 		goto fail;
 	}
 
-	mdss->vbif = msm_ioremap(pdev, "vbif_phys", "VBIF");
-	if (IS_ERR(mdss->vbif)) {
-		ret = PTR_ERR(mdss->vbif);
+	mdp5_mdss->vbif = msm_ioremap(pdev, "vbif_phys", "VBIF");
+	if (IS_ERR(mdp5_mdss->vbif)) {
+		ret = PTR_ERR(mdp5_mdss->vbif);
 		goto fail;
 	}
 
-	ret = msm_mdss_get_clocks(mdss);
+	ret = msm_mdss_get_clocks(mdp5_mdss);
 	if (ret) {
 		dev_err(dev->dev, "failed to get clocks: %d\n", ret);
 		goto fail;
 	}
 
 	/* Regulator to enable GDSCs in downstream kernels */
-	mdss->vdd = devm_regulator_get(dev->dev, "vdd");
-	if (IS_ERR(mdss->vdd)) {
-		ret = PTR_ERR(mdss->vdd);
+	mdp5_mdss->vdd = devm_regulator_get(dev->dev, "vdd");
+	if (IS_ERR(mdp5_mdss->vdd)) {
+		ret = PTR_ERR(mdp5_mdss->vdd);
 		goto fail;
 	}
 
-	ret = regulator_enable(mdss->vdd);
+	ret = regulator_enable(mdp5_mdss->vdd);
 	if (ret) {
 		dev_err(dev->dev, "failed to enable regulator vdd: %d\n",
 			ret);
@@ -258,25 +265,26 @@ int msm_mdss_init(struct drm_device *dev)
 	}
 
 	ret = devm_request_irq(dev->dev, platform_get_irq(pdev, 0),
-			       mdss_irq, 0, "mdss_isr", mdss);
+			       mdss_irq, 0, "mdss_isr", mdp5_mdss);
 	if (ret) {
 		dev_err(dev->dev, "failed to init irq: %d\n", ret);
 		goto fail_irq;
 	}
 
-	ret = mdss_irq_domain_init(mdss);
+	ret = mdss_irq_domain_init(mdp5_mdss);
 	if (ret) {
 		dev_err(dev->dev, "failed to init sub-block irqs: %d\n", ret);
 		goto fail_irq;
 	}
 
-	priv->mdss = mdss;
+	mdp5_mdss->base.funcs = &mdss_funcs;
+	priv->mdss = &mdp5_mdss->base;
 
 	pm_runtime_enable(dev->dev);
 
 	return 0;
 fail_irq:
-	regulator_disable(mdss->vdd);
+	regulator_disable(mdp5_mdss->vdd);
 fail:
 	return ret;
 }
diff --git a/drivers/gpu/drm/msm/disp/mdp_common.xml.h b/drivers/gpu/drm/msm/disp/mdp_common.xml.h
index 1494c407be44..d420c8044e23 100644
--- a/drivers/gpu/drm/msm/disp/mdp_common.xml.h
+++ b/drivers/gpu/drm/msm/disp/mdp_common.xml.h
@@ -8,19 +8,19 @@ http://github.com/freedreno/envytools/
 git clone https://github.com/freedreno/envytools.git
 
 The rules-ng-ng source files this header was generated from are:
-- /home/robclark/src/freedreno/envytools/rnndb/msm.xml                 (    676 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml (   1572 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp4.xml            (  20915 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp_common.xml      (   2849 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp5.xml            (  37411 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/dsi/dsi.xml             (  33004 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/dsi/sfpb.xml            (    602 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/dsi/mmss_cc.xml         (   1686 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/hdmi/qfprom.xml         (    600 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/hdmi/hdmi.xml           (  41799 bytes, from 2017-06-16 12:32:42)
-- /home/robclark/src/freedreno/envytools/rnndb/edp/edp.xml             (  10416 bytes, from 2017-05-17 13:21:27)
-
-Copyright (C) 2013-2017 by the following authors:
+- /home/robclark/src/envytools/rnndb/msm.xml                 (    676 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/freedreno_copyright.xml (   1572 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/mdp/mdp4.xml            (  20915 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/mdp/mdp_common.xml      (   2849 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/mdp/mdp5.xml            (  37411 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/dsi/dsi.xml             (  37239 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/dsi/sfpb.xml            (    602 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/dsi/mmss_cc.xml         (   1686 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/hdmi/qfprom.xml         (    600 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/hdmi/hdmi.xml           (  41799 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/edp/edp.xml             (  10416 bytes, from 2018-07-03 19:37:13)
+
+Copyright (C) 2013-2018 by the following authors:
 - Rob Clark <robdclark@gmail.com> (robclark)
 - Ilia Mirkin <imirkin@alum.mit.edu> (imirkin)
 
diff --git a/drivers/gpu/drm/msm/dsi/dsi.c b/drivers/gpu/drm/msm/dsi/dsi.c
index b744bcc7d8ad..ff8164cc6738 100644
--- a/drivers/gpu/drm/msm/dsi/dsi.c
+++ b/drivers/gpu/drm/msm/dsi/dsi.c
@@ -208,6 +208,9 @@ int msm_dsi_modeset_init(struct msm_dsi *msm_dsi, struct drm_device *dev,
 		goto fail;
 	}
 
+	if (!msm_dsi_manager_validate_current_config(msm_dsi->id))
+		goto fail;
+
 	msm_dsi->encoder = encoder;
 
 	msm_dsi->bridge = msm_dsi_manager_bridge_init(msm_dsi->id);
diff --git a/drivers/gpu/drm/msm/dsi/dsi.h b/drivers/gpu/drm/msm/dsi/dsi.h
index 70d9a9a47acd..08f3fc6771b7 100644
--- a/drivers/gpu/drm/msm/dsi/dsi.h
+++ b/drivers/gpu/drm/msm/dsi/dsi.h
@@ -100,6 +100,7 @@ bool msm_dsi_manager_cmd_xfer_trigger(int id, u32 dma_base, u32 len);
 void msm_dsi_manager_attach_dsi_device(int id, u32 device_flags);
 int msm_dsi_manager_register(struct msm_dsi *msm_dsi);
 void msm_dsi_manager_unregister(struct msm_dsi *msm_dsi);
+bool msm_dsi_manager_validate_current_config(u8 id);
 
 /* msm dsi */
 static inline bool msm_dsi_device_connected(struct msm_dsi *msm_dsi)
@@ -149,6 +150,7 @@ static inline int msm_dsi_pll_set_usecase(struct msm_dsi_pll *pll,
 #endif
 
 /* dsi host */
+struct msm_dsi_host;
 int msm_dsi_host_xfer_prepare(struct mipi_dsi_host *host,
 					const struct mipi_dsi_msg *msg);
 void msm_dsi_host_xfer_restore(struct mipi_dsi_host *host,
@@ -162,7 +164,8 @@ void msm_dsi_host_cmd_xfer_commit(struct mipi_dsi_host *host,
 int msm_dsi_host_enable(struct mipi_dsi_host *host);
 int msm_dsi_host_disable(struct mipi_dsi_host *host);
 int msm_dsi_host_power_on(struct mipi_dsi_host *host,
-			struct msm_dsi_phy_shared_timings *phy_shared_timings);
+			struct msm_dsi_phy_shared_timings *phy_shared_timings,
+			bool is_dual_dsi);
 int msm_dsi_host_power_off(struct mipi_dsi_host *host);
 int msm_dsi_host_set_display_mode(struct mipi_dsi_host *host,
 					struct drm_display_mode *mode);
@@ -175,13 +178,29 @@ int msm_dsi_host_set_src_pll(struct mipi_dsi_host *host,
 			struct msm_dsi_pll *src_pll);
 void msm_dsi_host_reset_phy(struct mipi_dsi_host *host);
 void msm_dsi_host_get_phy_clk_req(struct mipi_dsi_host *host,
-	struct msm_dsi_phy_clk_request *clk_req);
+	struct msm_dsi_phy_clk_request *clk_req,
+	bool is_dual_dsi);
 void msm_dsi_host_destroy(struct mipi_dsi_host *host);
 int msm_dsi_host_modeset_init(struct mipi_dsi_host *host,
 					struct drm_device *dev);
 int msm_dsi_host_init(struct msm_dsi *msm_dsi);
 int msm_dsi_runtime_suspend(struct device *dev);
 int msm_dsi_runtime_resume(struct device *dev);
+int dsi_link_clk_enable_6g(struct msm_dsi_host *msm_host);
+int dsi_link_clk_enable_v2(struct msm_dsi_host *msm_host);
+void dsi_link_clk_disable_6g(struct msm_dsi_host *msm_host);
+void dsi_link_clk_disable_v2(struct msm_dsi_host *msm_host);
+int dsi_tx_buf_alloc_6g(struct msm_dsi_host *msm_host, int size);
+int dsi_tx_buf_alloc_v2(struct msm_dsi_host *msm_host, int size);
+void *dsi_tx_buf_get_6g(struct msm_dsi_host *msm_host);
+void *dsi_tx_buf_get_v2(struct msm_dsi_host *msm_host);
+void dsi_tx_buf_put_6g(struct msm_dsi_host *msm_host);
+int dsi_dma_base_get_6g(struct msm_dsi_host *msm_host, uint64_t *iova);
+int dsi_dma_base_get_v2(struct msm_dsi_host *msm_host, uint64_t *iova);
+int dsi_clk_init_v2(struct msm_dsi_host *msm_host);
+int dsi_clk_init_6g_v2(struct msm_dsi_host *msm_host);
+int dsi_calc_clk_rate_v2(struct msm_dsi_host *msm_host, bool is_dual_dsi);
+int dsi_calc_clk_rate_6g(struct msm_dsi_host *msm_host, bool is_dual_dsi);
 
 /* dsi phy */
 struct msm_dsi_phy;
diff --git a/drivers/gpu/drm/msm/dsi/dsi.xml.h b/drivers/gpu/drm/msm/dsi/dsi.xml.h
index f6a9471b70c8..21f489a737d7 100644
--- a/drivers/gpu/drm/msm/dsi/dsi.xml.h
+++ b/drivers/gpu/drm/msm/dsi/dsi.xml.h
@@ -8,8 +8,17 @@ http://github.com/freedreno/envytools/
 git clone https://github.com/freedreno/envytools.git
 
 The rules-ng-ng source files this header was generated from are:
-- /local/mnt/workspace/source_trees/envytools/rnndb/../rnndb/dsi/dsi.xml    (  37239 bytes, from 2018-01-12 09:09:22)
-- /local/mnt/workspace/source_trees/envytools/rnndb/freedreno_copyright.xml (   1572 bytes, from 2016-05-09 06:32:54)
+- /home/robclark/src/envytools/rnndb/msm.xml                 (    676 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/freedreno_copyright.xml (   1572 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/mdp/mdp4.xml            (  20915 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/mdp/mdp_common.xml      (   2849 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/mdp/mdp5.xml            (  37411 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/dsi/dsi.xml             (  37239 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/dsi/sfpb.xml            (    602 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/dsi/mmss_cc.xml         (   1686 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/hdmi/qfprom.xml         (    600 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/hdmi/hdmi.xml           (  41799 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/edp/edp.xml             (  10416 bytes, from 2018-07-03 19:37:13)
 
 Copyright (C) 2013-2018 by the following authors:
 - Rob Clark <robdclark@gmail.com> (robclark)
diff --git a/drivers/gpu/drm/msm/dsi/dsi_cfg.c b/drivers/gpu/drm/msm/dsi/dsi_cfg.c
index 0327bb54b01b..dcdfb1bb54f9 100644
--- a/drivers/gpu/drm/msm/dsi/dsi_cfg.c
+++ b/drivers/gpu/drm/msm/dsi/dsi_cfg.c
@@ -136,20 +136,58 @@ static const struct msm_dsi_config sdm845_dsi_cfg = {
 	.num_dsi = 2,
 };
 
+const static struct msm_dsi_host_cfg_ops msm_dsi_v2_host_ops = {
+	.link_clk_enable = dsi_link_clk_enable_v2,
+	.link_clk_disable = dsi_link_clk_disable_v2,
+	.clk_init_ver = dsi_clk_init_v2,
+	.tx_buf_alloc = dsi_tx_buf_alloc_v2,
+	.tx_buf_get = dsi_tx_buf_get_v2,
+	.tx_buf_put = NULL,
+	.dma_base_get = dsi_dma_base_get_v2,
+	.calc_clk_rate = dsi_calc_clk_rate_v2,
+};
+
+const static struct msm_dsi_host_cfg_ops msm_dsi_6g_host_ops = {
+	.link_clk_enable = dsi_link_clk_enable_6g,
+	.link_clk_disable = dsi_link_clk_disable_6g,
+	.clk_init_ver = NULL,
+	.tx_buf_alloc = dsi_tx_buf_alloc_6g,
+	.tx_buf_get = dsi_tx_buf_get_6g,
+	.tx_buf_put = dsi_tx_buf_put_6g,
+	.dma_base_get = dsi_dma_base_get_6g,
+	.calc_clk_rate = dsi_calc_clk_rate_6g,
+};
+
+const static struct msm_dsi_host_cfg_ops msm_dsi_6g_v2_host_ops = {
+	.link_clk_enable = dsi_link_clk_enable_6g,
+	.link_clk_disable = dsi_link_clk_disable_6g,
+	.clk_init_ver = dsi_clk_init_6g_v2,
+	.tx_buf_alloc = dsi_tx_buf_alloc_6g,
+	.tx_buf_get = dsi_tx_buf_get_6g,
+	.tx_buf_put = dsi_tx_buf_put_6g,
+	.dma_base_get = dsi_dma_base_get_6g,
+	.calc_clk_rate = dsi_calc_clk_rate_6g,
+};
+
 static const struct msm_dsi_cfg_handler dsi_cfg_handlers[] = {
-	{MSM_DSI_VER_MAJOR_V2, MSM_DSI_V2_VER_MINOR_8064, &apq8064_dsi_cfg},
+	{MSM_DSI_VER_MAJOR_V2, MSM_DSI_V2_VER_MINOR_8064,
+		&apq8064_dsi_cfg, &msm_dsi_v2_host_ops},
 	{MSM_DSI_VER_MAJOR_6G, MSM_DSI_6G_VER_MINOR_V1_0,
-						&msm8974_apq8084_dsi_cfg},
+		&msm8974_apq8084_dsi_cfg, &msm_dsi_6g_host_ops},
 	{MSM_DSI_VER_MAJOR_6G, MSM_DSI_6G_VER_MINOR_V1_1,
-						&msm8974_apq8084_dsi_cfg},
+		&msm8974_apq8084_dsi_cfg, &msm_dsi_6g_host_ops},
 	{MSM_DSI_VER_MAJOR_6G, MSM_DSI_6G_VER_MINOR_V1_1_1,
-						&msm8974_apq8084_dsi_cfg},
+		&msm8974_apq8084_dsi_cfg, &msm_dsi_6g_host_ops},
 	{MSM_DSI_VER_MAJOR_6G, MSM_DSI_6G_VER_MINOR_V1_2,
-						&msm8974_apq8084_dsi_cfg},
-	{MSM_DSI_VER_MAJOR_6G, MSM_DSI_6G_VER_MINOR_V1_3, &msm8994_dsi_cfg},
-	{MSM_DSI_VER_MAJOR_6G, MSM_DSI_6G_VER_MINOR_V1_3_1, &msm8916_dsi_cfg},
-	{MSM_DSI_VER_MAJOR_6G, MSM_DSI_6G_VER_MINOR_V1_4_1, &msm8996_dsi_cfg},
-	{MSM_DSI_VER_MAJOR_6G, MSM_DSI_6G_VER_MINOR_V2_2_1, &sdm845_dsi_cfg},
+		&msm8974_apq8084_dsi_cfg, &msm_dsi_6g_host_ops},
+	{MSM_DSI_VER_MAJOR_6G, MSM_DSI_6G_VER_MINOR_V1_3,
+		&msm8994_dsi_cfg, &msm_dsi_6g_host_ops},
+	{MSM_DSI_VER_MAJOR_6G, MSM_DSI_6G_VER_MINOR_V1_3_1,
+		&msm8916_dsi_cfg, &msm_dsi_6g_host_ops},
+	{MSM_DSI_VER_MAJOR_6G, MSM_DSI_6G_VER_MINOR_V1_4_1,
+		&msm8996_dsi_cfg, &msm_dsi_6g_host_ops},
+	{MSM_DSI_VER_MAJOR_6G, MSM_DSI_6G_VER_MINOR_V2_2_1,
+		&sdm845_dsi_cfg, &msm_dsi_6g_v2_host_ops},
 };
 
 const struct msm_dsi_cfg_handler *msm_dsi_cfg_get(u32 major, u32 minor)
diff --git a/drivers/gpu/drm/msm/dsi/dsi_cfg.h b/drivers/gpu/drm/msm/dsi/dsi_cfg.h
index 9cfdcf1c95d5..16c507911110 100644
--- a/drivers/gpu/drm/msm/dsi/dsi_cfg.h
+++ b/drivers/gpu/drm/msm/dsi/dsi_cfg.h
@@ -40,10 +40,22 @@ struct msm_dsi_config {
 	const int num_dsi;
 };
 
+struct msm_dsi_host_cfg_ops {
+	int (*link_clk_enable)(struct msm_dsi_host *msm_host);
+	void (*link_clk_disable)(struct msm_dsi_host *msm_host);
+	int (*clk_init_ver)(struct msm_dsi_host *msm_host);
+	int (*tx_buf_alloc)(struct msm_dsi_host *msm_host, int size);
+	void* (*tx_buf_get)(struct msm_dsi_host *msm_host);
+	void (*tx_buf_put)(struct msm_dsi_host *msm_host);
+	int (*dma_base_get)(struct msm_dsi_host *msm_host, uint64_t *iova);
+	int (*calc_clk_rate)(struct msm_dsi_host *msm_host, bool is_dual_dsi);
+};
+
 struct msm_dsi_cfg_handler {
 	u32 major;
 	u32 minor;
 	const struct msm_dsi_config *cfg;
+	const struct msm_dsi_host_cfg_ops *ops;
 };
 
 const struct msm_dsi_cfg_handler *msm_dsi_cfg_get(u32 major, u32 minor);
diff --git a/drivers/gpu/drm/msm/dsi/dsi_host.c b/drivers/gpu/drm/msm/dsi/dsi_host.c
index 29841f440111..96fb5f635314 100644
--- a/drivers/gpu/drm/msm/dsi/dsi_host.c
+++ b/drivers/gpu/drm/msm/dsi/dsi_host.c
@@ -118,6 +118,7 @@ struct msm_dsi_host {
 	struct clk *byte_intf_clk;
 
 	u32 byte_clk_rate;
+	u32 pixel_clk_rate;
 	u32 esc_clk_rate;
 
 	/* DSI v2 specific clocks */
@@ -332,6 +333,54 @@ static int dsi_regulator_init(struct msm_dsi_host *msm_host)
 	return 0;
 }
 
+int dsi_clk_init_v2(struct msm_dsi_host *msm_host)
+{
+	struct platform_device *pdev = msm_host->pdev;
+	int ret = 0;
+
+	msm_host->src_clk = msm_clk_get(pdev, "src");
+
+	if (IS_ERR(msm_host->src_clk)) {
+		ret = PTR_ERR(msm_host->src_clk);
+		pr_err("%s: can't find src clock. ret=%d\n",
+			__func__, ret);
+		msm_host->src_clk = NULL;
+		return ret;
+	}
+
+	msm_host->esc_clk_src = clk_get_parent(msm_host->esc_clk);
+	if (!msm_host->esc_clk_src) {
+		ret = -ENODEV;
+		pr_err("%s: can't get esc clock parent. ret=%d\n",
+			__func__, ret);
+		return ret;
+	}
+
+	msm_host->dsi_clk_src = clk_get_parent(msm_host->src_clk);
+	if (!msm_host->dsi_clk_src) {
+		ret = -ENODEV;
+		pr_err("%s: can't get src clock parent. ret=%d\n",
+			__func__, ret);
+	}
+
+	return ret;
+}
+
+int dsi_clk_init_6g_v2(struct msm_dsi_host *msm_host)
+{
+	struct platform_device *pdev = msm_host->pdev;
+	int ret = 0;
+
+	msm_host->byte_intf_clk = msm_clk_get(pdev, "byte_intf");
+	if (IS_ERR(msm_host->byte_intf_clk)) {
+		ret = PTR_ERR(msm_host->byte_intf_clk);
+		pr_err("%s: can't find byte_intf clock. ret=%d\n",
+			__func__, ret);
+	}
+
+	return ret;
+}
+
 static int dsi_clk_init(struct msm_dsi_host *msm_host)
 {
 	struct platform_device *pdev = msm_host->pdev;
@@ -379,19 +428,6 @@ static int dsi_clk_init(struct msm_dsi_host *msm_host)
 		goto exit;
 	}
 
-	if (cfg_hnd->major == MSM_DSI_VER_MAJOR_6G &&
-	    cfg_hnd->minor >= MSM_DSI_6G_VER_MINOR_V2_2_1) {
-		msm_host->byte_intf_clk = msm_clk_get(pdev, "byte_intf");
-		if (IS_ERR(msm_host->byte_intf_clk)) {
-			ret = PTR_ERR(msm_host->byte_intf_clk);
-			pr_err("%s: can't find byte_intf clock. ret=%d\n",
-			        __func__, ret);
-			goto exit;
-		}
-	} else {
-		msm_host->byte_intf_clk = NULL;
-	}
-
 	msm_host->byte_clk_src = clk_get_parent(msm_host->byte_clk);
 	if (!msm_host->byte_clk_src) {
 		ret = -ENODEV;
@@ -406,31 +442,8 @@ static int dsi_clk_init(struct msm_dsi_host *msm_host)
 		goto exit;
 	}
 
-	if (cfg_hnd->major == MSM_DSI_VER_MAJOR_V2) {
-		msm_host->src_clk = msm_clk_get(pdev, "src");
-		if (IS_ERR(msm_host->src_clk)) {
-			ret = PTR_ERR(msm_host->src_clk);
-			pr_err("%s: can't find src clock. ret=%d\n",
-				__func__, ret);
-			msm_host->src_clk = NULL;
-			goto exit;
-		}
-
-		msm_host->esc_clk_src = clk_get_parent(msm_host->esc_clk);
-		if (!msm_host->esc_clk_src) {
-			ret = -ENODEV;
-			pr_err("%s: can't get esc clock parent. ret=%d\n",
-				__func__, ret);
-			goto exit;
-		}
-
-		msm_host->dsi_clk_src = clk_get_parent(msm_host->src_clk);
-		if (!msm_host->dsi_clk_src) {
-			ret = -ENODEV;
-			pr_err("%s: can't get src clock parent. ret=%d\n",
-				__func__, ret);
-		}
-	}
+	if (cfg_hnd->ops->clk_init_ver)
+		ret = cfg_hnd->ops->clk_init_ver(msm_host);
 exit:
 	return ret;
 }
@@ -498,7 +511,7 @@ int msm_dsi_runtime_resume(struct device *dev)
 	return dsi_bus_clk_enable(msm_host);
 }
 
-static int dsi_link_clk_enable_6g(struct msm_dsi_host *msm_host)
+int dsi_link_clk_enable_6g(struct msm_dsi_host *msm_host)
 {
 	int ret;
 
@@ -511,7 +524,7 @@ static int dsi_link_clk_enable_6g(struct msm_dsi_host *msm_host)
 		goto error;
 	}
 
-	ret = clk_set_rate(msm_host->pixel_clk, msm_host->mode->clock * 1000);
+	ret = clk_set_rate(msm_host->pixel_clk, msm_host->pixel_clk_rate);
 	if (ret) {
 		pr_err("%s: Failed to set rate pixel clk, %d\n", __func__, ret);
 		goto error;
@@ -566,7 +579,7 @@ error:
 	return ret;
 }
 
-static int dsi_link_clk_enable_v2(struct msm_dsi_host *msm_host)
+int dsi_link_clk_enable_v2(struct msm_dsi_host *msm_host)
 {
 	int ret;
 
@@ -592,7 +605,7 @@ static int dsi_link_clk_enable_v2(struct msm_dsi_host *msm_host)
 		goto error;
 	}
 
-	ret = clk_set_rate(msm_host->pixel_clk, msm_host->mode->clock * 1000);
+	ret = clk_set_rate(msm_host->pixel_clk, msm_host->pixel_clk_rate);
 	if (ret) {
 		pr_err("%s: Failed to set rate pixel clk, %d\n", __func__, ret);
 		goto error;
@@ -634,98 +647,121 @@ error:
 	return ret;
 }
 
-static int dsi_link_clk_enable(struct msm_dsi_host *msm_host)
+void dsi_link_clk_disable_6g(struct msm_dsi_host *msm_host)
 {
-	const struct msm_dsi_cfg_handler *cfg_hnd = msm_host->cfg_hnd;
+	clk_disable_unprepare(msm_host->esc_clk);
+	clk_disable_unprepare(msm_host->pixel_clk);
+	if (msm_host->byte_intf_clk)
+		clk_disable_unprepare(msm_host->byte_intf_clk);
+	clk_disable_unprepare(msm_host->byte_clk);
+}
 
-	if (cfg_hnd->major == MSM_DSI_VER_MAJOR_6G)
-		return dsi_link_clk_enable_6g(msm_host);
-	else
-		return dsi_link_clk_enable_v2(msm_host);
+void dsi_link_clk_disable_v2(struct msm_dsi_host *msm_host)
+{
+	clk_disable_unprepare(msm_host->pixel_clk);
+	clk_disable_unprepare(msm_host->src_clk);
+	clk_disable_unprepare(msm_host->esc_clk);
+	clk_disable_unprepare(msm_host->byte_clk);
 }
 
-static void dsi_link_clk_disable(struct msm_dsi_host *msm_host)
+static u32 dsi_get_pclk_rate(struct msm_dsi_host *msm_host, bool is_dual_dsi)
 {
-	const struct msm_dsi_cfg_handler *cfg_hnd = msm_host->cfg_hnd;
+	struct drm_display_mode *mode = msm_host->mode;
+	u32 pclk_rate;
 
-	if (cfg_hnd->major == MSM_DSI_VER_MAJOR_6G) {
-		clk_disable_unprepare(msm_host->esc_clk);
-		clk_disable_unprepare(msm_host->pixel_clk);
-		if (msm_host->byte_intf_clk)
-			clk_disable_unprepare(msm_host->byte_intf_clk);
-		clk_disable_unprepare(msm_host->byte_clk);
-	} else {
-		clk_disable_unprepare(msm_host->pixel_clk);
-		clk_disable_unprepare(msm_host->src_clk);
-		clk_disable_unprepare(msm_host->esc_clk);
-		clk_disable_unprepare(msm_host->byte_clk);
-	}
+	pclk_rate = mode->clock * 1000;
+
+	/*
+	 * For dual DSI mode, the current DRM mode has the complete width of the
+	 * panel. Since, the complete panel is driven by two DSI controllers,
+	 * the clock rates have to be split between the two dsi controllers.
+	 * Adjust the byte and pixel clock rates for each dsi host accordingly.
+	 */
+	if (is_dual_dsi)
+		pclk_rate /= 2;
+
+	return pclk_rate;
 }
 
-static int dsi_calc_clk_rate(struct msm_dsi_host *msm_host)
+static void dsi_calc_pclk(struct msm_dsi_host *msm_host, bool is_dual_dsi)
 {
-	struct drm_display_mode *mode = msm_host->mode;
-	const struct msm_dsi_cfg_handler *cfg_hnd = msm_host->cfg_hnd;
 	u8 lanes = msm_host->lanes;
 	u32 bpp = dsi_get_bpp(msm_host->format);
-	u32 pclk_rate;
+	u32 pclk_rate = dsi_get_pclk_rate(msm_host, is_dual_dsi);
+	u64 pclk_bpp = (u64)pclk_rate * bpp;
 
-	if (!mode) {
-		pr_err("%s: mode not set\n", __func__);
-		return -EINVAL;
-	}
-
-	pclk_rate = mode->clock * 1000;
-	if (lanes > 0) {
-		msm_host->byte_clk_rate = (pclk_rate * bpp) / (8 * lanes);
-	} else {
+	if (lanes == 0) {
 		pr_err("%s: forcing mdss_dsi lanes to 1\n", __func__);
-		msm_host->byte_clk_rate = (pclk_rate * bpp) / 8;
+		lanes = 1;
 	}
 
-	DBG("pclk=%d, bclk=%d", pclk_rate, msm_host->byte_clk_rate);
+	do_div(pclk_bpp, (8 * lanes));
 
-	msm_host->esc_clk_rate = clk_get_rate(msm_host->esc_clk);
+	msm_host->pixel_clk_rate = pclk_rate;
+	msm_host->byte_clk_rate = pclk_bpp;
 
-	if (cfg_hnd->major == MSM_DSI_VER_MAJOR_V2) {
-		unsigned int esc_mhz, esc_div;
-		unsigned long byte_mhz;
+	DBG("pclk=%d, bclk=%d", msm_host->pixel_clk_rate,
+				msm_host->byte_clk_rate);
 
-		msm_host->src_clk_rate = (pclk_rate * bpp) / 8;
+}
 
-		/*
-		 * esc clock is byte clock followed by a 4 bit divider,
-		 * we need to find an escape clock frequency within the
-		 * mipi DSI spec range within the maximum divider limit
-		 * We iterate here between an escape clock frequencey
-		 * between 20 Mhz to 5 Mhz and pick up the first one
-		 * that can be supported by our divider
-		 */
+int dsi_calc_clk_rate_6g(struct msm_dsi_host *msm_host, bool is_dual_dsi)
+{
+	if (!msm_host->mode) {
+		pr_err("%s: mode not set\n", __func__);
+		return -EINVAL;
+	}
+
+	dsi_calc_pclk(msm_host, is_dual_dsi);
+	msm_host->esc_clk_rate = clk_get_rate(msm_host->esc_clk);
+	return 0;
+}
 
-		byte_mhz = msm_host->byte_clk_rate / 1000000;
+int dsi_calc_clk_rate_v2(struct msm_dsi_host *msm_host, bool is_dual_dsi)
+{
+	u32 bpp = dsi_get_bpp(msm_host->format);
+	u64 pclk_bpp;
+	unsigned int esc_mhz, esc_div;
+	unsigned long byte_mhz;
 
-		for (esc_mhz = 20; esc_mhz >= 5; esc_mhz--) {
-			esc_div = DIV_ROUND_UP(byte_mhz, esc_mhz);
+	dsi_calc_pclk(msm_host, is_dual_dsi);
 
-			/*
-			 * TODO: Ideally, we shouldn't know what sort of divider
-			 * is available in mmss_cc, we're just assuming that
-			 * it'll always be a 4 bit divider. Need to come up with
-			 * a better way here.
-			 */
-			if (esc_div >= 1 && esc_div <= 16)
-				break;
-		}
+	pclk_bpp = (u64)dsi_get_pclk_rate(msm_host, is_dual_dsi) * bpp;
+	do_div(pclk_bpp, 8);
+	msm_host->src_clk_rate = pclk_bpp;
 
-		if (esc_mhz < 5)
-			return -EINVAL;
+	/*
+	 * esc clock is byte clock followed by a 4 bit divider,
+	 * we need to find an escape clock frequency within the
+	 * mipi DSI spec range within the maximum divider limit
+	 * We iterate here between an escape clock frequencey
+	 * between 20 Mhz to 5 Mhz and pick up the first one
+	 * that can be supported by our divider
+	 */
+
+	byte_mhz = msm_host->byte_clk_rate / 1000000;
 
-		msm_host->esc_clk_rate = msm_host->byte_clk_rate / esc_div;
+	for (esc_mhz = 20; esc_mhz >= 5; esc_mhz--) {
+		esc_div = DIV_ROUND_UP(byte_mhz, esc_mhz);
 
-		DBG("esc=%d, src=%d", msm_host->esc_clk_rate,
-			msm_host->src_clk_rate);
+		/*
+		 * TODO: Ideally, we shouldn't know what sort of divider
+		 * is available in mmss_cc, we're just assuming that
+		 * it'll always be a 4 bit divider. Need to come up with
+		 * a better way here.
+		 */
+		if (esc_div >= 1 && esc_div <= 16)
+			break;
 	}
 
+	if (esc_mhz < 5)
+		return -EINVAL;
+
+	msm_host->esc_clk_rate = msm_host->byte_clk_rate / esc_div;
+
+	DBG("esc=%d, src=%d", msm_host->esc_clk_rate,
+		msm_host->src_clk_rate);
+
 	return 0;
 }
 
@@ -885,7 +921,7 @@ static void dsi_ctrl_config(struct msm_dsi_host *msm_host, bool enable,
 	dsi_write(msm_host, REG_DSI_CTRL, data);
 }
 
-static void dsi_timing_setup(struct msm_dsi_host *msm_host)
+static void dsi_timing_setup(struct msm_dsi_host *msm_host, bool is_dual_dsi)
 {
 	struct drm_display_mode *mode = msm_host->mode;
 	u32 hs_start = 0, vs_start = 0; /* take sync start as 0 */
@@ -897,10 +933,26 @@ static void dsi_timing_setup(struct msm_dsi_host *msm_host)
 	u32 ha_end = ha_start + mode->hdisplay;
 	u32 va_start = v_total - mode->vsync_start;
 	u32 va_end = va_start + mode->vdisplay;
+	u32 hdisplay = mode->hdisplay;
 	u32 wc;
 
 	DBG("");
 
+	/*
+	 * For dual DSI mode, the current DRM mode has
+	 * the complete width of the panel. Since, the complete
+	 * panel is driven by two DSI controllers, the horizontal
+	 * timings have to be split between the two dsi controllers.
+	 * Adjust the DSI host timing values accordingly.
+	 */
+	if (is_dual_dsi) {
+		h_total /= 2;
+		hs_end /= 2;
+		ha_start /= 2;
+		ha_end /= 2;
+		hdisplay /= 2;
+	}
+
 	if (msm_host->mode_flags & MIPI_DSI_MODE_VIDEO) {
 		dsi_write(msm_host, REG_DSI_ACTIVE_H,
 			DSI_ACTIVE_H_START(ha_start) |
@@ -921,7 +973,7 @@ static void dsi_timing_setup(struct msm_dsi_host *msm_host)
 			DSI_ACTIVE_VSYNC_VPOS_END(vs_end));
 	} else {		/* command mode */
 		/* image data and 1 byte write_memory_start cmd */
-		wc = mode->hdisplay * dsi_get_bpp(msm_host->format) / 8 + 1;
+		wc = hdisplay * dsi_get_bpp(msm_host->format) / 8 + 1;
 
 		dsi_write(msm_host, REG_DSI_CMD_MDP_STREAM_CTRL,
 			DSI_CMD_MDP_STREAM_CTRL_WORD_COUNT(wc) |
@@ -931,7 +983,7 @@ static void dsi_timing_setup(struct msm_dsi_host *msm_host)
 					MIPI_DSI_DCS_LONG_WRITE));
 
 		dsi_write(msm_host, REG_DSI_CMD_MDP_STREAM_TOTAL,
-			DSI_CMD_MDP_STREAM_TOTAL_H_TOTAL(mode->hdisplay) |
+			DSI_CMD_MDP_STREAM_TOTAL_H_TOTAL(hdisplay) |
 			DSI_CMD_MDP_STREAM_TOTAL_V_TOTAL(mode->vdisplay));
 	}
 }
@@ -1015,50 +1067,37 @@ static void dsi_wait4video_eng_busy(struct msm_dsi_host *msm_host)
 	}
 }
 
-/* dsi_cmd */
-static int dsi_tx_buf_alloc(struct msm_dsi_host *msm_host, int size)
+int dsi_tx_buf_alloc_6g(struct msm_dsi_host *msm_host, int size)
 {
 	struct drm_device *dev = msm_host->dev;
 	struct msm_drm_private *priv = dev->dev_private;
-	const struct msm_dsi_cfg_handler *cfg_hnd = msm_host->cfg_hnd;
-	int ret;
 	uint64_t iova;
+	u8 *data;
 
-	if (cfg_hnd->major == MSM_DSI_VER_MAJOR_6G) {
-		msm_host->tx_gem_obj = msm_gem_new(dev, size, MSM_BO_UNCACHED);
-		if (IS_ERR(msm_host->tx_gem_obj)) {
-			ret = PTR_ERR(msm_host->tx_gem_obj);
-			pr_err("%s: failed to allocate gem, %d\n",
-				__func__, ret);
-			msm_host->tx_gem_obj = NULL;
-			return ret;
-		}
+	data = msm_gem_kernel_new(dev, size, MSM_BO_UNCACHED,
+					priv->kms->aspace,
+					&msm_host->tx_gem_obj, &iova);
 
-		ret = msm_gem_get_iova(msm_host->tx_gem_obj,
-				priv->kms->aspace, &iova);
-		if (ret) {
-			pr_err("%s: failed to get iova, %d\n", __func__, ret);
-			return ret;
-		}
+	if (IS_ERR(data)) {
+		msm_host->tx_gem_obj = NULL;
+		return PTR_ERR(data);
+	}
 
-		if (iova & 0x07) {
-			pr_err("%s: buf NOT 8 bytes aligned\n", __func__);
-			return -EINVAL;
-		}
+	msm_host->tx_size = msm_host->tx_gem_obj->size;
 
-		msm_host->tx_size = msm_host->tx_gem_obj->size;
-	} else {
-		msm_host->tx_buf = dma_alloc_coherent(dev->dev, size,
+	return 0;
+}
+
+int dsi_tx_buf_alloc_v2(struct msm_dsi_host *msm_host, int size)
+{
+	struct drm_device *dev = msm_host->dev;
+
+	msm_host->tx_buf = dma_alloc_coherent(dev->dev, size,
 					&msm_host->tx_buf_paddr, GFP_KERNEL);
-		if (!msm_host->tx_buf) {
-			ret = -ENOMEM;
-			pr_err("%s: failed to allocate tx buf, %d\n",
-				__func__, ret);
-			return ret;
-		}
+	if (!msm_host->tx_buf)
+		return -ENOMEM;
 
-		msm_host->tx_size = size;
-	}
+	msm_host->tx_size = size;
 
 	return 0;
 }
@@ -1089,6 +1128,21 @@ static void dsi_tx_buf_free(struct msm_dsi_host *msm_host)
 			msm_host->tx_buf_paddr);
 }
 
+void *dsi_tx_buf_get_6g(struct msm_dsi_host *msm_host)
+{
+	return msm_gem_get_vaddr(msm_host->tx_gem_obj);
+}
+
+void *dsi_tx_buf_get_v2(struct msm_dsi_host *msm_host)
+{
+	return msm_host->tx_buf;
+}
+
+void dsi_tx_buf_put_6g(struct msm_dsi_host *msm_host)
+{
+	msm_gem_put_vaddr(msm_host->tx_gem_obj);
+}
+
 /*
  * prepare cmd buffer to be txed
  */
@@ -1113,15 +1167,11 @@ static int dsi_cmd_dma_add(struct msm_dsi_host *msm_host,
 		return -EINVAL;
 	}
 
-	if (cfg_hnd->major == MSM_DSI_VER_MAJOR_6G) {
-		data = msm_gem_get_vaddr(msm_host->tx_gem_obj);
-		if (IS_ERR(data)) {
-			ret = PTR_ERR(data);
-			pr_err("%s: get vaddr failed, %d\n", __func__, ret);
-			return ret;
-		}
-	} else {
-		data = msm_host->tx_buf;
+	data = cfg_hnd->ops->tx_buf_get(msm_host);
+	if (IS_ERR(data)) {
+		ret = PTR_ERR(data);
+		pr_err("%s: get vaddr failed, %d\n", __func__, ret);
+		return ret;
 	}
 
 	/* MSM specific command format in memory */
@@ -1142,8 +1192,8 @@ static int dsi_cmd_dma_add(struct msm_dsi_host *msm_host,
 	if (packet.size < len)
 		memset(data + packet.size, 0xff, len - packet.size);
 
-	if (cfg_hnd->major == MSM_DSI_VER_MAJOR_6G)
-		msm_gem_put_vaddr(msm_host->tx_gem_obj);
+	if (cfg_hnd->ops->tx_buf_put)
+		cfg_hnd->ops->tx_buf_put(msm_host);
 
 	return len;
 }
@@ -1190,24 +1240,38 @@ static int dsi_long_read_resp(u8 *buf, const struct mipi_dsi_msg *msg)
 	return msg->rx_len;
 }
 
-static int dsi_cmd_dma_tx(struct msm_dsi_host *msm_host, int len)
+int dsi_dma_base_get_6g(struct msm_dsi_host *msm_host, uint64_t *dma_base)
 {
-	const struct msm_dsi_cfg_handler *cfg_hnd = msm_host->cfg_hnd;
 	struct drm_device *dev = msm_host->dev;
 	struct msm_drm_private *priv = dev->dev_private;
+
+	if (!dma_base)
+		return -EINVAL;
+
+	return msm_gem_get_iova(msm_host->tx_gem_obj,
+				priv->kms->aspace, dma_base);
+}
+
+int dsi_dma_base_get_v2(struct msm_dsi_host *msm_host, uint64_t *dma_base)
+{
+	if (!dma_base)
+		return -EINVAL;
+
+	*dma_base = msm_host->tx_buf_paddr;
+	return 0;
+}
+
+static int dsi_cmd_dma_tx(struct msm_dsi_host *msm_host, int len)
+{
+	const struct msm_dsi_cfg_handler *cfg_hnd = msm_host->cfg_hnd;
 	int ret;
 	uint64_t dma_base;
 	bool triggered;
 
-	if (cfg_hnd->major == MSM_DSI_VER_MAJOR_6G) {
-		ret = msm_gem_get_iova(msm_host->tx_gem_obj,
-				priv->kms->aspace, &dma_base);
-		if (ret) {
-			pr_err("%s: failed to get iova: %d\n", __func__, ret);
-			return ret;
-		}
-	} else {
-		dma_base = msm_host->tx_buf_paddr;
+	ret = cfg_hnd->ops->dma_base_get(msm_host, &dma_base);
+	if (ret) {
+		pr_err("%s: failed to get iova: %d\n", __func__, ret);
+		return ret;
 	}
 
 	reinit_completion(&msm_host->dma_comp);
@@ -1845,6 +1909,7 @@ int msm_dsi_host_modeset_init(struct mipi_dsi_host *host,
 					struct drm_device *dev)
 {
 	struct msm_dsi_host *msm_host = to_msm_dsi_host(host);
+	const struct msm_dsi_cfg_handler *cfg_hnd = msm_host->cfg_hnd;
 	struct platform_device *pdev = msm_host->pdev;
 	int ret;
 
@@ -1865,7 +1930,7 @@ int msm_dsi_host_modeset_init(struct mipi_dsi_host *host,
 	}
 
 	msm_host->dev = dev;
-	ret = dsi_tx_buf_alloc(msm_host, SZ_4K);
+	ret = cfg_hnd->ops->tx_buf_alloc(msm_host, SZ_4K);
 	if (ret) {
 		pr_err("%s: alloc tx gem obj failed, %d\n", __func__, ret);
 		return ret;
@@ -1923,6 +1988,7 @@ int msm_dsi_host_xfer_prepare(struct mipi_dsi_host *host,
 				const struct mipi_dsi_msg *msg)
 {
 	struct msm_dsi_host *msm_host = to_msm_dsi_host(host);
+	const struct msm_dsi_cfg_handler *cfg_hnd = msm_host->cfg_hnd;
 
 	/* TODO: make sure dsi_cmd_mdp is idle.
 	 * Since DSI6G v1.2.0, we can set DSI_TRIG_CTRL.BLOCK_DMA_WITHIN_FRAME
@@ -1935,7 +2001,7 @@ int msm_dsi_host_xfer_prepare(struct mipi_dsi_host *host,
 	 * mdp clock need to be enabled to receive dsi interrupt
 	 */
 	pm_runtime_get_sync(&msm_host->pdev->dev);
-	dsi_link_clk_enable(msm_host);
+	cfg_hnd->ops->link_clk_enable(msm_host);
 
 	/* TODO: vote for bus bandwidth */
 
@@ -1956,6 +2022,7 @@ void msm_dsi_host_xfer_restore(struct mipi_dsi_host *host,
 				const struct mipi_dsi_msg *msg)
 {
 	struct msm_dsi_host *msm_host = to_msm_dsi_host(host);
+	const struct msm_dsi_cfg_handler *cfg_hnd = msm_host->cfg_hnd;
 
 	dsi_intr_ctrl(msm_host, DSI_IRQ_MASK_CMD_DMA_DONE, 0);
 	dsi_write(msm_host, REG_DSI_CTRL, msm_host->dma_cmd_ctrl_restore);
@@ -1965,7 +2032,7 @@ void msm_dsi_host_xfer_restore(struct mipi_dsi_host *host,
 
 	/* TODO: unvote for bus bandwidth */
 
-	dsi_link_clk_disable(msm_host);
+	cfg_hnd->ops->link_clk_disable(msm_host);
 	pm_runtime_put_autosuspend(&msm_host->pdev->dev);
 }
 
@@ -2129,7 +2196,6 @@ int msm_dsi_host_set_src_pll(struct mipi_dsi_host *host,
 	struct msm_dsi_pll *src_pll)
 {
 	struct msm_dsi_host *msm_host = to_msm_dsi_host(host);
-	const struct msm_dsi_cfg_handler *cfg_hnd = msm_host->cfg_hnd;
 	struct clk *byte_clk_provider, *pixel_clk_provider;
 	int ret;
 
@@ -2155,14 +2221,16 @@ int msm_dsi_host_set_src_pll(struct mipi_dsi_host *host,
 		goto exit;
 	}
 
-	if (cfg_hnd->major == MSM_DSI_VER_MAJOR_V2) {
+	if (msm_host->dsi_clk_src) {
 		ret = clk_set_parent(msm_host->dsi_clk_src, pixel_clk_provider);
 		if (ret) {
 			pr_err("%s: can't set parent to dsi_clk_src. ret=%d\n",
 				__func__, ret);
 			goto exit;
 		}
+	}
 
+	if (msm_host->esc_clk_src) {
 		ret = clk_set_parent(msm_host->esc_clk_src, byte_clk_provider);
 		if (ret) {
 			pr_err("%s: can't set parent to esc_clk_src. ret=%d\n",
@@ -2189,12 +2257,14 @@ void msm_dsi_host_reset_phy(struct mipi_dsi_host *host)
 }
 
 void msm_dsi_host_get_phy_clk_req(struct mipi_dsi_host *host,
-	struct msm_dsi_phy_clk_request *clk_req)
+			struct msm_dsi_phy_clk_request *clk_req,
+			bool is_dual_dsi)
 {
 	struct msm_dsi_host *msm_host = to_msm_dsi_host(host);
+	const struct msm_dsi_cfg_handler *cfg_hnd = msm_host->cfg_hnd;
 	int ret;
 
-	ret = dsi_calc_clk_rate(msm_host);
+	ret = cfg_hnd->ops->calc_clk_rate(msm_host, is_dual_dsi);
 	if (ret) {
 		pr_err("%s: unable to calc clk rate, %d\n", __func__, ret);
 		return;
@@ -2256,9 +2326,11 @@ static void msm_dsi_sfpb_config(struct msm_dsi_host *msm_host, bool enable)
 }
 
 int msm_dsi_host_power_on(struct mipi_dsi_host *host,
-			struct msm_dsi_phy_shared_timings *phy_shared_timings)
+			struct msm_dsi_phy_shared_timings *phy_shared_timings,
+			bool is_dual_dsi)
 {
 	struct msm_dsi_host *msm_host = to_msm_dsi_host(host);
+	const struct msm_dsi_cfg_handler *cfg_hnd = msm_host->cfg_hnd;
 	int ret = 0;
 
 	mutex_lock(&msm_host->dev_mutex);
@@ -2277,7 +2349,7 @@ int msm_dsi_host_power_on(struct mipi_dsi_host *host,
 	}
 
 	pm_runtime_get_sync(&msm_host->pdev->dev);
-	ret = dsi_link_clk_enable(msm_host);
+	ret = cfg_hnd->ops->link_clk_enable(msm_host);
 	if (ret) {
 		pr_err("%s: failed to enable link clocks. ret=%d\n",
 		       __func__, ret);
@@ -2291,7 +2363,7 @@ int msm_dsi_host_power_on(struct mipi_dsi_host *host,
 		goto fail_disable_clk;
 	}
 
-	dsi_timing_setup(msm_host);
+	dsi_timing_setup(msm_host, is_dual_dsi);
 	dsi_sw_reset(msm_host);
 	dsi_ctrl_config(msm_host, true, phy_shared_timings);
 
@@ -2304,7 +2376,7 @@ int msm_dsi_host_power_on(struct mipi_dsi_host *host,
 	return 0;
 
 fail_disable_clk:
-	dsi_link_clk_disable(msm_host);
+	cfg_hnd->ops->link_clk_disable(msm_host);
 	pm_runtime_put_autosuspend(&msm_host->pdev->dev);
 fail_disable_reg:
 	dsi_host_regulator_disable(msm_host);
@@ -2316,6 +2388,7 @@ unlock_ret:
 int msm_dsi_host_power_off(struct mipi_dsi_host *host)
 {
 	struct msm_dsi_host *msm_host = to_msm_dsi_host(host);
+	const struct msm_dsi_cfg_handler *cfg_hnd = msm_host->cfg_hnd;
 
 	mutex_lock(&msm_host->dev_mutex);
 	if (!msm_host->power_on) {
@@ -2330,7 +2403,7 @@ int msm_dsi_host_power_off(struct mipi_dsi_host *host)
 
 	pinctrl_pm_select_sleep_state(&msm_host->pdev->dev);
 
-	dsi_link_clk_disable(msm_host);
+	cfg_hnd->ops->link_clk_disable(msm_host);
 	pm_runtime_put_autosuspend(&msm_host->pdev->dev);
 
 	dsi_host_regulator_disable(msm_host);
diff --git a/drivers/gpu/drm/msm/dsi/dsi_manager.c b/drivers/gpu/drm/msm/dsi/dsi_manager.c
index d5006d6923e0..5224010d90e4 100644
--- a/drivers/gpu/drm/msm/dsi/dsi_manager.c
+++ b/drivers/gpu/drm/msm/dsi/dsi_manager.c
@@ -134,8 +134,9 @@ static int enable_phy(struct msm_dsi *msm_dsi, int src_pll_id,
 {
 	struct msm_dsi_phy_clk_request clk_req;
 	int ret;
+	bool is_dual_dsi = IS_DUAL_DSI();
 
-	msm_dsi_host_get_phy_clk_req(msm_dsi->host, &clk_req);
+	msm_dsi_host_get_phy_clk_req(msm_dsi->host, &clk_req, is_dual_dsi);
 
 	ret = msm_dsi_phy_enable(msm_dsi->phy, src_pll_id, &clk_req);
 	msm_dsi_phy_get_shared_timings(msm_dsi->phy, shared_timings);
@@ -305,102 +306,25 @@ static void dsi_mgr_connector_destroy(struct drm_connector *connector)
 	kfree(dsi_connector);
 }
 
-static void dsi_dual_connector_fix_modes(struct drm_connector *connector)
-{
-	struct drm_display_mode *mode, *m;
-
-	/* Only support left-right mode */
-	list_for_each_entry_safe(mode, m, &connector->probed_modes, head) {
-		mode->clock >>= 1;
-		mode->hdisplay >>= 1;
-		mode->hsync_start >>= 1;
-		mode->hsync_end >>= 1;
-		mode->htotal >>= 1;
-		drm_mode_set_name(mode);
-	}
-}
-
-static int dsi_dual_connector_tile_init(
-			struct drm_connector *connector, int id)
-{
-	struct drm_display_mode *mode;
-	/* Fake topology id */
-	char topo_id[8] = {'M', 'S', 'M', 'D', 'U', 'D', 'S', 'I'};
-
-	if (connector->tile_group) {
-		DBG("Tile property has been initialized");
-		return 0;
-	}
-
-	/* Use the first mode only for now */
-	mode = list_first_entry(&connector->probed_modes,
-				struct drm_display_mode,
-				head);
-	if (!mode)
-		return -EINVAL;
-
-	connector->tile_group = drm_mode_get_tile_group(
-					connector->dev, topo_id);
-	if (!connector->tile_group)
-		connector->tile_group = drm_mode_create_tile_group(
-					connector->dev, topo_id);
-	if (!connector->tile_group) {
-		pr_err("%s: failed to create tile group\n", __func__);
-		return -ENOMEM;
-	}
-
-	connector->has_tile = true;
-	connector->tile_is_single_monitor = true;
-
-	/* mode has been fixed */
-	connector->tile_h_size = mode->hdisplay;
-	connector->tile_v_size = mode->vdisplay;
-
-	/* Only support left-right mode */
-	connector->num_h_tile = 2;
-	connector->num_v_tile = 1;
-
-	connector->tile_v_loc = 0;
-	connector->tile_h_loc = (id == DSI_RIGHT) ? 1 : 0;
-
-	return 0;
-}
-
 static int dsi_mgr_connector_get_modes(struct drm_connector *connector)
 {
 	int id = dsi_mgr_connector_get_id(connector);
 	struct msm_dsi *msm_dsi = dsi_mgr_get_dsi(id);
 	struct drm_panel *panel = msm_dsi->panel;
-	int ret, num;
+	int num;
 
 	if (!panel)
 		return 0;
 
-	/* Since we have 2 connectors, but only 1 drm_panel in dual DSI mode,
-	 * panel should not attach to any connector.
-	 * Only temporarily attach panel to the current connector here,
-	 * to let panel set mode to this connector.
+	/*
+	 * In dual DSI mode, we have one connector that can be
+	 * attached to the drm_panel.
 	 */
 	drm_panel_attach(panel, connector);
 	num = drm_panel_get_modes(panel);
-	drm_panel_detach(panel);
 	if (!num)
 		return 0;
 
-	if (IS_DUAL_DSI()) {
-		/* report half resolution to user */
-		dsi_dual_connector_fix_modes(connector);
-		ret = dsi_dual_connector_tile_init(connector, id);
-		if (ret)
-			return ret;
-		ret = drm_connector_set_tile_property(connector);
-		if (ret) {
-			pr_err("%s: set tile property failed, %d\n",
-					__func__, ret);
-			return ret;
-		}
-	}
-
 	return num;
 }
 
@@ -454,11 +378,11 @@ static void dsi_mgr_bridge_pre_enable(struct drm_bridge *bridge)
 	if (ret)
 		goto phy_en_fail;
 
-	/* Do nothing with the host if it is DSI 1 in case of dual DSI */
-	if (is_dual_dsi && (DSI_1 == id))
+	/* Do nothing with the host if it is slave-DSI in case of dual DSI */
+	if (is_dual_dsi && !IS_MASTER_DSI_LINK(id))
 		return;
 
-	ret = msm_dsi_host_power_on(host, &phy_shared_timings[id]);
+	ret = msm_dsi_host_power_on(host, &phy_shared_timings[id], is_dual_dsi);
 	if (ret) {
 		pr_err("%s: power on host %d failed, %d\n", __func__, id, ret);
 		goto host_on_fail;
@@ -466,7 +390,7 @@ static void dsi_mgr_bridge_pre_enable(struct drm_bridge *bridge)
 
 	if (is_dual_dsi && msm_dsi1) {
 		ret = msm_dsi_host_power_on(msm_dsi1->host,
-					    &phy_shared_timings[DSI_1]);
+				&phy_shared_timings[DSI_1], is_dual_dsi);
 		if (ret) {
 			pr_err("%s: power on host1 failed, %d\n",
 							__func__, ret);
@@ -556,11 +480,11 @@ static void dsi_mgr_bridge_post_disable(struct drm_bridge *bridge)
 		return;
 
 	/*
-	 * Do nothing with the host if it is DSI 1 in case of dual DSI.
+	 * Do nothing with the host if it is slave-DSI in case of dual DSI.
 	 * It is safe to call dsi_mgr_phy_disable() here because a single PHY
 	 * won't be diabled until both PHYs request disable.
 	 */
-	if (is_dual_dsi && (DSI_1 == id))
+	if (is_dual_dsi && !IS_MASTER_DSI_LINK(id))
 		goto disable_phy;
 
 	if (panel) {
@@ -621,7 +545,7 @@ static void dsi_mgr_bridge_mode_set(struct drm_bridge *bridge,
 			mode->vsync_end, mode->vtotal,
 			mode->type, mode->flags);
 
-	if (is_dual_dsi && (DSI_1 == id))
+	if (is_dual_dsi && !IS_MASTER_DSI_LINK(id))
 		return;
 
 	msm_dsi_host_set_display_mode(host, adjusted_mode);
@@ -689,6 +613,23 @@ struct drm_connector *msm_dsi_manager_connector_init(u8 id)
 	return connector;
 }
 
+bool msm_dsi_manager_validate_current_config(u8 id)
+{
+	bool is_dual_dsi = IS_DUAL_DSI();
+
+	/*
+	 * For dual DSI, we only have one drm panel. For this
+	 * use case, we register only one bridge/connector.
+	 * Skip bridge/connector initialisation if it is
+	 * slave-DSI for dual DSI configuration.
+	 */
+	if (is_dual_dsi && !IS_MASTER_DSI_LINK(id)) {
+		DBG("Skip bridge registration for slave DSI->id: %d\n", id);
+		return false;
+	}
+	return true;
+}
+
 /* initialize bridge */
 struct drm_bridge *msm_dsi_manager_bridge_init(u8 id)
 {
@@ -832,6 +773,7 @@ void msm_dsi_manager_attach_dsi_device(int id, u32 device_flags)
 	struct msm_drm_private *priv;
 	struct msm_kms *kms;
 	struct drm_encoder *encoder;
+	bool cmd_mode;
 
 	/*
 	 * drm_device pointer is assigned to msm_dsi only in the modeset_init
@@ -846,10 +788,11 @@ void msm_dsi_manager_attach_dsi_device(int id, u32 device_flags)
 	priv = dev->dev_private;
 	kms = priv->kms;
 	encoder = msm_dsi_get_encoder(msm_dsi);
+	cmd_mode = !(device_flags &
+				 MIPI_DSI_MODE_VIDEO);
 
 	if (encoder && kms->funcs->set_encoder_mode)
-		if (!(device_flags & MIPI_DSI_MODE_VIDEO))
-			kms->funcs->set_encoder_mode(kms, encoder, true);
+		kms->funcs->set_encoder_mode(kms, encoder, cmd_mode);
 }
 
 int msm_dsi_manager_register(struct msm_dsi *msm_dsi)
diff --git a/drivers/gpu/drm/msm/dsi/mmss_cc.xml.h b/drivers/gpu/drm/msm/dsi/mmss_cc.xml.h
index 57cf7fa7f1c4..874265314413 100644
--- a/drivers/gpu/drm/msm/dsi/mmss_cc.xml.h
+++ b/drivers/gpu/drm/msm/dsi/mmss_cc.xml.h
@@ -8,19 +8,19 @@ http://github.com/freedreno/envytools/
 git clone https://github.com/freedreno/envytools.git
 
 The rules-ng-ng source files this header was generated from are:
-- /home/robclark/src/freedreno/envytools/rnndb/msm.xml                 (    676 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml (   1572 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp4.xml            (  20915 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp_common.xml      (   2849 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp5.xml            (  37411 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/dsi/dsi.xml             (  33004 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/dsi/sfpb.xml            (    602 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/dsi/mmss_cc.xml         (   1686 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/hdmi/qfprom.xml         (    600 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/hdmi/hdmi.xml           (  41799 bytes, from 2017-06-16 12:32:42)
-- /home/robclark/src/freedreno/envytools/rnndb/edp/edp.xml             (  10416 bytes, from 2017-05-17 13:21:27)
-
-Copyright (C) 2013-2017 by the following authors:
+- /home/robclark/src/envytools/rnndb/msm.xml                 (    676 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/freedreno_copyright.xml (   1572 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/mdp/mdp4.xml            (  20915 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/mdp/mdp_common.xml      (   2849 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/mdp/mdp5.xml            (  37411 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/dsi/dsi.xml             (  37239 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/dsi/sfpb.xml            (    602 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/dsi/mmss_cc.xml         (   1686 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/hdmi/qfprom.xml         (    600 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/hdmi/hdmi.xml           (  41799 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/edp/edp.xml             (  10416 bytes, from 2018-07-03 19:37:13)
+
+Copyright (C) 2013-2018 by the following authors:
 - Rob Clark <robdclark@gmail.com> (robclark)
 - Ilia Mirkin <imirkin@alum.mit.edu> (imirkin)
 
diff --git a/drivers/gpu/drm/msm/dsi/pll/dsi_pll_10nm.c b/drivers/gpu/drm/msm/dsi/pll/dsi_pll_10nm.c
index c4c37a7df637..4c03f0b7343e 100644
--- a/drivers/gpu/drm/msm/dsi/pll/dsi_pll_10nm.c
+++ b/drivers/gpu/drm/msm/dsi/pll/dsi_pll_10nm.c
@@ -798,6 +798,8 @@ struct msm_dsi_pll *msm_dsi_pll_10nm_init(struct platform_device *pdev, int id)
 		return ERR_PTR(-ENOMEM);
 	}
 
+	spin_lock_init(&pll_10nm->postdiv_lock);
+
 	pll = &pll_10nm->base;
 	pll->min_rate = 1000000000UL;
 	pll->max_rate = 3500000000UL;
diff --git a/drivers/gpu/drm/msm/dsi/sfpb.xml.h b/drivers/gpu/drm/msm/dsi/sfpb.xml.h
index 9d4d1feaefd7..07c48ddb5301 100644
--- a/drivers/gpu/drm/msm/dsi/sfpb.xml.h
+++ b/drivers/gpu/drm/msm/dsi/sfpb.xml.h
@@ -8,19 +8,19 @@ http://github.com/freedreno/envytools/
 git clone https://github.com/freedreno/envytools.git
 
 The rules-ng-ng source files this header was generated from are:
-- /home/robclark/src/freedreno/envytools/rnndb/msm.xml                 (    676 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml (   1572 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp4.xml            (  20915 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp_common.xml      (   2849 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp5.xml            (  37411 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/dsi/dsi.xml             (  33004 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/dsi/sfpb.xml            (    602 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/dsi/mmss_cc.xml         (   1686 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/hdmi/qfprom.xml         (    600 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/hdmi/hdmi.xml           (  41799 bytes, from 2017-06-16 12:32:42)
-- /home/robclark/src/freedreno/envytools/rnndb/edp/edp.xml             (  10416 bytes, from 2017-05-17 13:21:27)
-
-Copyright (C) 2013-2017 by the following authors:
+- /home/robclark/src/envytools/rnndb/msm.xml                 (    676 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/freedreno_copyright.xml (   1572 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/mdp/mdp4.xml            (  20915 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/mdp/mdp_common.xml      (   2849 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/mdp/mdp5.xml            (  37411 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/dsi/dsi.xml             (  37239 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/dsi/sfpb.xml            (    602 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/dsi/mmss_cc.xml         (   1686 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/hdmi/qfprom.xml         (    600 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/hdmi/hdmi.xml           (  41799 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/edp/edp.xml             (  10416 bytes, from 2018-07-03 19:37:13)
+
+Copyright (C) 2013-2018 by the following authors:
 - Rob Clark <robdclark@gmail.com> (robclark)
 - Ilia Mirkin <imirkin@alum.mit.edu> (imirkin)
 
diff --git a/drivers/gpu/drm/msm/edp/edp.xml.h b/drivers/gpu/drm/msm/edp/edp.xml.h
index f150d4a47707..9cb6e6fe9810 100644
--- a/drivers/gpu/drm/msm/edp/edp.xml.h
+++ b/drivers/gpu/drm/msm/edp/edp.xml.h
@@ -8,19 +8,19 @@ http://github.com/freedreno/envytools/
 git clone https://github.com/freedreno/envytools.git
 
 The rules-ng-ng source files this header was generated from are:
-- /home/robclark/src/freedreno/envytools/rnndb/msm.xml                 (    676 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml (   1572 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp4.xml            (  20915 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp_common.xml      (   2849 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp5.xml            (  37411 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/dsi/dsi.xml             (  33004 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/dsi/sfpb.xml            (    602 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/dsi/mmss_cc.xml         (   1686 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/hdmi/qfprom.xml         (    600 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/hdmi/hdmi.xml           (  41799 bytes, from 2017-06-16 12:32:42)
-- /home/robclark/src/freedreno/envytools/rnndb/edp/edp.xml             (  10416 bytes, from 2017-05-17 13:21:27)
-
-Copyright (C) 2013-2017 by the following authors:
+- /home/robclark/src/envytools/rnndb/msm.xml                 (    676 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/freedreno_copyright.xml (   1572 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/mdp/mdp4.xml            (  20915 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/mdp/mdp_common.xml      (   2849 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/mdp/mdp5.xml            (  37411 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/dsi/dsi.xml             (  37239 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/dsi/sfpb.xml            (    602 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/dsi/mmss_cc.xml         (   1686 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/hdmi/qfprom.xml         (    600 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/hdmi/hdmi.xml           (  41799 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/edp/edp.xml             (  10416 bytes, from 2018-07-03 19:37:13)
+
+Copyright (C) 2013-2018 by the following authors:
 - Rob Clark <robdclark@gmail.com> (robclark)
 - Ilia Mirkin <imirkin@alum.mit.edu> (imirkin)
 
diff --git a/drivers/gpu/drm/msm/hdmi/hdmi.xml.h b/drivers/gpu/drm/msm/hdmi/hdmi.xml.h
index ecebf8b623ab..3eff3ea3b271 100644
--- a/drivers/gpu/drm/msm/hdmi/hdmi.xml.h
+++ b/drivers/gpu/drm/msm/hdmi/hdmi.xml.h
@@ -8,19 +8,19 @@ http://github.com/freedreno/envytools/
 git clone https://github.com/freedreno/envytools.git
 
 The rules-ng-ng source files this header was generated from are:
-- /home/robclark/src/freedreno/envytools/rnndb/msm.xml                 (    676 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml (   1572 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp4.xml            (  20915 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp_common.xml      (   2849 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp5.xml            (  37411 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/dsi/dsi.xml             (  33004 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/dsi/sfpb.xml            (    602 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/dsi/mmss_cc.xml         (   1686 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/hdmi/qfprom.xml         (    600 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/hdmi/hdmi.xml           (  41799 bytes, from 2017-06-16 12:32:42)
-- /home/robclark/src/freedreno/envytools/rnndb/edp/edp.xml             (  10416 bytes, from 2017-05-17 13:21:27)
-
-Copyright (C) 2013-2017 by the following authors:
+- /home/robclark/src/envytools/rnndb/msm.xml                 (    676 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/freedreno_copyright.xml (   1572 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/mdp/mdp4.xml            (  20915 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/mdp/mdp_common.xml      (   2849 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/mdp/mdp5.xml            (  37411 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/dsi/dsi.xml             (  37239 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/dsi/sfpb.xml            (    602 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/dsi/mmss_cc.xml         (   1686 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/hdmi/qfprom.xml         (    600 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/hdmi/hdmi.xml           (  41799 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/edp/edp.xml             (  10416 bytes, from 2018-07-03 19:37:13)
+
+Copyright (C) 2013-2018 by the following authors:
 - Rob Clark <robdclark@gmail.com> (robclark)
 - Ilia Mirkin <imirkin@alum.mit.edu> (imirkin)
 
diff --git a/drivers/gpu/drm/msm/hdmi/qfprom.xml.h b/drivers/gpu/drm/msm/hdmi/qfprom.xml.h
index da646deedf4b..7717d4269662 100644
--- a/drivers/gpu/drm/msm/hdmi/qfprom.xml.h
+++ b/drivers/gpu/drm/msm/hdmi/qfprom.xml.h
@@ -8,19 +8,19 @@ http://github.com/freedreno/envytools/
 git clone https://github.com/freedreno/envytools.git
 
 The rules-ng-ng source files this header was generated from are:
-- /home/robclark/src/freedreno/envytools/rnndb/msm.xml                 (    676 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml (   1572 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp4.xml            (  20915 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp_common.xml      (   2849 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp5.xml            (  37411 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/dsi/dsi.xml             (  33004 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/dsi/sfpb.xml            (    602 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/dsi/mmss_cc.xml         (   1686 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/hdmi/qfprom.xml         (    600 bytes, from 2017-05-17 13:21:27)
-- /home/robclark/src/freedreno/envytools/rnndb/hdmi/hdmi.xml           (  41799 bytes, from 2017-06-16 12:32:42)
-- /home/robclark/src/freedreno/envytools/rnndb/edp/edp.xml             (  10416 bytes, from 2017-05-17 13:21:27)
-
-Copyright (C) 2013-2017 by the following authors:
+- /home/robclark/src/envytools/rnndb/msm.xml                 (    676 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/freedreno_copyright.xml (   1572 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/mdp/mdp4.xml            (  20915 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/mdp/mdp_common.xml      (   2849 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/mdp/mdp5.xml            (  37411 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/dsi/dsi.xml             (  37239 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/dsi/sfpb.xml            (    602 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/dsi/mmss_cc.xml         (   1686 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/hdmi/qfprom.xml         (    600 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/hdmi/hdmi.xml           (  41799 bytes, from 2018-07-03 19:37:13)
+- /home/robclark/src/envytools/rnndb/edp/edp.xml             (  10416 bytes, from 2018-07-03 19:37:13)
+
+Copyright (C) 2013-2018 by the following authors:
 - Rob Clark <robdclark@gmail.com> (robclark)
 - Ilia Mirkin <imirkin@alum.mit.edu> (imirkin)
 
diff --git a/drivers/gpu/drm/msm/msm_atomic.c b/drivers/gpu/drm/msm/msm_atomic.c
index f0635c3da7f4..c1f1779c980f 100644
--- a/drivers/gpu/drm/msm/msm_atomic.c
+++ b/drivers/gpu/drm/msm/msm_atomic.c
@@ -71,12 +71,15 @@ void msm_atomic_commit_tail(struct drm_atomic_state *state)
 
 	drm_atomic_helper_commit_modeset_enables(dev, state);
 
+	if (kms->funcs->commit) {
+		DRM_DEBUG_ATOMIC("triggering commit\n");
+		kms->funcs->commit(kms, state);
+	}
+
 	msm_atomic_wait_for_commit_done(dev, state);
 
 	kms->funcs->complete_commit(kms, state);
 
-	drm_atomic_helper_wait_for_vblanks(dev, state);
-
 	drm_atomic_helper_commit_hw_done(state);
 
 	drm_atomic_helper_cleanup_planes(dev, state);
diff --git a/drivers/gpu/drm/msm/msm_debugfs.c b/drivers/gpu/drm/msm/msm_debugfs.c
index 1ff3fda245d1..f0da0d3c8a80 100644
--- a/drivers/gpu/drm/msm/msm_debugfs.c
+++ b/drivers/gpu/drm/msm/msm_debugfs.c
@@ -16,26 +16,101 @@
  */
 
 #ifdef CONFIG_DEBUG_FS
+#include <linux/debugfs.h>
 #include "msm_drv.h"
 #include "msm_gpu.h"
 #include "msm_kms.h"
 #include "msm_debugfs.h"
 
-static int msm_gpu_show(struct drm_device *dev, struct seq_file *m)
+struct msm_gpu_show_priv {
+	struct msm_gpu_state *state;
+	struct drm_device *dev;
+};
+
+static int msm_gpu_show(struct seq_file *m, void *arg)
+{
+	struct drm_printer p = drm_seq_file_printer(m);
+	struct msm_gpu_show_priv *show_priv = m->private;
+	struct msm_drm_private *priv = show_priv->dev->dev_private;
+	struct msm_gpu *gpu = priv->gpu;
+	int ret;
+
+	ret = mutex_lock_interruptible(&show_priv->dev->struct_mutex);
+	if (ret)
+		return ret;
+
+	drm_printf(&p, "%s Status:\n", gpu->name);
+	gpu->funcs->show(gpu, show_priv->state, &p);
+
+	mutex_unlock(&show_priv->dev->struct_mutex);
+
+	return 0;
+}
+
+static int msm_gpu_release(struct inode *inode, struct file *file)
 {
+	struct seq_file *m = file->private_data;
+	struct msm_gpu_show_priv *show_priv = m->private;
+	struct msm_drm_private *priv = show_priv->dev->dev_private;
+	struct msm_gpu *gpu = priv->gpu;
+	int ret;
+
+	ret = mutex_lock_interruptible(&show_priv->dev->struct_mutex);
+	if (ret)
+		return ret;
+
+	gpu->funcs->gpu_state_put(show_priv->state);
+	mutex_unlock(&show_priv->dev->struct_mutex);
+
+	kfree(show_priv);
+
+	return single_release(inode, file);
+}
+
+static int msm_gpu_open(struct inode *inode, struct file *file)
+{
+	struct drm_device *dev = inode->i_private;
 	struct msm_drm_private *priv = dev->dev_private;
 	struct msm_gpu *gpu = priv->gpu;
+	struct msm_gpu_show_priv *show_priv;
+	int ret;
 
-	if (gpu) {
-		seq_printf(m, "%s Status:\n", gpu->name);
-		pm_runtime_get_sync(&gpu->pdev->dev);
-		gpu->funcs->show(gpu, m);
-		pm_runtime_put_sync(&gpu->pdev->dev);
+	if (!gpu)
+		return -ENODEV;
+
+	show_priv = kmalloc(sizeof(*show_priv), GFP_KERNEL);
+	if (!show_priv)
+		return -ENOMEM;
+
+	ret = mutex_lock_interruptible(&dev->struct_mutex);
+	if (ret)
+		return ret;
+
+	pm_runtime_get_sync(&gpu->pdev->dev);
+	show_priv->state = gpu->funcs->gpu_state_get(gpu);
+	pm_runtime_put_sync(&gpu->pdev->dev);
+
+	mutex_unlock(&dev->struct_mutex);
+
+	if (IS_ERR(show_priv->state)) {
+		ret = PTR_ERR(show_priv->state);
+		kfree(show_priv);
+		return ret;
 	}
 
-	return 0;
+	show_priv->dev = dev;
+
+	return single_open(file, msm_gpu_show, show_priv);
 }
 
+static const struct file_operations msm_gpu_fops = {
+	.owner = THIS_MODULE,
+	.open = msm_gpu_open,
+	.read = seq_read,
+	.llseek = seq_lseek,
+	.release = msm_gpu_release,
+};
+
 static int msm_gem_show(struct drm_device *dev, struct seq_file *m)
 {
 	struct msm_drm_private *priv = dev->dev_private;
@@ -105,7 +180,6 @@ static int show_locked(struct seq_file *m, void *arg)
 }
 
 static struct drm_info_list msm_debugfs_list[] = {
-		{"gpu", show_locked, 0, msm_gpu_show},
 		{"gem", show_locked, 0, msm_gem_show},
 		{ "mm", show_locked, 0, msm_mm_show },
 		{ "fb", show_locked, 0, msm_fb_show },
@@ -158,6 +232,9 @@ int msm_debugfs_init(struct drm_minor *minor)
 		return ret;
 	}
 
+	debugfs_create_file("gpu", S_IRUSR, minor->debugfs_root,
+		dev, &msm_gpu_fops);
+
 	if (priv->kms->funcs->debugfs_init) {
 		ret = priv->kms->funcs->debugfs_init(priv->kms, minor);
 		if (ret)
diff --git a/drivers/gpu/drm/msm/msm_drv.c b/drivers/gpu/drm/msm/msm_drv.c
index 021a0b6f9a59..c1abad8a8612 100644
--- a/drivers/gpu/drm/msm/msm_drv.c
+++ b/drivers/gpu/drm/msm/msm_drv.c
@@ -1,4 +1,5 @@
 /*
+ * Copyright (c) 2016-2018, The Linux Foundation. All rights reserved.
  * Copyright (C) 2013 Red Hat
  * Author: Rob Clark <robdclark@gmail.com>
  *
@@ -15,6 +16,8 @@
  * this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include <linux/kthread.h>
+#include <uapi/linux/sched/types.h>
 #include <drm/drm_of.h>
 
 #include "msm_drv.h"
@@ -78,6 +81,63 @@ module_param(modeset, bool, 0600);
  * Util/helpers:
  */
 
+int msm_clk_bulk_get(struct device *dev, struct clk_bulk_data **bulk)
+{
+	struct property *prop;
+	const char *name;
+	struct clk_bulk_data *local;
+	int i = 0, ret, count;
+
+	count = of_property_count_strings(dev->of_node, "clock-names");
+	if (count < 1)
+		return 0;
+
+	local = devm_kcalloc(dev, sizeof(struct clk_bulk_data *),
+		count, GFP_KERNEL);
+	if (!local)
+		return -ENOMEM;
+
+	of_property_for_each_string(dev->of_node, "clock-names", prop, name) {
+		local[i].id = devm_kstrdup(dev, name, GFP_KERNEL);
+		if (!local[i].id) {
+			devm_kfree(dev, local);
+			return -ENOMEM;
+		}
+
+		i++;
+	}
+
+	ret = devm_clk_bulk_get(dev, count, local);
+
+	if (ret) {
+		for (i = 0; i < count; i++)
+			devm_kfree(dev, (void *) local[i].id);
+		devm_kfree(dev, local);
+
+		return ret;
+	}
+
+	*bulk = local;
+	return count;
+}
+
+struct clk *msm_clk_bulk_get_clock(struct clk_bulk_data *bulk, int count,
+		const char *name)
+{
+	int i;
+	char n[32];
+
+	snprintf(n, sizeof(n), "%s_clk", name);
+
+	for (i = 0; bulk && i < count; i++) {
+		if (!strcmp(bulk[i].id, name) || !strcmp(bulk[i].id, n))
+			return bulk[i].clk;
+	}
+
+
+	return NULL;
+}
+
 struct clk *msm_clk_get(struct platform_device *pdev, const char *name)
 {
 	struct clk *clk;
@@ -149,7 +209,7 @@ struct vblank_event {
 	bool enable;
 };
 
-static void vblank_ctrl_worker(struct work_struct *work)
+static void vblank_ctrl_worker(struct kthread_work *work)
 {
 	struct msm_vblank_ctrl *vbl_ctrl = container_of(work,
 						struct msm_vblank_ctrl, work);
@@ -197,7 +257,8 @@ static int vblank_ctrl_queue_work(struct msm_drm_private *priv,
 	list_add_tail(&vbl_ev->node, &vbl_ctrl->event_list);
 	spin_unlock_irqrestore(&vbl_ctrl->lock, flags);
 
-	queue_work(priv->wq, &vbl_ctrl->work);
+	kthread_queue_work(&priv->disp_thread[crtc_id].worker,
+			&vbl_ctrl->work);
 
 	return 0;
 }
@@ -208,19 +269,36 @@ static int msm_drm_uninit(struct device *dev)
 	struct drm_device *ddev = platform_get_drvdata(pdev);
 	struct msm_drm_private *priv = ddev->dev_private;
 	struct msm_kms *kms = priv->kms;
+	struct msm_mdss *mdss = priv->mdss;
 	struct msm_vblank_ctrl *vbl_ctrl = &priv->vblank_ctrl;
 	struct vblank_event *vbl_ev, *tmp;
+	int i;
 
 	/* We must cancel and cleanup any pending vblank enable/disable
 	 * work before drm_irq_uninstall() to avoid work re-enabling an
 	 * irq after uninstall has disabled it.
 	 */
-	cancel_work_sync(&vbl_ctrl->work);
+	kthread_flush_work(&vbl_ctrl->work);
 	list_for_each_entry_safe(vbl_ev, tmp, &vbl_ctrl->event_list, node) {
 		list_del(&vbl_ev->node);
 		kfree(vbl_ev);
 	}
 
+	/* clean up display commit/event worker threads */
+	for (i = 0; i < priv->num_crtcs; i++) {
+		if (priv->disp_thread[i].thread) {
+			kthread_flush_worker(&priv->disp_thread[i].worker);
+			kthread_stop(priv->disp_thread[i].thread);
+			priv->disp_thread[i].thread = NULL;
+		}
+
+		if (priv->event_thread[i].thread) {
+			kthread_flush_worker(&priv->event_thread[i].worker);
+			kthread_stop(priv->event_thread[i].thread);
+			priv->event_thread[i].thread = NULL;
+		}
+	}
+
 	msm_gem_shrinker_cleanup(ddev);
 
 	drm_kms_helper_poll_fini(ddev);
@@ -243,9 +321,6 @@ static int msm_drm_uninit(struct device *dev)
 	flush_workqueue(priv->wq);
 	destroy_workqueue(priv->wq);
 
-	flush_workqueue(priv->atomic_wq);
-	destroy_workqueue(priv->atomic_wq);
-
 	if (kms && kms->funcs)
 		kms->funcs->destroy(kms);
 
@@ -258,7 +333,8 @@ static int msm_drm_uninit(struct device *dev)
 
 	component_unbind_all(dev, ddev);
 
-	msm_mdss_destroy(ddev);
+	if (mdss && mdss->funcs)
+		mdss->funcs->destroy(ddev);
 
 	ddev->dev_private = NULL;
 	drm_dev_unref(ddev);
@@ -268,6 +344,10 @@ static int msm_drm_uninit(struct device *dev)
 	return 0;
 }
 
+#define KMS_MDP4 4
+#define KMS_MDP5 5
+#define KMS_DPU  3
+
 static int get_mdp_ver(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
@@ -357,7 +437,9 @@ static int msm_drm_init(struct device *dev, struct drm_driver *drv)
 	struct drm_device *ddev;
 	struct msm_drm_private *priv;
 	struct msm_kms *kms;
-	int ret;
+	struct msm_mdss *mdss;
+	int ret, i;
+	struct sched_param param;
 
 	ddev = drm_dev_alloc(drv, dev);
 	if (IS_ERR(ddev)) {
@@ -369,53 +451,61 @@ static int msm_drm_init(struct device *dev, struct drm_driver *drv)
 
 	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
 	if (!priv) {
-		drm_dev_unref(ddev);
-		return -ENOMEM;
+		ret = -ENOMEM;
+		goto err_unref_drm_dev;
 	}
 
 	ddev->dev_private = priv;
 	priv->dev = ddev;
 
-	ret = msm_mdss_init(ddev);
-	if (ret) {
-		kfree(priv);
-		drm_dev_unref(ddev);
-		return ret;
+	switch (get_mdp_ver(pdev)) {
+	case KMS_MDP5:
+		ret = mdp5_mdss_init(ddev);
+		break;
+	case KMS_DPU:
+		ret = dpu_mdss_init(ddev);
+		break;
+	default:
+		ret = 0;
+		break;
 	}
+	if (ret)
+		goto err_free_priv;
+
+	mdss = priv->mdss;
 
 	priv->wq = alloc_ordered_workqueue("msm", 0);
-	priv->atomic_wq = alloc_ordered_workqueue("msm:atomic", 0);
 
 	INIT_LIST_HEAD(&priv->inactive_list);
 	INIT_LIST_HEAD(&priv->vblank_ctrl.event_list);
-	INIT_WORK(&priv->vblank_ctrl.work, vblank_ctrl_worker);
+	kthread_init_work(&priv->vblank_ctrl.work, vblank_ctrl_worker);
 	spin_lock_init(&priv->vblank_ctrl.lock);
 
 	drm_mode_config_init(ddev);
 
 	/* Bind all our sub-components: */
 	ret = component_bind_all(dev, ddev);
-	if (ret) {
-		msm_mdss_destroy(ddev);
-		kfree(priv);
-		drm_dev_unref(ddev);
-		return ret;
-	}
+	if (ret)
+		goto err_destroy_mdss;
 
 	ret = msm_init_vram(ddev);
 	if (ret)
-		goto fail;
+		goto err_msm_uninit;
 
 	msm_gem_shrinker_init(ddev);
 
 	switch (get_mdp_ver(pdev)) {
-	case 4:
+	case KMS_MDP4:
 		kms = mdp4_kms_init(ddev);
 		priv->kms = kms;
 		break;
-	case 5:
+	case KMS_MDP5:
 		kms = mdp5_kms_init(ddev);
 		break;
+	case KMS_DPU:
+		kms = dpu_kms_init(ddev);
+		priv->kms = kms;
+		break;
 	default:
 		kms = ERR_PTR(-ENODEV);
 		break;
@@ -430,24 +520,100 @@ static int msm_drm_init(struct device *dev, struct drm_driver *drv)
 		 */
 		dev_err(dev, "failed to load kms\n");
 		ret = PTR_ERR(kms);
-		goto fail;
+		goto err_msm_uninit;
 	}
 
+	/* Enable normalization of plane zpos */
+	ddev->mode_config.normalize_zpos = true;
+
 	if (kms) {
 		ret = kms->funcs->hw_init(kms);
 		if (ret) {
 			dev_err(dev, "kms hw init failed: %d\n", ret);
-			goto fail;
+			goto err_msm_uninit;
 		}
 	}
 
 	ddev->mode_config.funcs = &mode_config_funcs;
 	ddev->mode_config.helper_private = &mode_config_helper_funcs;
 
+	/**
+	 * this priority was found during empiric testing to have appropriate
+	 * realtime scheduling to process display updates and interact with
+	 * other real time and normal priority task
+	 */
+	param.sched_priority = 16;
+	for (i = 0; i < priv->num_crtcs; i++) {
+
+		/* initialize display thread */
+		priv->disp_thread[i].crtc_id = priv->crtcs[i]->base.id;
+		kthread_init_worker(&priv->disp_thread[i].worker);
+		priv->disp_thread[i].dev = ddev;
+		priv->disp_thread[i].thread =
+			kthread_run(kthread_worker_fn,
+				&priv->disp_thread[i].worker,
+				"crtc_commit:%d", priv->disp_thread[i].crtc_id);
+		ret = sched_setscheduler(priv->disp_thread[i].thread,
+							SCHED_FIFO, &param);
+		if (ret)
+			pr_warn("display thread priority update failed: %d\n",
+									ret);
+
+		if (IS_ERR(priv->disp_thread[i].thread)) {
+			dev_err(dev, "failed to create crtc_commit kthread\n");
+			priv->disp_thread[i].thread = NULL;
+		}
+
+		/* initialize event thread */
+		priv->event_thread[i].crtc_id = priv->crtcs[i]->base.id;
+		kthread_init_worker(&priv->event_thread[i].worker);
+		priv->event_thread[i].dev = ddev;
+		priv->event_thread[i].thread =
+			kthread_run(kthread_worker_fn,
+				&priv->event_thread[i].worker,
+				"crtc_event:%d", priv->event_thread[i].crtc_id);
+		/**
+		 * event thread should also run at same priority as disp_thread
+		 * because it is handling frame_done events. A lower priority
+		 * event thread and higher priority disp_thread can causes
+		 * frame_pending counters beyond 2. This can lead to commit
+		 * failure at crtc commit level.
+		 */
+		ret = sched_setscheduler(priv->event_thread[i].thread,
+							SCHED_FIFO, &param);
+		if (ret)
+			pr_warn("display event thread priority update failed: %d\n",
+									ret);
+
+		if (IS_ERR(priv->event_thread[i].thread)) {
+			dev_err(dev, "failed to create crtc_event kthread\n");
+			priv->event_thread[i].thread = NULL;
+		}
+
+		if ((!priv->disp_thread[i].thread) ||
+				!priv->event_thread[i].thread) {
+			/* clean up previously created threads if any */
+			for ( ; i >= 0; i--) {
+				if (priv->disp_thread[i].thread) {
+					kthread_stop(
+						priv->disp_thread[i].thread);
+					priv->disp_thread[i].thread = NULL;
+				}
+
+				if (priv->event_thread[i].thread) {
+					kthread_stop(
+						priv->event_thread[i].thread);
+					priv->event_thread[i].thread = NULL;
+				}
+			}
+			goto err_msm_uninit;
+		}
+	}
+
 	ret = drm_vblank_init(ddev, priv->num_crtcs);
 	if (ret < 0) {
 		dev_err(dev, "failed to initialize vblank\n");
-		goto fail;
+		goto err_msm_uninit;
 	}
 
 	if (kms) {
@@ -456,13 +622,13 @@ static int msm_drm_init(struct device *dev, struct drm_driver *drv)
 		pm_runtime_put_sync(dev);
 		if (ret < 0) {
 			dev_err(dev, "failed to install IRQ handler\n");
-			goto fail;
+			goto err_msm_uninit;
 		}
 	}
 
 	ret = drm_dev_register(ddev, 0);
 	if (ret)
-		goto fail;
+		goto err_msm_uninit;
 
 	drm_mode_config_reset(ddev);
 
@@ -473,15 +639,23 @@ static int msm_drm_init(struct device *dev, struct drm_driver *drv)
 
 	ret = msm_debugfs_late_init(ddev);
 	if (ret)
-		goto fail;
+		goto err_msm_uninit;
 
 	drm_kms_helper_poll_init(ddev);
 
 	return 0;
 
-fail:
+err_msm_uninit:
 	msm_drm_uninit(dev);
 	return ret;
+err_destroy_mdss:
+	if (mdss && mdss->funcs)
+		mdss->funcs->destroy(ddev);
+err_free_priv:
+	kfree(priv);
+err_unref_drm_dev:
+	drm_dev_unref(ddev);
+	return ret;
 }
 
 /*
@@ -894,16 +1068,35 @@ static struct drm_driver msm_driver = {
 static int msm_pm_suspend(struct device *dev)
 {
 	struct drm_device *ddev = dev_get_drvdata(dev);
+	struct msm_drm_private *priv = ddev->dev_private;
+	struct msm_kms *kms = priv->kms;
+
+	/* TODO: Use atomic helper suspend/resume */
+	if (kms && kms->funcs && kms->funcs->pm_suspend)
+		return kms->funcs->pm_suspend(dev);
 
 	drm_kms_helper_poll_disable(ddev);
 
+	priv->pm_state = drm_atomic_helper_suspend(ddev);
+	if (IS_ERR(priv->pm_state)) {
+		drm_kms_helper_poll_enable(ddev);
+		return PTR_ERR(priv->pm_state);
+	}
+
 	return 0;
 }
 
 static int msm_pm_resume(struct device *dev)
 {
 	struct drm_device *ddev = dev_get_drvdata(dev);
+	struct msm_drm_private *priv = ddev->dev_private;
+	struct msm_kms *kms = priv->kms;
+
+	/* TODO: Use atomic helper suspend/resume */
+	if (kms && kms->funcs && kms->funcs->pm_resume)
+		return kms->funcs->pm_resume(dev);
 
+	drm_atomic_helper_resume(ddev, priv->pm_state);
 	drm_kms_helper_poll_enable(ddev);
 
 	return 0;
@@ -915,11 +1108,12 @@ static int msm_runtime_suspend(struct device *dev)
 {
 	struct drm_device *ddev = dev_get_drvdata(dev);
 	struct msm_drm_private *priv = ddev->dev_private;
+	struct msm_mdss *mdss = priv->mdss;
 
 	DBG("");
 
-	if (priv->mdss)
-		return msm_mdss_disable(priv->mdss);
+	if (mdss && mdss->funcs)
+		return mdss->funcs->disable(mdss);
 
 	return 0;
 }
@@ -928,11 +1122,12 @@ static int msm_runtime_resume(struct device *dev)
 {
 	struct drm_device *ddev = dev_get_drvdata(dev);
 	struct msm_drm_private *priv = ddev->dev_private;
+	struct msm_mdss *mdss = priv->mdss;
 
 	DBG("");
 
-	if (priv->mdss)
-		return msm_mdss_enable(priv->mdss);
+	if (mdss && mdss->funcs)
+		return mdss->funcs->enable(mdss);
 
 	return 0;
 }
@@ -1031,12 +1226,13 @@ static int add_display_components(struct device *dev,
 	int ret;
 
 	/*
-	 * MDP5 based devices don't have a flat hierarchy. There is a top level
-	 * parent: MDSS, and children: MDP5, DSI, HDMI, eDP etc. Populate the
-	 * children devices, find the MDP5 node, and then add the interfaces
-	 * to our components list.
+	 * MDP5/DPU based devices don't have a flat hierarchy. There is a top
+	 * level parent: MDSS, and children: MDP5/DPU, DSI, HDMI, eDP etc.
+	 * Populate the children devices, find the MDP5/DPU node, and then add
+	 * the interfaces to our components list.
 	 */
-	if (of_device_is_compatible(dev->of_node, "qcom,mdss")) {
+	if (of_device_is_compatible(dev->of_node, "qcom,mdss") ||
+	    of_device_is_compatible(dev->of_node, "qcom,sdm845-mdss")) {
 		ret = of_platform_populate(dev->of_node, NULL, NULL, dev);
 		if (ret) {
 			dev_err(dev, "failed to populate children devices\n");
@@ -1146,8 +1342,9 @@ static int msm_pdev_remove(struct platform_device *pdev)
 }
 
 static const struct of_device_id dt_match[] = {
-	{ .compatible = "qcom,mdp4", .data = (void *)4 },	/* MDP4 */
-	{ .compatible = "qcom,mdss", .data = (void *)5 },	/* MDP5 MDSS */
+	{ .compatible = "qcom,mdp4", .data = (void *)KMS_MDP4 },
+	{ .compatible = "qcom,mdss", .data = (void *)KMS_MDP5 },
+	{ .compatible = "qcom,sdm845-mdss", .data = (void *)KMS_DPU },
 	{}
 };
 MODULE_DEVICE_TABLE(of, dt_match);
@@ -1169,6 +1366,7 @@ static int __init msm_drm_register(void)
 
 	DBG("init");
 	msm_mdp_register();
+	msm_dpu_register();
 	msm_dsi_register();
 	msm_edp_register();
 	msm_hdmi_register();
@@ -1185,6 +1383,7 @@ static void __exit msm_drm_unregister(void)
 	msm_edp_unregister();
 	msm_dsi_unregister();
 	msm_mdp_unregister();
+	msm_dpu_unregister();
 }
 
 module_init(msm_drm_register);
diff --git a/drivers/gpu/drm/msm/msm_drv.h b/drivers/gpu/drm/msm/msm_drv.h
index b2da1fbf81e0..8e510d5c758a 100644
--- a/drivers/gpu/drm/msm/msm_drv.h
+++ b/drivers/gpu/drm/msm/msm_drv.h
@@ -1,4 +1,5 @@
 /*
+ * Copyright (c) 2016-2018, The Linux Foundation. All rights reserved.
  * Copyright (C) 2013 Red Hat
  * Author: Rob Clark <robdclark@gmail.com>
  *
@@ -33,6 +34,7 @@
 #include <linux/of_graph.h>
 #include <linux/of_device.h>
 #include <asm/sizes.h>
+#include <linux/kthread.h>
 
 #include <drm/drmP.h>
 #include <drm/drm_atomic.h>
@@ -54,6 +56,12 @@ struct msm_fence_context;
 struct msm_gem_address_space;
 struct msm_gem_vma;
 
+#define MAX_CRTCS      8
+#define MAX_PLANES     20
+#define MAX_ENCODERS   8
+#define MAX_BRIDGES    8
+#define MAX_CONNECTORS 8
+
 struct msm_file_private {
 	rwlock_t queuelock;
 	struct list_head submitqueues;
@@ -68,12 +76,77 @@ enum msm_mdp_plane_property {
 };
 
 struct msm_vblank_ctrl {
-	struct work_struct work;
+	struct kthread_work work;
 	struct list_head event_list;
 	spinlock_t lock;
 };
 
 #define MSM_GPU_MAX_RINGS 4
+#define MAX_H_TILES_PER_DISPLAY 2
+
+/**
+ * enum msm_display_caps - features/capabilities supported by displays
+ * @MSM_DISPLAY_CAP_VID_MODE:           Video or "active" mode supported
+ * @MSM_DISPLAY_CAP_CMD_MODE:           Command mode supported
+ * @MSM_DISPLAY_CAP_HOT_PLUG:           Hot plug detection supported
+ * @MSM_DISPLAY_CAP_EDID:               EDID supported
+ */
+enum msm_display_caps {
+	MSM_DISPLAY_CAP_VID_MODE	= BIT(0),
+	MSM_DISPLAY_CAP_CMD_MODE	= BIT(1),
+	MSM_DISPLAY_CAP_HOT_PLUG	= BIT(2),
+	MSM_DISPLAY_CAP_EDID		= BIT(3),
+};
+
+/**
+ * enum msm_event_wait - type of HW events to wait for
+ * @MSM_ENC_COMMIT_DONE - wait for the driver to flush the registers to HW
+ * @MSM_ENC_TX_COMPLETE - wait for the HW to transfer the frame to panel
+ * @MSM_ENC_VBLANK - wait for the HW VBLANK event (for driver-internal waiters)
+ */
+enum msm_event_wait {
+	MSM_ENC_COMMIT_DONE = 0,
+	MSM_ENC_TX_COMPLETE,
+	MSM_ENC_VBLANK,
+};
+
+/**
+ * struct msm_display_topology - defines a display topology pipeline
+ * @num_lm:       number of layer mixers used
+ * @num_enc:      number of compression encoder blocks used
+ * @num_intf:     number of interfaces the panel is mounted on
+ */
+struct msm_display_topology {
+	u32 num_lm;
+	u32 num_enc;
+	u32 num_intf;
+};
+
+/**
+ * struct msm_display_info - defines display properties
+ * @intf_type:          DRM_MODE_CONNECTOR_ display type
+ * @capabilities:       Bitmask of display flags
+ * @num_of_h_tiles:     Number of horizontal tiles in case of split interface
+ * @h_tile_instance:    Controller instance used per tile. Number of elements is
+ *                      based on num_of_h_tiles
+ * @is_te_using_watchdog_timer:  Boolean to indicate watchdog TE is
+ *				 used instead of panel TE in cmd mode panels
+ */
+struct msm_display_info {
+	int intf_type;
+	uint32_t capabilities;
+	uint32_t num_of_h_tiles;
+	uint32_t h_tile_instance[MAX_H_TILES_PER_DISPLAY];
+	bool is_te_using_watchdog_timer;
+};
+
+/* Commit/Event thread specific structure */
+struct msm_drm_thread {
+	struct drm_device *dev;
+	struct task_struct *thread;
+	unsigned int crtc_id;
+	struct kthread_worker worker;
+};
 
 struct msm_drm_private {
 
@@ -84,7 +157,7 @@ struct msm_drm_private {
 	/* subordinate devices, if present: */
 	struct platform_device *gpu_pdev;
 
-	/* top level MDSS wrapper device (for MDP5 only) */
+	/* top level MDSS wrapper device (for MDP5/DPU only) */
 	struct msm_mdss *mdss;
 
 	/* possibly this should be in the kms component, but it is
@@ -115,22 +188,24 @@ struct msm_drm_private {
 	struct list_head inactive_list;
 
 	struct workqueue_struct *wq;
-	struct workqueue_struct *atomic_wq;
 
 	unsigned int num_planes;
-	struct drm_plane *planes[16];
+	struct drm_plane *planes[MAX_PLANES];
 
 	unsigned int num_crtcs;
-	struct drm_crtc *crtcs[8];
+	struct drm_crtc *crtcs[MAX_CRTCS];
+
+	struct msm_drm_thread disp_thread[MAX_CRTCS];
+	struct msm_drm_thread event_thread[MAX_CRTCS];
 
 	unsigned int num_encoders;
-	struct drm_encoder *encoders[8];
+	struct drm_encoder *encoders[MAX_ENCODERS];
 
 	unsigned int num_bridges;
-	struct drm_bridge *bridges[8];
+	struct drm_bridge *bridges[MAX_BRIDGES];
 
 	unsigned int num_connectors;
-	struct drm_connector *connectors[8];
+	struct drm_connector *connectors[MAX_CONNECTORS];
 
 	/* Properties */
 	struct drm_property *plane_property[PLANE_PROP_MAX_NUM];
@@ -150,6 +225,7 @@ struct msm_drm_private {
 	struct shrinker shrinker;
 
 	struct msm_vblank_ctrl vblank_ctrl;
+	struct drm_atomic_state *pm_state;
 };
 
 struct msm_format {
@@ -174,6 +250,9 @@ struct msm_gem_address_space *
 msm_gem_address_space_create(struct device *dev, struct iommu_domain *domain,
 		const char *name);
 
+int msm_register_mmu(struct drm_device *dev, struct msm_mmu *mmu);
+void msm_unregister_mmu(struct drm_device *dev, struct msm_mmu *mmu);
+
 void msm_gem_submit_free(struct msm_gem_submit *submit);
 int msm_ioctl_gem_submit(struct drm_device *dev, void *data,
 		struct drm_file *file);
@@ -184,7 +263,7 @@ void msm_gem_shrinker_cleanup(struct drm_device *dev);
 int msm_gem_mmap_obj(struct drm_gem_object *obj,
 			struct vm_area_struct *vma);
 int msm_gem_mmap(struct file *filp, struct vm_area_struct *vma);
-int msm_gem_fault(struct vm_fault *vmf);
+vm_fault_t msm_gem_fault(struct vm_fault *vmf);
 uint64_t msm_gem_mmap_offset(struct drm_gem_object *obj);
 int msm_gem_get_iova(struct drm_gem_object *obj,
 		struct msm_gem_address_space *aspace, uint64_t *iova);
@@ -285,6 +364,8 @@ static inline int msm_dsi_modeset_init(struct msm_dsi *msm_dsi,
 
 void __init msm_mdp_register(void);
 void __exit msm_mdp_unregister(void);
+void __init msm_dpu_register(void);
+void __exit msm_dpu_unregister(void);
 
 #ifdef CONFIG_DEBUG_FS
 void msm_gem_describe(struct drm_gem_object *obj, struct seq_file *m);
@@ -306,6 +387,10 @@ static inline void msm_perf_debugfs_cleanup(struct msm_drm_private *priv) {}
 #endif
 
 struct clk *msm_clk_get(struct platform_device *pdev, const char *name);
+int msm_clk_bulk_get(struct device *dev, struct clk_bulk_data **bulk);
+
+struct clk *msm_clk_bulk_get_clock(struct clk_bulk_data *bulk, int count,
+	const char *name);
 void __iomem *msm_ioremap(struct platform_device *pdev, const char *name,
 		const char *dbgname);
 void msm_writel(u32 data, void __iomem *addr);
diff --git a/drivers/gpu/drm/msm/msm_gem.c b/drivers/gpu/drm/msm/msm_gem.c
index f583bb4222f9..f59ca27a4a35 100644
--- a/drivers/gpu/drm/msm/msm_gem.c
+++ b/drivers/gpu/drm/msm/msm_gem.c
@@ -219,7 +219,7 @@ int msm_gem_mmap(struct file *filp, struct vm_area_struct *vma)
 	return msm_gem_mmap_obj(vma->vm_private_data, vma);
 }
 
-int msm_gem_fault(struct vm_fault *vmf)
+vm_fault_t msm_gem_fault(struct vm_fault *vmf)
 {
 	struct vm_area_struct *vma = vmf->vma;
 	struct drm_gem_object *obj = vma->vm_private_data;
@@ -227,15 +227,18 @@ int msm_gem_fault(struct vm_fault *vmf)
 	struct page **pages;
 	unsigned long pfn;
 	pgoff_t pgoff;
-	int ret;
+	int err;
+	vm_fault_t ret;
 
 	/*
 	 * vm_ops.open/drm_gem_mmap_obj and close get and put
 	 * a reference on obj. So, we dont need to hold one here.
 	 */
-	ret = mutex_lock_interruptible(&msm_obj->lock);
-	if (ret)
+	err = mutex_lock_interruptible(&msm_obj->lock);
+	if (err) {
+		ret = VM_FAULT_NOPAGE;
 		goto out;
+	}
 
 	if (WARN_ON(msm_obj->madv != MSM_MADV_WILLNEED)) {
 		mutex_unlock(&msm_obj->lock);
@@ -245,7 +248,7 @@ int msm_gem_fault(struct vm_fault *vmf)
 	/* make sure we have pages attached now */
 	pages = get_pages(obj);
 	if (IS_ERR(pages)) {
-		ret = PTR_ERR(pages);
+		ret = vmf_error(PTR_ERR(pages));
 		goto out_unlock;
 	}
 
@@ -257,27 +260,11 @@ int msm_gem_fault(struct vm_fault *vmf)
 	VERB("Inserting %p pfn %lx, pa %lx", (void *)vmf->address,
 			pfn, pfn << PAGE_SHIFT);
 
-	ret = vm_insert_mixed(vma, vmf->address, __pfn_to_pfn_t(pfn, PFN_DEV));
-
+	ret = vmf_insert_mixed(vma, vmf->address, __pfn_to_pfn_t(pfn, PFN_DEV));
 out_unlock:
 	mutex_unlock(&msm_obj->lock);
 out:
-	switch (ret) {
-	case -EAGAIN:
-	case 0:
-	case -ERESTARTSYS:
-	case -EINTR:
-	case -EBUSY:
-		/*
-		 * EBUSY is ok: this just means that another thread
-		 * already did the job.
-		 */
-		return VM_FAULT_NOPAGE;
-	case -ENOMEM:
-		return VM_FAULT_OOM;
-	default:
-		return VM_FAULT_SIGBUS;
-	}
+	return ret;
 }
 
 /** get mmap offset */
diff --git a/drivers/gpu/drm/msm/msm_gpu.c b/drivers/gpu/drm/msm/msm_gpu.c
index 1c09acfb4028..5e808cfec345 100644
--- a/drivers/gpu/drm/msm/msm_gpu.c
+++ b/drivers/gpu/drm/msm/msm_gpu.c
@@ -20,10 +20,11 @@
 #include "msm_mmu.h"
 #include "msm_fence.h"
 
+#include <generated/utsrelease.h>
 #include <linux/string_helpers.h>
 #include <linux/pm_opp.h>
 #include <linux/devfreq.h>
-
+#include <linux/devcoredump.h>
 
 /*
  * Power Management:
@@ -87,7 +88,7 @@ static struct devfreq_dev_profile msm_devfreq_profile = {
 static void msm_devfreq_init(struct msm_gpu *gpu)
 {
 	/* We need target support to do devfreq */
-	if (!gpu->funcs->gpu_busy)
+	if (!gpu->funcs->gpu_busy || !gpu->core_clk)
 		return;
 
 	msm_devfreq_profile.initial_freq = gpu->fast_rate;
@@ -141,8 +142,6 @@ static int disable_pwrrail(struct msm_gpu *gpu)
 
 static int enable_clk(struct msm_gpu *gpu)
 {
-	int i;
-
 	if (gpu->core_clk && gpu->fast_rate)
 		clk_set_rate(gpu->core_clk, gpu->fast_rate);
 
@@ -150,28 +149,12 @@ static int enable_clk(struct msm_gpu *gpu)
 	if (gpu->rbbmtimer_clk)
 		clk_set_rate(gpu->rbbmtimer_clk, 19200000);
 
-	for (i = gpu->nr_clocks - 1; i >= 0; i--)
-		if (gpu->grp_clks[i])
-			clk_prepare(gpu->grp_clks[i]);
-
-	for (i = gpu->nr_clocks - 1; i >= 0; i--)
-		if (gpu->grp_clks[i])
-			clk_enable(gpu->grp_clks[i]);
-
-	return 0;
+	return clk_bulk_prepare_enable(gpu->nr_clocks, gpu->grp_clks);
 }
 
 static int disable_clk(struct msm_gpu *gpu)
 {
-	int i;
-
-	for (i = gpu->nr_clocks - 1; i >= 0; i--)
-		if (gpu->grp_clks[i])
-			clk_disable(gpu->grp_clks[i]);
-
-	for (i = gpu->nr_clocks - 1; i >= 0; i--)
-		if (gpu->grp_clks[i])
-			clk_unprepare(gpu->grp_clks[i]);
+	clk_bulk_disable_unprepare(gpu->nr_clocks, gpu->grp_clks);
 
 	/*
 	 * Set the clock to a deliberately low rate. On older targets the clock
@@ -273,6 +256,123 @@ int msm_gpu_hw_init(struct msm_gpu *gpu)
 	return ret;
 }
 
+#ifdef CONFIG_DEV_COREDUMP
+static ssize_t msm_gpu_devcoredump_read(char *buffer, loff_t offset,
+		size_t count, void *data, size_t datalen)
+{
+	struct msm_gpu *gpu = data;
+	struct drm_print_iterator iter;
+	struct drm_printer p;
+	struct msm_gpu_state *state;
+
+	state = msm_gpu_crashstate_get(gpu);
+	if (!state)
+		return 0;
+
+	iter.data = buffer;
+	iter.offset = 0;
+	iter.start = offset;
+	iter.remain = count;
+
+	p = drm_coredump_printer(&iter);
+
+	drm_printf(&p, "---\n");
+	drm_printf(&p, "kernel: " UTS_RELEASE "\n");
+	drm_printf(&p, "module: " KBUILD_MODNAME "\n");
+	drm_printf(&p, "time: %lld.%09ld\n",
+		state->time.tv_sec, state->time.tv_nsec);
+	if (state->comm)
+		drm_printf(&p, "comm: %s\n", state->comm);
+	if (state->cmd)
+		drm_printf(&p, "cmdline: %s\n", state->cmd);
+
+	gpu->funcs->show(gpu, state, &p);
+
+	msm_gpu_crashstate_put(gpu);
+
+	return count - iter.remain;
+}
+
+static void msm_gpu_devcoredump_free(void *data)
+{
+	struct msm_gpu *gpu = data;
+
+	msm_gpu_crashstate_put(gpu);
+}
+
+static void msm_gpu_crashstate_get_bo(struct msm_gpu_state *state,
+		struct msm_gem_object *obj, u64 iova, u32 flags)
+{
+	struct msm_gpu_state_bo *state_bo = &state->bos[state->nr_bos];
+
+	/* Don't record write only objects */
+
+	state_bo->size = obj->base.size;
+	state_bo->iova = iova;
+
+	/* Only store the data for buffer objects marked for read */
+	if ((flags & MSM_SUBMIT_BO_READ)) {
+		void *ptr;
+
+		state_bo->data = kvmalloc(obj->base.size, GFP_KERNEL);
+		if (!state_bo->data)
+			return;
+
+		ptr = msm_gem_get_vaddr_active(&obj->base);
+		if (IS_ERR(ptr)) {
+			kvfree(state_bo->data);
+			return;
+		}
+
+		memcpy(state_bo->data, ptr, obj->base.size);
+		msm_gem_put_vaddr(&obj->base);
+	}
+
+	state->nr_bos++;
+}
+
+static void msm_gpu_crashstate_capture(struct msm_gpu *gpu,
+		struct msm_gem_submit *submit, char *comm, char *cmd)
+{
+	struct msm_gpu_state *state;
+
+	/* Only save one crash state at a time */
+	if (gpu->crashstate)
+		return;
+
+	state = gpu->funcs->gpu_state_get(gpu);
+	if (IS_ERR_OR_NULL(state))
+		return;
+
+	/* Fill in the additional crash state information */
+	state->comm = kstrdup(comm, GFP_KERNEL);
+	state->cmd = kstrdup(cmd, GFP_KERNEL);
+
+	if (submit) {
+		int i;
+
+		state->bos = kcalloc(submit->nr_bos,
+			sizeof(struct msm_gpu_state_bo), GFP_KERNEL);
+
+		for (i = 0; state->bos && i < submit->nr_bos; i++)
+			msm_gpu_crashstate_get_bo(state, submit->bos[i].obj,
+				submit->bos[i].iova, submit->bos[i].flags);
+	}
+
+	/* Set the active crash state to be dumped on failure */
+	gpu->crashstate = state;
+
+	/* FIXME: Release the crashstate if this errors out? */
+	dev_coredumpm(gpu->dev->dev, THIS_MODULE, gpu, 0, GFP_KERNEL,
+		msm_gpu_devcoredump_read, msm_gpu_devcoredump_free);
+}
+#else
+static void msm_gpu_crashstate_capture(struct msm_gpu *gpu, char *comm,
+		char *cmd)
+{
+}
+#endif
+
 /*
  * Hangcheck detection for locked gpu:
  */
@@ -314,6 +414,7 @@ static void recover_worker(struct work_struct *work)
 	struct msm_drm_private *priv = dev->dev_private;
 	struct msm_gem_submit *submit;
 	struct msm_ringbuffer *cur_ring = gpu->funcs->active_ring(gpu);
+	char *comm = NULL, *cmd = NULL;
 	int i;
 
 	mutex_lock(&dev->struct_mutex);
@@ -327,7 +428,7 @@ static void recover_worker(struct work_struct *work)
 		rcu_read_lock();
 		task = pid_task(submit->pid, PIDTYPE_PID);
 		if (task) {
-			char *cmd;
+			comm = kstrdup(task->comm, GFP_ATOMIC);
 
 			/*
 			 * So slightly annoying, in other paths like
@@ -340,22 +441,28 @@ static void recover_worker(struct work_struct *work)
 			 * about the submit going away.
 			 */
 			mutex_unlock(&dev->struct_mutex);
-			cmd = kstrdup_quotable_cmdline(task, GFP_KERNEL);
+			cmd = kstrdup_quotable_cmdline(task, GFP_ATOMIC);
 			mutex_lock(&dev->struct_mutex);
+		}
+		rcu_read_unlock();
 
+		if (comm && cmd) {
 			dev_err(dev->dev, "%s: offending task: %s (%s)\n",
-				gpu->name, task->comm, cmd);
+				gpu->name, comm, cmd);
 
 			msm_rd_dump_submit(priv->hangrd, submit,
-				"offending task: %s (%s)", task->comm, cmd);
-
-			kfree(cmd);
-		} else {
+				"offending task: %s (%s)", comm, cmd);
+		} else
 			msm_rd_dump_submit(priv->hangrd, submit, NULL);
-		}
-		rcu_read_unlock();
 	}
 
+	/* Record the crash state */
+	pm_runtime_get_sync(&gpu->pdev->dev);
+	msm_gpu_crashstate_capture(gpu, submit, comm, cmd);
+	pm_runtime_put_sync(&gpu->pdev->dev);
+
+	kfree(cmd);
+	kfree(comm);
 
 	/*
 	 * Update all the rings with the latest and greatest fence.. this
@@ -660,44 +767,22 @@ static irqreturn_t irq_handler(int irq, void *data)
 	return gpu->funcs->irq(gpu);
 }
 
-static struct clk *get_clock(struct device *dev, const char *name)
-{
-	struct clk *clk = devm_clk_get(dev, name);
-
-	return IS_ERR(clk) ? NULL : clk;
-}
-
 static int get_clocks(struct platform_device *pdev, struct msm_gpu *gpu)
 {
-	struct device *dev = &pdev->dev;
-	struct property *prop;
-	const char *name;
-	int i = 0;
+	int ret = msm_clk_bulk_get(&pdev->dev, &gpu->grp_clks);
 
-	gpu->nr_clocks = of_property_count_strings(dev->of_node, "clock-names");
-	if (gpu->nr_clocks < 1) {
+	if (ret < 1) {
 		gpu->nr_clocks = 0;
-		return 0;
-	}
-
-	gpu->grp_clks = devm_kcalloc(dev, sizeof(struct clk *), gpu->nr_clocks,
-		GFP_KERNEL);
-	if (!gpu->grp_clks) {
-		gpu->nr_clocks = 0;
-		return -ENOMEM;
+		return ret;
 	}
 
-	of_property_for_each_string(dev->of_node, "clock-names", prop, name) {
-		gpu->grp_clks[i] = get_clock(dev, name);
+	gpu->nr_clocks = ret;
 
-		/* Remember the key clocks that we need to control later */
-		if (!strcmp(name, "core") || !strcmp(name, "core_clk"))
-			gpu->core_clk = gpu->grp_clks[i];
-		else if (!strcmp(name, "rbbmtimer") || !strcmp(name, "rbbmtimer_clk"))
-			gpu->rbbmtimer_clk = gpu->grp_clks[i];
+	gpu->core_clk = msm_clk_bulk_get_clock(gpu->grp_clks,
+		gpu->nr_clocks, "core");
 
-		++i;
-	}
+	gpu->rbbmtimer_clk = msm_clk_bulk_get_clock(gpu->grp_clks,
+		gpu->nr_clocks, "rbbmtimer");
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/msm/msm_gpu.h b/drivers/gpu/drm/msm/msm_gpu.h
index b8241179175a..9122ee6e55e4 100644
--- a/drivers/gpu/drm/msm/msm_gpu.h
+++ b/drivers/gpu/drm/msm/msm_gpu.h
@@ -27,6 +27,7 @@
 
 struct msm_gem_submit;
 struct msm_gpu_perfcntr;
+struct msm_gpu_state;
 
 struct msm_gpu_config {
 	const char *ioname;
@@ -64,11 +65,14 @@ struct msm_gpu_funcs {
 	void (*destroy)(struct msm_gpu *gpu);
 #ifdef CONFIG_DEBUG_FS
 	/* show GPU status in debugfs: */
-	void (*show)(struct msm_gpu *gpu, struct seq_file *m);
+	void (*show)(struct msm_gpu *gpu, struct msm_gpu_state *state,
+			struct drm_printer *p);
 	/* for generation specific debugfs: */
 	int (*debugfs_init)(struct msm_gpu *gpu, struct drm_minor *minor);
 #endif
 	int (*gpu_busy)(struct msm_gpu *gpu, uint64_t *value);
+	struct msm_gpu_state *(*gpu_state_get)(struct msm_gpu *gpu);
+	int (*gpu_state_put)(struct msm_gpu_state *state);
 };
 
 struct msm_gpu {
@@ -108,7 +112,7 @@ struct msm_gpu {
 
 	/* Power Control: */
 	struct regulator *gpu_reg, *gpu_cx;
-	struct clk **grp_clks;
+	struct clk_bulk_data *grp_clks;
 	int nr_clocks;
 	struct clk *ebi1_clk, *core_clk, *rbbmtimer_clk;
 	uint32_t fast_rate;
@@ -129,6 +133,8 @@ struct msm_gpu {
 		u64 busy_cycles;
 		ktime_t time;
 	} devfreq;
+
+	struct msm_gpu_state *crashstate;
 };
 
 /* It turns out that all targets use the same ringbuffer size */
@@ -175,6 +181,38 @@ struct msm_gpu_submitqueue {
 	struct kref ref;
 };
 
+struct msm_gpu_state_bo {
+	u64 iova;
+	size_t size;
+	void *data;
+};
+
+struct msm_gpu_state {
+	struct kref ref;
+	struct timespec64 time;
+
+	struct {
+		u64 iova;
+		u32 fence;
+		u32 seqno;
+		u32 rptr;
+		u32 wptr;
+		void *data;
+		int data_size;
+	} ring[MSM_GPU_MAX_RINGS];
+
+	int nr_registers;
+	u32 *registers;
+
+	u32 rbbm_status;
+
+	char *comm;
+	char *cmd;
+
+	int nr_bos;
+	struct msm_gpu_state_bo *bos;
+};
+
 static inline void gpu_write(struct msm_gpu *gpu, u32 reg, u32 data)
 {
 	msm_writel(data, gpu->mmio + (reg << 2));
@@ -254,4 +292,32 @@ static inline void msm_submitqueue_put(struct msm_gpu_submitqueue *queue)
 		kref_put(&queue->ref, msm_submitqueue_destroy);
 }
 
+static inline struct msm_gpu_state *msm_gpu_crashstate_get(struct msm_gpu *gpu)
+{
+	struct msm_gpu_state *state = NULL;
+
+	mutex_lock(&gpu->dev->struct_mutex);
+
+	if (gpu->crashstate) {
+		kref_get(&gpu->crashstate->ref);
+		state = gpu->crashstate;
+	}
+
+	mutex_unlock(&gpu->dev->struct_mutex);
+
+	return state;
+}
+
+static inline void msm_gpu_crashstate_put(struct msm_gpu *gpu)
+{
+	mutex_lock(&gpu->dev->struct_mutex);
+
+	if (gpu->crashstate) {
+		if (gpu->funcs->gpu_state_put(gpu->crashstate))
+			gpu->crashstate = NULL;
+	}
+
+	mutex_unlock(&gpu->dev->struct_mutex);
+}
+
 #endif /* __MSM_GPU_H__ */
diff --git a/drivers/gpu/drm/msm/msm_kms.h b/drivers/gpu/drm/msm/msm_kms.h
index dfd92947de2c..fd88cebb6adb 100644
--- a/drivers/gpu/drm/msm/msm_kms.h
+++ b/drivers/gpu/drm/msm/msm_kms.h
@@ -1,4 +1,5 @@
 /*
+ * Copyright (c) 2016-2018, The Linux Foundation. All rights reserved.
  * Copyright (C) 2013 Red Hat
  * Author: Rob Clark <robdclark@gmail.com>
  *
@@ -42,6 +43,7 @@ struct msm_kms_funcs {
 	void (*disable_vblank)(struct msm_kms *kms, struct drm_crtc *crtc);
 	/* modeset, bracketing atomic_commit(): */
 	void (*prepare_commit)(struct msm_kms *kms, struct drm_atomic_state *state);
+	void (*commit)(struct msm_kms *kms, struct drm_atomic_state *state);
 	void (*complete_commit)(struct msm_kms *kms, struct drm_atomic_state *state);
 	/* functions to wait for atomic commit completed on each CRTC */
 	void (*wait_for_crtc_commit_done)(struct msm_kms *kms,
@@ -50,6 +52,11 @@ struct msm_kms_funcs {
 	const struct msm_format *(*get_format)(struct msm_kms *kms,
 					const uint32_t format,
 					const uint64_t modifiers);
+	/* do format checking on format modified through fb_cmd2 modifiers */
+	int (*check_modified_format)(const struct msm_kms *kms,
+			const struct msm_format *msm_fmt,
+			const struct drm_mode_fb_cmd2 *cmd,
+			struct drm_gem_object **bos);
 	/* misc: */
 	long (*round_pixclk)(struct msm_kms *kms, unsigned long rate,
 			struct drm_encoder *encoder);
@@ -60,6 +67,9 @@ struct msm_kms_funcs {
 	void (*set_encoder_mode)(struct msm_kms *kms,
 				 struct drm_encoder *encoder,
 				 bool cmd_mode);
+	/* pm suspend/resume hooks */
+	int (*pm_suspend)(struct device *dev);
+	int (*pm_resume)(struct device *dev);
 	/* cleanup: */
 	void (*destroy)(struct msm_kms *kms);
 #ifdef CONFIG_DEBUG_FS
@@ -86,9 +96,20 @@ static inline void msm_kms_init(struct msm_kms *kms,
 
 struct msm_kms *mdp4_kms_init(struct drm_device *dev);
 struct msm_kms *mdp5_kms_init(struct drm_device *dev);
-int msm_mdss_init(struct drm_device *dev);
-void msm_mdss_destroy(struct drm_device *dev);
-int msm_mdss_enable(struct msm_mdss *mdss);
-int msm_mdss_disable(struct msm_mdss *mdss);
+struct msm_kms *dpu_kms_init(struct drm_device *dev);
+
+struct msm_mdss_funcs {
+	int (*enable)(struct msm_mdss *mdss);
+	int (*disable)(struct msm_mdss *mdss);
+	void (*destroy)(struct drm_device *dev);
+};
+
+struct msm_mdss {
+	struct drm_device *dev;
+	const struct msm_mdss_funcs *funcs;
+};
+
+int mdp5_mdss_init(struct drm_device *dev);
+int dpu_mdss_init(struct drm_device *dev);
 
 #endif /* __MSM_KMS_H__ */
diff --git a/drivers/gpu/drm/nouveau/dispnv04/crtc.c b/drivers/gpu/drm/nouveau/dispnv04/crtc.c
index 6aa6ee16dcbd..2c569e264df3 100644
--- a/drivers/gpu/drm/nouveau/dispnv04/crtc.c
+++ b/drivers/gpu/drm/nouveau/dispnv04/crtc.c
@@ -1017,7 +1017,7 @@ nv04_crtc_cursor_set(struct drm_crtc *crtc, struct drm_file *file_priv,
 	nv_crtc->cursor.set_offset(nv_crtc, nv_crtc->cursor.offset);
 	nv_crtc->cursor.show(nv_crtc, true);
 out:
-	drm_gem_object_unreference_unlocked(gem);
+	drm_gem_object_put_unlocked(gem);
 	return ret;
 }
 
diff --git a/drivers/gpu/drm/nouveau/dispnv04/disp.c b/drivers/gpu/drm/nouveau/dispnv04/disp.c
index 501d2d290e9c..70dce544984e 100644
--- a/drivers/gpu/drm/nouveau/dispnv04/disp.c
+++ b/drivers/gpu/drm/nouveau/dispnv04/disp.c
@@ -55,6 +55,9 @@ nv04_display_create(struct drm_device *dev)
 	nouveau_display(dev)->init = nv04_display_init;
 	nouveau_display(dev)->fini = nv04_display_fini;
 
+	/* Pre-nv50 doesn't support atomic, so don't expose the ioctls */
+	dev->driver->driver_features &= ~DRIVER_ATOMIC;
+
 	nouveau_hw_save_vga_fonts(dev, 1);
 
 	nv04_crtc_create(dev, 0);
diff --git a/drivers/gpu/drm/nouveau/dispnv50/disp.c b/drivers/gpu/drm/nouveau/dispnv50/disp.c
index 5a247eb71899..8412119bd940 100644
--- a/drivers/gpu/drm/nouveau/dispnv50/disp.c
+++ b/drivers/gpu/drm/nouveau/dispnv50/disp.c
@@ -136,12 +136,24 @@ nv50_dmac_create(struct nvif_device *device, struct nvif_object *disp,
 {
 	struct nouveau_cli *cli = (void *)device->object.client;
 	struct nv50_disp_core_channel_dma_v0 *args = data;
+	u8 type = NVIF_MEM_COHERENT;
 	int ret;
 
 	mutex_init(&dmac->lock);
 
-	ret = nvif_mem_init_map(&cli->mmu, NVIF_MEM_COHERENT, 0x1000,
-				&dmac->push);
+	/* Pascal added support for 47-bit physical addresses, but some
+	 * parts of EVO still only accept 40-bit PAs.
+	 *
+	 * To avoid issues on systems with large amounts of RAM, and on
+	 * systems where an IOMMU maps pages at a high address, we need
+	 * to allocate push buffers in VRAM instead.
+	 *
+	 * This appears to match NVIDIA's behaviour on Pascal.
+	 */
+	if (device->info.family == NV_DEVICE_INFO_V0_PASCAL)
+		type |= NVIF_MEM_VRAM;
+
+	ret = nvif_mem_init_map(&cli->mmu, type, 0x1000, &dmac->push);
 	if (ret)
 		return ret;
 
@@ -216,6 +228,19 @@ void
 evo_kick(u32 *push, struct nv50_dmac *evoc)
 {
 	struct nv50_dmac *dmac = evoc;
+
+	/* Push buffer fetches are not coherent with BAR1, we need to ensure
+	 * writes have been flushed right through to VRAM before writing PUT.
+	 */
+	if (dmac->push.type & NVIF_MEM_VRAM) {
+		struct nvif_device *device = dmac->base.device;
+		nvif_wr32(&device->object, 0x070000, 0x00000001);
+		nvif_msec(device, 2000,
+			if (!(nvif_rd32(&device->object, 0x070000) & 0x00000002))
+				break;
+		);
+	}
+
 	nvif_wr32(&dmac->base.user, 0x0000, (push - dmac->ptr) << 2);
 	mutex_unlock(&dmac->lock);
 }
@@ -1007,7 +1032,7 @@ nv50_mstm_destroy_connector(struct drm_dp_mst_topology_mgr *mgr,
 	mstc->port = NULL;
 	drm_modeset_unlock(&drm->dev->mode_config.connection_mutex);
 
-	drm_connector_unreference(&mstc->connector);
+	drm_connector_put(&mstc->connector);
 }
 
 static void
@@ -1585,8 +1610,9 @@ nv50_pior_create(struct drm_connector *connector, struct dcb_output *dcbe)
  *****************************************************************************/
 
 static void
-nv50_disp_atomic_commit_core(struct nouveau_drm *drm, u32 *interlock)
+nv50_disp_atomic_commit_core(struct drm_atomic_state *state, u32 *interlock)
 {
+	struct nouveau_drm *drm = nouveau_drm(state->dev);
 	struct nv50_disp *disp = nv50_disp(drm->dev);
 	struct nv50_core *core = disp->core;
 	struct nv50_mstm *mstm;
@@ -1618,6 +1644,22 @@ nv50_disp_atomic_commit_core(struct nouveau_drm *drm, u32 *interlock)
 }
 
 static void
+nv50_disp_atomic_commit_wndw(struct drm_atomic_state *state, u32 *interlock)
+{
+	struct drm_plane_state *new_plane_state;
+	struct drm_plane *plane;
+	int i;
+
+	for_each_new_plane_in_state(state, plane, new_plane_state, i) {
+		struct nv50_wndw *wndw = nv50_wndw(plane);
+		if (interlock[wndw->interlock.type] & wndw->interlock.data) {
+			if (wndw->func->update)
+				wndw->func->update(wndw, interlock);
+		}
+	}
+}
+
+static void
 nv50_disp_atomic_commit_tail(struct drm_atomic_state *state)
 {
 	struct drm_device *dev = state->dev;
@@ -1684,7 +1726,8 @@ nv50_disp_atomic_commit_tail(struct drm_atomic_state *state)
 			help->disable(encoder);
 			interlock[NV50_DISP_INTERLOCK_CORE] |= 1;
 			if (outp->flush_disable) {
-				nv50_disp_atomic_commit_core(drm, interlock);
+				nv50_disp_atomic_commit_wndw(state, interlock);
+				nv50_disp_atomic_commit_core(state, interlock);
 				memset(interlock, 0x00, sizeof(interlock));
 			}
 		}
@@ -1693,15 +1736,8 @@ nv50_disp_atomic_commit_tail(struct drm_atomic_state *state)
 	/* Flush disable. */
 	if (interlock[NV50_DISP_INTERLOCK_CORE]) {
 		if (atom->flush_disable) {
-			for_each_new_plane_in_state(state, plane, new_plane_state, i) {
-				struct nv50_wndw *wndw = nv50_wndw(plane);
-				if (interlock[wndw->interlock.type] & wndw->interlock.data) {
-					if (wndw->func->update)
-						wndw->func->update(wndw, interlock);
-				}
-			}
-
-			nv50_disp_atomic_commit_core(drm, interlock);
+			nv50_disp_atomic_commit_wndw(state, interlock);
+			nv50_disp_atomic_commit_core(state, interlock);
 			memset(interlock, 0x00, sizeof(interlock));
 		}
 	}
@@ -1762,18 +1798,14 @@ nv50_disp_atomic_commit_tail(struct drm_atomic_state *state)
 	}
 
 	/* Flush update. */
-	for_each_new_plane_in_state(state, plane, new_plane_state, i) {
-		struct nv50_wndw *wndw = nv50_wndw(plane);
-		if (interlock[wndw->interlock.type] & wndw->interlock.data) {
-			if (wndw->func->update)
-				wndw->func->update(wndw, interlock);
-		}
-	}
+	nv50_disp_atomic_commit_wndw(state, interlock);
 
 	if (interlock[NV50_DISP_INTERLOCK_CORE]) {
 		if (interlock[NV50_DISP_INTERLOCK_BASE] ||
+		    interlock[NV50_DISP_INTERLOCK_OVLY] ||
+		    interlock[NV50_DISP_INTERLOCK_WNDW] ||
 		    !atom->state.legacy_cursor_update)
-			nv50_disp_atomic_commit_core(drm, interlock);
+			nv50_disp_atomic_commit_core(state, interlock);
 		else
 			disp->core->func->update(disp->core, interlock, false);
 	}
@@ -1871,7 +1903,7 @@ nv50_disp_atomic_commit(struct drm_device *dev,
 		nv50_disp_atomic_commit_tail(state);
 
 	drm_for_each_crtc(crtc, dev) {
-		if (crtc->state->enable) {
+		if (crtc->state->active) {
 			if (!drm->have_disp_power_ref) {
 				drm->have_disp_power_ref = true;
 				return 0;
@@ -2119,10 +2151,6 @@ nv50_display_destroy(struct drm_device *dev)
 	kfree(disp);
 }
 
-MODULE_PARM_DESC(atomic, "Expose atomic ioctl (default: disabled)");
-static int nouveau_atomic = 0;
-module_param_named(atomic, nouveau_atomic, int, 0400);
-
 int
 nv50_display_create(struct drm_device *dev)
 {
@@ -2147,8 +2175,6 @@ nv50_display_create(struct drm_device *dev)
 	disp->disp = &nouveau_display(dev)->disp;
 	dev->mode_config.funcs = &nv50_disp_func;
 	dev->driver->driver_features |= DRIVER_PREFER_XBGR_30BPP;
-	if (nouveau_atomic)
-		dev->driver->driver_features |= DRIVER_ATOMIC;
 
 	/* small shared memory area we use for notifiers and semaphores */
 	ret = nouveau_bo_new(&drm->client, 4096, 0x1000, TTM_PL_FLAG_VRAM,
@@ -2231,6 +2257,9 @@ nv50_display_create(struct drm_device *dev)
 		connector->funcs->destroy(connector);
 	}
 
+	/* Disable vblank irqs aggressively for power-saving, safe on nv50+ */
+	dev->vblank_disable_immediate = true;
+
 out:
 	if (ret)
 		nv50_display_destroy(dev);
diff --git a/drivers/gpu/drm/nouveau/dispnv50/wndw.c b/drivers/gpu/drm/nouveau/dispnv50/wndw.c
index c5a9bc1af5af..2187922e8dc2 100644
--- a/drivers/gpu/drm/nouveau/dispnv50/wndw.c
+++ b/drivers/gpu/drm/nouveau/dispnv50/wndw.c
@@ -586,7 +586,6 @@ nv50_wndw_new_(const struct nv50_wndw_func *func, struct drm_device *dev,
 	wndw->id = index;
 	wndw->interlock.type = interlock_type;
 	wndw->interlock.data = interlock_data;
-	wndw->ctxdma.parent = &wndw->wndw.base.user;
 
 	wndw->ctxdma.parent = &wndw->wndw.base.user;
 	INIT_LIST_HEAD(&wndw->ctxdma.list);
diff --git a/drivers/gpu/drm/nouveau/include/nvif/object.h b/drivers/gpu/drm/nouveau/include/nvif/object.h
index 20754d9e6883..8407651f6ac6 100644
--- a/drivers/gpu/drm/nouveau/include/nvif/object.h
+++ b/drivers/gpu/drm/nouveau/include/nvif/object.h
@@ -78,7 +78,7 @@ struct nvif_mclass {
 #define nvif_mclass(o,m) ({                                                    \
 	struct nvif_object *object = (o);                                      \
 	struct nvif_sclass *sclass;                                            \
-	const typeof(m[0]) *mclass = (m);                                      \
+	typeof(m[0]) *mclass = (m);                                            \
 	int ret = -ENODEV;                                                     \
 	int cnt, i, j;                                                         \
                                                                                \
diff --git a/drivers/gpu/drm/nouveau/nouveau_abi16.c b/drivers/gpu/drm/nouveau/nouveau_abi16.c
index e2211bb2cf79..e67a471331b5 100644
--- a/drivers/gpu/drm/nouveau/nouveau_abi16.c
+++ b/drivers/gpu/drm/nouveau/nouveau_abi16.c
@@ -139,7 +139,7 @@ nouveau_abi16_chan_fini(struct nouveau_abi16 *abi16,
 	if (chan->ntfy) {
 		nouveau_vma_del(&chan->ntfy_vma);
 		nouveau_bo_unpin(chan->ntfy);
-		drm_gem_object_unreference_unlocked(&chan->ntfy->gem);
+		drm_gem_object_put_unlocked(&chan->ntfy->gem);
 	}
 
 	if (chan->heap.block_size)
diff --git a/drivers/gpu/drm/nouveau/nouveau_backlight.c b/drivers/gpu/drm/nouveau/nouveau_backlight.c
index debbbf0fd4bd..408b955e5c39 100644
--- a/drivers/gpu/drm/nouveau/nouveau_backlight.c
+++ b/drivers/gpu/drm/nouveau/nouveau_backlight.c
@@ -267,6 +267,7 @@ nouveau_backlight_init(struct drm_device *dev)
 	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct nvif_device *device = &drm->client.device;
 	struct drm_connector *connector;
+	struct drm_connector_list_iter conn_iter;
 
 	INIT_LIST_HEAD(&drm->bl_connectors);
 
@@ -275,7 +276,8 @@ nouveau_backlight_init(struct drm_device *dev)
 		return 0;
 	}
 
-	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
+	drm_connector_list_iter_begin(dev, &conn_iter);
+	drm_for_each_connector_iter(connector, &conn_iter) {
 		if (connector->connector_type != DRM_MODE_CONNECTOR_LVDS &&
 		    connector->connector_type != DRM_MODE_CONNECTOR_eDP)
 			continue;
@@ -292,7 +294,7 @@ nouveau_backlight_init(struct drm_device *dev)
 			break;
 		}
 	}
-
+	drm_connector_list_iter_end(&conn_iter);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/nouveau/nouveau_connector.c b/drivers/gpu/drm/nouveau/nouveau_connector.c
index 22a15478d23d..51932c72334e 100644
--- a/drivers/gpu/drm/nouveau/nouveau_connector.c
+++ b/drivers/gpu/drm/nouveau/nouveau_connector.c
@@ -1193,14 +1193,19 @@ nouveau_connector_create(struct drm_device *dev, int index)
 	struct nouveau_display *disp = nouveau_display(dev);
 	struct nouveau_connector *nv_connector = NULL;
 	struct drm_connector *connector;
+	struct drm_connector_list_iter conn_iter;
 	int type, ret = 0;
 	bool dummy;
 
-	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
+	drm_connector_list_iter_begin(dev, &conn_iter);
+	nouveau_for_each_non_mst_connector_iter(connector, &conn_iter) {
 		nv_connector = nouveau_connector(connector);
-		if (nv_connector->index == index)
+		if (nv_connector->index == index) {
+			drm_connector_list_iter_end(&conn_iter);
 			return connector;
+		}
 	}
+	drm_connector_list_iter_end(&conn_iter);
 
 	nv_connector = kzalloc(sizeof(*nv_connector), GFP_KERNEL);
 	if (!nv_connector)
diff --git a/drivers/gpu/drm/nouveau/nouveau_connector.h b/drivers/gpu/drm/nouveau/nouveau_connector.h
index a4d1a059bd3d..dc7454e7f19a 100644
--- a/drivers/gpu/drm/nouveau/nouveau_connector.h
+++ b/drivers/gpu/drm/nouveau/nouveau_connector.h
@@ -33,6 +33,7 @@
 #include <drm/drm_encoder.h>
 #include <drm/drm_dp_helper.h>
 #include "nouveau_crtc.h"
+#include "nouveau_encoder.h"
 
 struct nvkm_i2c_port;
 
@@ -60,19 +61,46 @@ static inline struct nouveau_connector *nouveau_connector(
 	return container_of(con, struct nouveau_connector, base);
 }
 
+static inline bool
+nouveau_connector_is_mst(struct drm_connector *connector)
+{
+	const struct nouveau_encoder *nv_encoder;
+	const struct drm_encoder *encoder;
+
+	if (connector->connector_type != DRM_MODE_CONNECTOR_DisplayPort)
+		return false;
+
+	nv_encoder = find_encoder(connector, DCB_OUTPUT_ANY);
+	if (!nv_encoder)
+		return false;
+
+	encoder = &nv_encoder->base.base;
+	return encoder->encoder_type == DRM_MODE_ENCODER_DPMST;
+}
+
+#define nouveau_for_each_non_mst_connector_iter(connector, iter) \
+	drm_for_each_connector_iter(connector, iter) \
+		for_each_if(!nouveau_connector_is_mst(connector))
+
 static inline struct nouveau_connector *
 nouveau_crtc_connector_get(struct nouveau_crtc *nv_crtc)
 {
 	struct drm_device *dev = nv_crtc->base.dev;
 	struct drm_connector *connector;
+	struct drm_connector_list_iter conn_iter;
+	struct nouveau_connector *nv_connector = NULL;
 	struct drm_crtc *crtc = to_drm_crtc(nv_crtc);
 
-	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
-		if (connector->encoder && connector->encoder->crtc == crtc)
-			return nouveau_connector(connector);
+	drm_connector_list_iter_begin(dev, &conn_iter);
+	nouveau_for_each_non_mst_connector_iter(connector, &conn_iter) {
+		if (connector->encoder && connector->encoder->crtc == crtc) {
+			nv_connector = nouveau_connector(connector);
+			break;
+		}
 	}
+	drm_connector_list_iter_end(&conn_iter);
 
-	return NULL;
+	return nv_connector;
 }
 
 struct drm_connector *
diff --git a/drivers/gpu/drm/nouveau/nouveau_debugfs.c b/drivers/gpu/drm/nouveau/nouveau_debugfs.c
index 963a4dba8213..9109b69cd052 100644
--- a/drivers/gpu/drm/nouveau/nouveau_debugfs.c
+++ b/drivers/gpu/drm/nouveau/nouveau_debugfs.c
@@ -160,7 +160,11 @@ nouveau_debugfs_pstate_set(struct file *file, const char __user *ubuf,
 		args.ustate = value;
 	}
 
+	ret = pm_runtime_get_sync(drm->dev);
+	if (IS_ERR_VALUE(ret) && ret != -EACCES)
+		return ret;
 	ret = nvif_mthd(ctrl, NVIF_CONTROL_PSTATE_USER, &args, sizeof(args));
+	pm_runtime_put_autosuspend(drm->dev);
 	if (ret < 0)
 		return ret;
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_display.c b/drivers/gpu/drm/nouveau/nouveau_display.c
index 774b429142bc..139368b31916 100644
--- a/drivers/gpu/drm/nouveau/nouveau_display.c
+++ b/drivers/gpu/drm/nouveau/nouveau_display.c
@@ -205,7 +205,7 @@ nouveau_user_framebuffer_destroy(struct drm_framebuffer *drm_fb)
 	struct nouveau_framebuffer *fb = nouveau_framebuffer(drm_fb);
 
 	if (fb->nvbo)
-		drm_gem_object_unreference_unlocked(&fb->nvbo->gem);
+		drm_gem_object_put_unlocked(&fb->nvbo->gem);
 
 	drm_framebuffer_cleanup(drm_fb);
 	kfree(fb);
@@ -287,7 +287,7 @@ nouveau_user_framebuffer_create(struct drm_device *dev,
 	if (ret == 0)
 		return &fb->base;
 
-	drm_gem_object_unreference_unlocked(gem);
+	drm_gem_object_put_unlocked(gem);
 	return ERR_PTR(ret);
 }
 
@@ -404,6 +404,7 @@ nouveau_display_init(struct drm_device *dev)
 	struct nouveau_display *disp = nouveau_display(dev);
 	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct drm_connector *connector;
+	struct drm_connector_list_iter conn_iter;
 	int ret;
 
 	ret = disp->init(dev);
@@ -411,10 +412,12 @@ nouveau_display_init(struct drm_device *dev)
 		return ret;
 
 	/* enable hotplug interrupts */
-	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
+	drm_connector_list_iter_begin(dev, &conn_iter);
+	nouveau_for_each_non_mst_connector_iter(connector, &conn_iter) {
 		struct nouveau_connector *conn = nouveau_connector(connector);
 		nvif_notify_get(&conn->hpd);
 	}
+	drm_connector_list_iter_end(&conn_iter);
 
 	/* enable flip completion events */
 	nvif_notify_get(&drm->flip);
@@ -427,6 +430,7 @@ nouveau_display_fini(struct drm_device *dev, bool suspend)
 	struct nouveau_display *disp = nouveau_display(dev);
 	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct drm_connector *connector;
+	struct drm_connector_list_iter conn_iter;
 
 	if (!suspend) {
 		if (drm_drv_uses_atomic_modeset(dev))
@@ -439,10 +443,12 @@ nouveau_display_fini(struct drm_device *dev, bool suspend)
 	nvif_notify_put(&drm->flip);
 
 	/* disable hotplug interrupts */
-	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
+	drm_connector_list_iter_begin(dev, &conn_iter);
+	nouveau_for_each_non_mst_connector_iter(connector, &conn_iter) {
 		struct nouveau_connector *conn = nouveau_connector(connector);
 		nvif_notify_put(&conn->hpd);
 	}
+	drm_connector_list_iter_end(&conn_iter);
 
 	drm_kms_helper_poll_disable(dev);
 	disp->fini(dev);
@@ -939,7 +945,7 @@ nouveau_display_dumb_create(struct drm_file *file_priv, struct drm_device *dev,
 		return ret;
 
 	ret = drm_gem_handle_create(file_priv, &bo->gem, &args->handle);
-	drm_gem_object_unreference_unlocked(&bo->gem);
+	drm_gem_object_put_unlocked(&bo->gem);
 	return ret;
 }
 
@@ -954,7 +960,7 @@ nouveau_display_dumb_map_offset(struct drm_file *file_priv,
 	if (gem) {
 		struct nouveau_bo *bo = nouveau_gem_object(gem);
 		*poffset = drm_vma_node_offset_addr(&bo->bo.vma_node);
-		drm_gem_object_unreference_unlocked(gem);
+		drm_gem_object_put_unlocked(gem);
 		return 0;
 	}
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.c b/drivers/gpu/drm/nouveau/nouveau_drm.c
index 775443c9af94..c7ec86d6c3c9 100644
--- a/drivers/gpu/drm/nouveau/nouveau_drm.c
+++ b/drivers/gpu/drm/nouveau/nouveau_drm.c
@@ -81,6 +81,10 @@ MODULE_PARM_DESC(modeset, "enable driver (default: auto, "
 int nouveau_modeset = -1;
 module_param_named(modeset, nouveau_modeset, int, 0400);
 
+MODULE_PARM_DESC(atomic, "Expose atomic ioctl (default: disabled)");
+static int nouveau_atomic = 0;
+module_param_named(atomic, nouveau_atomic, int, 0400);
+
 MODULE_PARM_DESC(runpm, "disable (0), force enable (1), optimus only default (-1)");
 static int nouveau_runtime_pm = -1;
 module_param_named(runpm, nouveau_runtime_pm, int, 0400);
@@ -509,6 +513,9 @@ static int nouveau_drm_probe(struct pci_dev *pdev,
 
 	pci_set_master(pdev);
 
+	if (nouveau_atomic)
+		driver_pci.driver_features |= DRIVER_ATOMIC;
+
 	ret = drm_get_pci_dev(pdev, pent, &driver_pci);
 	if (ret) {
 		nvkm_device_del(&device);
@@ -874,22 +881,11 @@ nouveau_pmops_runtime_resume(struct device *dev)
 static int
 nouveau_pmops_runtime_idle(struct device *dev)
 {
-	struct pci_dev *pdev = to_pci_dev(dev);
-	struct drm_device *drm_dev = pci_get_drvdata(pdev);
-	struct nouveau_drm *drm = nouveau_drm(drm_dev);
-	struct drm_crtc *crtc;
-
 	if (!nouveau_pmops_runtime()) {
 		pm_runtime_forbid(dev);
 		return -EBUSY;
 	}
 
-	list_for_each_entry(crtc, &drm->dev->mode_config.crtc_list, head) {
-		if (crtc->enabled) {
-			DRM_DEBUG_DRIVER("failing to power off - crtc active\n");
-			return -EBUSY;
-		}
-	}
 	pm_runtime_mark_last_busy(dev);
 	pm_runtime_autosuspend(dev);
 	/* we don't want the main rpm_idle to call suspend - we want to autosuspend */
@@ -912,8 +908,10 @@ nouveau_drm_open(struct drm_device *dev, struct drm_file *fpriv)
 	get_task_comm(tmpname, current);
 	snprintf(name, sizeof(name), "%s[%d]", tmpname, pid_nr(fpriv->pid));
 
-	if (!(cli = kzalloc(sizeof(*cli), GFP_KERNEL)))
-		return ret;
+	if (!(cli = kzalloc(sizeof(*cli), GFP_KERNEL))) {
+		ret = -ENOMEM;
+		goto done;
+	}
 
 	ret = nouveau_cli_init(drm, name, cli);
 	if (ret)
diff --git a/drivers/gpu/drm/nouveau/nouveau_fbcon.c b/drivers/gpu/drm/nouveau/nouveau_fbcon.c
index 85c1f10bc2b6..844498c4267c 100644
--- a/drivers/gpu/drm/nouveau/nouveau_fbcon.c
+++ b/drivers/gpu/drm/nouveau/nouveau_fbcon.c
@@ -429,7 +429,7 @@ nouveau_fbcon_destroy(struct drm_device *dev, struct nouveau_fbdev *fbcon)
 		nouveau_vma_del(&nouveau_fb->vma);
 		nouveau_bo_unmap(nouveau_fb->nvbo);
 		nouveau_bo_unpin(nouveau_fb->nvbo);
-		drm_framebuffer_unreference(&nouveau_fb->base);
+		drm_framebuffer_put(&nouveau_fb->base);
 	}
 
 	return 0;
diff --git a/drivers/gpu/drm/nouveau/nouveau_gem.c b/drivers/gpu/drm/nouveau/nouveau_gem.c
index 300daee74209..b56524d343c3 100644
--- a/drivers/gpu/drm/nouveau/nouveau_gem.c
+++ b/drivers/gpu/drm/nouveau/nouveau_gem.c
@@ -274,7 +274,7 @@ nouveau_gem_ioctl_new(struct drm_device *dev, void *data,
 	}
 
 	/* drop reference from allocate - handle holds it now */
-	drm_gem_object_unreference_unlocked(&nvbo->gem);
+	drm_gem_object_put_unlocked(&nvbo->gem);
 	return ret;
 }
 
@@ -354,7 +354,7 @@ validate_fini_no_ticket(struct validate_op *op, struct nouveau_fence *fence,
 		list_del(&nvbo->entry);
 		nvbo->reserved_by = NULL;
 		ttm_bo_unreserve(&nvbo->bo);
-		drm_gem_object_unreference_unlocked(&nvbo->gem);
+		drm_gem_object_put_unlocked(&nvbo->gem);
 	}
 }
 
@@ -400,14 +400,14 @@ retry:
 		nvbo = nouveau_gem_object(gem);
 		if (nvbo == res_bo) {
 			res_bo = NULL;
-			drm_gem_object_unreference_unlocked(gem);
+			drm_gem_object_put_unlocked(gem);
 			continue;
 		}
 
 		if (nvbo->reserved_by && nvbo->reserved_by == file_priv) {
 			NV_PRINTK(err, cli, "multiple instances of buffer %d on "
 				      "validation list\n", b->handle);
-			drm_gem_object_unreference_unlocked(gem);
+			drm_gem_object_put_unlocked(gem);
 			ret = -EINVAL;
 			break;
 		}
@@ -616,7 +616,7 @@ nouveau_gem_pushbuf_reloc_apply(struct nouveau_cli *cli,
 		struct nouveau_bo *nvbo;
 		uint32_t data;
 
-		if (unlikely(r->bo_index > req->nr_buffers)) {
+		if (unlikely(r->bo_index >= req->nr_buffers)) {
 			NV_PRINTK(err, cli, "reloc bo index invalid\n");
 			ret = -EINVAL;
 			break;
@@ -626,7 +626,7 @@ nouveau_gem_pushbuf_reloc_apply(struct nouveau_cli *cli,
 		if (b->presumed.valid)
 			continue;
 
-		if (unlikely(r->reloc_bo_index > req->nr_buffers)) {
+		if (unlikely(r->reloc_bo_index >= req->nr_buffers)) {
 			NV_PRINTK(err, cli, "reloc container bo index invalid\n");
 			ret = -EINVAL;
 			break;
@@ -894,7 +894,7 @@ nouveau_gem_ioctl_cpu_prep(struct drm_device *dev, void *data,
 		ret = lret;
 
 	nouveau_bo_sync_for_cpu(nvbo);
-	drm_gem_object_unreference_unlocked(gem);
+	drm_gem_object_put_unlocked(gem);
 
 	return ret;
 }
@@ -913,7 +913,7 @@ nouveau_gem_ioctl_cpu_fini(struct drm_device *dev, void *data,
 	nvbo = nouveau_gem_object(gem);
 
 	nouveau_bo_sync_for_device(nvbo);
-	drm_gem_object_unreference_unlocked(gem);
+	drm_gem_object_put_unlocked(gem);
 	return 0;
 }
 
@@ -930,7 +930,7 @@ nouveau_gem_ioctl_info(struct drm_device *dev, void *data,
 		return -ENOENT;
 
 	ret = nouveau_gem_info(file_priv, gem, req);
-	drm_gem_object_unreference_unlocked(gem);
+	drm_gem_object_put_unlocked(gem);
 	return ret;
 }
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_hwmon.c b/drivers/gpu/drm/nouveau/nouveau_hwmon.c
index 44178b4c3599..08a1ab6b150d 100644
--- a/drivers/gpu/drm/nouveau/nouveau_hwmon.c
+++ b/drivers/gpu/drm/nouveau/nouveau_hwmon.c
@@ -69,8 +69,8 @@ nouveau_hwmon_set_temp1_auto_point1_temp(struct device *d,
 	struct nvkm_therm *therm = nvxx_therm(&drm->client.device);
 	long value;
 
-	if (kstrtol(buf, 10, &value) == -EINVAL)
-		return count;
+	if (kstrtol(buf, 10, &value))
+		return -EINVAL;
 
 	therm->attr_set(therm, NVKM_THERM_ATTR_THRS_FAN_BOOST,
 			value / 1000);
@@ -102,8 +102,8 @@ nouveau_hwmon_set_temp1_auto_point1_temp_hyst(struct device *d,
 	struct nvkm_therm *therm = nvxx_therm(&drm->client.device);
 	long value;
 
-	if (kstrtol(buf, 10, &value) == -EINVAL)
-		return count;
+	if (kstrtol(buf, 10, &value))
+		return -EINVAL;
 
 	therm->attr_set(therm, NVKM_THERM_ATTR_THRS_FAN_BOOST_HYST,
 			value / 1000);
@@ -156,7 +156,7 @@ nouveau_hwmon_set_pwm1_min(struct device *d, struct device_attribute *a,
 	long value;
 	int ret;
 
-	if (kstrtol(buf, 10, &value) == -EINVAL)
+	if (kstrtol(buf, 10, &value))
 		return -EINVAL;
 
 	ret = therm->attr_set(therm, NVKM_THERM_ATTR_FAN_MIN_DUTY, value);
@@ -179,7 +179,7 @@ nouveau_hwmon_set_pwm1_max(struct device *d, struct device_attribute *a,
 	long value;
 	int ret;
 
-	if (kstrtol(buf, 10, &value) == -EINVAL)
+	if (kstrtol(buf, 10, &value))
 		return -EINVAL;
 
 	ret = therm->attr_set(therm, NVKM_THERM_ATTR_FAN_MAX_DUTY, value);
diff --git a/drivers/gpu/drm/nouveau/nouveau_platform.c b/drivers/gpu/drm/nouveau/nouveau_platform.c
index 1ada186fab77..039e23548e08 100644
--- a/drivers/gpu/drm/nouveau/nouveau_platform.c
+++ b/drivers/gpu/drm/nouveau/nouveau_platform.c
@@ -36,7 +36,7 @@ static int nouveau_platform_probe(struct platform_device *pdev)
 
 	ret = drm_dev_register(drm, 0);
 	if (ret < 0) {
-		drm_dev_unref(drm);
+		drm_dev_put(drm);
 		return ret;
 	}
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/core/engine.c b/drivers/gpu/drm/nouveau/nvkm/core/engine.c
index d0322ce85172..1a47c40e171b 100644
--- a/drivers/gpu/drm/nouveau/nvkm/core/engine.c
+++ b/drivers/gpu/drm/nouveau/nvkm/core/engine.c
@@ -87,11 +87,12 @@ nvkm_engine_info(struct nvkm_subdev *subdev, u64 mthd, u64 *data)
 {
 	struct nvkm_engine *engine = nvkm_engine(subdev);
 	if (engine->func->info) {
-		if ((engine = nvkm_engine_ref(engine))) {
+		if (!IS_ERR((engine = nvkm_engine_ref(engine)))) {
 			int ret = engine->func->info(engine, mthd, data);
 			nvkm_engine_unref(&engine);
 			return ret;
 		}
+		return PTR_ERR(engine);
 	}
 	return -ENOSYS;
 }
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c
index 78597da6313a..0e372a190d3f 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c
@@ -23,6 +23,10 @@
 #ifdef CONFIG_NOUVEAU_PLATFORM_DRIVER
 #include "priv.h"
 
+#if IS_ENABLED(CONFIG_ARM_DMA_USE_IOMMU)
+#include <asm/dma-iommu.h>
+#endif
+
 static int
 nvkm_device_tegra_power_up(struct nvkm_device_tegra *tdev)
 {
@@ -105,6 +109,15 @@ nvkm_device_tegra_probe_iommu(struct nvkm_device_tegra *tdev)
 	unsigned long pgsize_bitmap;
 	int ret;
 
+#if IS_ENABLED(CONFIG_ARM_DMA_USE_IOMMU)
+	if (dev->archdata.mapping) {
+		struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
+
+		arm_iommu_detach_device(dev);
+		arm_iommu_release_mapping(mapping);
+	}
+#endif
+
 	if (!tdev->func->iommu_bit)
 		return;
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/changf119.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/changf119.c
index 29e6dd58ac48..525f95d06429 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/changf119.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/changf119.c
@@ -52,7 +52,7 @@ void
 gf119_disp_chan_intr(struct nv50_disp_chan *chan, bool en)
 {
 	struct nvkm_device *device = chan->disp->base.engine.subdev.device;
-	const u64 mask = 0x00000001 << chan->chid.user;
+	const u32 mask = 0x00000001 << chan->chid.user;
 	if (!en) {
 		nvkm_mask(device, 0x610090, mask, 0x00000000);
 		nvkm_mask(device, 0x6100a0, mask, 0x00000000);
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.c
index 57719f675eec..bcf32d92ee5a 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.c
@@ -166,8 +166,8 @@ void
 nv50_disp_chan_intr(struct nv50_disp_chan *chan, bool en)
 {
 	struct nvkm_device *device = chan->disp->base.engine.subdev.device;
-	const u64 mask = 0x00010001 << chan->chid.user;
-	const u64 data = en ? 0x00010000 : 0x00000000;
+	const u32 mask = 0x00010001 << chan->chid.user;
+	const u32 data = en ? 0x00010000 << chan->chid.user : 0x00000000;
 	nvkm_mask(device, 0x610028, mask, data);
 }
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gv100.c
index 19173ea19096..3b3327789ae7 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gv100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gv100.c
@@ -25,24 +25,31 @@
 #include <nvif/class.h>
 
 static void
-gv100_gr_trap_mp(struct gf100_gr *gr, int gpc, int tpc)
+gv100_gr_trap_sm(struct gf100_gr *gr, int gpc, int tpc, int sm)
 {
 	struct nvkm_subdev *subdev = &gr->base.engine.subdev;
 	struct nvkm_device *device = subdev->device;
-	u32 werr = nvkm_rd32(device, TPC_UNIT(gpc, tpc, 0x730));
-	u32 gerr = nvkm_rd32(device, TPC_UNIT(gpc, tpc, 0x734));
+	u32 werr = nvkm_rd32(device, TPC_UNIT(gpc, tpc, 0x730 + (sm * 0x80)));
+	u32 gerr = nvkm_rd32(device, TPC_UNIT(gpc, tpc, 0x734 + (sm * 0x80)));
 	const struct nvkm_enum *warp;
 	char glob[128];
 
 	nvkm_snprintbf(glob, sizeof(glob), gf100_mp_global_error, gerr);
 	warp = nvkm_enum_find(gf100_mp_warp_error, werr & 0xffff);
 
-	nvkm_error(subdev, "GPC%i/TPC%i/MP trap: "
+	nvkm_error(subdev, "GPC%i/TPC%i/SM%d trap: "
 			   "global %08x [%s] warp %04x [%s]\n",
-		   gpc, tpc, gerr, glob, werr, warp ? warp->name : "");
+		   gpc, tpc, sm, gerr, glob, werr, warp ? warp->name : "");
+
+	nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x730 + sm * 0x80), 0x00000000);
+	nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x734 + sm * 0x80), gerr);
+}
 
-	nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x730), 0x00000000);
-	nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x734), gerr);
+static void
+gv100_gr_trap_mp(struct gf100_gr *gr, int gpc, int tpc)
+{
+	gv100_gr_trap_sm(gr, gpc, tpc, 0);
+	gv100_gr_trap_sm(gr, gpc, tpc, 1);
 }
 
 static void
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/vpstate.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/vpstate.c
index 20b6fc8243e0..71524548de32 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/vpstate.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/vpstate.c
@@ -58,8 +58,14 @@ nvbios_vpstate_parse(struct nvkm_bios *b, struct nvbios_vpstate_header *h)
 		h->ecount   = nvbios_rd08(b, h->offset + 0x5);
 
 		h->base_id  = nvbios_rd08(b, h->offset + 0x0f);
-		h->boost_id = nvbios_rd08(b, h->offset + 0x10);
-		h->tdp_id   = nvbios_rd08(b, h->offset + 0x11);
+		if (h->hlen > 0x10)
+			h->boost_id = nvbios_rd08(b, h->offset + 0x10);
+		else
+			h->boost_id = 0xff;
+		if (h->hlen > 0x11)
+			h->tdp_id = nvbios_rd08(b, h->offset + 0x11);
+		else
+			h->tdp_id = 0xff;
 		return 0;
 	default:
 		return -EINVAL;
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/base.c
index 007bf4af33b9..16ad91c91a7b 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/base.c
@@ -133,8 +133,14 @@ nvkm_fault_oneinit(struct nvkm_subdev *subdev)
 		}
 	}
 
-	return nvkm_event_init(&nvkm_fault_ntfy, 1, fault->buffer_nr,
-			       &fault->event);
+	ret = nvkm_event_init(&nvkm_fault_ntfy, 1, fault->buffer_nr,
+			      &fault->event);
+	if (ret)
+		return ret;
+
+	if (fault->func->oneinit)
+		ret = fault->func->oneinit(fault);
+	return ret;
 }
 
 static void *
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/gv100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/gv100.c
index 73c7728b5969..3cd610d7deb5 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/gv100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/gv100.c
@@ -176,8 +176,17 @@ gv100_fault_init(struct nvkm_fault *fault)
 	nvkm_notify_get(&fault->nrpfb);
 }
 
+static int
+gv100_fault_oneinit(struct nvkm_fault *fault)
+{
+	return nvkm_notify_init(&fault->buffer[0]->object, &fault->event,
+				gv100_fault_ntfy_nrpfb, false, NULL, 0, 0,
+				&fault->nrpfb);
+}
+
 static const struct nvkm_fault_func
 gv100_fault = {
+	.oneinit = gv100_fault_oneinit,
 	.init = gv100_fault_init,
 	.fini = gv100_fault_fini,
 	.intr = gv100_fault_intr,
@@ -192,15 +201,5 @@ int
 gv100_fault_new(struct nvkm_device *device, int index,
 		struct nvkm_fault **pfault)
 {
-	struct nvkm_fault *fault;
-	int ret;
-
-	ret = nvkm_fault_new_(&gv100_fault, device, index, &fault);
-	*pfault = fault;
-	if (ret)
-		return ret;
-
-	return nvkm_notify_init(&fault->buffer[0]->object, &fault->event,
-				gv100_fault_ntfy_nrpfb, false, NULL, 0, 0,
-				&fault->nrpfb);
+	return nvkm_fault_new_(&gv100_fault, device, index, pfault);
 }
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/priv.h
index 44843ecf12b0..e4d2f5234fd1 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/priv.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/priv.h
@@ -20,6 +20,7 @@ int nvkm_fault_new_(const struct nvkm_fault_func *, struct nvkm_device *,
 		    int index, struct nvkm_fault **);
 
 struct nvkm_fault_func {
+	int (*oneinit)(struct nvkm_fault *);
 	void (*init)(struct nvkm_fault *);
 	void (*fini)(struct nvkm_fault *);
 	void (*intr)(struct nvkm_fault *);
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c
index 73b5d46104bd..434d2fc5bb1c 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c
@@ -140,6 +140,9 @@ nvkm_fb_init(struct nvkm_subdev *subdev)
 	if (fb->func->init)
 		fb->func->init(fb);
 
+	if (fb->func->init_remapper)
+		fb->func->init_remapper(fb);
+
 	if (fb->func->init_page) {
 		ret = fb->func->init_page(fb);
 		if (WARN_ON(ret))
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp100.c
index dffe1f5e1071..8205ce436b3e 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp100.c
@@ -37,6 +37,14 @@ gp100_fb_init_unkn(struct nvkm_fb *base)
 }
 
 void
+gp100_fb_init_remapper(struct nvkm_fb *fb)
+{
+	struct nvkm_device *device = fb->subdev.device;
+	/* Disable address remapper. */
+	nvkm_mask(device, 0x100c14, 0x00040000, 0x00000000);
+}
+
+void
 gp100_fb_init(struct nvkm_fb *base)
 {
 	struct gf100_fb *fb = gf100_fb(base);
@@ -56,6 +64,7 @@ gp100_fb = {
 	.dtor = gf100_fb_dtor,
 	.oneinit = gf100_fb_oneinit,
 	.init = gp100_fb_init,
+	.init_remapper = gp100_fb_init_remapper,
 	.init_page = gm200_fb_init_page,
 	.init_unkn = gp100_fb_init_unkn,
 	.ram_new = gp100_ram_new,
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp102.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp102.c
index b84b9861ef26..b4d74e815674 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp102.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp102.c
@@ -31,6 +31,7 @@ gp102_fb = {
 	.dtor = gf100_fb_dtor,
 	.oneinit = gf100_fb_oneinit,
 	.init = gp100_fb_init,
+	.init_remapper = gp100_fb_init_remapper,
 	.init_page = gm200_fb_init_page,
 	.ram_new = gp100_ram_new,
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/priv.h
index 2857f31466bf..1e4ad61c19e1 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/priv.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/priv.h
@@ -11,6 +11,7 @@ struct nvkm_fb_func {
 	u32 (*tags)(struct nvkm_fb *);
 	int (*oneinit)(struct nvkm_fb *);
 	void (*init)(struct nvkm_fb *);
+	void (*init_remapper)(struct nvkm_fb *);
 	int (*init_page)(struct nvkm_fb *);
 	void (*init_unkn)(struct nvkm_fb *);
 	void (*intr)(struct nvkm_fb *);
@@ -69,5 +70,6 @@ int gf100_fb_init_page(struct nvkm_fb *);
 
 int gm200_fb_init_page(struct nvkm_fb *);
 
+void gp100_fb_init_remapper(struct nvkm_fb *);
 void gp100_fb_init_unkn(struct nvkm_fb *);
 #endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b. b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.
deleted file mode 100644
index e69de29bb2d1..000000000000
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/gp10b.
+++ /dev/null
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/acr_r352.c b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/acr_r352.c
index a721354249ce..d02e183717dc 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/acr_r352.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/acr_r352.c
@@ -414,6 +414,20 @@ acr_r352_ls_write_wpr(struct acr_r352 *acr, struct list_head *imgs,
 {
 	struct ls_ucode_img *_img;
 	u32 pos = 0;
+	u32 max_desc_size = 0;
+	u8 *gdesc;
+
+	/* Figure out how large we need gdesc to be. */
+	list_for_each_entry(_img, imgs, node) {
+		const struct acr_r352_ls_func *ls_func =
+					    acr->func->ls_func[_img->falcon_id];
+
+		max_desc_size = max(max_desc_size, ls_func->bl_desc_size);
+	}
+
+	gdesc = kmalloc(max_desc_size, GFP_KERNEL);
+	if (!gdesc)
+		return -ENOMEM;
 
 	nvkm_kmap(wpr_blob);
 
@@ -421,7 +435,6 @@ acr_r352_ls_write_wpr(struct acr_r352 *acr, struct list_head *imgs,
 		struct ls_ucode_img_r352 *img = ls_ucode_img_r352(_img);
 		const struct acr_r352_ls_func *ls_func =
 					    acr->func->ls_func[_img->falcon_id];
-		u8 gdesc[ls_func->bl_desc_size];
 
 		nvkm_gpuobj_memcpy_to(wpr_blob, pos, &img->wpr_header,
 				      sizeof(img->wpr_header));
@@ -447,6 +460,8 @@ acr_r352_ls_write_wpr(struct acr_r352 *acr, struct list_head *imgs,
 
 	nvkm_done(wpr_blob);
 
+	kfree(gdesc);
+
 	return 0;
 }
 
@@ -771,7 +786,11 @@ acr_r352_load(struct nvkm_acr *_acr, struct nvkm_falcon *falcon,
 	struct fw_bl_desc *hsbl_desc;
 	void *bl, *blob_data, *hsbl_code, *hsbl_data;
 	u32 code_size;
-	u8 bl_desc[bl_desc_size];
+	u8 *bl_desc;
+
+	bl_desc = kzalloc(bl_desc_size, GFP_KERNEL);
+	if (!bl_desc)
+		return -ENOMEM;
 
 	/* Find the bootloader descriptor for our blob and copy it */
 	if (blob == acr->load_blob) {
@@ -802,7 +821,6 @@ acr_r352_load(struct nvkm_acr *_acr, struct nvkm_falcon *falcon,
 			      code_size, hsbl_desc->start_tag, 0, false);
 
 	/* Generate the BL header */
-	memset(bl_desc, 0, bl_desc_size);
 	acr->func->generate_hs_bl_desc(load_hdr, bl_desc, offset);
 
 	/*
@@ -811,6 +829,7 @@ acr_r352_load(struct nvkm_acr *_acr, struct nvkm_falcon *falcon,
 	nvkm_falcon_load_dmem(falcon, bl_desc, hsbl_desc->dmem_load_off,
 			      bl_desc_size, 0);
 
+	kfree(bl_desc);
 	return hsbl_desc->start_tag << 8;
 }
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/acr_r367.c b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/acr_r367.c
index 866877b88797..978ad0790367 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/acr_r367.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/acr_r367.c
@@ -265,6 +265,19 @@ acr_r367_ls_write_wpr(struct acr_r352 *acr, struct list_head *imgs,
 {
 	struct ls_ucode_img *_img;
 	u32 pos = 0;
+	u32 max_desc_size = 0;
+	u8 *gdesc;
+
+	list_for_each_entry(_img, imgs, node) {
+		const struct acr_r352_ls_func *ls_func =
+					    acr->func->ls_func[_img->falcon_id];
+
+		max_desc_size = max(max_desc_size, ls_func->bl_desc_size);
+	}
+
+	gdesc = kmalloc(max_desc_size, GFP_KERNEL);
+	if (!gdesc)
+		return -ENOMEM;
 
 	nvkm_kmap(wpr_blob);
 
@@ -272,7 +285,6 @@ acr_r367_ls_write_wpr(struct acr_r352 *acr, struct list_head *imgs,
 		struct ls_ucode_img_r367 *img = ls_ucode_img_r367(_img);
 		const struct acr_r352_ls_func *ls_func =
 					    acr->func->ls_func[_img->falcon_id];
-		u8 gdesc[ls_func->bl_desc_size];
 
 		nvkm_gpuobj_memcpy_to(wpr_blob, pos, &img->wpr_header,
 				      sizeof(img->wpr_header));
@@ -298,6 +310,8 @@ acr_r367_ls_write_wpr(struct acr_r352 *acr, struct list_head *imgs,
 
 	nvkm_done(wpr_blob);
 
+	kfree(gdesc);
+
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm20b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm20b.c
index 30491d132d59..df8b919dcf09 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm20b.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm20b.c
@@ -129,6 +129,7 @@ gm20b_secboot_new(struct nvkm_device *device, int index,
 	return 0;
 }
 
+#if IS_ENABLED(CONFIG_ARCH_TEGRA_210_SOC)
 MODULE_FIRMWARE("nvidia/gm20b/acr/bl.bin");
 MODULE_FIRMWARE("nvidia/gm20b/acr/ucode_load.bin");
 MODULE_FIRMWARE("nvidia/gm20b/gr/fecs_bl.bin");
@@ -144,3 +145,4 @@ MODULE_FIRMWARE("nvidia/gm20b/gr/sw_method_init.bin");
 MODULE_FIRMWARE("nvidia/gm20b/pmu/desc.bin");
 MODULE_FIRMWARE("nvidia/gm20b/pmu/image.bin");
 MODULE_FIRMWARE("nvidia/gm20b/pmu/sig.bin");
+#endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gp10b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gp10b.c
index 632e9545e292..28ca29d0eeee 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gp10b.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gp10b.c
@@ -74,6 +74,7 @@ gp10b_secboot_new(struct nvkm_device *device, int index,
 	return 0;
 }
 
+#if IS_ENABLED(CONFIG_ARCH_TEGRA_186_SOC)
 MODULE_FIRMWARE("nvidia/gp10b/acr/bl.bin");
 MODULE_FIRMWARE("nvidia/gp10b/acr/ucode_load.bin");
 MODULE_FIRMWARE("nvidia/gp10b/gr/fecs_bl.bin");
@@ -91,3 +92,4 @@ MODULE_FIRMWARE("nvidia/gp10b/gr/sw_method_init.bin");
 MODULE_FIRMWARE("nvidia/gp10b/pmu/desc.bin");
 MODULE_FIRMWARE("nvidia/gp10b/pmu/image.bin");
 MODULE_FIRMWARE("nvidia/gp10b/pmu/sig.bin");
+#endif
diff --git a/drivers/gpu/drm/panel/panel-simple.c b/drivers/gpu/drm/panel/panel-simple.c
index 5b5d0a24e713..97964f7f2ace 100644
--- a/drivers/gpu/drm/panel/panel-simple.c
+++ b/drivers/gpu/drm/panel/panel-simple.c
@@ -1385,6 +1385,9 @@ static const struct panel_desc innolux_tv123wam = {
 		.width = 259,
 		.height = 173,
 	},
+	.delay = {
+		.unprepare = 500,
+	},
 };
 
 static const struct drm_display_mode innolux_zj070na_01p_mode = {
diff --git a/drivers/gpu/drm/radeon/ci_dpm.c b/drivers/gpu/drm/radeon/ci_dpm.c
index b9302c918271..d587779a80b4 100644
--- a/drivers/gpu/drm/radeon/ci_dpm.c
+++ b/drivers/gpu/drm/radeon/ci_dpm.c
@@ -5676,19 +5676,29 @@ int ci_dpm_init(struct radeon_device *rdev)
 	u16 data_offset, size;
 	u8 frev, crev;
 	struct ci_power_info *pi;
+	enum pci_bus_speed speed_cap;
+	struct pci_dev *root = rdev->pdev->bus->self;
 	int ret;
-	u32 mask;
 
 	pi = kzalloc(sizeof(struct ci_power_info), GFP_KERNEL);
 	if (pi == NULL)
 		return -ENOMEM;
 	rdev->pm.dpm.priv = pi;
 
-	ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
-	if (ret)
+	speed_cap = pcie_get_speed_cap(root);
+	if (speed_cap == PCI_SPEED_UNKNOWN) {
 		pi->sys_pcie_mask = 0;
-	else
-		pi->sys_pcie_mask = mask;
+	} else {
+		if (speed_cap == PCIE_SPEED_8_0GT)
+			pi->sys_pcie_mask = RADEON_PCIE_SPEED_25 |
+				RADEON_PCIE_SPEED_50 |
+				RADEON_PCIE_SPEED_80;
+		else if (speed_cap == PCIE_SPEED_5_0GT)
+			pi->sys_pcie_mask = RADEON_PCIE_SPEED_25 |
+				RADEON_PCIE_SPEED_50;
+		else
+			pi->sys_pcie_mask = RADEON_PCIE_SPEED_25;
+	}
 	pi->force_pcie_gen = RADEON_PCIE_GEN_INVALID;
 
 	pi->pcie_gen_performance.max = RADEON_PCIE_GEN1;
diff --git a/drivers/gpu/drm/radeon/cik.c b/drivers/gpu/drm/radeon/cik.c
index 7c73bc7e2f85..ebce4601a305 100644
--- a/drivers/gpu/drm/radeon/cik.c
+++ b/drivers/gpu/drm/radeon/cik.c
@@ -9499,9 +9499,10 @@ int cik_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
 static void cik_pcie_gen3_enable(struct radeon_device *rdev)
 {
 	struct pci_dev *root = rdev->pdev->bus->self;
+	enum pci_bus_speed speed_cap;
 	int bridge_pos, gpu_pos;
-	u32 speed_cntl, mask, current_data_rate;
-	int ret, i;
+	u32 speed_cntl, current_data_rate;
+	int i;
 	u16 tmp16;
 
 	if (pci_is_root_bus(rdev->pdev->bus))
@@ -9516,23 +9517,24 @@ static void cik_pcie_gen3_enable(struct radeon_device *rdev)
 	if (!(rdev->flags & RADEON_IS_PCIE))
 		return;
 
-	ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
-	if (ret != 0)
+	speed_cap = pcie_get_speed_cap(root);
+	if (speed_cap == PCI_SPEED_UNKNOWN)
 		return;
 
-	if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
+	if ((speed_cap != PCIE_SPEED_8_0GT) &&
+	    (speed_cap != PCIE_SPEED_5_0GT))
 		return;
 
 	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
 	current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
 		LC_CURRENT_DATA_RATE_SHIFT;
-	if (mask & DRM_PCIE_SPEED_80) {
+	if (speed_cap == PCIE_SPEED_8_0GT) {
 		if (current_data_rate == 2) {
 			DRM_INFO("PCIE gen 3 link speeds already enabled\n");
 			return;
 		}
 		DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
-	} else if (mask & DRM_PCIE_SPEED_50) {
+	} else if (speed_cap == PCIE_SPEED_5_0GT) {
 		if (current_data_rate == 1) {
 			DRM_INFO("PCIE gen 2 link speeds already enabled\n");
 			return;
@@ -9548,7 +9550,7 @@ static void cik_pcie_gen3_enable(struct radeon_device *rdev)
 	if (!gpu_pos)
 		return;
 
-	if (mask & DRM_PCIE_SPEED_80) {
+	if (speed_cap == PCIE_SPEED_8_0GT) {
 		/* re-try equalization if gen3 is not already enabled */
 		if (current_data_rate != 2) {
 			u16 bridge_cfg, gpu_cfg;
@@ -9636,9 +9638,9 @@ static void cik_pcie_gen3_enable(struct radeon_device *rdev)
 
 	pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
 	tmp16 &= ~0xf;
-	if (mask & DRM_PCIE_SPEED_80)
+	if (speed_cap == PCIE_SPEED_8_0GT)
 		tmp16 |= 3; /* gen3 */
-	else if (mask & DRM_PCIE_SPEED_50)
+	else if (speed_cap == PCIE_SPEED_5_0GT)
 		tmp16 |= 2; /* gen2 */
 	else
 		tmp16 |= 1; /* gen1 */
diff --git a/drivers/gpu/drm/radeon/r600_dpm.c b/drivers/gpu/drm/radeon/r600_dpm.c
index 73d4c5348116..5e044c98fca2 100644
--- a/drivers/gpu/drm/radeon/r600_dpm.c
+++ b/drivers/gpu/drm/radeon/r600_dpm.c
@@ -1327,9 +1327,9 @@ enum radeon_pcie_gen r600_get_pcie_gen_support(struct radeon_device *rdev,
 	case RADEON_PCIE_GEN3:
 		return RADEON_PCIE_GEN3;
 	default:
-		if ((sys_mask & DRM_PCIE_SPEED_80) && (default_gen == RADEON_PCIE_GEN3))
+		if ((sys_mask & RADEON_PCIE_SPEED_80) && (default_gen == RADEON_PCIE_GEN3))
 			return RADEON_PCIE_GEN3;
-		else if ((sys_mask & DRM_PCIE_SPEED_50) && (default_gen == RADEON_PCIE_GEN2))
+		else if ((sys_mask & RADEON_PCIE_SPEED_50) && (default_gen == RADEON_PCIE_GEN2))
 			return RADEON_PCIE_GEN2;
 		else
 			return RADEON_PCIE_GEN1;
diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h
index 4a2eb409aacc..1a6f6edb3515 100644
--- a/drivers/gpu/drm/radeon/radeon.h
+++ b/drivers/gpu/drm/radeon/radeon.h
@@ -1653,6 +1653,10 @@ struct radeon_pm {
 	struct radeon_dpm       dpm;
 };
 
+#define RADEON_PCIE_SPEED_25 1
+#define RADEON_PCIE_SPEED_50 2
+#define RADEON_PCIE_SPEED_80 4
+
 int radeon_pm_get_type_index(struct radeon_device *rdev,
 			     enum radeon_pm_state_type ps_type,
 			     int instance);
diff --git a/drivers/gpu/drm/radeon/radeon_mn.c b/drivers/gpu/drm/radeon/radeon_mn.c
index abd24975c9b1..f8b35df44c60 100644
--- a/drivers/gpu/drm/radeon/radeon_mn.c
+++ b/drivers/gpu/drm/radeon/radeon_mn.c
@@ -118,19 +118,27 @@ static void radeon_mn_release(struct mmu_notifier *mn,
  * We block for all BOs between start and end to be idle and
  * unmap them by move them into system domain again.
  */
-static void radeon_mn_invalidate_range_start(struct mmu_notifier *mn,
+static int radeon_mn_invalidate_range_start(struct mmu_notifier *mn,
 					     struct mm_struct *mm,
 					     unsigned long start,
-					     unsigned long end)
+					     unsigned long end,
+					     bool blockable)
 {
 	struct radeon_mn *rmn = container_of(mn, struct radeon_mn, mn);
 	struct ttm_operation_ctx ctx = { false, false };
 	struct interval_tree_node *it;
+	int ret = 0;
 
 	/* notification is exclusive, but interval is inclusive */
 	end -= 1;
 
-	mutex_lock(&rmn->lock);
+	/* TODO we should be able to split locking for interval tree and
+	 * the tear down.
+	 */
+	if (blockable)
+		mutex_lock(&rmn->lock);
+	else if (!mutex_trylock(&rmn->lock))
+		return -EAGAIN;
 
 	it = interval_tree_iter_first(&rmn->objects, start, end);
 	while (it) {
@@ -138,6 +146,11 @@ static void radeon_mn_invalidate_range_start(struct mmu_notifier *mn,
 		struct radeon_bo *bo;
 		long r;
 
+		if (!blockable) {
+			ret = -EAGAIN;
+			goto out_unlock;
+		}
+
 		node = container_of(it, struct radeon_mn_node, it);
 		it = interval_tree_iter_next(it, start, end);
 
@@ -166,7 +179,10 @@ static void radeon_mn_invalidate_range_start(struct mmu_notifier *mn,
 		}
 	}
 	
+out_unlock:
 	mutex_unlock(&rmn->lock);
+
+	return ret;
 }
 
 static const struct mmu_notifier_ops radeon_mn_ops = {
diff --git a/drivers/gpu/drm/radeon/radeon_object.c b/drivers/gpu/drm/radeon/radeon_object.c
index edbb4cd519fd..ba2fd295697f 100644
--- a/drivers/gpu/drm/radeon/radeon_object.c
+++ b/drivers/gpu/drm/radeon/radeon_object.c
@@ -307,7 +307,7 @@ struct radeon_bo *radeon_bo_ref(struct radeon_bo *bo)
 	if (bo == NULL)
 		return NULL;
 
-	ttm_bo_reference(&bo->tbo);
+	ttm_bo_get(&bo->tbo);
 	return bo;
 }
 
@@ -320,9 +320,8 @@ void radeon_bo_unref(struct radeon_bo **bo)
 		return;
 	rdev = (*bo)->rdev;
 	tbo = &((*bo)->tbo);
-	ttm_bo_unref(&tbo);
-	if (tbo == NULL)
-		*bo = NULL;
+	ttm_bo_put(tbo);
+	*bo = NULL;
 }
 
 int radeon_bo_pin_restricted(struct radeon_bo *bo, u32 domain, u64 max_offset,
diff --git a/drivers/gpu/drm/radeon/radeon_test.c b/drivers/gpu/drm/radeon/radeon_test.c
index 48f4b273e316..0c7f228db6e3 100644
--- a/drivers/gpu/drm/radeon/radeon_test.c
+++ b/drivers/gpu/drm/radeon/radeon_test.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
 /*
  * Copyright 2009 VMware, Inc.
  *
diff --git a/drivers/gpu/drm/radeon/si.c b/drivers/gpu/drm/radeon/si.c
index 1907c950d76f..85c604d29235 100644
--- a/drivers/gpu/drm/radeon/si.c
+++ b/drivers/gpu/drm/radeon/si.c
@@ -7082,9 +7082,10 @@ int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
 static void si_pcie_gen3_enable(struct radeon_device *rdev)
 {
 	struct pci_dev *root = rdev->pdev->bus->self;
+	enum pci_bus_speed speed_cap;
 	int bridge_pos, gpu_pos;
-	u32 speed_cntl, mask, current_data_rate;
-	int ret, i;
+	u32 speed_cntl, current_data_rate;
+	int i;
 	u16 tmp16;
 
 	if (pci_is_root_bus(rdev->pdev->bus))
@@ -7099,23 +7100,24 @@ static void si_pcie_gen3_enable(struct radeon_device *rdev)
 	if (!(rdev->flags & RADEON_IS_PCIE))
 		return;
 
-	ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
-	if (ret != 0)
+	speed_cap = pcie_get_speed_cap(root);
+	if (speed_cap == PCI_SPEED_UNKNOWN)
 		return;
 
-	if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
+	if ((speed_cap != PCIE_SPEED_8_0GT) &&
+	    (speed_cap != PCIE_SPEED_5_0GT))
 		return;
 
 	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
 	current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
 		LC_CURRENT_DATA_RATE_SHIFT;
-	if (mask & DRM_PCIE_SPEED_80) {
+	if (speed_cap == PCIE_SPEED_8_0GT) {
 		if (current_data_rate == 2) {
 			DRM_INFO("PCIE gen 3 link speeds already enabled\n");
 			return;
 		}
 		DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
-	} else if (mask & DRM_PCIE_SPEED_50) {
+	} else if (speed_cap == PCIE_SPEED_5_0GT) {
 		if (current_data_rate == 1) {
 			DRM_INFO("PCIE gen 2 link speeds already enabled\n");
 			return;
@@ -7131,7 +7133,7 @@ static void si_pcie_gen3_enable(struct radeon_device *rdev)
 	if (!gpu_pos)
 		return;
 
-	if (mask & DRM_PCIE_SPEED_80) {
+	if (speed_cap == PCIE_SPEED_8_0GT) {
 		/* re-try equalization if gen3 is not already enabled */
 		if (current_data_rate != 2) {
 			u16 bridge_cfg, gpu_cfg;
@@ -7219,9 +7221,9 @@ static void si_pcie_gen3_enable(struct radeon_device *rdev)
 
 	pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
 	tmp16 &= ~0xf;
-	if (mask & DRM_PCIE_SPEED_80)
+	if (speed_cap == PCIE_SPEED_8_0GT)
 		tmp16 |= 3; /* gen3 */
-	else if (mask & DRM_PCIE_SPEED_50)
+	else if (speed_cap == PCIE_SPEED_5_0GT)
 		tmp16 |= 2; /* gen2 */
 	else
 		tmp16 |= 1; /* gen1 */
diff --git a/drivers/gpu/drm/radeon/si_dpm.c b/drivers/gpu/drm/radeon/si_dpm.c
index fea88078cf8e..8fb60b3af015 100644
--- a/drivers/gpu/drm/radeon/si_dpm.c
+++ b/drivers/gpu/drm/radeon/si_dpm.c
@@ -6899,8 +6899,9 @@ int si_dpm_init(struct radeon_device *rdev)
 	struct ni_power_info *ni_pi;
 	struct si_power_info *si_pi;
 	struct atom_clock_dividers dividers;
+	enum pci_bus_speed speed_cap;
+	struct pci_dev *root = rdev->pdev->bus->self;
 	int ret;
-	u32 mask;
 
 	si_pi = kzalloc(sizeof(struct si_power_info), GFP_KERNEL);
 	if (si_pi == NULL)
@@ -6910,11 +6911,20 @@ int si_dpm_init(struct radeon_device *rdev)
 	eg_pi = &ni_pi->eg;
 	pi = &eg_pi->rv7xx;
 
-	ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
-	if (ret)
+	speed_cap = pcie_get_speed_cap(root);
+	if (speed_cap == PCI_SPEED_UNKNOWN) {
 		si_pi->sys_pcie_mask = 0;
-	else
-		si_pi->sys_pcie_mask = mask;
+	} else {
+		if (speed_cap == PCIE_SPEED_8_0GT)
+			si_pi->sys_pcie_mask = RADEON_PCIE_SPEED_25 |
+				RADEON_PCIE_SPEED_50 |
+				RADEON_PCIE_SPEED_80;
+		else if (speed_cap == PCIE_SPEED_5_0GT)
+			si_pi->sys_pcie_mask = RADEON_PCIE_SPEED_25 |
+				RADEON_PCIE_SPEED_50;
+		else
+			si_pi->sys_pcie_mask = RADEON_PCIE_SPEED_25;
+	}
 	si_pi->force_pcie_gen = RADEON_PCIE_GEN_INVALID;
 	si_pi->boot_pcie_gen = si_get_current_pcie_speed(rdev);
 
diff --git a/drivers/gpu/drm/scheduler/Makefile b/drivers/gpu/drm/scheduler/Makefile
index bd0377c0d2ee..7665883f81d4 100644
--- a/drivers/gpu/drm/scheduler/Makefile
+++ b/drivers/gpu/drm/scheduler/Makefile
@@ -20,7 +20,6 @@
 # OTHER DEALINGS IN THE SOFTWARE.
 #
 #
-ccflags-y := -Iinclude/drm
 gpu-sched-y := gpu_scheduler.o sched_fence.o
 
 obj-$(CONFIG_DRM_SCHED) += gpu-sched.o
diff --git a/drivers/gpu/drm/scheduler/gpu_scheduler.c b/drivers/gpu/drm/scheduler/gpu_scheduler.c
index 6a316701da73..4fc211e19d6e 100644
--- a/drivers/gpu/drm/scheduler/gpu_scheduler.c
+++ b/drivers/gpu/drm/scheduler/gpu_scheduler.c
@@ -69,11 +69,13 @@ static void drm_sched_process_job(struct dma_fence *f, struct dma_fence_cb *cb);
  *
  * Initializes a scheduler runqueue.
  */
-static void drm_sched_rq_init(struct drm_sched_rq *rq)
+static void drm_sched_rq_init(struct drm_gpu_scheduler *sched,
+			      struct drm_sched_rq *rq)
 {
 	spin_lock_init(&rq->lock);
 	INIT_LIST_HEAD(&rq->entities);
 	rq->current_entity = NULL;
+	rq->sched = sched;
 }
 
 /**
@@ -160,26 +162,29 @@ drm_sched_rq_select_entity(struct drm_sched_rq *rq)
  * drm_sched_entity_init - Init a context entity used by scheduler when
  * submit to HW ring.
  *
- * @sched: scheduler instance
  * @entity: scheduler entity to init
- * @rq: the run queue this entity belongs
+ * @rq_list: the list of run queue on which jobs from this
+ *           entity can be submitted
+ * @num_rq_list: number of run queue in rq_list
  * @guilty: atomic_t set to 1 when a job on this queue
  *          is found to be guilty causing a timeout
  *
+ * Note: the rq_list should have atleast one element to schedule
+ *       the entity
+ *
  * Returns 0 on success or a negative error code on failure.
 */
-int drm_sched_entity_init(struct drm_gpu_scheduler *sched,
-			  struct drm_sched_entity *entity,
-			  struct drm_sched_rq *rq,
+int drm_sched_entity_init(struct drm_sched_entity *entity,
+			  struct drm_sched_rq **rq_list,
+			  unsigned int num_rq_list,
 			  atomic_t *guilty)
 {
-	if (!(sched && entity && rq))
+	if (!(entity && rq_list && num_rq_list > 0 && rq_list[0]))
 		return -EINVAL;
 
 	memset(entity, 0, sizeof(struct drm_sched_entity));
 	INIT_LIST_HEAD(&entity->list);
-	entity->rq = rq;
-	entity->sched = sched;
+	entity->rq = rq_list[0];
 	entity->guilty = guilty;
 	entity->last_scheduled = NULL;
 
@@ -194,21 +199,6 @@ int drm_sched_entity_init(struct drm_gpu_scheduler *sched,
 EXPORT_SYMBOL(drm_sched_entity_init);
 
 /**
- * drm_sched_entity_is_initialized - Query if entity is initialized
- *
- * @sched: Pointer to scheduler instance
- * @entity: The pointer to a valid scheduler entity
- *
- * return true if entity is initialized, false otherwise
-*/
-static bool drm_sched_entity_is_initialized(struct drm_gpu_scheduler *sched,
-					    struct drm_sched_entity *entity)
-{
-	return entity->sched == sched &&
-		entity->rq != NULL;
-}
-
-/**
  * drm_sched_entity_is_idle - Check if entity is idle
  *
  * @entity: scheduler entity
@@ -219,7 +209,8 @@ static bool drm_sched_entity_is_idle(struct drm_sched_entity *entity)
 {
 	rmb();
 
-	if (!entity->rq || spsc_queue_peek(&entity->job_queue) == NULL)
+	if (list_empty(&entity->list) ||
+	    spsc_queue_peek(&entity->job_queue) == NULL)
 		return true;
 
 	return false;
@@ -256,9 +247,8 @@ static void drm_sched_entity_kill_jobs_cb(struct dma_fence *f,
 
 
 /**
- * drm_sched_entity_do_release - Destroy a context entity
+ * drm_sched_entity_flush - Flush a context entity
  *
- * @sched: scheduler instance
  * @entity: scheduler entity
  * @timeout: time to wait in for Q to become empty in jiffies.
  *
@@ -267,13 +257,13 @@ static void drm_sched_entity_kill_jobs_cb(struct dma_fence *f,
  *
  * Returns the remaining time in jiffies left from the input timeout
  */
-long drm_sched_entity_do_release(struct drm_gpu_scheduler *sched,
-			   struct drm_sched_entity *entity, long timeout)
+long drm_sched_entity_flush(struct drm_sched_entity *entity, long timeout)
 {
+	struct drm_gpu_scheduler *sched;
+	struct task_struct *last_user;
 	long ret = timeout;
 
-	if (!drm_sched_entity_is_initialized(sched, entity))
-		return ret;
+	sched = entity->rq->sched;
 	/**
 	 * The client will not queue more IBs during this fini, consume existing
 	 * queued IBs or discard them on SIGKILL
@@ -289,28 +279,30 @@ long drm_sched_entity_do_release(struct drm_gpu_scheduler *sched,
 
 
 	/* For killed process disable any more IBs enqueue right now */
-	if ((current->flags & PF_EXITING) && (current->exit_code == SIGKILL))
-		drm_sched_entity_set_rq(entity, NULL);
+	last_user = cmpxchg(&entity->last_user, current->group_leader, NULL);
+	if ((!last_user || last_user == current->group_leader) &&
+	    (current->flags & PF_EXITING) && (current->exit_code == SIGKILL))
+		drm_sched_rq_remove_entity(entity->rq, entity);
 
 	return ret;
 }
-EXPORT_SYMBOL(drm_sched_entity_do_release);
+EXPORT_SYMBOL(drm_sched_entity_flush);
 
 /**
  * drm_sched_entity_cleanup - Destroy a context entity
  *
- * @sched: scheduler instance
  * @entity: scheduler entity
  *
  * This should be called after @drm_sched_entity_do_release. It goes over the
  * entity and signals all jobs with an error code if the process was killed.
  *
  */
-void drm_sched_entity_cleanup(struct drm_gpu_scheduler *sched,
-			   struct drm_sched_entity *entity)
+void drm_sched_entity_fini(struct drm_sched_entity *entity)
 {
+	struct drm_gpu_scheduler *sched;
 
-	drm_sched_entity_set_rq(entity, NULL);
+	sched = entity->rq->sched;
+	drm_sched_rq_remove_entity(entity->rq, entity);
 
 	/* Consumption of existing IBs wasn't completed. Forcefully
 	 * remove them here.
@@ -357,23 +349,21 @@ void drm_sched_entity_cleanup(struct drm_gpu_scheduler *sched,
 	dma_fence_put(entity->last_scheduled);
 	entity->last_scheduled = NULL;
 }
-EXPORT_SYMBOL(drm_sched_entity_cleanup);
+EXPORT_SYMBOL(drm_sched_entity_fini);
 
 /**
  * drm_sched_entity_fini - Destroy a context entity
  *
- * @sched: scheduler instance
  * @entity: scheduler entity
  *
  * Calls drm_sched_entity_do_release() and drm_sched_entity_cleanup()
  */
-void drm_sched_entity_fini(struct drm_gpu_scheduler *sched,
-				struct drm_sched_entity *entity)
+void drm_sched_entity_destroy(struct drm_sched_entity *entity)
 {
-	drm_sched_entity_do_release(sched, entity, MAX_WAIT_SCHED_ENTITY_Q_EMPTY);
-	drm_sched_entity_cleanup(sched, entity);
+	drm_sched_entity_flush(entity, MAX_WAIT_SCHED_ENTITY_Q_EMPTY);
+	drm_sched_entity_fini(entity);
 }
-EXPORT_SYMBOL(drm_sched_entity_fini);
+EXPORT_SYMBOL(drm_sched_entity_destroy);
 
 static void drm_sched_entity_wakeup(struct dma_fence *f, struct dma_fence_cb *cb)
 {
@@ -381,7 +371,7 @@ static void drm_sched_entity_wakeup(struct dma_fence *f, struct dma_fence_cb *cb
 		container_of(cb, struct drm_sched_entity, cb);
 	entity->dependency = NULL;
 	dma_fence_put(f);
-	drm_sched_wakeup(entity->sched);
+	drm_sched_wakeup(entity->rq->sched);
 }
 
 static void drm_sched_entity_clear_dep(struct dma_fence *f, struct dma_fence_cb *cb)
@@ -407,15 +397,12 @@ void drm_sched_entity_set_rq(struct drm_sched_entity *entity,
 	if (entity->rq == rq)
 		return;
 
-	spin_lock(&entity->rq_lock);
-
-	if (entity->rq)
-		drm_sched_rq_remove_entity(entity->rq, entity);
+	BUG_ON(!rq);
 
+	spin_lock(&entity->rq_lock);
+	drm_sched_rq_remove_entity(entity->rq, entity);
 	entity->rq = rq;
-	if (rq)
-		drm_sched_rq_add_entity(rq, entity);
-
+	drm_sched_rq_add_entity(rq, entity);
 	spin_unlock(&entity->rq_lock);
 }
 EXPORT_SYMBOL(drm_sched_entity_set_rq);
@@ -431,7 +418,7 @@ EXPORT_SYMBOL(drm_sched_entity_set_rq);
 bool drm_sched_dependency_optimized(struct dma_fence* fence,
 				    struct drm_sched_entity *entity)
 {
-	struct drm_gpu_scheduler *sched = entity->sched;
+	struct drm_gpu_scheduler *sched = entity->rq->sched;
 	struct drm_sched_fence *s_fence;
 
 	if (!fence || dma_fence_is_signaled(fence))
@@ -448,7 +435,7 @@ EXPORT_SYMBOL(drm_sched_dependency_optimized);
 
 static bool drm_sched_entity_add_dependency_cb(struct drm_sched_entity *entity)
 {
-	struct drm_gpu_scheduler *sched = entity->sched;
+	struct drm_gpu_scheduler *sched = entity->rq->sched;
 	struct dma_fence * fence = entity->dependency;
 	struct drm_sched_fence *s_fence;
 
@@ -493,7 +480,7 @@ static bool drm_sched_entity_add_dependency_cb(struct drm_sched_entity *entity)
 static struct drm_sched_job *
 drm_sched_entity_pop_job(struct drm_sched_entity *entity)
 {
-	struct drm_gpu_scheduler *sched = entity->sched;
+	struct drm_gpu_scheduler *sched = entity->rq->sched;
 	struct drm_sched_job *sched_job = to_drm_sched_job(
 						spsc_queue_peek(&entity->job_queue));
 
@@ -535,12 +522,18 @@ void drm_sched_entity_push_job(struct drm_sched_job *sched_job,
 
 	trace_drm_sched_job(sched_job, entity);
 
+	WRITE_ONCE(entity->last_user, current->group_leader);
 	first = spsc_queue_push(&entity->job_queue, &sched_job->queue_node);
 
 	/* first job wakes up scheduler */
 	if (first) {
 		/* Add the entity to the run queue */
 		spin_lock(&entity->rq_lock);
+		if (!entity->rq) {
+			DRM_ERROR("Trying to push to a killed entity\n");
+			spin_unlock(&entity->rq_lock);
+			return;
+		}
 		drm_sched_rq_add_entity(entity->rq, entity);
 		spin_unlock(&entity->rq_lock);
 		drm_sched_wakeup(sched);
@@ -556,24 +549,28 @@ static void drm_sched_job_finish(struct work_struct *work)
 						   finish_work);
 	struct drm_gpu_scheduler *sched = s_job->sched;
 
-	/* remove job from ring_mirror_list */
-	spin_lock(&sched->job_list_lock);
-	list_del_init(&s_job->node);
-	if (sched->timeout != MAX_SCHEDULE_TIMEOUT) {
-		struct drm_sched_job *next;
-
-		spin_unlock(&sched->job_list_lock);
-		cancel_delayed_work_sync(&s_job->work_tdr);
-		spin_lock(&sched->job_list_lock);
+	/*
+	 * Canceling the timeout without removing our job from the ring mirror
+	 * list is safe, as we will only end up in this worker if our jobs
+	 * finished fence has been signaled. So even if some another worker
+	 * manages to find this job as the next job in the list, the fence
+	 * signaled check below will prevent the timeout to be restarted.
+	 */
+	cancel_delayed_work_sync(&s_job->work_tdr);
 
-		/* queue TDR for next job */
-		next = list_first_entry_or_null(&sched->ring_mirror_list,
-						struct drm_sched_job, node);
+	spin_lock(&sched->job_list_lock);
+	/* queue TDR for next job */
+	if (sched->timeout != MAX_SCHEDULE_TIMEOUT &&
+	    !list_is_last(&s_job->node, &sched->ring_mirror_list)) {
+		struct drm_sched_job *next = list_next_entry(s_job, node);
 
-		if (next)
+		if (!dma_fence_is_signaled(&next->s_fence->finished))
 			schedule_delayed_work(&next->work_tdr, sched->timeout);
 	}
+	/* remove job from ring_mirror_list */
+	list_del(&s_job->node);
 	spin_unlock(&sched->job_list_lock);
+
 	dma_fence_put(&s_job->s_fence->finished);
 	sched->ops->free_job(s_job);
 }
@@ -719,7 +716,6 @@ EXPORT_SYMBOL(drm_sched_job_recovery);
  * drm_sched_job_init - init a scheduler job
  *
  * @job: scheduler job to init
- * @sched: scheduler instance
  * @entity: scheduler entity to use
  * @owner: job owner for debugging
  *
@@ -729,10 +725,11 @@ EXPORT_SYMBOL(drm_sched_job_recovery);
  * Returns 0 for success, negative error code otherwise.
  */
 int drm_sched_job_init(struct drm_sched_job *job,
-		       struct drm_gpu_scheduler *sched,
 		       struct drm_sched_entity *entity,
 		       void *owner)
 {
+	struct drm_gpu_scheduler *sched = entity->rq->sched;
+
 	job->sched = sched;
 	job->entity = entity;
 	job->s_priority = entity->rq - sched->sched_rq;
@@ -926,7 +923,7 @@ int drm_sched_init(struct drm_gpu_scheduler *sched,
 	sched->timeout = timeout;
 	sched->hang_limit = hang_limit;
 	for (i = DRM_SCHED_PRIORITY_MIN; i < DRM_SCHED_PRIORITY_MAX; i++)
-		drm_sched_rq_init(&sched->sched_rq[i]);
+		drm_sched_rq_init(sched, &sched->sched_rq[i]);
 
 	init_waitqueue_head(&sched->wake_up_worker);
 	init_waitqueue_head(&sched->job_scheduled);
diff --git a/drivers/gpu/drm/scheduler/sched_fence.c b/drivers/gpu/drm/scheduler/sched_fence.c
index 45d9c3affbea..d8d2dff9ea2f 100644
--- a/drivers/gpu/drm/scheduler/sched_fence.c
+++ b/drivers/gpu/drm/scheduler/sched_fence.c
@@ -161,7 +161,7 @@ struct drm_sched_fence *drm_sched_fence_create(struct drm_sched_entity *entity,
 		return NULL;
 
 	fence->owner = owner;
-	fence->sched = entity->sched;
+	fence->sched = entity->rq->sched;
 	spin_lock_init(&fence->lock);
 
 	seq = atomic_inc_return(&entity->fence_seq);
diff --git a/drivers/gpu/drm/sun4i/Makefile b/drivers/gpu/drm/sun4i/Makefile
index b04ea0f3da75..0eb38ac8e86e 100644
--- a/drivers/gpu/drm/sun4i/Makefile
+++ b/drivers/gpu/drm/sun4i/Makefile
@@ -32,7 +32,10 @@ obj-$(CONFIG_DRM_SUN4I)		+= sun4i-tcon.o
 obj-$(CONFIG_DRM_SUN4I)		+= sun4i_tv.o
 obj-$(CONFIG_DRM_SUN4I)		+= sun6i_drc.o
 
-obj-$(CONFIG_DRM_SUN4I_BACKEND)	+= sun4i-backend.o sun4i-frontend.o
+obj-$(CONFIG_DRM_SUN4I_BACKEND)	+= sun4i-backend.o
+ifdef CONFIG_DRM_SUN4I_BACKEND
+obj-$(CONFIG_DRM_SUN4I)		+= sun4i-frontend.o
+endif
 obj-$(CONFIG_DRM_SUN4I_HDMI)	+= sun4i-drm-hdmi.o
 obj-$(CONFIG_DRM_SUN6I_DSI)	+= sun6i-dsi.o
 obj-$(CONFIG_DRM_SUN8I_DW_HDMI)	+= sun8i-drm-hdmi.o
diff --git a/drivers/gpu/drm/sun4i/sun6i_drc.c b/drivers/gpu/drm/sun4i/sun6i_drc.c
index b5e071a49045..88eb268fdf73 100644
--- a/drivers/gpu/drm/sun4i/sun6i_drc.c
+++ b/drivers/gpu/drm/sun4i/sun6i_drc.c
@@ -12,6 +12,7 @@
 #include <linux/clk.h>
 #include <linux/component.h>
 #include <linux/module.h>
+#include <linux/mod_devicetable.h>
 #include <linux/platform_device.h>
 #include <linux/regmap.h>
 #include <linux/reset.h>
diff --git a/drivers/gpu/drm/tegra/dc.c b/drivers/gpu/drm/tegra/dc.c
index c3afe7b2237e..965088afcfad 100644
--- a/drivers/gpu/drm/tegra/dc.c
+++ b/drivers/gpu/drm/tegra/dc.c
@@ -2312,7 +2312,7 @@ static int tegra_dc_couple(struct tegra_dc *dc)
 	 * POWER_CONTROL registers during CRTC enabling.
 	 */
 	if (dc->soc->coupled_pm && dc->pipe == 1) {
-		u32 flags = DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE;
+		u32 flags = DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_CONSUMER;
 		struct device_link *link;
 		struct device *partner;
 
diff --git a/drivers/gpu/drm/tegra/drm.c b/drivers/gpu/drm/tegra/drm.c
index 776c1513e582..a2bd5876c633 100644
--- a/drivers/gpu/drm/tegra/drm.c
+++ b/drivers/gpu/drm/tegra/drm.c
@@ -398,7 +398,7 @@ int tegra_drm_submit(struct tegra_drm_context *context,
 		 * unaligned offset is malformed and cause commands stream
 		 * corruption on the buffer address relocation.
 		 */
-		if (offset & 3 || offset >= obj->gem.size) {
+		if (offset & 3 || offset > obj->gem.size) {
 			err = -EINVAL;
 			goto fail;
 		}
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_drv.c b/drivers/gpu/drm/tilcdc/tilcdc_drv.c
index b8a5e4ed22e6..0fb300d41a09 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_drv.c
+++ b/drivers/gpu/drm/tilcdc/tilcdc_drv.c
@@ -378,7 +378,7 @@ static int tilcdc_init(struct drm_driver *ddrv, struct device *dev)
 	if (!priv->external_connector &&
 	    ((priv->num_encoders == 0) || (priv->num_connectors == 0))) {
 		dev_err(dev, "no encoders/connectors found\n");
-		ret = -ENXIO;
+		ret = -EPROBE_DEFER;
 		goto init_failed;
 	}
 
diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c
index 5d8688e522d1..7c484729f9b2 100644
--- a/drivers/gpu/drm/ttm/ttm_bo.c
+++ b/drivers/gpu/drm/ttm/ttm_bo.c
@@ -287,12 +287,9 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
 
 	if (ret) {
 		if (bdev->driver->move_notify) {
-			struct ttm_mem_reg tmp_mem = *mem;
-			*mem = bo->mem;
-			bo->mem = tmp_mem;
+			swap(*mem, bo->mem);
 			bdev->driver->move_notify(bo, false, mem);
-			bo->mem = *mem;
-			*mem = tmp_mem;
+			swap(*mem, bo->mem);
 		}
 
 		goto out_err;
@@ -590,12 +587,18 @@ static void ttm_bo_release(struct kref *kref)
 	kref_put(&bo->list_kref, ttm_bo_release_list);
 }
 
+void ttm_bo_put(struct ttm_buffer_object *bo)
+{
+	kref_put(&bo->kref, ttm_bo_release);
+}
+EXPORT_SYMBOL(ttm_bo_put);
+
 void ttm_bo_unref(struct ttm_buffer_object **p_bo)
 {
 	struct ttm_buffer_object *bo = *p_bo;
 
 	*p_bo = NULL;
-	kref_put(&bo->kref, ttm_bo_release);
+	ttm_bo_put(bo);
 }
 EXPORT_SYMBOL(ttm_bo_unref);
 
@@ -1201,7 +1204,7 @@ int ttm_bo_init_reserved(struct ttm_bo_device *bdev,
 		if (!resv)
 			ttm_bo_unreserve(bo);
 
-		ttm_bo_unref(&bo);
+		ttm_bo_put(bo);
 		return ret;
 	}
 
diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c
index f2c167702eef..046a6dda690a 100644
--- a/drivers/gpu/drm/ttm/ttm_bo_util.c
+++ b/drivers/gpu/drm/ttm/ttm_bo_util.c
@@ -463,7 +463,7 @@ static void ttm_transfered_destroy(struct ttm_buffer_object *bo)
 	struct ttm_transfer_obj *fbo;
 
 	fbo = container_of(bo, struct ttm_transfer_obj, base);
-	ttm_bo_unref(&fbo->bo);
+	ttm_bo_put(fbo->bo);
 	kfree(fbo);
 }
 
@@ -492,8 +492,9 @@ static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo,
 	if (!fbo)
 		return -ENOMEM;
 
+	ttm_bo_get(bo);
 	fbo->base = *bo;
-	fbo->bo = ttm_bo_reference(bo);
+	fbo->bo = bo;
 
 	/**
 	 * Fix up members that we shouldn't copy directly:
@@ -730,7 +731,7 @@ int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo,
 			bo->ttm = NULL;
 
 		ttm_bo_unreserve(ghost_obj);
-		ttm_bo_unref(&ghost_obj);
+		ttm_bo_put(ghost_obj);
 	}
 
 	*old_mem = *new_mem;
@@ -786,7 +787,7 @@ int ttm_bo_pipeline_move(struct ttm_buffer_object *bo,
 			bo->ttm = NULL;
 
 		ttm_bo_unreserve(ghost_obj);
-		ttm_bo_unref(&ghost_obj);
+		ttm_bo_put(ghost_obj);
 
 	} else if (from->flags & TTM_MEMTYPE_FLAG_FIXED) {
 
@@ -851,7 +852,7 @@ int ttm_bo_pipeline_gutting(struct ttm_buffer_object *bo)
 	bo->ttm = NULL;
 
 	ttm_bo_unreserve(ghost);
-	ttm_bo_unref(&ghost);
+	ttm_bo_put(ghost);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c
index 0ca0ec47334e..6fe91c1b692d 100644
--- a/drivers/gpu/drm/ttm/ttm_bo_vm.c
+++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c
@@ -68,11 +68,11 @@ static vm_fault_t ttm_bo_vm_fault_idle(struct ttm_buffer_object *bo,
 		if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT)
 			goto out_unlock;
 
-		ttm_bo_reference(bo);
+		ttm_bo_get(bo);
 		up_read(&vmf->vma->vm_mm->mmap_sem);
 		(void) dma_fence_wait(bo->moving, true);
 		ttm_bo_unreserve(bo);
-		ttm_bo_unref(&bo);
+		ttm_bo_put(bo);
 		goto out_unlock;
 	}
 
@@ -138,10 +138,10 @@ static vm_fault_t ttm_bo_vm_fault(struct vm_fault *vmf)
 
 		if (vmf->flags & FAULT_FLAG_ALLOW_RETRY) {
 			if (!(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) {
-				ttm_bo_reference(bo);
+				ttm_bo_get(bo);
 				up_read(&vmf->vma->vm_mm->mmap_sem);
 				(void) ttm_bo_wait_unreserved(bo);
-				ttm_bo_unref(&bo);
+				ttm_bo_put(bo);
 			}
 
 			return VM_FAULT_RETRY;
@@ -302,14 +302,14 @@ static void ttm_bo_vm_open(struct vm_area_struct *vma)
 
 	WARN_ON(bo->bdev->dev_mapping != vma->vm_file->f_mapping);
 
-	(void)ttm_bo_reference(bo);
+	ttm_bo_get(bo);
 }
 
 static void ttm_bo_vm_close(struct vm_area_struct *vma)
 {
 	struct ttm_buffer_object *bo = (struct ttm_buffer_object *)vma->vm_private_data;
 
-	ttm_bo_unref(&bo);
+	ttm_bo_put(bo);
 	vma->vm_private_data = NULL;
 }
 
@@ -461,7 +461,7 @@ int ttm_bo_mmap(struct file *filp, struct vm_area_struct *vma,
 	vma->vm_flags |= VM_IO | VM_DONTEXPAND | VM_DONTDUMP;
 	return 0;
 out_unref:
-	ttm_bo_unref(&bo);
+	ttm_bo_put(bo);
 	return ret;
 }
 EXPORT_SYMBOL(ttm_bo_mmap);
@@ -471,8 +471,10 @@ int ttm_fbdev_mmap(struct vm_area_struct *vma, struct ttm_buffer_object *bo)
 	if (vma->vm_pgoff != 0)
 		return -EACCES;
 
+	ttm_bo_get(bo);
+
 	vma->vm_ops = &ttm_bo_vm_ops;
-	vma->vm_private_data = ttm_bo_reference(bo);
+	vma->vm_private_data = bo;
 	vma->vm_flags |= VM_MIXEDMAP;
 	vma->vm_flags |= VM_IO | VM_DONTEXPAND;
 	return 0;
diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc.c b/drivers/gpu/drm/ttm/ttm_page_alloc.c
index 6e2d1300b457..f841accc2c00 100644
--- a/drivers/gpu/drm/ttm/ttm_page_alloc.c
+++ b/drivers/gpu/drm/ttm/ttm_page_alloc.c
@@ -47,13 +47,7 @@
 
 #include <drm/ttm/ttm_bo_driver.h>
 #include <drm/ttm/ttm_page_alloc.h>
-
-#if IS_ENABLED(CONFIG_AGP)
-#include <asm/agp.h>
-#endif
-#ifdef CONFIG_X86
-#include <asm/set_memory.h>
-#endif
+#include <drm/ttm/ttm_set_memory.h>
 
 #define NUM_PAGES_TO_ALLOC		(PAGE_SIZE/sizeof(struct page *))
 #define SMALL_ALLOCATION		16
@@ -222,52 +216,6 @@ static struct kobj_type ttm_pool_kobj_type = {
 
 static struct ttm_pool_manager *_manager;
 
-#ifndef CONFIG_X86
-static int set_pages_wb(struct page *page, int numpages)
-{
-#if IS_ENABLED(CONFIG_AGP)
-	int i;
-
-	for (i = 0; i < numpages; i++)
-		unmap_page_from_agp(page++);
-#endif
-	return 0;
-}
-
-static int set_pages_array_wb(struct page **pages, int addrinarray)
-{
-#if IS_ENABLED(CONFIG_AGP)
-	int i;
-
-	for (i = 0; i < addrinarray; i++)
-		unmap_page_from_agp(pages[i]);
-#endif
-	return 0;
-}
-
-static int set_pages_array_wc(struct page **pages, int addrinarray)
-{
-#if IS_ENABLED(CONFIG_AGP)
-	int i;
-
-	for (i = 0; i < addrinarray; i++)
-		map_page_into_agp(pages[i]);
-#endif
-	return 0;
-}
-
-static int set_pages_array_uc(struct page **pages, int addrinarray)
-{
-#if IS_ENABLED(CONFIG_AGP)
-	int i;
-
-	for (i = 0; i < addrinarray; i++)
-		map_page_into_agp(pages[i]);
-#endif
-	return 0;
-}
-#endif
-
 /**
  * Select the right pool or requested caching state and ttm flags. */
 static struct ttm_page_pool *ttm_get_pool(int flags, bool huge,
@@ -302,13 +250,13 @@ static void ttm_pages_put(struct page *pages[], unsigned npages,
 	unsigned int i, pages_nr = (1 << order);
 
 	if (order == 0) {
-		if (set_pages_array_wb(pages, npages))
+		if (ttm_set_pages_array_wb(pages, npages))
 			pr_err("Failed to set %d pages to wb!\n", npages);
 	}
 
 	for (i = 0; i < npages; ++i) {
 		if (order > 0) {
-			if (set_pages_wb(pages[i], pages_nr))
+			if (ttm_set_pages_wb(pages[i], pages_nr))
 				pr_err("Failed to set %d pages to wb!\n", pages_nr);
 		}
 		__free_pages(pages[i], order);
@@ -498,12 +446,12 @@ static int ttm_set_pages_caching(struct page **pages,
 	/* Set page caching */
 	switch (cstate) {
 	case tt_uncached:
-		r = set_pages_array_uc(pages, cpages);
+		r = ttm_set_pages_array_uc(pages, cpages);
 		if (r)
 			pr_err("Failed to set %d pages to uc!\n", cpages);
 		break;
 	case tt_wc:
-		r = set_pages_array_wc(pages, cpages);
+		r = ttm_set_pages_array_wc(pages, cpages);
 		if (r)
 			pr_err("Failed to set %d pages to wc!\n", cpages);
 		break;
diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c b/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c
index 3f14c1cc0789..507be7ac1165 100644
--- a/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c
+++ b/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c
@@ -50,12 +50,7 @@
 #include <linux/kthread.h>
 #include <drm/ttm/ttm_bo_driver.h>
 #include <drm/ttm/ttm_page_alloc.h>
-#if IS_ENABLED(CONFIG_AGP)
-#include <asm/agp.h>
-#endif
-#ifdef CONFIG_X86
-#include <asm/set_memory.h>
-#endif
+#include <drm/ttm/ttm_set_memory.h>
 
 #define NUM_PAGES_TO_ALLOC		(PAGE_SIZE/sizeof(struct page *))
 #define SMALL_ALLOCATION		4
@@ -268,54 +263,19 @@ static struct kobj_type ttm_pool_kobj_type = {
 	.default_attrs = ttm_pool_attrs,
 };
 
-#ifndef CONFIG_X86
-static int set_pages_array_wb(struct page **pages, int addrinarray)
-{
-#if IS_ENABLED(CONFIG_AGP)
-	int i;
-
-	for (i = 0; i < addrinarray; i++)
-		unmap_page_from_agp(pages[i]);
-#endif
-	return 0;
-}
-
-static int set_pages_array_wc(struct page **pages, int addrinarray)
-{
-#if IS_ENABLED(CONFIG_AGP)
-	int i;
-
-	for (i = 0; i < addrinarray; i++)
-		map_page_into_agp(pages[i]);
-#endif
-	return 0;
-}
-
-static int set_pages_array_uc(struct page **pages, int addrinarray)
-{
-#if IS_ENABLED(CONFIG_AGP)
-	int i;
-
-	for (i = 0; i < addrinarray; i++)
-		map_page_into_agp(pages[i]);
-#endif
-	return 0;
-}
-#endif /* for !CONFIG_X86 */
-
 static int ttm_set_pages_caching(struct dma_pool *pool,
 				 struct page **pages, unsigned cpages)
 {
 	int r = 0;
 	/* Set page caching */
 	if (pool->type & IS_UC) {
-		r = set_pages_array_uc(pages, cpages);
+		r = ttm_set_pages_array_uc(pages, cpages);
 		if (r)
 			pr_err("%s: Failed to set %d pages to uc!\n",
 			       pool->dev_name, cpages);
 	}
 	if (pool->type & IS_WC) {
-		r = set_pages_array_wc(pages, cpages);
+		r = ttm_set_pages_array_wc(pages, cpages);
 		if (r)
 			pr_err("%s: Failed to set %d pages to wc!\n",
 			       pool->dev_name, cpages);
@@ -389,17 +349,14 @@ static void ttm_pool_update_free_locked(struct dma_pool *pool,
 static void ttm_dma_page_put(struct dma_pool *pool, struct dma_page *d_page)
 {
 	struct page *page = d_page->p;
-	unsigned i, num_pages;
+	unsigned num_pages;
 
 	/* Don't set WB on WB page pool. */
 	if (!(pool->type & IS_CACHED)) {
 		num_pages = pool->size / PAGE_SIZE;
-		for (i = 0; i < num_pages; ++i, ++page) {
-			if (set_pages_array_wb(&page, 1)) {
-				pr_err("%s: Failed to set %d pages to wb!\n",
-				       pool->dev_name, 1);
-			}
-		}
+		if (ttm_set_pages_wb(page, num_pages))
+			pr_err("%s: Failed to set %d pages to wb!\n",
+			       pool->dev_name, num_pages);
 	}
 
 	list_del(&d_page->page_list);
@@ -420,7 +377,7 @@ static void ttm_dma_pages_put(struct dma_pool *pool, struct list_head *d_pages,
 
 	/* Don't set WB on WB page pool. */
 	if (npages && !(pool->type & IS_CACHED) &&
-	    set_pages_array_wb(pages, npages))
+	    ttm_set_pages_array_wb(pages, npages))
 		pr_err("%s: Failed to set %d pages to wb!\n",
 		       pool->dev_name, npages);
 
diff --git a/drivers/gpu/drm/ttm/ttm_tt.c b/drivers/gpu/drm/ttm/ttm_tt.c
index a1e543972ca7..e3a0691582ff 100644
--- a/drivers/gpu/drm/ttm/ttm_tt.c
+++ b/drivers/gpu/drm/ttm/ttm_tt.c
@@ -38,9 +38,7 @@
 #include <drm/drm_cache.h>
 #include <drm/ttm/ttm_bo_driver.h>
 #include <drm/ttm/ttm_page_alloc.h>
-#ifdef CONFIG_X86
-#include <asm/set_memory.h>
-#endif
+#include <drm/ttm/ttm_set_memory.h>
 
 /**
  * Allocates a ttm structure for the given BO.
@@ -115,10 +113,9 @@ static int ttm_sg_tt_alloc_page_directory(struct ttm_dma_tt *ttm)
 	return 0;
 }
 
-#ifdef CONFIG_X86
-static inline int ttm_tt_set_page_caching(struct page *p,
-					  enum ttm_caching_state c_old,
-					  enum ttm_caching_state c_new)
+static int ttm_tt_set_page_caching(struct page *p,
+				   enum ttm_caching_state c_old,
+				   enum ttm_caching_state c_new)
 {
 	int ret = 0;
 
@@ -129,26 +126,18 @@ static inline int ttm_tt_set_page_caching(struct page *p,
 		/* p isn't in the default caching state, set it to
 		 * writeback first to free its current memtype. */
 
-		ret = set_pages_wb(p, 1);
+		ret = ttm_set_pages_wb(p, 1);
 		if (ret)
 			return ret;
 	}
 
 	if (c_new == tt_wc)
-		ret = set_memory_wc((unsigned long) page_address(p), 1);
+		ret = ttm_set_pages_wc(p, 1);
 	else if (c_new == tt_uncached)
-		ret = set_pages_uc(p, 1);
+		ret = ttm_set_pages_uc(p, 1);
 
 	return ret;
 }
-#else /* CONFIG_X86 */
-static inline int ttm_tt_set_page_caching(struct page *p,
-					  enum ttm_caching_state c_old,
-					  enum ttm_caching_state c_new)
-{
-	return 0;
-}
-#endif /* CONFIG_X86 */
 
 /*
  * Change caching policy for the linear kernel map
diff --git a/drivers/gpu/drm/udl/udl_drv.h b/drivers/gpu/drm/udl/udl_drv.h
index 072582570a4f..e9e9b1ff678e 100644
--- a/drivers/gpu/drm/udl/udl_drv.h
+++ b/drivers/gpu/drm/udl/udl_drv.h
@@ -113,7 +113,7 @@ udl_fb_user_fb_create(struct drm_device *dev,
 		      struct drm_file *file,
 		      const struct drm_mode_fb_cmd2 *mode_cmd);
 
-int udl_render_hline(struct drm_device *dev, int bpp, struct urb **urb_ptr,
+int udl_render_hline(struct drm_device *dev, int log_bpp, struct urb **urb_ptr,
 		     const char *front, char **urb_buf_ptr,
 		     u32 byte_offset, u32 device_byte_offset, u32 byte_width,
 		     int *ident_ptr, int *sent_ptr);
diff --git a/drivers/gpu/drm/udl/udl_fb.c b/drivers/gpu/drm/udl/udl_fb.c
index 2ebdc6d5a76e..dbb62f6eb48a 100644
--- a/drivers/gpu/drm/udl/udl_fb.c
+++ b/drivers/gpu/drm/udl/udl_fb.c
@@ -90,7 +90,10 @@ int udl_handle_damage(struct udl_framebuffer *fb, int x, int y,
 	int bytes_identical = 0;
 	struct urb *urb;
 	int aligned_x;
-	int bpp = fb->base.format->cpp[0];
+	int log_bpp;
+
+	BUG_ON(!is_power_of_2(fb->base.format->cpp[0]));
+	log_bpp = __ffs(fb->base.format->cpp[0]);
 
 	if (!fb->active_16)
 		return 0;
@@ -125,19 +128,22 @@ int udl_handle_damage(struct udl_framebuffer *fb, int x, int y,
 
 	for (i = y; i < y + height ; i++) {
 		const int line_offset = fb->base.pitches[0] * i;
-		const int byte_offset = line_offset + (x * bpp);
-		const int dev_byte_offset = (fb->base.width * bpp * i) + (x * bpp);
-		if (udl_render_hline(dev, bpp, &urb,
+		const int byte_offset = line_offset + (x << log_bpp);
+		const int dev_byte_offset = (fb->base.width * i + x) << log_bpp;
+		if (udl_render_hline(dev, log_bpp, &urb,
 				     (char *) fb->obj->vmapping,
 				     &cmd, byte_offset, dev_byte_offset,
-				     width * bpp,
+				     width << log_bpp,
 				     &bytes_identical, &bytes_sent))
 			goto error;
 	}
 
 	if (cmd > (char *) urb->transfer_buffer) {
 		/* Send partial buffer remaining before exiting */
-		int len = cmd - (char *) urb->transfer_buffer;
+		int len;
+		if (cmd < (char *) urb->transfer_buffer + urb->transfer_buffer_length)
+			*cmd++ = 0xAF;
+		len = cmd - (char *) urb->transfer_buffer;
 		ret = udl_submit_urb(dev, urb, len);
 		bytes_sent += len;
 	} else
@@ -146,7 +152,7 @@ int udl_handle_damage(struct udl_framebuffer *fb, int x, int y,
 error:
 	atomic_add(bytes_sent, &udl->bytes_sent);
 	atomic_add(bytes_identical, &udl->bytes_identical);
-	atomic_add(width*height*bpp, &udl->bytes_rendered);
+	atomic_add((width * height) << log_bpp, &udl->bytes_rendered);
 	end_cycles = get_cycles();
 	atomic_add(((unsigned int) ((end_cycles - start_cycles)
 		    >> 10)), /* Kcycles */
@@ -172,7 +178,7 @@ static int udl_fb_mmap(struct fb_info *info, struct vm_area_struct *vma)
 
 	pos = (unsigned long)info->fix.smem_start + offset;
 
-	pr_notice("mmap() framebuffer addr:%lu size:%lu\n",
+	pr_debug("mmap() framebuffer addr:%lu size:%lu\n",
 		  pos, size);
 
 	/* We don't want the framebuffer to be mapped encrypted */
@@ -218,7 +224,7 @@ static int udl_fb_open(struct fb_info *info, int user)
 
 		struct fb_deferred_io *fbdefio;
 
-		fbdefio = kmalloc(sizeof(struct fb_deferred_io), GFP_KERNEL);
+		fbdefio = kzalloc(sizeof(struct fb_deferred_io), GFP_KERNEL);
 
 		if (fbdefio) {
 			fbdefio->delay = DL_DEFIO_WRITE_DELAY;
@@ -230,7 +236,7 @@ static int udl_fb_open(struct fb_info *info, int user)
 	}
 #endif
 
-	pr_notice("open /dev/fb%d user=%d fb_info=%p count=%d\n",
+	pr_debug("open /dev/fb%d user=%d fb_info=%p count=%d\n",
 		  info->node, user, info, ufbdev->fb_count);
 
 	return 0;
@@ -255,7 +261,7 @@ static int udl_fb_release(struct fb_info *info, int user)
 	}
 #endif
 
-	pr_warn("released /dev/fb%d user=%d count=%d\n",
+	pr_debug("released /dev/fb%d user=%d count=%d\n",
 		info->node, user, ufbdev->fb_count);
 
 	return 0;
diff --git a/drivers/gpu/drm/udl/udl_main.c b/drivers/gpu/drm/udl/udl_main.c
index d518de8f496b..f455f095a146 100644
--- a/drivers/gpu/drm/udl/udl_main.c
+++ b/drivers/gpu/drm/udl/udl_main.c
@@ -170,25 +170,19 @@ static void udl_free_urb_list(struct drm_device *dev)
 	struct list_head *node;
 	struct urb_node *unode;
 	struct urb *urb;
-	int ret;
-	unsigned long flags;
 
 	DRM_DEBUG("Waiting for completes and freeing all render urbs\n");
 
 	/* keep waiting and freeing, until we've got 'em all */
 	while (count--) {
+		down(&udl->urbs.limit_sem);
 
-		/* Getting interrupted means a leak, but ok at shutdown*/
-		ret = down_interruptible(&udl->urbs.limit_sem);
-		if (ret)
-			break;
-
-		spin_lock_irqsave(&udl->urbs.lock, flags);
+		spin_lock_irq(&udl->urbs.lock);
 
 		node = udl->urbs.list.next; /* have reserved one with sem */
 		list_del_init(node);
 
-		spin_unlock_irqrestore(&udl->urbs.lock, flags);
+		spin_unlock_irq(&udl->urbs.lock);
 
 		unode = list_entry(node, struct urb_node, entry);
 		urb = unode->urb;
@@ -205,17 +199,22 @@ static void udl_free_urb_list(struct drm_device *dev)
 static int udl_alloc_urb_list(struct drm_device *dev, int count, size_t size)
 {
 	struct udl_device *udl = dev->dev_private;
-	int i = 0;
 	struct urb *urb;
 	struct urb_node *unode;
 	char *buf;
+	size_t wanted_size = count * size;
 
 	spin_lock_init(&udl->urbs.lock);
 
+retry:
 	udl->urbs.size = size;
 	INIT_LIST_HEAD(&udl->urbs.list);
 
-	while (i < count) {
+	sema_init(&udl->urbs.limit_sem, 0);
+	udl->urbs.count = 0;
+	udl->urbs.available = 0;
+
+	while (udl->urbs.count * size < wanted_size) {
 		unode = kzalloc(sizeof(struct urb_node), GFP_KERNEL);
 		if (!unode)
 			break;
@@ -231,11 +230,16 @@ static int udl_alloc_urb_list(struct drm_device *dev, int count, size_t size)
 		}
 		unode->urb = urb;
 
-		buf = usb_alloc_coherent(udl->udev, MAX_TRANSFER, GFP_KERNEL,
+		buf = usb_alloc_coherent(udl->udev, size, GFP_KERNEL,
 					 &urb->transfer_dma);
 		if (!buf) {
 			kfree(unode);
 			usb_free_urb(urb);
+			if (size > PAGE_SIZE) {
+				size /= 2;
+				udl_free_urb_list(dev);
+				goto retry;
+			}
 			break;
 		}
 
@@ -246,16 +250,14 @@ static int udl_alloc_urb_list(struct drm_device *dev, int count, size_t size)
 
 		list_add_tail(&unode->entry, &udl->urbs.list);
 
-		i++;
+		up(&udl->urbs.limit_sem);
+		udl->urbs.count++;
+		udl->urbs.available++;
 	}
 
-	sema_init(&udl->urbs.limit_sem, i);
-	udl->urbs.count = i;
-	udl->urbs.available = i;
-
-	DRM_DEBUG("allocated %d %d byte urbs\n", i, (int) size);
+	DRM_DEBUG("allocated %d %d byte urbs\n", udl->urbs.count, (int) size);
 
-	return i;
+	return udl->urbs.count;
 }
 
 struct urb *udl_get_urb(struct drm_device *dev)
@@ -265,7 +267,6 @@ struct urb *udl_get_urb(struct drm_device *dev)
 	struct list_head *entry;
 	struct urb_node *unode;
 	struct urb *urb = NULL;
-	unsigned long flags;
 
 	/* Wait for an in-flight buffer to complete and get re-queued */
 	ret = down_timeout(&udl->urbs.limit_sem, GET_URB_TIMEOUT);
@@ -276,14 +277,14 @@ struct urb *udl_get_urb(struct drm_device *dev)
 		goto error;
 	}
 
-	spin_lock_irqsave(&udl->urbs.lock, flags);
+	spin_lock_irq(&udl->urbs.lock);
 
 	BUG_ON(list_empty(&udl->urbs.list)); /* reserved one with limit_sem */
 	entry = udl->urbs.list.next;
 	list_del_init(entry);
 	udl->urbs.available--;
 
-	spin_unlock_irqrestore(&udl->urbs.lock, flags);
+	spin_unlock_irq(&udl->urbs.lock);
 
 	unode = list_entry(entry, struct urb_node, entry);
 	urb = unode->urb;
diff --git a/drivers/gpu/drm/udl/udl_modeset.c b/drivers/gpu/drm/udl/udl_modeset.c
index 5bcae7649795..7e37765cf5ac 100644
--- a/drivers/gpu/drm/udl/udl_modeset.c
+++ b/drivers/gpu/drm/udl/udl_modeset.c
@@ -243,7 +243,7 @@ static int udl_crtc_write_mode_to_hw(struct drm_crtc *crtc)
 
 	memcpy(buf, udl->mode_buf, udl->mode_buf_len);
 	retval = udl_submit_urb(dev, urb, udl->mode_buf_len);
-	DRM_INFO("write mode info %d\n", udl->mode_buf_len);
+	DRM_DEBUG("write mode info %d\n", udl->mode_buf_len);
 	return retval;
 }
 
@@ -366,7 +366,6 @@ static int udl_crtc_page_flip(struct drm_crtc *crtc,
 {
 	struct udl_framebuffer *ufb = to_udl_fb(fb);
 	struct drm_device *dev = crtc->dev;
-	unsigned long flags;
 
 	struct drm_framebuffer *old_fb = crtc->primary->fb;
 	if (old_fb) {
@@ -377,10 +376,10 @@ static int udl_crtc_page_flip(struct drm_crtc *crtc,
 
 	udl_handle_damage(ufb, 0, 0, fb->width, fb->height);
 
-	spin_lock_irqsave(&dev->event_lock, flags);
+	spin_lock_irq(&dev->event_lock);
 	if (event)
 		drm_crtc_send_vblank_event(crtc, event);
-	spin_unlock_irqrestore(&dev->event_lock, flags);
+	spin_unlock_irq(&dev->event_lock);
 	crtc->primary->fb = fb;
 
 	return 0;
diff --git a/drivers/gpu/drm/udl/udl_transfer.c b/drivers/gpu/drm/udl/udl_transfer.c
index 0c87b1ac6b68..ce87661e544f 100644
--- a/drivers/gpu/drm/udl/udl_transfer.c
+++ b/drivers/gpu/drm/udl/udl_transfer.c
@@ -13,7 +13,6 @@
 #include <linux/module.h>
 #include <linux/slab.h>
 #include <linux/fb.h>
-#include <linux/prefetch.h>
 #include <asm/unaligned.h>
 
 #include <drm/drmP.h>
@@ -51,9 +50,6 @@ static int udl_trim_hline(const u8 *bback, const u8 **bfront, int *width_bytes)
 	int start = width;
 	int end = width;
 
-	prefetch((void *) front);
-	prefetch((void *) back);
-
 	for (j = 0; j < width; j++) {
 		if (back[j] != front[j]) {
 			start = j;
@@ -83,12 +79,12 @@ static inline u16 pixel32_to_be16(const uint32_t pixel)
 		((pixel >> 8) & 0xf800));
 }
 
-static inline u16 get_pixel_val16(const uint8_t *pixel, int bpp)
+static inline u16 get_pixel_val16(const uint8_t *pixel, int log_bpp)
 {
-	u16 pixel_val16 = 0;
-	if (bpp == 2)
+	u16 pixel_val16;
+	if (log_bpp == 1)
 		pixel_val16 = *(const uint16_t *)pixel;
-	else if (bpp == 4)
+	else
 		pixel_val16 = pixel32_to_be16(*(const uint32_t *)pixel);
 	return pixel_val16;
 }
@@ -125,8 +121,9 @@ static void udl_compress_hline16(
 	const u8 *const pixel_end,
 	uint32_t *device_address_ptr,
 	uint8_t **command_buffer_ptr,
-	const uint8_t *const cmd_buffer_end, int bpp)
+	const uint8_t *const cmd_buffer_end, int log_bpp)
 {
+	const int bpp = 1 << log_bpp;
 	const u8 *pixel = *pixel_start_ptr;
 	uint32_t dev_addr  = *device_address_ptr;
 	uint8_t *cmd = *command_buffer_ptr;
@@ -139,8 +136,6 @@ static void udl_compress_hline16(
 		const u8 *cmd_pixel_start, *cmd_pixel_end = NULL;
 		uint16_t pixel_val16;
 
-		prefetchw((void *) cmd); /* pull in one cache line at least */
-
 		*cmd++ = 0xaf;
 		*cmd++ = 0x6b;
 		*cmd++ = (uint8_t) ((dev_addr >> 16) & 0xFF);
@@ -153,12 +148,11 @@ static void udl_compress_hline16(
 		raw_pixels_count_byte = cmd++; /*  we'll know this later */
 		raw_pixel_start = pixel;
 
-		cmd_pixel_end = pixel + (min(MAX_CMD_PIXELS + 1,
-			min((int)(pixel_end - pixel) / bpp,
-			    (int)(cmd_buffer_end - cmd) / 2))) * bpp;
+		cmd_pixel_end = pixel + (min3(MAX_CMD_PIXELS + 1UL,
+					(unsigned long)(pixel_end - pixel) >> log_bpp,
+					(unsigned long)(cmd_buffer_end - 1 - cmd) / 2) << log_bpp);
 
-		prefetch_range((void *) pixel, (cmd_pixel_end - pixel) * bpp);
-		pixel_val16 = get_pixel_val16(pixel, bpp);
+		pixel_val16 = get_pixel_val16(pixel, log_bpp);
 
 		while (pixel < cmd_pixel_end) {
 			const u8 *const start = pixel;
@@ -170,7 +164,7 @@ static void udl_compress_hline16(
 			pixel += bpp;
 
 			while (pixel < cmd_pixel_end) {
-				pixel_val16 = get_pixel_val16(pixel, bpp);
+				pixel_val16 = get_pixel_val16(pixel, log_bpp);
 				if (pixel_val16 != repeating_pixel_val16)
 					break;
 				pixel += bpp;
@@ -179,10 +173,10 @@ static void udl_compress_hline16(
 			if (unlikely(pixel > start + bpp)) {
 				/* go back and fill in raw pixel count */
 				*raw_pixels_count_byte = (((start -
-						raw_pixel_start) / bpp) + 1) & 0xFF;
+						raw_pixel_start) >> log_bpp) + 1) & 0xFF;
 
 				/* immediately after raw data is repeat byte */
-				*cmd++ = (((pixel - start) / bpp) - 1) & 0xFF;
+				*cmd++ = (((pixel - start) >> log_bpp) - 1) & 0xFF;
 
 				/* Then start another raw pixel span */
 				raw_pixel_start = pixel;
@@ -192,11 +186,14 @@ static void udl_compress_hline16(
 
 		if (pixel > raw_pixel_start) {
 			/* finalize last RAW span */
-			*raw_pixels_count_byte = ((pixel-raw_pixel_start) / bpp) & 0xFF;
+			*raw_pixels_count_byte = ((pixel - raw_pixel_start) >> log_bpp) & 0xFF;
+		} else {
+			/* undo unused byte */
+			cmd--;
 		}
 
-		*cmd_pixels_count_byte = ((pixel - cmd_pixel_start) / bpp) & 0xFF;
-		dev_addr += ((pixel - cmd_pixel_start) / bpp) * 2;
+		*cmd_pixels_count_byte = ((pixel - cmd_pixel_start) >> log_bpp) & 0xFF;
+		dev_addr += ((pixel - cmd_pixel_start) >> log_bpp) * 2;
 	}
 
 	if (cmd_buffer_end <= MIN_RLX_CMD_BYTES + cmd) {
@@ -219,19 +216,19 @@ static void udl_compress_hline16(
  * (that we can only write to, slowly, and can never read), and (optionally)
  * our shadow copy that tracks what's been sent to that hardware buffer.
  */
-int udl_render_hline(struct drm_device *dev, int bpp, struct urb **urb_ptr,
+int udl_render_hline(struct drm_device *dev, int log_bpp, struct urb **urb_ptr,
 		     const char *front, char **urb_buf_ptr,
 		     u32 byte_offset, u32 device_byte_offset,
 		     u32 byte_width,
 		     int *ident_ptr, int *sent_ptr)
 {
 	const u8 *line_start, *line_end, *next_pixel;
-	u32 base16 = 0 + (device_byte_offset / bpp) * 2;
+	u32 base16 = 0 + (device_byte_offset >> log_bpp) * 2;
 	struct urb *urb = *urb_ptr;
 	u8 *cmd = *urb_buf_ptr;
 	u8 *cmd_end = (u8 *) urb->transfer_buffer + urb->transfer_buffer_length;
 
-	BUG_ON(!(bpp == 2 || bpp == 4));
+	BUG_ON(!(log_bpp == 1 || log_bpp == 2));
 
 	line_start = (u8 *) (front + byte_offset);
 	next_pixel = line_start;
@@ -241,7 +238,7 @@ int udl_render_hline(struct drm_device *dev, int bpp, struct urb **urb_ptr,
 
 		udl_compress_hline16(&next_pixel,
 			     line_end, &base16,
-			     (u8 **) &cmd, (u8 *) cmd_end, bpp);
+			     (u8 **) &cmd, (u8 *) cmd_end, log_bpp);
 
 		if (cmd >= cmd_end) {
 			int len = cmd - (u8 *) urb->transfer_buffer;
diff --git a/drivers/gpu/drm/v3d/v3d_drv.c b/drivers/gpu/drm/v3d/v3d_drv.c
index cdb582043b4f..2a85fa68ffea 100644
--- a/drivers/gpu/drm/v3d/v3d_drv.c
+++ b/drivers/gpu/drm/v3d/v3d_drv.c
@@ -123,6 +123,7 @@ v3d_open(struct drm_device *dev, struct drm_file *file)
 {
 	struct v3d_dev *v3d = to_v3d_dev(dev);
 	struct v3d_file_priv *v3d_priv;
+	struct drm_sched_rq *rq;
 	int i;
 
 	v3d_priv = kzalloc(sizeof(*v3d_priv), GFP_KERNEL);
@@ -132,10 +133,8 @@ v3d_open(struct drm_device *dev, struct drm_file *file)
 	v3d_priv->v3d = v3d;
 
 	for (i = 0; i < V3D_MAX_QUEUES; i++) {
-		drm_sched_entity_init(&v3d->queue[i].sched,
-				      &v3d_priv->sched_entity[i],
-				      &v3d->queue[i].sched.sched_rq[DRM_SCHED_PRIORITY_NORMAL],
-				      NULL);
+		rq = &v3d->queue[i].sched.sched_rq[DRM_SCHED_PRIORITY_NORMAL];
+		drm_sched_entity_init(&v3d_priv->sched_entity[i], &rq, 1, NULL);
 	}
 
 	file->driver_priv = v3d_priv;
@@ -146,13 +145,11 @@ v3d_open(struct drm_device *dev, struct drm_file *file)
 static void
 v3d_postclose(struct drm_device *dev, struct drm_file *file)
 {
-	struct v3d_dev *v3d = to_v3d_dev(dev);
 	struct v3d_file_priv *v3d_priv = file->driver_priv;
 	enum v3d_queue q;
 
 	for (q = 0; q < V3D_MAX_QUEUES; q++) {
-		drm_sched_entity_fini(&v3d->queue[q].sched,
-				      &v3d_priv->sched_entity[q]);
+		drm_sched_entity_destroy(&v3d_priv->sched_entity[q]);
 	}
 
 	kfree(v3d_priv);
diff --git a/drivers/gpu/drm/v3d/v3d_gem.c b/drivers/gpu/drm/v3d/v3d_gem.c
index e1fcbb4cd0ae..5ce24098a5fd 100644
--- a/drivers/gpu/drm/v3d/v3d_gem.c
+++ b/drivers/gpu/drm/v3d/v3d_gem.c
@@ -553,7 +553,6 @@ v3d_submit_cl_ioctl(struct drm_device *dev, void *data,
 	mutex_lock(&v3d->sched_lock);
 	if (exec->bin.start != exec->bin.end) {
 		ret = drm_sched_job_init(&exec->bin.base,
-					 &v3d->queue[V3D_BIN].sched,
 					 &v3d_priv->sched_entity[V3D_BIN],
 					 v3d_priv);
 		if (ret)
@@ -568,7 +567,6 @@ v3d_submit_cl_ioctl(struct drm_device *dev, void *data,
 	}
 
 	ret = drm_sched_job_init(&exec->render.base,
-				 &v3d->queue[V3D_RENDER].sched,
 				 &v3d_priv->sched_entity[V3D_RENDER],
 				 v3d_priv);
 	if (ret)
diff --git a/drivers/gpu/drm/vc4/vc4_plane.c b/drivers/gpu/drm/vc4/vc4_plane.c
index 688ad9bb0f08..cf78f74bb87f 100644
--- a/drivers/gpu/drm/vc4/vc4_plane.c
+++ b/drivers/gpu/drm/vc4/vc4_plane.c
@@ -318,6 +318,9 @@ static int vc4_plane_setup_clipping_and_scaling(struct drm_plane_state *state)
 			vc4_state->x_scaling[0] = VC4_SCALING_TPZ;
 		if (vc4_state->y_scaling[0] == VC4_SCALING_NONE)
 			vc4_state->y_scaling[0] = VC4_SCALING_TPZ;
+	} else {
+		vc4_state->x_scaling[1] = VC4_SCALING_NONE;
+		vc4_state->y_scaling[1] = VC4_SCALING_NONE;
 	}
 
 	vc4_state->is_unity = (vc4_state->x_scaling[0] == VC4_SCALING_NONE &&
diff --git a/drivers/gpu/drm/vkms/vkms_drv.c b/drivers/gpu/drm/vkms/vkms_drv.c
index 5d78bd97e69c..bd9d4b2389bd 100644
--- a/drivers/gpu/drm/vkms/vkms_drv.c
+++ b/drivers/gpu/drm/vkms/vkms_drv.c
@@ -44,6 +44,7 @@ static void vkms_release(struct drm_device *dev)
 	struct vkms_device *vkms = container_of(dev, struct vkms_device, drm);
 
 	platform_device_unregister(vkms->platform);
+	drm_atomic_helper_shutdown(&vkms->drm);
 	drm_mode_config_cleanup(&vkms->drm);
 	drm_dev_fini(&vkms->drm);
 	destroy_workqueue(vkms->output.crc_workq);
diff --git a/drivers/gpu/drm/vmwgfx/Makefile b/drivers/gpu/drm/vmwgfx/Makefile
index 794cc9d5c9b0..09b2aa08363e 100644
--- a/drivers/gpu/drm/vmwgfx/Makefile
+++ b/drivers/gpu/drm/vmwgfx/Makefile
@@ -1,9 +1,9 @@
 # SPDX-License-Identifier: GPL-2.0
 vmwgfx-y := vmwgfx_execbuf.o vmwgfx_gmr.o vmwgfx_kms.o vmwgfx_drv.o \
-	    vmwgfx_fb.o vmwgfx_ioctl.o vmwgfx_resource.o vmwgfx_buffer.o \
+	    vmwgfx_fb.o vmwgfx_ioctl.o vmwgfx_resource.o vmwgfx_ttm_buffer.o \
 	    vmwgfx_fifo.o vmwgfx_irq.o vmwgfx_ldu.o vmwgfx_ttm_glue.o \
 	    vmwgfx_overlay.o vmwgfx_marker.o vmwgfx_gmrid_manager.o \
-	    vmwgfx_fence.o vmwgfx_dmabuf.o vmwgfx_scrn.o vmwgfx_context.o \
+	    vmwgfx_fence.o vmwgfx_bo.o vmwgfx_scrn.o vmwgfx_context.o \
 	    vmwgfx_surface.o vmwgfx_prime.o vmwgfx_mob.o vmwgfx_shader.o \
 	    vmwgfx_cmdbuf_res.o vmwgfx_cmdbuf.o vmwgfx_stdu.o \
 	    vmwgfx_cotable.o vmwgfx_so.o vmwgfx_binding.o vmwgfx_msg.o \
diff --git a/drivers/gpu/drm/vmwgfx/device_include/svga3d_cmd.h b/drivers/gpu/drm/vmwgfx/device_include/svga3d_cmd.h
index 3bf75978e155..9cbba0e8ce6a 100644
--- a/drivers/gpu/drm/vmwgfx/device_include/svga3d_cmd.h
+++ b/drivers/gpu/drm/vmwgfx/device_include/svga3d_cmd.h
@@ -47,10 +47,10 @@
  * the SVGA3D protocol and remain reserved; they should not be used in the
  * future.
  *
- * IDs between 1040 and 1999 (inclusive) are available for use by the
+ * IDs between 1040 and 2999 (inclusive) are available for use by the
  * current SVGA3D protocol.
  *
- * FIFO clients other than SVGA3D should stay below 1000, or at 2000
+ * FIFO clients other than SVGA3D should stay below 1000, or at 3000
  * and up.
  */
 
@@ -90,19 +90,19 @@ typedef enum {
    SVGA_3D_CMD_BLIT_SURFACE_TO_SCREEN                     = 1069,
    SVGA_3D_CMD_SURFACE_DEFINE_V2                          = 1070,
    SVGA_3D_CMD_GENERATE_MIPMAPS                           = 1071,
-   SVGA_3D_CMD_VIDEO_CREATE_DECODER                       = 1072,
-   SVGA_3D_CMD_VIDEO_DESTROY_DECODER                      = 1073,
-   SVGA_3D_CMD_VIDEO_CREATE_PROCESSOR                     = 1074,
-   SVGA_3D_CMD_VIDEO_DESTROY_PROCESSOR                    = 1075,
-   SVGA_3D_CMD_VIDEO_DECODE_START_FRAME                   = 1076,
-   SVGA_3D_CMD_VIDEO_DECODE_RENDER                        = 1077,
-   SVGA_3D_CMD_VIDEO_DECODE_END_FRAME                     = 1078,
-   SVGA_3D_CMD_VIDEO_PROCESS_FRAME                        = 1079,
+   SVGA_3D_CMD_DEAD4                                      = 1072,
+   SVGA_3D_CMD_DEAD5                                      = 1073,
+   SVGA_3D_CMD_DEAD6                                      = 1074,
+   SVGA_3D_CMD_DEAD7                                      = 1075,
+   SVGA_3D_CMD_DEAD8                                      = 1076,
+   SVGA_3D_CMD_DEAD9                                      = 1077,
+   SVGA_3D_CMD_DEAD10                                     = 1078,
+   SVGA_3D_CMD_DEAD11                                     = 1079,
    SVGA_3D_CMD_ACTIVATE_SURFACE                           = 1080,
    SVGA_3D_CMD_DEACTIVATE_SURFACE                         = 1081,
    SVGA_3D_CMD_SCREEN_DMA                                 = 1082,
-   SVGA_3D_CMD_SET_UNITY_SURFACE_COOKIE                   = 1083,
-   SVGA_3D_CMD_OPEN_CONTEXT_SURFACE                       = 1084,
+   SVGA_3D_CMD_DEAD1                                      = 1083,
+   SVGA_3D_CMD_DEAD2                                      = 1084,
 
    SVGA_3D_CMD_LOGICOPS_BITBLT                            = 1085,
    SVGA_3D_CMD_LOGICOPS_TRANSBLT                          = 1086,
@@ -218,7 +218,7 @@ typedef enum {
    SVGA_3D_CMD_DX_CLEAR_DEPTHSTENCIL_VIEW                 = 1177,
    SVGA_3D_CMD_DX_PRED_COPY_REGION                        = 1178,
    SVGA_3D_CMD_DX_PRED_COPY                               = 1179,
-   SVGA_3D_CMD_DX_STRETCHBLT                              = 1180,
+   SVGA_3D_CMD_DX_PRESENTBLT                              = 1180,
    SVGA_3D_CMD_DX_GENMIPS                                 = 1181,
    SVGA_3D_CMD_DX_UPDATE_SUBRESOURCE                      = 1182,
    SVGA_3D_CMD_DX_READBACK_SUBRESOURCE                    = 1183,
@@ -255,7 +255,7 @@ typedef enum {
    SVGA_3D_CMD_DX_READBACK_ALL_QUERY                      = 1214,
    SVGA_3D_CMD_DX_PRED_TRANSFER_FROM_BUFFER               = 1215,
    SVGA_3D_CMD_DX_MOB_FENCE_64                            = 1216,
-   SVGA_3D_CMD_DX_BIND_SHADER_ON_CONTEXT                  = 1217,
+   SVGA_3D_CMD_DX_BIND_ALL_SHADER                         = 1217,
    SVGA_3D_CMD_DX_HINT                                    = 1218,
    SVGA_3D_CMD_DX_BUFFER_UPDATE                           = 1219,
    SVGA_3D_CMD_DX_SET_VS_CONSTANT_BUFFER_OFFSET           = 1220,
@@ -263,17 +263,47 @@ typedef enum {
    SVGA_3D_CMD_DX_SET_GS_CONSTANT_BUFFER_OFFSET           = 1222,
 
    /*
-    * Reserve some IDs to be used for the DX11 shader types.
+    * Reserve some IDs to be used for the SM5 shader types.
     */
    SVGA_3D_CMD_DX_RESERVED1                               = 1223,
    SVGA_3D_CMD_DX_RESERVED2                               = 1224,
    SVGA_3D_CMD_DX_RESERVED3                               = 1225,
 
-   SVGA_3D_CMD_DX_MAX                                     = 1226,
-   SVGA_3D_CMD_MAX                                        = 1226,
+   SVGA_3D_CMD_DX_COND_BIND_ALL_SHADER                    = 1226,
+   SVGA_3D_CMD_DX_MAX                                     = 1227,
+
+   SVGA_3D_CMD_SCREEN_COPY                                = 1227,
+
+   /*
+    * Reserve some IDs to be used for video.
+    */
+   SVGA_3D_CMD_VIDEO_RESERVED1                            = 1228,
+   SVGA_3D_CMD_VIDEO_RESERVED2                            = 1229,
+   SVGA_3D_CMD_VIDEO_RESERVED3                            = 1230,
+   SVGA_3D_CMD_VIDEO_RESERVED4                            = 1231,
+   SVGA_3D_CMD_VIDEO_RESERVED5                            = 1232,
+   SVGA_3D_CMD_VIDEO_RESERVED6                            = 1233,
+   SVGA_3D_CMD_VIDEO_RESERVED7                            = 1234,
+   SVGA_3D_CMD_VIDEO_RESERVED8                            = 1235,
+
+   SVGA_3D_CMD_GROW_OTABLE                                = 1236,
+   SVGA_3D_CMD_DX_GROW_COTABLE                            = 1237,
+   SVGA_3D_CMD_INTRA_SURFACE_COPY                         = 1238,
+
+   SVGA_3D_CMD_DEFINE_GB_SURFACE_V3                       = 1239,
+
+   SVGA_3D_CMD_DX_RESOLVE_COPY                            = 1240,
+   SVGA_3D_CMD_DX_PRED_RESOLVE_COPY                       = 1241,
+   SVGA_3D_CMD_DX_PRED_CONVERT_REGION                     = 1242,
+   SVGA_3D_CMD_DX_PRED_CONVERT                            = 1243,
+   SVGA_3D_CMD_WHOLE_SURFACE_COPY                         = 1244,
+
+   SVGA_3D_CMD_MAX                                        = 1245,
    SVGA_3D_CMD_FUTURE_MAX                                 = 3000
 } SVGAFifo3dCmdId;
 
+#define SVGA_NUM_3D_CMD (SVGA_3D_CMD_MAX - SVGA_3D_CMD_BASE)
+
 /*
  * FIFO command format definitions:
  */
@@ -302,7 +332,7 @@ typedef
 #include "vmware_pack_begin.h"
 struct {
    uint32                      sid;
-   SVGA3dSurfaceFlags          surfaceFlags;
+   SVGA3dSurface1Flags         surfaceFlags;
    SVGA3dSurfaceFormat         format;
    /*
     * If surfaceFlags has SVGA3D_SURFACE_CUBEMAP bit set, all SVGA3dSurfaceFace
@@ -328,7 +358,7 @@ typedef
 #include "vmware_pack_begin.h"
 struct {
    uint32                      sid;
-   SVGA3dSurfaceFlags          surfaceFlags;
+   SVGA3dSurface1Flags         surfaceFlags;
    SVGA3dSurfaceFormat         format;
    /*
     * If surfaceFlags has SVGA3D_SURFACE_CUBEMAP bit set, all SVGA3dSurfaceFace
@@ -460,6 +490,28 @@ struct {
 #include "vmware_pack_end.h"
 SVGA3dCmdSurfaceCopy;               /* SVGA_3D_CMD_SURFACE_COPY */
 
+/*
+ * Perform a surface copy within the same image.
+ * The src/dest boxes are allowed to overlap.
+ */
+typedef
+#include "vmware_pack_begin.h"
+struct {
+   SVGA3dSurfaceImageId  surface;
+   SVGA3dCopyBox box;
+}
+#include "vmware_pack_end.h"
+SVGA3dCmdIntraSurfaceCopy;               /* SVGA_3D_CMD_INTRA_SURFACE_COPY */
+
+typedef
+#include "vmware_pack_begin.h"
+struct {
+   uint32 srcSid;
+   uint32 destSid;
+}
+#include "vmware_pack_end.h"
+SVGA3dCmdWholeSurfaceCopy;               /* SVGA_3D_CMD_WHOLE_SURFACE_COPY */
+
 typedef
 #include "vmware_pack_begin.h"
 struct {
@@ -773,6 +825,17 @@ struct {
 #include "vmware_pack_end.h"
 SVGA3dVertexElement;
 
+/*
+ * Should the vertex element respect the stream value?  The high bit of the
+ * stream should be set to indicate that the stream should be respected.  If
+ * the high bit is not set, the stream will be ignored and replaced by the index
+ * of the position of the currently considered vertex element.
+ *
+ * All guests should set this bit and correctly specify the stream going
+ * forward.
+ */
+#define SVGA3D_VERTEX_ELEMENT_RESPECT_STREAM (1 << 7)
+
 typedef
 #include "vmware_pack_begin.h"
 struct {
@@ -1103,8 +1166,6 @@ struct {
 #include "vmware_pack_end.h"
 SVGA3dCmdGenerateMipmaps;             /* SVGA_3D_CMD_GENERATE_MIPMAPS */
 
-
-
 typedef
 #include "vmware_pack_begin.h"
 struct {
@@ -1148,38 +1209,6 @@ struct SVGA3dCmdScreenDMA {
 SVGA3dCmdScreenDMA;        /* SVGA_3D_CMD_SCREEN_DMA */
 
 /*
- * Set Unity Surface Cookie
- *
- * Associates the supplied cookie with the surface id for use with
- * Unity.  This cookie is a hint from guest to host, there is no way
- * for the guest to readback the cookie and the host is free to drop
- * the cookie association at will.  The default value for the cookie
- * on all surfaces is 0.
- */
-
-typedef
-#include "vmware_pack_begin.h"
-struct SVGA3dCmdSetUnitySurfaceCookie {
-   uint32 sid;
-   uint64 cookie;
-}
-#include "vmware_pack_end.h"
-SVGA3dCmdSetUnitySurfaceCookie;   /* SVGA_3D_CMD_SET_UNITY_SURFACE_COOKIE */
-
-/*
- * Open a context-specific surface in a non-context-specific manner.
- */
-
-typedef
-#include "vmware_pack_begin.h"
-struct SVGA3dCmdOpenContextSurface {
-   uint32 sid;
-}
-#include "vmware_pack_end.h"
-SVGA3dCmdOpenContextSurface;   /* SVGA_3D_CMD_OPEN_CONTEXT_SURFACE */
-
-
-/*
  * Logic ops
  */
 
@@ -1325,7 +1354,7 @@ typedef
 #include "vmware_pack_begin.h"
 struct {
    SVGA3dSurfaceFormat format;
-   SVGA3dSurfaceFlags surfaceFlags;
+   SVGA3dSurface1Flags surface1Flags;
    uint32 numMipLevels;
    uint32 multisampleCount;
    SVGA3dTextureFilter autogenFilter;
@@ -1333,7 +1362,11 @@ struct {
    SVGAMobId mobid;
    uint32 arraySize;
    uint32 mobPitch;
-   uint32 pad[5];
+   SVGA3dSurface2Flags surface2Flags;
+   uint8 multisamplePattern;
+   uint8 qualityLevel;
+   uint8  pad0[2];
+   uint32 pad1[3];
 }
 #include "vmware_pack_end.h"
 SVGAOTableSurfaceEntry;
@@ -1361,7 +1394,8 @@ struct {
 SVGAOTableShaderEntry;
 #define SVGA3D_OTABLE_SHADER_ENTRY_SIZE (sizeof(SVGAOTableShaderEntry))
 
-#define SVGA_STFLAG_PRIMARY (1 << 0)
+#define SVGA_STFLAG_PRIMARY  (1 << 0)
+#define SVGA_STFLAG_RESERVED (1 << 1) /* Added with cap SVGA_CAP_HP_CMD_QUEUE */
 typedef uint32 SVGAScreenTargetFlags;
 
 typedef
@@ -1529,6 +1563,25 @@ struct {
 #include "vmware_pack_end.h"
 SVGA3dCmdSetOTableBase64;  /* SVGA_3D_CMD_SET_OTABLE_BASE64 */
 
+/*
+ * Guests using SVGA_3D_CMD_GROW_OTABLE are promising that
+ * the new OTable contains the same contents as the old one, except possibly
+ * for some new invalid entries at the end.
+ *
+ * (Otherwise, guests should use one of the SetOTableBase commands.)
+ */
+typedef
+#include "vmware_pack_begin.h"
+struct {
+   SVGAOTableType type;
+   PPN64 baseAddress;
+   uint32 sizeInBytes;
+   uint32 validSizeInBytes;
+   SVGAMobFormat ptDepth;
+}
+#include "vmware_pack_end.h"
+SVGA3dCmdGrowOTable;  /* SVGA_3D_CMD_GROW_OTABLE */
+
 typedef
 #include "vmware_pack_begin.h"
 struct {
@@ -1616,7 +1669,7 @@ typedef
 #include "vmware_pack_begin.h"
 struct SVGA3dCmdDefineGBSurface {
    uint32 sid;
-   SVGA3dSurfaceFlags surfaceFlags;
+   SVGA3dSurface1Flags surfaceFlags;
    SVGA3dSurfaceFormat format;
    uint32 numMipLevels;
    uint32 multisampleCount;
@@ -1627,6 +1680,45 @@ struct SVGA3dCmdDefineGBSurface {
 SVGA3dCmdDefineGBSurface;   /* SVGA_3D_CMD_DEFINE_GB_SURFACE */
 
 /*
+ * Defines a guest-backed surface, adding the arraySize field.
+ */
+typedef
+#include "vmware_pack_begin.h"
+struct SVGA3dCmdDefineGBSurface_v2 {
+   uint32 sid;
+   SVGA3dSurface1Flags surfaceFlags;
+   SVGA3dSurfaceFormat format;
+   uint32 numMipLevels;
+   uint32 multisampleCount;
+   SVGA3dTextureFilter autogenFilter;
+   SVGA3dSize size;
+   uint32 arraySize;
+   uint32 pad;
+}
+#include "vmware_pack_end.h"
+SVGA3dCmdDefineGBSurface_v2;   /* SVGA_3D_CMD_DEFINE_GB_SURFACE_V2 */
+
+/*
+ * Defines a guest-backed surface, adding the larger flags.
+ */
+typedef
+#include "vmware_pack_begin.h"
+struct SVGA3dCmdDefineGBSurface_v3 {
+   uint32 sid;
+   SVGA3dSurfaceAllFlags surfaceFlags;
+   SVGA3dSurfaceFormat format;
+   uint32 numMipLevels;
+   uint32 multisampleCount;
+   SVGA3dMSPattern multisamplePattern;
+   SVGA3dMSQualityLevel qualityLevel;
+   SVGA3dTextureFilter autogenFilter;
+   SVGA3dSize size;
+   uint32 arraySize;
+}
+#include "vmware_pack_end.h"
+SVGA3dCmdDefineGBSurface_v3;   /* SVGA_3D_CMD_DEFINE_GB_SURFACE_V3 */
+
+/*
  * Destroy a guest-backed surface.
  */
 
@@ -1673,7 +1765,7 @@ SVGA3dCmdBindGBSurfaceWithPitch;   /* SVGA_3D_CMD_BIND_GB_SURFACE_WITH_PITCH */
 
 typedef
 #include "vmware_pack_begin.h"
-struct{
+struct SVGA3dCmdCondBindGBSurface {
    uint32 sid;
    SVGAMobId testMobid;
    SVGAMobId mobid;
@@ -2067,6 +2159,26 @@ struct {
    uint32 mobOffset;
 }
 #include "vmware_pack_end.h"
-SVGA3dCmdGBMobFence;  /* SVGA_3D_CMD_GB_MOB_FENCE*/
+SVGA3dCmdGBMobFence;  /* SVGA_3D_CMD_GB_MOB_FENCE */
+
+typedef
+#include "vmware_pack_begin.h"
+struct {
+   uint32 stid;
+   SVGA3dSurfaceImageId dest;
+
+   uint32 statusMobId;
+   uint32 statusMobOffset;
+
+   /* Reserved fields */
+   uint32 mustBeInvalidId;
+   uint32 mustBeZero;
+}
+#include "vmware_pack_end.h"
+SVGA3dCmdScreenCopy;  /* SVGA_3D_CMD_SCREEN_COPY */
+
+#define SVGA_SCREEN_COPY_STATUS_FAILURE 0x00
+#define SVGA_SCREEN_COPY_STATUS_SUCCESS 0x01
+#define SVGA_SCREEN_COPY_STATUS_INVALID 0xFFFFFFFF
 
 #endif /* _SVGA3D_CMD_H_ */
diff --git a/drivers/gpu/drm/vmwgfx/device_include/svga3d_devcaps.h b/drivers/gpu/drm/vmwgfx/device_include/svga3d_devcaps.h
index e1c0d94cc1e9..f256560049bf 100644
--- a/drivers/gpu/drm/vmwgfx/device_include/svga3d_devcaps.h
+++ b/drivers/gpu/drm/vmwgfx/device_include/svga3d_devcaps.h
@@ -230,9 +230,9 @@ typedef enum {
    SVGA3D_DEVCAP_DEAD2                             = 94,
 
    /*
-    * Does the device support the DX commands?
+    * Does the device support DXContexts?
     */
-   SVGA3D_DEVCAP_DX                                = 95,
+   SVGA3D_DEVCAP_DXCONTEXT                         = 95,
 
    /*
     * What is the maximum size of a texture array?
@@ -242,21 +242,47 @@ typedef enum {
    SVGA3D_DEVCAP_MAX_TEXTURE_ARRAY_SIZE            = 96,
 
    /*
-    * What is the maximum number of vertex buffers that can
-    * be used in the DXContext inputAssembly?
+    * What is the maximum number of vertex buffers or vertex input registers
+    * that can be expected to work correctly with a DXContext?
+    *
+    * The guest is allowed to set up to SVGA3D_DX_MAX_VERTEXBUFFERS, but
+    * anything in excess of this cap is not guaranteed to render correctly.
+    *
+    * Similarly, the guest can set up to SVGA3D_DX_MAX_VERTEXINPUTREGISTERS
+    * input registers without the SVGA3D_DEVCAP_SM4_1 cap, or
+    * SVGA3D_DX_SM41_MAX_VERTEXINPUTREGISTERS with the SVGA3D_DEVCAP_SM4_1,
+    * but only the registers up to this cap value are guaranteed to render
+    * correctly.
+    *
+    * If guest-drivers are able to expose a lower-limit, it's recommended
+    * that they clamp to this value.  Otherwise, the host will make a
+    * best-effort on case-by-case basis if guests exceed this.
     */
    SVGA3D_DEVCAP_DX_MAX_VERTEXBUFFERS              = 97,
 
    /*
-    * What is the maximum number of constant buffers
-    * that can be expected to work correctly with a
-    * DX context?
+    * What is the maximum number of constant buffers that can be expected to
+    * work correctly with a DX context?
+    *
+    * The guest is allowed to set up to SVGA3D_DX_MAX_CONSTBUFFERS, but
+    * anything in excess of this cap is not guaranteed to render correctly.
+    *
+    * If guest-drivers are able to expose a lower-limit, it's recommended
+    * that they clamp to this value.  Otherwise, the host will make a
+    * best-effort on case-by-case basis if guests exceed this.
     */
    SVGA3D_DEVCAP_DX_MAX_CONSTANT_BUFFERS           = 98,
 
    /*
     * Does the device support provoking vertex control?
-    * If zero, the first vertex will always be the provoking vertex.
+    *
+    * If this cap is present, the provokingVertexLast field in the
+    * rasterizer state is enabled.  (Guests can then set it to FALSE,
+    * meaning that the first vertex is the provoking vertex, or TRUE,
+    * meaning that the last verteix is the provoking vertex.)
+    *
+    * If this cap is FALSE, then guests should set the provokingVertexLast
+    * to FALSE, otherwise rendering behavior is undefined.
     */
    SVGA3D_DEVCAP_DX_PROVOKING_VERTEX               = 99,
 
@@ -282,7 +308,7 @@ typedef enum {
    SVGA3D_DEVCAP_DXFMT_BUMPU8V8                    = 119,
    SVGA3D_DEVCAP_DXFMT_BUMPL6V5U5                  = 120,
    SVGA3D_DEVCAP_DXFMT_BUMPX8L8V8U8                = 121,
-   SVGA3D_DEVCAP_DXFMT_BUMPL8V8U8                  = 122,
+   SVGA3D_DEVCAP_DXFMT_FORMAT_DEAD1                = 122,
    SVGA3D_DEVCAP_DXFMT_ARGB_S10E5                  = 123,
    SVGA3D_DEVCAP_DXFMT_ARGB_S23E8                  = 124,
    SVGA3D_DEVCAP_DXFMT_A2R10G10B10                 = 125,
@@ -321,8 +347,8 @@ typedef enum {
    SVGA3D_DEVCAP_DXFMT_R32G32_SINT                 = 158,
    SVGA3D_DEVCAP_DXFMT_R32G8X24_TYPELESS           = 159,
    SVGA3D_DEVCAP_DXFMT_D32_FLOAT_S8X24_UINT        = 160,
-   SVGA3D_DEVCAP_DXFMT_R32_FLOAT_X8X24_TYPELESS    = 161,
-   SVGA3D_DEVCAP_DXFMT_X32_TYPELESS_G8X24_UINT     = 162,
+   SVGA3D_DEVCAP_DXFMT_R32_FLOAT_X8X24             = 161,
+   SVGA3D_DEVCAP_DXFMT_X32_G8X24_UINT              = 162,
    SVGA3D_DEVCAP_DXFMT_R10G10B10A2_TYPELESS        = 163,
    SVGA3D_DEVCAP_DXFMT_R10G10B10A2_UINT            = 164,
    SVGA3D_DEVCAP_DXFMT_R11G11B10_FLOAT             = 165,
@@ -340,8 +366,8 @@ typedef enum {
    SVGA3D_DEVCAP_DXFMT_R32_SINT                    = 177,
    SVGA3D_DEVCAP_DXFMT_R24G8_TYPELESS              = 178,
    SVGA3D_DEVCAP_DXFMT_D24_UNORM_S8_UINT           = 179,
-   SVGA3D_DEVCAP_DXFMT_R24_UNORM_X8_TYPELESS       = 180,
-   SVGA3D_DEVCAP_DXFMT_X24_TYPELESS_G8_UINT        = 181,
+   SVGA3D_DEVCAP_DXFMT_R24_UNORM_X8                = 180,
+   SVGA3D_DEVCAP_DXFMT_X24_G8_UINT                 = 181,
    SVGA3D_DEVCAP_DXFMT_R8G8_TYPELESS               = 182,
    SVGA3D_DEVCAP_DXFMT_R8G8_UNORM                  = 183,
    SVGA3D_DEVCAP_DXFMT_R8G8_UINT                   = 184,
@@ -405,6 +431,17 @@ typedef enum {
    SVGA3D_DEVCAP_DXFMT_BC4_UNORM                   = 242,
    SVGA3D_DEVCAP_DXFMT_BC5_UNORM                   = 243,
 
+   /*
+    * Advertises shaderModel 4.1 support, independent blend-states,
+    * cube-map arrays, and a higher vertex input registers limit.
+    *
+    * (See documentation on SVGA3D_DEVCAP_DX_MAX_VERTEXBUFFERS.)
+    */
+   SVGA3D_DEVCAP_SM41                              = 244,
+
+   SVGA3D_DEVCAP_MULTISAMPLE_2X                    = 245,
+   SVGA3D_DEVCAP_MULTISAMPLE_4X                    = 246,
+
    SVGA3D_DEVCAP_MAX                       /* This must be the last index. */
 } SVGA3dDevCapIndex;
 
@@ -420,9 +457,7 @@ typedef enum {
  * MIPS: Does the format support mip levels?
  * ARRAY: Does the format support texture arrays?
  * VOLUME: Does the format support having volume?
- * MULTISAMPLE_2: Does the format support 2x multisample?
- * MULTISAMPLE_4: Does the format support 4x multisample?
- * MULTISAMPLE_8: Does the format support 8x multisample?
+ * MULTISAMPLE: Does the format support multisample?
  */
 #define SVGA3D_DXFMT_SUPPORTED                (1 <<  0)
 #define SVGA3D_DXFMT_SHADER_SAMPLE            (1 <<  1)
@@ -433,20 +468,8 @@ typedef enum {
 #define SVGA3D_DXFMT_ARRAY                    (1 <<  6)
 #define SVGA3D_DXFMT_VOLUME                   (1 <<  7)
 #define SVGA3D_DXFMT_DX_VERTEX_BUFFER         (1 <<  8)
-#define SVGADX_DXFMT_MULTISAMPLE_2            (1 <<  9)
-#define SVGADX_DXFMT_MULTISAMPLE_4            (1 << 10)
-#define SVGADX_DXFMT_MULTISAMPLE_8            (1 << 11)
-#define SVGADX_DXFMT_MAX                      (1 << 12)
-
-/*
- * Convenience mask for any multisample capability.
- *
- * The multisample bits imply both load and render capability.
- */
-#define SVGA3D_DXFMT_MULTISAMPLE ( \
-           SVGADX_DXFMT_MULTISAMPLE_2 | \
-           SVGADX_DXFMT_MULTISAMPLE_4 | \
-           SVGADX_DXFMT_MULTISAMPLE_8 )
+#define SVGA3D_DXFMT_MULTISAMPLE              (1 <<  9)
+#define SVGA3D_DXFMT_MAX                      (1 << 10)
 
 typedef union {
    Bool   b;
diff --git a/drivers/gpu/drm/vmwgfx/device_include/svga3d_dx.h b/drivers/gpu/drm/vmwgfx/device_include/svga3d_dx.h
index f4606d04556b..7a49c94df221 100644
--- a/drivers/gpu/drm/vmwgfx/device_include/svga3d_dx.h
+++ b/drivers/gpu/drm/vmwgfx/device_include/svga3d_dx.h
@@ -57,6 +57,16 @@ typedef uint32 SVGA3dInputClassification;
 #define SVGA3D_RESOURCE_TYPE_MAX      7
 typedef uint32 SVGA3dResourceType;
 
+#define SVGA3D_COLOR_WRITE_ENABLE_RED     (1 << 0)
+#define SVGA3D_COLOR_WRITE_ENABLE_GREEN   (1 << 1)
+#define SVGA3D_COLOR_WRITE_ENABLE_BLUE    (1 << 2)
+#define SVGA3D_COLOR_WRITE_ENABLE_ALPHA   (1 << 3)
+#define SVGA3D_COLOR_WRITE_ENABLE_ALL     (SVGA3D_COLOR_WRITE_ENABLE_RED |   \
+                                           SVGA3D_COLOR_WRITE_ENABLE_GREEN | \
+                                           SVGA3D_COLOR_WRITE_ENABLE_BLUE |  \
+                                           SVGA3D_COLOR_WRITE_ENABLE_ALPHA)
+typedef uint8 SVGA3dColorWriteEnable;
+
 #define SVGA3D_DEPTH_WRITE_MASK_ZERO   0
 #define SVGA3D_DEPTH_WRITE_MASK_ALL    1
 typedef uint8 SVGA3dDepthWriteMask;
@@ -89,17 +99,28 @@ typedef uint8 SVGA3dCullMode;
 #define SVGA3D_COMPARISON_MAX             9
 typedef uint8 SVGA3dComparisonFunc;
 
+/*
+ * SVGA3D_MULTISAMPLE_RAST_DISABLE disables MSAA for all primitives.
+ * SVGA3D_MULTISAMPLE_RAST_DISABLE_LINE, which is supported in SM41,
+ * disables MSAA for lines only.
+ */
+#define SVGA3D_MULTISAMPLE_RAST_DISABLE        0
+#define SVGA3D_MULTISAMPLE_RAST_ENABLE         1
+#define SVGA3D_MULTISAMPLE_RAST_DX_MAX         1
+#define SVGA3D_MULTISAMPLE_RAST_DISABLE_LINE   2
+#define SVGA3D_MULTISAMPLE_RAST_MAX            2
+typedef uint8 SVGA3dMultisampleRastEnable;
+
 #define SVGA3D_DX_MAX_VERTEXBUFFERS 32
+#define SVGA3D_DX_MAX_VERTEXINPUTREGISTERS 16
+#define SVGA3D_DX_SM41_MAX_VERTEXINPUTREGISTERS 32
 #define SVGA3D_DX_MAX_SOTARGETS 4
 #define SVGA3D_DX_MAX_SRVIEWS 128
 #define SVGA3D_DX_MAX_CONSTBUFFERS 16
 #define SVGA3D_DX_MAX_SAMPLERS 16
 
-/* Id limits */
-static const uint32 SVGA3dBlendObjectCountPerContext = 4096;
-static const uint32 SVGA3dDepthStencilObjectCountPerContext = 4096;
+#define SVGA3D_DX_MAX_CONSTBUF_BINDING_SIZE (4096 * 4 * (uint32)sizeof(uint32))
 
-typedef uint32 SVGA3dSurfaceId;
 typedef uint32 SVGA3dShaderResourceViewId;
 typedef uint32 SVGA3dRenderTargetViewId;
 typedef uint32 SVGA3dDepthStencilViewId;
@@ -195,20 +216,6 @@ SVGA3dCmdDXInvalidateContext;   /* SVGA_3D_CMD_DX_INVALIDATE_CONTEXT */
 
 typedef
 #include "vmware_pack_begin.h"
-struct SVGA3dReplyFormatData {
-   uint32 formatSupport;
-   uint32 msaa2xQualityLevels:5;
-   uint32 msaa4xQualityLevels:5;
-   uint32 msaa8xQualityLevels:5;
-   uint32 msaa16xQualityLevels:5;
-   uint32 msaa32xQualityLevels:5;
-   uint32 pad:7;
-}
-#include "vmware_pack_end.h"
-SVGA3dReplyFormatData;
-
-typedef
-#include "vmware_pack_begin.h"
 struct SVGA3dCmdDXSetSingleConstantBuffer {
    uint32 slot;
    SVGA3dShaderType type;
@@ -625,6 +632,28 @@ SVGA3dCmdDXPredCopy; /* SVGA_3D_CMD_DX_PRED_COPY */
 
 typedef
 #include "vmware_pack_begin.h"
+struct SVGA3dCmdDXPredConvertRegion {
+   SVGA3dSurfaceId dstSid;
+   uint32 dstSubResource;
+   SVGA3dBox destBox;
+   SVGA3dSurfaceId srcSid;
+   uint32 srcSubResource;
+   SVGA3dBox srcBox;
+}
+#include "vmware_pack_end.h"
+SVGA3dCmdDXPredConvertRegion; /* SVGA_3D_CMD_DX_PRED_CONVERT_REGION */
+
+typedef
+#include "vmware_pack_begin.h"
+struct SVGA3dCmdDXPredConvert {
+   SVGA3dSurfaceId dstSid;
+   SVGA3dSurfaceId srcSid;
+}
+#include "vmware_pack_end.h"
+SVGA3dCmdDXPredConvert; /* SVGA_3D_CMD_DX_PRED_CONVERT */
+
+typedef
+#include "vmware_pack_begin.h"
 struct SVGA3dCmdDXBufferCopy {
    SVGA3dSurfaceId dest;
    SVGA3dSurfaceId src;
@@ -636,23 +665,57 @@ struct SVGA3dCmdDXBufferCopy {
 SVGA3dCmdDXBufferCopy;
 /* SVGA_3D_CMD_DX_BUFFER_COPY */
 
-typedef uint32 SVGA3dDXStretchBltMode;
-#define SVGADX_STRETCHBLT_LINEAR         (1 << 0)
-#define SVGADX_STRETCHBLT_FORCE_SRC_SRGB (1 << 1)
+/*
+ * Perform a surface copy between a multisample, and a non-multisampled
+ * surface.
+ */
+typedef
+#include "vmware_pack_begin.h"
+struct {
+   SVGA3dSurfaceId dstSid;
+   uint32 dstSubResource;
+   SVGA3dSurfaceId srcSid;
+   uint32 srcSubResource;
+   SVGA3dSurfaceFormat copyFormat;
+}
+#include "vmware_pack_end.h"
+SVGA3dCmdDXResolveCopy;               /* SVGA_3D_CMD_DX_RESOLVE_COPY */
+
+/*
+ * Perform a predicated surface copy between a multisample, and a
+ * non-multisampled surface.
+ */
+typedef
+#include "vmware_pack_begin.h"
+struct {
+   SVGA3dSurfaceId dstSid;
+   uint32 dstSubResource;
+   SVGA3dSurfaceId srcSid;
+   uint32 srcSubResource;
+   SVGA3dSurfaceFormat copyFormat;
+}
+#include "vmware_pack_end.h"
+SVGA3dCmdDXPredResolveCopy;           /* SVGA_3D_CMD_DX_PRED_RESOLVE_COPY */
+
+typedef uint32 SVGA3dDXPresentBltMode;
+#define SVGADX_PRESENTBLT_LINEAR           (1 << 0)
+#define SVGADX_PRESENTBLT_FORCE_SRC_SRGB   (1 << 1)
+#define SVGADX_PRESENTBLT_FORCE_SRC_XRBIAS (1 << 2)
+#define SVGADX_PRESENTBLT_MODE_MAX         (1 << 3)
 
 typedef
 #include "vmware_pack_begin.h"
-struct SVGA3dCmdDXStretchBlt {
+struct SVGA3dCmdDXPresentBlt {
    SVGA3dSurfaceId srcSid;
    uint32 srcSubResource;
    SVGA3dSurfaceId dstSid;
    uint32 destSubResource;
    SVGA3dBox boxSrc;
    SVGA3dBox boxDest;
-   SVGA3dDXStretchBltMode mode;
+   SVGA3dDXPresentBltMode mode;
 }
 #include "vmware_pack_end.h"
-SVGA3dCmdDXStretchBlt; /* SVGA_3D_CMD_DX_STRETCHBLT */
+SVGA3dCmdDXPresentBlt; /* SVGA_3D_CMD_DX_PRESENTBLT*/
 
 typedef
 #include "vmware_pack_begin.h"
@@ -663,26 +726,6 @@ struct SVGA3dCmdDXGenMips {
 SVGA3dCmdDXGenMips; /* SVGA_3D_CMD_DX_GENMIPS */
 
 /*
- * Defines a resource/DX surface.  Resources share the surfaceId namespace.
- *
- */
-typedef
-#include "vmware_pack_begin.h"
-struct SVGA3dCmdDefineGBSurface_v2 {
-   uint32 sid;
-   SVGA3dSurfaceFlags surfaceFlags;
-   SVGA3dSurfaceFormat format;
-   uint32 numMipLevels;
-   uint32 multisampleCount;
-   SVGA3dTextureFilter autogenFilter;
-   SVGA3dSize size;
-   uint32 arraySize;
-   uint32 pad;
-}
-#include "vmware_pack_end.h"
-SVGA3dCmdDefineGBSurface_v2;   /* SVGA_3D_CMD_DEFINE_GB_SURFACE_V2 */
-
-/*
  * Update a sub-resource in a guest-backed resource.
  * (Inform the device that the guest-contents have been updated.)
  */
@@ -725,7 +768,8 @@ SVGA3dCmdDXInvalidateSubResource;   /* SVGA_3D_CMD_DX_INVALIDATE_SUBRESOURCE */
 
 /*
  * Raw byte wise transfer from a buffer surface into another surface
- * of the requested box.
+ * of the requested box.  Supported if 3d is enabled and SVGA_CAP_DX
+ * is set.  This command does not take a context.
  */
 typedef
 #include "vmware_pack_begin.h"
@@ -774,6 +818,93 @@ struct SVGA3dCmdDXSurfaceCopyAndReadback {
 SVGA3dCmdDXSurfaceCopyAndReadback;
 /* SVGA_3D_CMD_DX_SURFACE_COPY_AND_READBACK */
 
+/*
+ * SVGA_DX_HINT_NONE: Does nothing.
+ *
+ * SVGA_DX_HINT_PREFETCH_OBJECT:
+ * SVGA_DX_HINT_PREEVICT_OBJECT:
+ *      Consumes a SVGAObjectRef, and hints that the host should consider
+ *      fetching/evicting the specified object.
+ *
+ *      An id of SVGA3D_INVALID_ID can be used if the guest isn't sure
+ *      what object was affected.  (For instance, if the guest knows that
+ *      it is about to evict a DXShader, but doesn't know precisely which one,
+ *      the device can still use this to help limit it's search, or track
+ *      how many page-outs have happened.)
+ *
+ * SVGA_DX_HINT_PREFETCH_COBJECT:
+ * SVGA_DX_HINT_PREEVICT_COBJECT:
+ *      Same as the above, except they consume an SVGACObjectRef.
+ */
+typedef uint32 SVGADXHintId;
+#define SVGA_DX_HINT_NONE              0
+#define SVGA_DX_HINT_PREFETCH_OBJECT   1
+#define SVGA_DX_HINT_PREEVICT_OBJECT   2
+#define SVGA_DX_HINT_PREFETCH_COBJECT  3
+#define SVGA_DX_HINT_PREEVICT_COBJECT  4
+#define SVGA_DX_HINT_MAX               5
+
+typedef
+#include "vmware_pack_begin.h"
+struct SVGAObjectRef {
+   SVGAOTableType type;
+   uint32 id;
+}
+#include "vmware_pack_end.h"
+SVGAObjectRef;
+
+typedef
+#include "vmware_pack_begin.h"
+struct SVGACObjectRef {
+   SVGACOTableType type;
+   uint32 cid;
+   uint32 id;
+}
+#include "vmware_pack_end.h"
+SVGACObjectRef;
+
+typedef
+#include "vmware_pack_begin.h"
+struct SVGA3dCmdDXHint {
+   SVGADXHintId hintId;
+
+   /*
+    * Followed by variable sized data depending on the hintId.
+    */
+}
+#include "vmware_pack_end.h"
+SVGA3dCmdDXHint;
+/* SVGA_3D_CMD_DX_HINT */
+
+typedef
+#include "vmware_pack_begin.h"
+struct SVGA3dCmdDXBufferUpdate {
+   SVGA3dSurfaceId sid;
+   uint32 x;
+   uint32 width;
+}
+#include "vmware_pack_end.h"
+SVGA3dCmdDXBufferUpdate;
+/* SVGA_3D_CMD_DX_BUFFER_UPDATE */
+
+typedef
+#include "vmware_pack_begin.h"
+struct SVGA3dCmdDXSetConstantBufferOffset {
+   uint32 slot;
+   uint32 offsetInBytes;
+}
+#include "vmware_pack_end.h"
+SVGA3dCmdDXSetConstantBufferOffset;
+
+typedef SVGA3dCmdDXSetConstantBufferOffset SVGA3dCmdDXSetVSConstantBufferOffset;
+/* SVGA_3D_CMD_DX_SET_VS_CONSTANT_BUFFER_OFFSET */
+
+typedef SVGA3dCmdDXSetConstantBufferOffset SVGA3dCmdDXSetPSConstantBufferOffset;
+/* SVGA_3D_CMD_DX_SET_PS_CONSTANT_BUFFER_OFFSET */
+
+typedef SVGA3dCmdDXSetConstantBufferOffset SVGA3dCmdDXSetGSConstantBufferOffset;
+/* SVGA_3D_CMD_DX_SET_GS_CONSTANT_BUFFER_OFFSET */
+
 
 typedef
 #include "vmware_pack_begin.h"
@@ -790,7 +921,7 @@ struct {
          uint32 firstArraySlice;
          uint32 mipLevels;
          uint32 arraySize;
-      } tex;
+      } tex; /* 1d, 2d, 3d, cube */
       struct {
          uint32 firstElement;
          uint32 numElements;
@@ -845,6 +976,7 @@ struct SVGA3dRenderTargetViewDesc {
       struct {
          uint32 firstElement;
          uint32 numElements;
+         uint32 padding0;
       } buffer;
       struct {
          uint32 mipSlice;
@@ -965,9 +1097,6 @@ SVGA3dInputElementDesc;
 typedef
 #include "vmware_pack_begin.h"
 struct {
-   /*
-    * XXX: How many of these can there be?
-    */
    uint32 elid;
    uint32 numDescs;
    SVGA3dInputElementDesc desc[32];
@@ -1008,7 +1137,7 @@ struct SVGA3dDXBlendStatePerRT {
       uint8 srcBlendAlpha;
       uint8 destBlendAlpha;
       uint8 blendOpAlpha;
-      uint8 renderTargetWriteMask;
+      SVGA3dColorWriteEnable renderTargetWriteMask;
       uint8 logicOpEnable;
       uint8 logicOp;
       uint16 pad0;
@@ -1126,7 +1255,7 @@ struct {
    float slopeScaledDepthBias;
    uint8 depthClipEnable;
    uint8 scissorEnable;
-   uint8 multisampleEnable;
+   SVGA3dMultisampleRastEnable multisampleEnable;
    uint8 antialiasedLineEnable;
    float lineWidth;
    uint8 lineStippleEnable;
@@ -1153,7 +1282,7 @@ struct SVGA3dCmdDXDefineRasterizerState {
    float slopeScaledDepthBias;
    uint8 depthClipEnable;
    uint8 scissorEnable;
-   uint8 multisampleEnable;
+   SVGA3dMultisampleRastEnable multisampleEnable;
    uint8 antialiasedLineEnable;
    float lineWidth;
    uint8 lineStippleEnable;
@@ -1223,21 +1352,6 @@ struct SVGA3dCmdDXDestroySamplerState {
 #include "vmware_pack_end.h"
 SVGA3dCmdDXDestroySamplerState; /* SVGA_3D_CMD_DX_DESTROY_SAMPLER_STATE */
 
-/*
- */
-typedef
-#include "vmware_pack_begin.h"
-struct SVGA3dSignatureEntry {
-   uint8 systemValue;
-   uint8 reg;                 /* register is a reserved word */
-   uint16 mask;
-   uint8 registerComponentType;
-   uint8 minPrecision;
-   uint16 pad0;
-}
-#include "vmware_pack_end.h"
-SVGA3dSignatureEntry;
-
 typedef
 #include "vmware_pack_begin.h"
 struct SVGA3dCmdDXDefineShader {
@@ -1255,12 +1369,7 @@ struct SVGACOTableDXShaderEntry {
    uint32 sizeInBytes;
    uint32 offsetInBytes;
    SVGAMobId mobid;
-   uint32 numInputSignatureEntries;
-   uint32 numOutputSignatureEntries;
-
-   uint32 numPatchConstantSignatureEntries;
-
-   uint32 pad;
+   uint32 pad[4];
 }
 #include "vmware_pack_end.h"
 SVGACOTableDXShaderEntry;
@@ -1284,6 +1393,25 @@ struct SVGA3dCmdDXBindShader {
 #include "vmware_pack_end.h"
 SVGA3dCmdDXBindShader;   /* SVGA_3D_CMD_DX_BIND_SHADER */
 
+typedef
+#include "vmware_pack_begin.h"
+struct SVGA3dCmdDXBindAllShader {
+   uint32 cid;
+   SVGAMobId mobid;
+}
+#include "vmware_pack_end.h"
+SVGA3dCmdDXBindAllShader;   /* SVGA_3D_CMD_DX_BIND_ALL_SHADER */
+
+typedef
+#include "vmware_pack_begin.h"
+struct SVGA3dCmdDXCondBindAllShader {
+   uint32 cid;
+   SVGAMobId testMobid;
+   SVGAMobId mobid;
+}
+#include "vmware_pack_end.h"
+SVGA3dCmdDXCondBindAllShader;   /* SVGA_3D_CMD_DX_COND_BIND_ALL_SHADER */
+
 /*
  * The maximum number of streamout decl's in each streamout entry.
  */
@@ -1357,7 +1485,6 @@ SVGA3dCmdDXMobFence64;  /* SVGA_3D_CMD_DX_MOB_FENCE_64 */
  *
  * This command allows the guest to bind a mob to a context-object table.
  */
-
 typedef
 #include "vmware_pack_begin.h"
 struct SVGA3dCmdDXSetCOTable {
@@ -1369,6 +1496,26 @@ struct SVGA3dCmdDXSetCOTable {
 #include "vmware_pack_end.h"
 SVGA3dCmdDXSetCOTable; /* SVGA_3D_CMD_DX_SET_COTABLE */
 
+/*
+ * Guests using SVGA_3D_CMD_DX_GROW_COTABLE are promising that
+ * the new COTable contains the same contents as the old one, except possibly
+ * for some new invalid entries at the end.
+ *
+ * If there is an old cotable mob bound, it also has to still be valid.
+ *
+ * (Otherwise, guests should use the DXSetCOTableBase command.)
+ */
+typedef
+#include "vmware_pack_begin.h"
+struct SVGA3dCmdDXGrowCOTable {
+   uint32 cid;
+   uint32 mobid;
+   SVGACOTableType type;
+   uint32 validSizeInBytes;
+}
+#include "vmware_pack_end.h"
+SVGA3dCmdDXGrowCOTable; /* SVGA_3D_CMD_DX_GROW_COTABLE */
+
 typedef
 #include "vmware_pack_begin.h"
 struct SVGA3dCmdDXReadbackCOTable {
@@ -1472,7 +1619,7 @@ struct SVGADXContextMobFormat {
    SVGA3dQueryId queryID[SVGA3D_MAX_QUERY];
 
    SVGA3dCOTableData cotables[SVGA_COTABLE_MAX];
-   uint32 pad7[381];
+   uint32 pad7[380];
 }
 #include "vmware_pack_end.h"
 SVGADXContextMobFormat;
diff --git a/drivers/gpu/drm/vmwgfx/device_include/svga3d_limits.h b/drivers/gpu/drm/vmwgfx/device_include/svga3d_limits.h
index 8732490d45d5..b22a67f15660 100644
--- a/drivers/gpu/drm/vmwgfx/device_include/svga3d_limits.h
+++ b/drivers/gpu/drm/vmwgfx/device_include/svga3d_limits.h
@@ -63,7 +63,9 @@
  * Maximum size in dwords of shader text the SVGA device will allow.
  * Currently 8 MB.
  */
-#define SVGA3D_MAX_SHADER_MEMORY  (8 * 1024 * 1024 / sizeof(uint32))
+#define SVGA3D_MAX_SHADER_MEMORY_BYTES (8 * 1024 * 1024)
+#define SVGA3D_MAX_SHADER_MEMORY  (SVGA3D_MAX_SHADER_MEMORY_BYTES / \
+                                   sizeof(uint32))
 
 #define SVGA3D_MAX_CLIP_PLANES    6
 
diff --git a/drivers/gpu/drm/vmwgfx/device_include/svga3d_surfacedefs.h b/drivers/gpu/drm/vmwgfx/device_include/svga3d_surfacedefs.h
index b45e96309233..f2bfd3d80598 100644
--- a/drivers/gpu/drm/vmwgfx/device_include/svga3d_surfacedefs.h
+++ b/drivers/gpu/drm/vmwgfx/device_include/svga3d_surfacedefs.h
@@ -25,189 +25,355 @@
  *
  **************************************************************************/
 
-#include <linux/kernel.h>
-
-#ifdef __KERNEL__
-
-#include <drm/vmwgfx_drm.h>
-#define surf_size_struct struct drm_vmw_size
-
-#else /* __KERNEL__ */
+/*
+ * svga3d_surfacedefs.h --
+ *
+ *      Surface definitions and inlineable utilities for SVGA3d.
+ */
 
-#ifndef ARRAY_SIZE
-#define ARRAY_SIZE(_A) (sizeof(_A) / sizeof((_A)[0]))
-#endif /* ARRAY_SIZE */
+#ifndef _SVGA3D_SURFACEDEFS_H_
+#define _SVGA3D_SURFACEDEFS_H_
 
-#define max_t(type, x, y)  ((x) > (y) ? (x) : (y))
-#define surf_size_struct SVGA3dSize
-#define u32 uint32
+#define INCLUDE_ALLOW_USERLEVEL
+#define INCLUDE_ALLOW_MODULE
+#include "includeCheck.h"
 
-#endif /* __KERNEL__ */
+#include <linux/kernel.h>
+#include <drm/vmwgfx_drm.h>
 
 #include "svga3d_reg.h"
 
+#define surf_size_struct struct drm_vmw_size
+
 /*
- * enum svga3d_block_desc describes the active data channels in a block.
- *
- * There can be at-most four active channels in a block:
- *    1. Red, bump W, luminance and depth are stored in the first channel.
- *    2. Green, bump V and stencil are stored in the second channel.
- *    3. Blue and bump U are stored in the third channel.
- *    4. Alpha and bump Q are stored in the fourth channel.
- *
- * Block channels can be used to store compressed and buffer data:
- *    1. For compressed formats, only the data channel is used and its size
- *       is equal to that of a singular block in the compression scheme.
- *    2. For buffer formats, only the data channel is used and its size is
- *       exactly one byte in length.
- *    3. In each case the bit depth represent the size of a singular block.
- *
- * Note: Compressed and IEEE formats do not use the bitMask structure.
+ * enum svga3d_block_desc - describes generic properties about formats.
  */
-
 enum svga3d_block_desc {
-	SVGA3DBLOCKDESC_NONE        = 0,         /* No channels are active */
-	SVGA3DBLOCKDESC_BLUE        = 1 << 0,    /* Block with red channel
-						    data */
-	SVGA3DBLOCKDESC_U           = 1 << 0,    /* Block with bump U channel
-						    data */
-	SVGA3DBLOCKDESC_UV_VIDEO    = 1 << 7,    /* Block with alternating video
-						    U and V */
-	SVGA3DBLOCKDESC_GREEN       = 1 << 1,    /* Block with green channel
-						    data */
-	SVGA3DBLOCKDESC_V           = 1 << 1,    /* Block with bump V channel
-						    data */
-	SVGA3DBLOCKDESC_STENCIL     = 1 << 1,    /* Block with a stencil
-						    channel */
-	SVGA3DBLOCKDESC_RED         = 1 << 2,    /* Block with blue channel
-						    data */
-	SVGA3DBLOCKDESC_W           = 1 << 2,    /* Block with bump W channel
-						    data */
-	SVGA3DBLOCKDESC_LUMINANCE   = 1 << 2,    /* Block with luminance channel
-						    data */
-	SVGA3DBLOCKDESC_Y           = 1 << 2,    /* Block with video luminance
-						    data */
-	SVGA3DBLOCKDESC_DEPTH       = 1 << 2,    /* Block with depth channel */
-	SVGA3DBLOCKDESC_ALPHA       = 1 << 3,    /* Block with an alpha
-						    channel */
-	SVGA3DBLOCKDESC_Q           = 1 << 3,    /* Block with bump Q channel
-						    data */
-	SVGA3DBLOCKDESC_BUFFER      = 1 << 4,    /* Block stores 1 byte of
-						    data */
-	SVGA3DBLOCKDESC_COMPRESSED  = 1 << 5,    /* Block stores n bytes of
-						    data depending on the
-						    compression method used */
-	SVGA3DBLOCKDESC_IEEE_FP     = 1 << 6,    /* Block stores data in an IEEE
-						    floating point
-						    representation in
-						    all channels */
-	SVGA3DBLOCKDESC_PLANAR_YUV  = 1 << 8,    /* Three separate blocks store
-						    data. */
-	SVGA3DBLOCKDESC_U_VIDEO     = 1 << 9,    /* Block with U video data */
-	SVGA3DBLOCKDESC_V_VIDEO     = 1 << 10,   /* Block with V video data */
-	SVGA3DBLOCKDESC_EXP         = 1 << 11,   /* Shared exponent */
-	SVGA3DBLOCKDESC_SRGB        = 1 << 12,   /* Data is in sRGB format */
-	SVGA3DBLOCKDESC_2PLANAR_YUV = 1 << 13,   /* 2 planes of Y, UV,
-						    e.g., NV12. */
-	SVGA3DBLOCKDESC_3PLANAR_YUV = 1 << 14,   /* 3 planes of separate
-						    Y, U, V, e.g., YV12. */
-
-	SVGA3DBLOCKDESC_RG         = SVGA3DBLOCKDESC_RED |
-	SVGA3DBLOCKDESC_GREEN,
-	SVGA3DBLOCKDESC_RGB        = SVGA3DBLOCKDESC_RG |
-	SVGA3DBLOCKDESC_BLUE,
-	SVGA3DBLOCKDESC_RGB_SRGB   = SVGA3DBLOCKDESC_RGB |
-	SVGA3DBLOCKDESC_SRGB,
-	SVGA3DBLOCKDESC_RGBA       = SVGA3DBLOCKDESC_RGB |
-	SVGA3DBLOCKDESC_ALPHA,
-	SVGA3DBLOCKDESC_RGBA_SRGB  = SVGA3DBLOCKDESC_RGBA |
-	SVGA3DBLOCKDESC_SRGB,
+	/* Nothing special can be said about this format. */
+	SVGA3DBLOCKDESC_NONE        = 0,
+
+	/* Format contains Blue/U data */
+	SVGA3DBLOCKDESC_BLUE        = 1 << 0,
+	SVGA3DBLOCKDESC_W           = 1 << 0,
+	SVGA3DBLOCKDESC_BUMP_L      = 1 << 0,
+
+	/* Format contains Green/V data */
+	SVGA3DBLOCKDESC_GREEN       = 1 << 1,
+	SVGA3DBLOCKDESC_V           = 1 << 1,
+
+	/* Format contains Red/W/Luminance data */
+	SVGA3DBLOCKDESC_RED         = 1 << 2,
+	SVGA3DBLOCKDESC_U           = 1 << 2,
+	SVGA3DBLOCKDESC_LUMINANCE   = 1 << 2,
+
+	/* Format contains Alpha/Q data */
+	SVGA3DBLOCKDESC_ALPHA       = 1 << 3,
+	SVGA3DBLOCKDESC_Q           = 1 << 3,
+
+	/* Format is a buffer */
+	SVGA3DBLOCKDESC_BUFFER      = 1 << 4,
+
+	/* Format is compressed */
+	SVGA3DBLOCKDESC_COMPRESSED  = 1 << 5,
+
+	/* Format uses IEEE floating point */
+	SVGA3DBLOCKDESC_FP          = 1 << 6,
+
+	/* Three separate blocks store data. */
+	SVGA3DBLOCKDESC_PLANAR_YUV  = 1 << 7,
+
+	/* 2 planes of Y, UV, e.g., NV12. */
+	SVGA3DBLOCKDESC_2PLANAR_YUV = 1 << 8,
+
+	/* 3 planes of separate Y, U, V, e.g., YV12. */
+	SVGA3DBLOCKDESC_3PLANAR_YUV = 1 << 9,
+
+	/* Block with a stencil channel */
+	SVGA3DBLOCKDESC_STENCIL     = 1 << 11,
+
+	/* Typeless format */
+	SVGA3DBLOCKDESC_TYPELESS    = 1 << 12,
+
+	/* Channels are signed integers */
+	SVGA3DBLOCKDESC_SINT        = 1 << 13,
+
+	/* Channels are unsigned integers */
+	SVGA3DBLOCKDESC_UINT        = 1 << 14,
+
+	/* Channels are normalized (when sampling) */
+	SVGA3DBLOCKDESC_NORM        = 1 << 15,
+
+	/* Channels are in SRGB */
+	SVGA3DBLOCKDESC_SRGB        = 1 << 16,
+
+	/* Shared exponent */
+	SVGA3DBLOCKDESC_EXP         = 1 << 17,
+
+	/* Format contains color data. */
+	SVGA3DBLOCKDESC_COLOR       = 1 << 18,
+	/* Format contains depth data. */
+	SVGA3DBLOCKDESC_DEPTH       = 1 << 19,
+	/* Format contains bump data. */
+	SVGA3DBLOCKDESC_BUMP        = 1 << 20,
+
+	/* Format contains YUV video data. */
+	SVGA3DBLOCKDESC_YUV_VIDEO   = 1 << 21,
+
+	/* For mixed unsigned/signed formats. */
+	SVGA3DBLOCKDESC_MIXED       = 1 << 22,
+
+	/* For distingushing CxV8U8. */
+	SVGA3DBLOCKDESC_CX          = 1 << 23,
+
+	/* Different compressed format groups. */
+	SVGA3DBLOCKDESC_BC1         = 1 << 24,
+	SVGA3DBLOCKDESC_BC2         = 1 << 25,
+	SVGA3DBLOCKDESC_BC3         = 1 << 26,
+	SVGA3DBLOCKDESC_BC4         = 1 << 27,
+	SVGA3DBLOCKDESC_BC5         = 1 << 28,
+
+	SVGA3DBLOCKDESC_A_UINT    = SVGA3DBLOCKDESC_ALPHA |
+				    SVGA3DBLOCKDESC_UINT |
+				    SVGA3DBLOCKDESC_COLOR,
+	SVGA3DBLOCKDESC_A_UNORM   = SVGA3DBLOCKDESC_A_UINT |
+				    SVGA3DBLOCKDESC_NORM,
+	SVGA3DBLOCKDESC_R_UINT    = SVGA3DBLOCKDESC_RED |
+				    SVGA3DBLOCKDESC_UINT |
+				    SVGA3DBLOCKDESC_COLOR,
+	SVGA3DBLOCKDESC_R_UNORM   = SVGA3DBLOCKDESC_R_UINT |
+				    SVGA3DBLOCKDESC_NORM,
+	SVGA3DBLOCKDESC_R_SINT    = SVGA3DBLOCKDESC_RED |
+				    SVGA3DBLOCKDESC_SINT |
+				    SVGA3DBLOCKDESC_COLOR,
+	SVGA3DBLOCKDESC_R_SNORM   = SVGA3DBLOCKDESC_R_SINT |
+				    SVGA3DBLOCKDESC_NORM,
+	SVGA3DBLOCKDESC_G_UINT    = SVGA3DBLOCKDESC_GREEN |
+				    SVGA3DBLOCKDESC_UINT |
+				    SVGA3DBLOCKDESC_COLOR,
+	SVGA3DBLOCKDESC_RG_UINT    = SVGA3DBLOCKDESC_RED |
+				     SVGA3DBLOCKDESC_GREEN |
+				     SVGA3DBLOCKDESC_UINT |
+				     SVGA3DBLOCKDESC_COLOR,
+	SVGA3DBLOCKDESC_RG_UNORM   = SVGA3DBLOCKDESC_RG_UINT |
+				     SVGA3DBLOCKDESC_NORM,
+	SVGA3DBLOCKDESC_RG_SINT    = SVGA3DBLOCKDESC_RED |
+				     SVGA3DBLOCKDESC_GREEN |
+				     SVGA3DBLOCKDESC_SINT |
+				     SVGA3DBLOCKDESC_COLOR,
+	SVGA3DBLOCKDESC_RG_SNORM   = SVGA3DBLOCKDESC_RG_SINT |
+				     SVGA3DBLOCKDESC_NORM,
+	SVGA3DBLOCKDESC_RGB_UINT   = SVGA3DBLOCKDESC_RED |
+				     SVGA3DBLOCKDESC_GREEN |
+				     SVGA3DBLOCKDESC_BLUE |
+				     SVGA3DBLOCKDESC_UINT |
+				     SVGA3DBLOCKDESC_COLOR,
+	SVGA3DBLOCKDESC_RGB_SINT   = SVGA3DBLOCKDESC_RED |
+				     SVGA3DBLOCKDESC_GREEN |
+				     SVGA3DBLOCKDESC_BLUE |
+				     SVGA3DBLOCKDESC_SINT |
+				     SVGA3DBLOCKDESC_COLOR,
+	SVGA3DBLOCKDESC_RGB_UNORM   = SVGA3DBLOCKDESC_RGB_UINT |
+				      SVGA3DBLOCKDESC_NORM,
+	SVGA3DBLOCKDESC_RGB_UNORM_SRGB = SVGA3DBLOCKDESC_RGB_UNORM |
+					 SVGA3DBLOCKDESC_SRGB,
+	SVGA3DBLOCKDESC_RGBA_UINT  = SVGA3DBLOCKDESC_RED |
+				     SVGA3DBLOCKDESC_GREEN |
+				     SVGA3DBLOCKDESC_BLUE |
+				     SVGA3DBLOCKDESC_ALPHA |
+				     SVGA3DBLOCKDESC_UINT |
+				     SVGA3DBLOCKDESC_COLOR,
+	SVGA3DBLOCKDESC_RGBA_UNORM = SVGA3DBLOCKDESC_RGBA_UINT |
+				     SVGA3DBLOCKDESC_NORM,
+	SVGA3DBLOCKDESC_RGBA_UNORM_SRGB = SVGA3DBLOCKDESC_RGBA_UNORM |
+					  SVGA3DBLOCKDESC_SRGB,
+	SVGA3DBLOCKDESC_RGBA_SINT  = SVGA3DBLOCKDESC_RED |
+				     SVGA3DBLOCKDESC_GREEN |
+				     SVGA3DBLOCKDESC_BLUE |
+				     SVGA3DBLOCKDESC_ALPHA |
+				     SVGA3DBLOCKDESC_SINT |
+				     SVGA3DBLOCKDESC_COLOR,
+	SVGA3DBLOCKDESC_RGBA_SNORM = SVGA3DBLOCKDESC_RGBA_SINT |
+				     SVGA3DBLOCKDESC_NORM,
+	SVGA3DBLOCKDESC_RGBA_FP    = SVGA3DBLOCKDESC_RED |
+				     SVGA3DBLOCKDESC_GREEN |
+				     SVGA3DBLOCKDESC_BLUE |
+				     SVGA3DBLOCKDESC_ALPHA |
+				     SVGA3DBLOCKDESC_FP |
+				     SVGA3DBLOCKDESC_COLOR,
 	SVGA3DBLOCKDESC_UV         = SVGA3DBLOCKDESC_U |
-	SVGA3DBLOCKDESC_V,
+				     SVGA3DBLOCKDESC_V |
+				     SVGA3DBLOCKDESC_BUMP,
 	SVGA3DBLOCKDESC_UVL        = SVGA3DBLOCKDESC_UV |
-	SVGA3DBLOCKDESC_LUMINANCE,
+				     SVGA3DBLOCKDESC_BUMP_L |
+				     SVGA3DBLOCKDESC_MIXED |
+				     SVGA3DBLOCKDESC_BUMP,
 	SVGA3DBLOCKDESC_UVW        = SVGA3DBLOCKDESC_UV |
-	SVGA3DBLOCKDESC_W,
+				     SVGA3DBLOCKDESC_W |
+				     SVGA3DBLOCKDESC_BUMP,
 	SVGA3DBLOCKDESC_UVWA       = SVGA3DBLOCKDESC_UVW |
-	SVGA3DBLOCKDESC_ALPHA,
+				     SVGA3DBLOCKDESC_ALPHA |
+				     SVGA3DBLOCKDESC_MIXED |
+				     SVGA3DBLOCKDESC_BUMP,
 	SVGA3DBLOCKDESC_UVWQ       = SVGA3DBLOCKDESC_U |
-	SVGA3DBLOCKDESC_V |
-	SVGA3DBLOCKDESC_W |
-	SVGA3DBLOCKDESC_Q,
-	SVGA3DBLOCKDESC_LA         = SVGA3DBLOCKDESC_LUMINANCE |
-	SVGA3DBLOCKDESC_ALPHA,
+				     SVGA3DBLOCKDESC_V |
+				     SVGA3DBLOCKDESC_W |
+				     SVGA3DBLOCKDESC_Q |
+				     SVGA3DBLOCKDESC_BUMP,
+	SVGA3DBLOCKDESC_L_UNORM    = SVGA3DBLOCKDESC_LUMINANCE |
+				     SVGA3DBLOCKDESC_UINT |
+				     SVGA3DBLOCKDESC_NORM |
+				     SVGA3DBLOCKDESC_COLOR,
+	SVGA3DBLOCKDESC_LA_UNORM   = SVGA3DBLOCKDESC_LUMINANCE |
+				     SVGA3DBLOCKDESC_ALPHA |
+				     SVGA3DBLOCKDESC_UINT |
+				     SVGA3DBLOCKDESC_NORM |
+				     SVGA3DBLOCKDESC_COLOR,
 	SVGA3DBLOCKDESC_R_FP       = SVGA3DBLOCKDESC_RED |
-	SVGA3DBLOCKDESC_IEEE_FP,
+				     SVGA3DBLOCKDESC_FP |
+				     SVGA3DBLOCKDESC_COLOR,
 	SVGA3DBLOCKDESC_RG_FP      = SVGA3DBLOCKDESC_R_FP |
-	SVGA3DBLOCKDESC_GREEN,
+				     SVGA3DBLOCKDESC_GREEN |
+				     SVGA3DBLOCKDESC_COLOR,
 	SVGA3DBLOCKDESC_RGB_FP     = SVGA3DBLOCKDESC_RG_FP |
-	SVGA3DBLOCKDESC_BLUE,
-	SVGA3DBLOCKDESC_RGBA_FP    = SVGA3DBLOCKDESC_RGB_FP |
-	SVGA3DBLOCKDESC_ALPHA,
-	SVGA3DBLOCKDESC_DS         = SVGA3DBLOCKDESC_DEPTH |
-	SVGA3DBLOCKDESC_STENCIL,
-	SVGA3DBLOCKDESC_YUV        = SVGA3DBLOCKDESC_UV_VIDEO |
-	SVGA3DBLOCKDESC_Y,
+				     SVGA3DBLOCKDESC_BLUE |
+				     SVGA3DBLOCKDESC_COLOR,
+	SVGA3DBLOCKDESC_YUV        = SVGA3DBLOCKDESC_YUV_VIDEO |
+				     SVGA3DBLOCKDESC_COLOR,
 	SVGA3DBLOCKDESC_AYUV       = SVGA3DBLOCKDESC_ALPHA |
-	SVGA3DBLOCKDESC_Y |
-	SVGA3DBLOCKDESC_U_VIDEO |
-	SVGA3DBLOCKDESC_V_VIDEO,
-	SVGA3DBLOCKDESC_RGBE       = SVGA3DBLOCKDESC_RGB |
-	SVGA3DBLOCKDESC_EXP,
-	SVGA3DBLOCKDESC_COMPRESSED_SRGB = SVGA3DBLOCKDESC_COMPRESSED |
-	SVGA3DBLOCKDESC_SRGB,
-	SVGA3DBLOCKDESC_NV12       = SVGA3DBLOCKDESC_PLANAR_YUV |
-	SVGA3DBLOCKDESC_2PLANAR_YUV,
-	SVGA3DBLOCKDESC_YV12       = SVGA3DBLOCKDESC_PLANAR_YUV |
-	SVGA3DBLOCKDESC_3PLANAR_YUV,
+				     SVGA3DBLOCKDESC_YUV_VIDEO |
+				     SVGA3DBLOCKDESC_COLOR,
+	SVGA3DBLOCKDESC_RGB_EXP       = SVGA3DBLOCKDESC_RED |
+					SVGA3DBLOCKDESC_GREEN |
+					SVGA3DBLOCKDESC_BLUE |
+					SVGA3DBLOCKDESC_EXP |
+					SVGA3DBLOCKDESC_COLOR,
+
+	SVGA3DBLOCKDESC_COMP_TYPELESS = SVGA3DBLOCKDESC_COMPRESSED |
+					SVGA3DBLOCKDESC_TYPELESS,
+	SVGA3DBLOCKDESC_COMP_UNORM = SVGA3DBLOCKDESC_COMPRESSED |
+				     SVGA3DBLOCKDESC_UINT |
+				     SVGA3DBLOCKDESC_NORM |
+				     SVGA3DBLOCKDESC_COLOR,
+	SVGA3DBLOCKDESC_COMP_SNORM = SVGA3DBLOCKDESC_COMPRESSED |
+				     SVGA3DBLOCKDESC_SINT |
+				     SVGA3DBLOCKDESC_NORM |
+				     SVGA3DBLOCKDESC_COLOR,
+	SVGA3DBLOCKDESC_COMP_UNORM_SRGB = SVGA3DBLOCKDESC_COMP_UNORM |
+					  SVGA3DBLOCKDESC_SRGB,
+	SVGA3DBLOCKDESC_BC1_COMP_TYPELESS = SVGA3DBLOCKDESC_BC1 |
+					    SVGA3DBLOCKDESC_COMP_TYPELESS,
+	SVGA3DBLOCKDESC_BC1_COMP_UNORM = SVGA3DBLOCKDESC_BC1 |
+					 SVGA3DBLOCKDESC_COMP_UNORM,
+	SVGA3DBLOCKDESC_BC1_COMP_UNORM_SRGB = SVGA3DBLOCKDESC_BC1_COMP_UNORM |
+					      SVGA3DBLOCKDESC_SRGB,
+	SVGA3DBLOCKDESC_BC2_COMP_TYPELESS = SVGA3DBLOCKDESC_BC2 |
+					    SVGA3DBLOCKDESC_COMP_TYPELESS,
+	SVGA3DBLOCKDESC_BC2_COMP_UNORM = SVGA3DBLOCKDESC_BC2 |
+					 SVGA3DBLOCKDESC_COMP_UNORM,
+	SVGA3DBLOCKDESC_BC2_COMP_UNORM_SRGB = SVGA3DBLOCKDESC_BC2_COMP_UNORM |
+					      SVGA3DBLOCKDESC_SRGB,
+	SVGA3DBLOCKDESC_BC3_COMP_TYPELESS = SVGA3DBLOCKDESC_BC3 |
+					    SVGA3DBLOCKDESC_COMP_TYPELESS,
+	SVGA3DBLOCKDESC_BC3_COMP_UNORM = SVGA3DBLOCKDESC_BC3 |
+					 SVGA3DBLOCKDESC_COMP_UNORM,
+	SVGA3DBLOCKDESC_BC3_COMP_UNORM_SRGB = SVGA3DBLOCKDESC_BC3_COMP_UNORM |
+					      SVGA3DBLOCKDESC_SRGB,
+	SVGA3DBLOCKDESC_BC4_COMP_TYPELESS = SVGA3DBLOCKDESC_BC4 |
+					    SVGA3DBLOCKDESC_COMP_TYPELESS,
+	SVGA3DBLOCKDESC_BC4_COMP_UNORM = SVGA3DBLOCKDESC_BC4 |
+					 SVGA3DBLOCKDESC_COMP_UNORM,
+	SVGA3DBLOCKDESC_BC4_COMP_SNORM = SVGA3DBLOCKDESC_BC4 |
+					 SVGA3DBLOCKDESC_COMP_SNORM,
+	SVGA3DBLOCKDESC_BC5_COMP_TYPELESS = SVGA3DBLOCKDESC_BC5 |
+					    SVGA3DBLOCKDESC_COMP_TYPELESS,
+	SVGA3DBLOCKDESC_BC5_COMP_UNORM = SVGA3DBLOCKDESC_BC5 |
+					 SVGA3DBLOCKDESC_COMP_UNORM,
+	SVGA3DBLOCKDESC_BC5_COMP_SNORM = SVGA3DBLOCKDESC_BC5 |
+					 SVGA3DBLOCKDESC_COMP_SNORM,
+
+	SVGA3DBLOCKDESC_NV12       = SVGA3DBLOCKDESC_YUV_VIDEO |
+				     SVGA3DBLOCKDESC_PLANAR_YUV |
+				     SVGA3DBLOCKDESC_2PLANAR_YUV |
+				     SVGA3DBLOCKDESC_COLOR,
+	SVGA3DBLOCKDESC_YV12       = SVGA3DBLOCKDESC_YUV_VIDEO |
+				     SVGA3DBLOCKDESC_PLANAR_YUV |
+				     SVGA3DBLOCKDESC_3PLANAR_YUV |
+				     SVGA3DBLOCKDESC_COLOR,
+
+	SVGA3DBLOCKDESC_DEPTH_UINT = SVGA3DBLOCKDESC_DEPTH |
+				     SVGA3DBLOCKDESC_UINT,
+	SVGA3DBLOCKDESC_DEPTH_UNORM = SVGA3DBLOCKDESC_DEPTH_UINT |
+				     SVGA3DBLOCKDESC_NORM,
+	SVGA3DBLOCKDESC_DS      =    SVGA3DBLOCKDESC_DEPTH |
+				     SVGA3DBLOCKDESC_STENCIL,
+	SVGA3DBLOCKDESC_DS_UINT =    SVGA3DBLOCKDESC_DEPTH |
+				     SVGA3DBLOCKDESC_STENCIL |
+				     SVGA3DBLOCKDESC_UINT,
+	SVGA3DBLOCKDESC_DS_UNORM =   SVGA3DBLOCKDESC_DS_UINT |
+				     SVGA3DBLOCKDESC_NORM,
+	SVGA3DBLOCKDESC_DEPTH_FP   = SVGA3DBLOCKDESC_DEPTH |
+				     SVGA3DBLOCKDESC_FP,
+
+	SVGA3DBLOCKDESC_UV_UINT    = SVGA3DBLOCKDESC_UV |
+				     SVGA3DBLOCKDESC_UINT,
+	SVGA3DBLOCKDESC_UV_SNORM   = SVGA3DBLOCKDESC_UV |
+				     SVGA3DBLOCKDESC_SINT |
+				     SVGA3DBLOCKDESC_NORM,
+	SVGA3DBLOCKDESC_UVCX_SNORM = SVGA3DBLOCKDESC_UV_SNORM |
+				     SVGA3DBLOCKDESC_CX,
+	SVGA3DBLOCKDESC_UVWQ_SNORM = SVGA3DBLOCKDESC_UVWQ |
+				     SVGA3DBLOCKDESC_SINT |
+				     SVGA3DBLOCKDESC_NORM,
 };
 
-/*
- * SVGA3dSurfaceDesc describes the actual pixel data.
- *
- * This structure provides the following information:
- *    1. Block description.
- *    2. Dimensions of a block in the surface.
- *    3. Size of block in bytes.
- *    4. Bit depth of the pixel data.
- *    5. Channel bit depths and masks (if applicable).
- */
 struct svga3d_channel_def {
 	union {
 		u8 blue;
-		u8 u;
+		u8 w_bump;
+		u8 l_bump;
 		u8 uv_video;
 		u8 u_video;
 	};
 	union {
 		u8 green;
-		u8 v;
 		u8 stencil;
+		u8 v_bump;
 		u8 v_video;
 	};
 	union {
 		u8 red;
-		u8 w;
+		u8 u_bump;
 		u8 luminance;
-		u8 y;
+		u8 y_video;
 		u8 depth;
 		u8 data;
 	};
 	union {
 		u8 alpha;
-		u8 q;
+		u8 q_bump;
 		u8 exp;
 	};
 };
 
+/*
+ * struct svga3d_surface_desc - describes the actual pixel data.
+ *
+ * @format: Format
+ * @block_desc: Block description
+ * @block_size: Dimensions in pixels of a block
+ * @bytes_per_block: Size of block in bytes
+ * @pitch_bytes_per_block: Size of a block in bytes for purposes of pitch
+ * @bit_depth: Channel bit depths
+ * @bit_offset: Channel bit masks (in bits offset from the start of the pointer)
+ */
 struct svga3d_surface_desc {
 	SVGA3dSurfaceFormat format;
 	enum svga3d_block_desc block_desc;
+
 	surf_size_struct block_size;
 	u32 bytes_per_block;
 	u32 pitch_bytes_per_block;
 
-	u32 total_bit_depth;
 	struct svga3d_channel_def bit_depth;
 	struct svga3d_channel_def bit_offset;
 };
@@ -215,729 +381,728 @@ struct svga3d_surface_desc {
 static const struct svga3d_surface_desc svga3d_surface_descs[] = {
    {SVGA3D_FORMAT_INVALID, SVGA3DBLOCKDESC_NONE,
       {1, 1, 1},  0, 0,
-      0, {{0}, {0}, {0}, {0}},
+      {{0}, {0}, {0}, {0}},
       {{0}, {0}, {0}, {0}}},
 
-   {SVGA3D_X8R8G8B8, SVGA3DBLOCKDESC_RGB,
+   {SVGA3D_X8R8G8B8, SVGA3DBLOCKDESC_RGB_UNORM,
       {1, 1, 1},  4, 4,
-      24, {{8}, {8}, {8}, {0}},
+      {{8}, {8}, {8}, {0}},
       {{0}, {8}, {16}, {24}}},
 
-   {SVGA3D_A8R8G8B8, SVGA3DBLOCKDESC_RGBA,
+   {SVGA3D_A8R8G8B8, SVGA3DBLOCKDESC_RGBA_UNORM,
       {1, 1, 1},  4, 4,
-      32, {{8}, {8}, {8}, {8}},
+      {{8}, {8}, {8}, {8}},
       {{0}, {8}, {16}, {24}}},
 
-   {SVGA3D_R5G6B5, SVGA3DBLOCKDESC_RGB,
+   {SVGA3D_R5G6B5, SVGA3DBLOCKDESC_RGB_UNORM,
       {1, 1, 1},  2, 2,
-      16, {{5}, {6}, {5}, {0}},
+      {{5}, {6}, {5}, {0}},
       {{0}, {5}, {11}, {0}}},
 
-   {SVGA3D_X1R5G5B5, SVGA3DBLOCKDESC_RGB,
+   {SVGA3D_X1R5G5B5, SVGA3DBLOCKDESC_RGB_UNORM,
       {1, 1, 1},  2, 2,
-      15, {{5}, {5}, {5}, {0}},
+      {{5}, {5}, {5}, {0}},
       {{0}, {5}, {10}, {0}}},
 
-   {SVGA3D_A1R5G5B5, SVGA3DBLOCKDESC_RGBA,
+   {SVGA3D_A1R5G5B5, SVGA3DBLOCKDESC_RGBA_UNORM,
       {1, 1, 1},  2, 2,
-      16, {{5}, {5}, {5}, {1}},
+      {{5}, {5}, {5}, {1}},
       {{0}, {5}, {10}, {15}}},
 
-   {SVGA3D_A4R4G4B4, SVGA3DBLOCKDESC_RGBA,
+   {SVGA3D_A4R4G4B4, SVGA3DBLOCKDESC_RGBA_UNORM,
       {1, 1, 1},  2, 2,
-      16, {{4}, {4}, {4}, {4}},
+      {{4}, {4}, {4}, {4}},
       {{0}, {4}, {8}, {12}}},
 
-   {SVGA3D_Z_D32, SVGA3DBLOCKDESC_DEPTH,
+   {SVGA3D_Z_D32, SVGA3DBLOCKDESC_DEPTH_UNORM,
       {1, 1, 1},  4, 4,
-      32, {{0}, {0}, {32}, {0}},
+      {{0}, {0}, {32}, {0}},
       {{0}, {0}, {0}, {0}}},
 
-   {SVGA3D_Z_D16, SVGA3DBLOCKDESC_DEPTH,
+   {SVGA3D_Z_D16, SVGA3DBLOCKDESC_DEPTH_UNORM,
       {1, 1, 1},  2, 2,
-      16, {{0}, {0}, {16}, {0}},
+      {{0}, {0}, {16}, {0}},
       {{0}, {0}, {0}, {0}}},
 
-   {SVGA3D_Z_D24S8, SVGA3DBLOCKDESC_DS,
+   {SVGA3D_Z_D24S8, SVGA3DBLOCKDESC_DS_UNORM,
       {1, 1, 1},  4, 4,
-      32, {{0}, {8}, {24}, {0}},
-      {{0}, {24}, {0}, {0}}},
+      {{0}, {8}, {24}, {0}},
+      {{0}, {0}, {8}, {0}}},
 
-   {SVGA3D_Z_D15S1, SVGA3DBLOCKDESC_DS,
+   {SVGA3D_Z_D15S1, SVGA3DBLOCKDESC_DS_UNORM,
       {1, 1, 1},  2, 2,
-      16, {{0}, {1}, {15}, {0}},
-      {{0}, {15}, {0}, {0}}},
+      {{0}, {1}, {15}, {0}},
+      {{0}, {0}, {1}, {0}}},
 
-   {SVGA3D_LUMINANCE8, SVGA3DBLOCKDESC_LUMINANCE,
+   {SVGA3D_LUMINANCE8, SVGA3DBLOCKDESC_L_UNORM,
       {1, 1, 1},  1, 1,
-      8, {{0}, {0}, {8}, {0}},
+      {{0}, {0}, {8}, {0}},
       {{0}, {0}, {0}, {0}}},
 
-   {SVGA3D_LUMINANCE4_ALPHA4, SVGA3DBLOCKDESC_LA,
-    {1  , 1, 1},  1, 1,
-      8, {{0}, {0}, {4}, {4}},
+   {SVGA3D_LUMINANCE4_ALPHA4, SVGA3DBLOCKDESC_LA_UNORM,
+      {1, 1, 1},  1, 1,
+      {{0}, {0}, {4}, {4}},
       {{0}, {0}, {0}, {4}}},
 
-   {SVGA3D_LUMINANCE16, SVGA3DBLOCKDESC_LUMINANCE,
+   {SVGA3D_LUMINANCE16, SVGA3DBLOCKDESC_L_UNORM,
       {1, 1, 1},  2, 2,
-      16, {{0}, {0}, {16}, {0}},
+      {{0}, {0}, {16}, {0}},
       {{0}, {0}, {0}, {0}}},
 
-   {SVGA3D_LUMINANCE8_ALPHA8, SVGA3DBLOCKDESC_LA,
+   {SVGA3D_LUMINANCE8_ALPHA8, SVGA3DBLOCKDESC_LA_UNORM,
       {1, 1, 1},  2, 2,
-      16, {{0}, {0}, {8}, {8}},
+      {{0}, {0}, {8}, {8}},
       {{0}, {0}, {0}, {8}}},
 
-   {SVGA3D_DXT1, SVGA3DBLOCKDESC_COMPRESSED,
+   {SVGA3D_DXT1, SVGA3DBLOCKDESC_BC1_COMP_UNORM,
       {4, 4, 1},  8, 8,
-      64, {{0}, {0}, {64}, {0}},
+      {{0}, {0}, {64}, {0}},
       {{0}, {0}, {0}, {0}}},
 
-   {SVGA3D_DXT2, SVGA3DBLOCKDESC_COMPRESSED,
+   {SVGA3D_DXT2, SVGA3DBLOCKDESC_BC2_COMP_UNORM,
       {4, 4, 1},  16, 16,
-      128, {{0}, {0}, {128}, {0}},
+      {{0}, {0}, {128}, {0}},
       {{0}, {0}, {0}, {0}}},
 
-   {SVGA3D_DXT3, SVGA3DBLOCKDESC_COMPRESSED,
+   {SVGA3D_DXT3, SVGA3DBLOCKDESC_BC2_COMP_UNORM,
       {4, 4, 1},  16, 16,
-      128, {{0}, {0}, {128}, {0}},
+      {{0}, {0}, {128}, {0}},
       {{0}, {0}, {0}, {0}}},
 
-   {SVGA3D_DXT4, SVGA3DBLOCKDESC_COMPRESSED,
+   {SVGA3D_DXT4, SVGA3DBLOCKDESC_BC3_COMP_UNORM,
       {4, 4, 1},  16, 16,
-      128, {{0}, {0}, {128}, {0}},
+      {{0}, {0}, {128}, {0}},
       {{0}, {0}, {0}, {0}}},
 
-   {SVGA3D_DXT5, SVGA3DBLOCKDESC_COMPRESSED,
+   {SVGA3D_DXT5, SVGA3DBLOCKDESC_BC3_COMP_UNORM,
       {4, 4, 1},  16, 16,
-      128, {{0}, {0}, {128}, {0}},
+      {{0}, {0}, {128}, {0}},
       {{0}, {0}, {0}, {0}}},
 
-   {SVGA3D_BUMPU8V8, SVGA3DBLOCKDESC_UV,
+   {SVGA3D_BUMPU8V8, SVGA3DBLOCKDESC_UV_SNORM,
       {1, 1, 1},  2, 2,
-      16, {{0}, {0}, {8}, {8}},
-      {{0}, {0}, {0}, {8}}},
+      {{0}, {8}, {8}, {0}},
+      {{0}, {8}, {0}, {0}}},
 
    {SVGA3D_BUMPL6V5U5, SVGA3DBLOCKDESC_UVL,
       {1, 1, 1},  2, 2,
-      16, {{5}, {5}, {6}, {0}},
-      {{11}, {6}, {0}, {0}}},
+      {{6}, {5}, {5}, {0}},
+      {{10}, {5}, {0}, {0}}},
 
    {SVGA3D_BUMPX8L8V8U8, SVGA3DBLOCKDESC_UVL,
       {1, 1, 1},  4, 4,
-      32, {{8}, {8}, {8}, {0}},
+      {{8}, {8}, {8}, {0}},
       {{16}, {8}, {0}, {0}}},
 
-   {SVGA3D_BUMPL8V8U8, SVGA3DBLOCKDESC_UVL,
+   {SVGA3D_FORMAT_DEAD1, SVGA3DBLOCKDESC_UVL,
       {1, 1, 1},  3, 3,
-      24, {{8}, {8}, {8}, {0}},
+      {{8}, {8}, {8}, {0}},
       {{16}, {8}, {0}, {0}}},
 
    {SVGA3D_ARGB_S10E5, SVGA3DBLOCKDESC_RGBA_FP,
       {1, 1, 1},  8, 8,
-      64, {{16}, {16}, {16}, {16}},
+      {{16}, {16}, {16}, {16}},
       {{32}, {16}, {0}, {48}}},
 
    {SVGA3D_ARGB_S23E8, SVGA3DBLOCKDESC_RGBA_FP,
       {1, 1, 1},  16, 16,
-      128, {{32}, {32}, {32}, {32}},
+      {{32}, {32}, {32}, {32}},
       {{64}, {32}, {0}, {96}}},
 
-   {SVGA3D_A2R10G10B10, SVGA3DBLOCKDESC_RGBA,
+   {SVGA3D_A2R10G10B10, SVGA3DBLOCKDESC_RGBA_UNORM,
       {1, 1, 1},  4, 4,
-      32, {{10}, {10}, {10}, {2}},
+      {{10}, {10}, {10}, {2}},
       {{0}, {10}, {20}, {30}}},
 
-   {SVGA3D_V8U8, SVGA3DBLOCKDESC_UV,
+   {SVGA3D_V8U8, SVGA3DBLOCKDESC_UV_SNORM,
       {1, 1, 1},  2, 2,
-      16, {{8}, {8}, {0}, {0}},
-      {{8}, {0}, {0}, {0}}},
+      {{0}, {8}, {8}, {0}},
+      {{0}, {8}, {0}, {0}}},
 
-   {SVGA3D_Q8W8V8U8, SVGA3DBLOCKDESC_UVWQ,
+   {SVGA3D_Q8W8V8U8, SVGA3DBLOCKDESC_UVWQ_SNORM,
       {1, 1, 1},  4, 4,
-      32, {{8}, {8}, {8}, {8}},
-      {{24}, {16}, {8}, {0}}},
+      {{8}, {8}, {8}, {8}},
+      {{16}, {8}, {0}, {24}}},
 
-   {SVGA3D_CxV8U8, SVGA3DBLOCKDESC_UV,
+   {SVGA3D_CxV8U8, SVGA3DBLOCKDESC_UVCX_SNORM,
       {1, 1, 1},  2, 2,
-      16, {{8}, {8}, {0}, {0}},
-      {{8}, {0}, {0}, {0}}},
+      {{0}, {8}, {8}, {0}},
+      {{0}, {8}, {0}, {0}}},
 
    {SVGA3D_X8L8V8U8, SVGA3DBLOCKDESC_UVL,
       {1, 1, 1},  4, 4,
-      24, {{8}, {8}, {8}, {0}},
+      {{8}, {8}, {8}, {0}},
       {{16}, {8}, {0}, {0}}},
 
    {SVGA3D_A2W10V10U10, SVGA3DBLOCKDESC_UVWA,
       {1, 1, 1},  4, 4,
-      32, {{10}, {10}, {10}, {2}},
-      {{0}, {10}, {20}, {30}}},
+      {{10}, {10}, {10}, {2}},
+      {{20}, {10}, {0}, {30}}},
 
-   {SVGA3D_ALPHA8, SVGA3DBLOCKDESC_ALPHA,
+   {SVGA3D_ALPHA8, SVGA3DBLOCKDESC_A_UNORM,
       {1, 1, 1},  1, 1,
-      8, {{0}, {0}, {0}, {8}},
+      {{0}, {0}, {0}, {8}},
       {{0}, {0}, {0}, {0}}},
 
    {SVGA3D_R_S10E5, SVGA3DBLOCKDESC_R_FP,
       {1, 1, 1},  2, 2,
-      16, {{0}, {0}, {16}, {0}},
+      {{0}, {0}, {16}, {0}},
       {{0}, {0}, {0}, {0}}},
 
    {SVGA3D_R_S23E8, SVGA3DBLOCKDESC_R_FP,
       {1, 1, 1},  4, 4,
-      32, {{0}, {0}, {32}, {0}},
+      {{0}, {0}, {32}, {0}},
       {{0}, {0}, {0}, {0}}},
 
    {SVGA3D_RG_S10E5, SVGA3DBLOCKDESC_RG_FP,
       {1, 1, 1},  4, 4,
-      32, {{0}, {16}, {16}, {0}},
+      {{0}, {16}, {16}, {0}},
       {{0}, {16}, {0}, {0}}},
 
    {SVGA3D_RG_S23E8, SVGA3DBLOCKDESC_RG_FP,
       {1, 1, 1},  8, 8,
-      64, {{0}, {32}, {32}, {0}},
+      {{0}, {32}, {32}, {0}},
       {{0}, {32}, {0}, {0}}},
 
    {SVGA3D_BUFFER, SVGA3DBLOCKDESC_BUFFER,
       {1, 1, 1},  1, 1,
-      8, {{0}, {0}, {8}, {0}},
+      {{0}, {0}, {8}, {0}},
       {{0}, {0}, {0}, {0}}},
 
-   {SVGA3D_Z_D24X8, SVGA3DBLOCKDESC_DEPTH,
+   {SVGA3D_Z_D24X8, SVGA3DBLOCKDESC_DEPTH_UNORM,
       {1, 1, 1},  4, 4,
-      32, {{0}, {0}, {24}, {0}},
-      {{0}, {24}, {0}, {0}}},
+      {{0}, {0}, {24}, {0}},
+      {{0}, {0}, {8}, {0}}},
 
-   {SVGA3D_V16U16, SVGA3DBLOCKDESC_UV,
+   {SVGA3D_V16U16, SVGA3DBLOCKDESC_UV_SNORM,
       {1, 1, 1},  4, 4,
-      32, {{16}, {16}, {0}, {0}},
-      {{16}, {0}, {0}, {0}}},
+      {{0}, {16}, {16}, {0}},
+      {{0}, {16}, {0}, {0}}},
 
-   {SVGA3D_G16R16, SVGA3DBLOCKDESC_RG,
+   {SVGA3D_G16R16, SVGA3DBLOCKDESC_RG_UNORM,
       {1, 1, 1},  4, 4,
-      32, {{0}, {16}, {16}, {0}},
-      {{0}, {0}, {16}, {0}}},
+      {{0}, {16}, {16}, {0}},
+      {{0}, {16}, {0}, {0}}},
 
-   {SVGA3D_A16B16G16R16, SVGA3DBLOCKDESC_RGBA,
+   {SVGA3D_A16B16G16R16, SVGA3DBLOCKDESC_RGBA_UNORM,
       {1, 1, 1},  8, 8,
-      64, {{16}, {16}, {16}, {16}},
+      {{16}, {16}, {16}, {16}},
       {{32}, {16}, {0}, {48}}},
 
    {SVGA3D_UYVY, SVGA3DBLOCKDESC_YUV,
-      {1, 1, 1},  2, 2,
-      16, {{8}, {0}, {8}, {0}},
+      {2, 1, 1},  4, 4,
+      {{8}, {0}, {8}, {0}},
       {{0}, {0}, {8}, {0}}},
 
    {SVGA3D_YUY2, SVGA3DBLOCKDESC_YUV,
-      {1, 1, 1},  2, 2,
-      16, {{8}, {0}, {8}, {0}},
+      {2, 1, 1},  4, 4,
+      {{8}, {0}, {8}, {0}},
       {{8}, {0}, {0}, {0}}},
 
    {SVGA3D_NV12, SVGA3DBLOCKDESC_NV12,
       {2, 2, 1},  6, 2,
-      48, {{0}, {0}, {48}, {0}},
+      {{0}, {0}, {48}, {0}},
       {{0}, {0}, {0}, {0}}},
 
    {SVGA3D_AYUV, SVGA3DBLOCKDESC_AYUV,
       {1, 1, 1},  4, 4,
-      32, {{8}, {8}, {8}, {8}},
+      {{8}, {8}, {8}, {8}},
       {{0}, {8}, {16}, {24}}},
 
-   {SVGA3D_R32G32B32A32_TYPELESS, SVGA3DBLOCKDESC_RGBA,
+   {SVGA3D_R32G32B32A32_TYPELESS, SVGA3DBLOCKDESC_TYPELESS,
       {1, 1, 1},  16, 16,
-      128, {{32}, {32}, {32}, {32}},
+      {{32}, {32}, {32}, {32}},
       {{64}, {32}, {0}, {96}}},
 
-   {SVGA3D_R32G32B32A32_UINT, SVGA3DBLOCKDESC_RGBA,
+   {SVGA3D_R32G32B32A32_UINT, SVGA3DBLOCKDESC_RGBA_UINT,
       {1, 1, 1},  16, 16,
-      128, {{32}, {32}, {32}, {32}},
+      {{32}, {32}, {32}, {32}},
       {{64}, {32}, {0}, {96}}},
 
-   {SVGA3D_R32G32B32A32_SINT, SVGA3DBLOCKDESC_UVWQ,
+   {SVGA3D_R32G32B32A32_SINT, SVGA3DBLOCKDESC_RGBA_SINT,
       {1, 1, 1},  16, 16,
-      128, {{32}, {32}, {32}, {32}},
+      {{32}, {32}, {32}, {32}},
       {{64}, {32}, {0}, {96}}},
 
-   {SVGA3D_R32G32B32_TYPELESS, SVGA3DBLOCKDESC_RGB,
+   {SVGA3D_R32G32B32_TYPELESS, SVGA3DBLOCKDESC_TYPELESS,
       {1, 1, 1},  12, 12,
-      96, {{32}, {32}, {32}, {0}},
+      {{32}, {32}, {32}, {0}},
       {{64}, {32}, {0}, {0}}},
 
    {SVGA3D_R32G32B32_FLOAT, SVGA3DBLOCKDESC_RGB_FP,
       {1, 1, 1},  12, 12,
-      96, {{32}, {32}, {32}, {0}},
+      {{32}, {32}, {32}, {0}},
       {{64}, {32}, {0}, {0}}},
 
-   {SVGA3D_R32G32B32_UINT, SVGA3DBLOCKDESC_RGB,
+   {SVGA3D_R32G32B32_UINT, SVGA3DBLOCKDESC_RGB_UINT,
       {1, 1, 1},  12, 12,
-      96, {{32}, {32}, {32}, {0}},
+      {{32}, {32}, {32}, {0}},
       {{64}, {32}, {0}, {0}}},
 
-   {SVGA3D_R32G32B32_SINT, SVGA3DBLOCKDESC_UVW,
+   {SVGA3D_R32G32B32_SINT, SVGA3DBLOCKDESC_RGB_SINT,
       {1, 1, 1},  12, 12,
-      96, {{32}, {32}, {32}, {0}},
+      {{32}, {32}, {32}, {0}},
       {{64}, {32}, {0}, {0}}},
 
-   {SVGA3D_R16G16B16A16_TYPELESS, SVGA3DBLOCKDESC_RGBA,
+   {SVGA3D_R16G16B16A16_TYPELESS, SVGA3DBLOCKDESC_TYPELESS,
       {1, 1, 1},  8, 8,
-      64, {{16}, {16}, {16}, {16}},
+      {{16}, {16}, {16}, {16}},
       {{32}, {16}, {0}, {48}}},
 
-   {SVGA3D_R16G16B16A16_UINT, SVGA3DBLOCKDESC_RGBA,
+   {SVGA3D_R16G16B16A16_UINT, SVGA3DBLOCKDESC_RGBA_UINT,
       {1, 1, 1},  8, 8,
-      64, {{16}, {16}, {16}, {16}},
+      {{16}, {16}, {16}, {16}},
       {{32}, {16}, {0}, {48}}},
 
-   {SVGA3D_R16G16B16A16_SNORM, SVGA3DBLOCKDESC_UVWQ,
+   {SVGA3D_R16G16B16A16_SNORM, SVGA3DBLOCKDESC_RGBA_SNORM,
       {1, 1, 1},  8, 8,
-      64, {{16}, {16}, {16}, {16}},
+      {{16}, {16}, {16}, {16}},
       {{32}, {16}, {0}, {48}}},
 
-   {SVGA3D_R16G16B16A16_SINT, SVGA3DBLOCKDESC_UVWQ,
+   {SVGA3D_R16G16B16A16_SINT, SVGA3DBLOCKDESC_RGBA_SINT,
       {1, 1, 1},  8, 8,
-      64, {{16}, {16}, {16}, {16}},
+      {{16}, {16}, {16}, {16}},
       {{32}, {16}, {0}, {48}}},
 
-   {SVGA3D_R32G32_TYPELESS, SVGA3DBLOCKDESC_RG,
+   {SVGA3D_R32G32_TYPELESS, SVGA3DBLOCKDESC_TYPELESS,
       {1, 1, 1},  8, 8,
-      64, {{0}, {32}, {32}, {0}},
+      {{0}, {32}, {32}, {0}},
       {{0}, {32}, {0}, {0}}},
 
-   {SVGA3D_R32G32_UINT, SVGA3DBLOCKDESC_RG,
+   {SVGA3D_R32G32_UINT, SVGA3DBLOCKDESC_RG_UINT,
       {1, 1, 1},  8, 8,
-      64, {{0}, {32}, {32}, {0}},
+      {{0}, {32}, {32}, {0}},
       {{0}, {32}, {0}, {0}}},
 
-   {SVGA3D_R32G32_SINT, SVGA3DBLOCKDESC_UV,
+   {SVGA3D_R32G32_SINT, SVGA3DBLOCKDESC_RG_SINT,
       {1, 1, 1},  8, 8,
-      64, {{0}, {32}, {32}, {0}},
+      {{0}, {32}, {32}, {0}},
       {{0}, {32}, {0}, {0}}},
 
-   {SVGA3D_R32G8X24_TYPELESS, SVGA3DBLOCKDESC_RG,
+   {SVGA3D_R32G8X24_TYPELESS, SVGA3DBLOCKDESC_TYPELESS,
       {1, 1, 1},  8, 8,
-      64, {{0}, {8}, {32}, {0}},
+      {{0}, {8}, {32}, {0}},
       {{0}, {32}, {0}, {0}}},
 
    {SVGA3D_D32_FLOAT_S8X24_UINT, SVGA3DBLOCKDESC_DS,
       {1, 1, 1},  8, 8,
-      64, {{0}, {8}, {32}, {0}},
+      {{0}, {8}, {32}, {0}},
       {{0}, {32}, {0}, {0}}},
 
-   {SVGA3D_R32_FLOAT_X8X24_TYPELESS, SVGA3DBLOCKDESC_R_FP,
+   {SVGA3D_R32_FLOAT_X8X24, SVGA3DBLOCKDESC_R_FP,
       {1, 1, 1},  8, 8,
-      64, {{0}, {0}, {32}, {0}},
+      {{0}, {0}, {32}, {0}},
       {{0}, {0}, {0}, {0}}},
 
-   {SVGA3D_X32_TYPELESS_G8X24_UINT, SVGA3DBLOCKDESC_GREEN,
+   {SVGA3D_X32_G8X24_UINT, SVGA3DBLOCKDESC_G_UINT,
       {1, 1, 1},  8, 8,
-      64, {{0}, {8}, {0}, {0}},
+      {{0}, {8}, {0}, {0}},
       {{0}, {32}, {0}, {0}}},
 
-   {SVGA3D_R10G10B10A2_TYPELESS, SVGA3DBLOCKDESC_RGBA,
+   {SVGA3D_R10G10B10A2_TYPELESS, SVGA3DBLOCKDESC_TYPELESS,
       {1, 1, 1},  4, 4,
-      32, {{10}, {10}, {10}, {2}},
-      {{0}, {10}, {20}, {30}}},
+      {{10}, {10}, {10}, {2}},
+      {{20}, {10}, {0}, {30}}},
 
-   {SVGA3D_R10G10B10A2_UINT, SVGA3DBLOCKDESC_RGBA,
+   {SVGA3D_R10G10B10A2_UINT, SVGA3DBLOCKDESC_RGBA_UINT,
       {1, 1, 1},  4, 4,
-      32, {{10}, {10}, {10}, {2}},
-      {{0}, {10}, {20}, {30}}},
+      {{10}, {10}, {10}, {2}},
+      {{20}, {10}, {0}, {30}}},
 
    {SVGA3D_R11G11B10_FLOAT, SVGA3DBLOCKDESC_RGB_FP,
       {1, 1, 1},  4, 4,
-      32, {{10}, {11}, {11}, {0}},
-      {{0}, {10}, {21}, {0}}},
+      {{10}, {11}, {11}, {0}},
+      {{22}, {11}, {0}, {0}}},
 
-   {SVGA3D_R8G8B8A8_TYPELESS, SVGA3DBLOCKDESC_RGBA,
+   {SVGA3D_R8G8B8A8_TYPELESS, SVGA3DBLOCKDESC_TYPELESS,
       {1, 1, 1},  4, 4,
-      32, {{8}, {8}, {8}, {8}},
+      {{8}, {8}, {8}, {8}},
       {{16}, {8}, {0}, {24}}},
 
-   {SVGA3D_R8G8B8A8_UNORM, SVGA3DBLOCKDESC_RGBA,
+   {SVGA3D_R8G8B8A8_UNORM, SVGA3DBLOCKDESC_RGBA_UNORM,
       {1, 1, 1},  4, 4,
-      32, {{8}, {8}, {8}, {8}},
+      {{8}, {8}, {8}, {8}},
       {{16}, {8}, {0}, {24}}},
 
-   {SVGA3D_R8G8B8A8_UNORM_SRGB, SVGA3DBLOCKDESC_RGBA_SRGB,
+   {SVGA3D_R8G8B8A8_UNORM_SRGB, SVGA3DBLOCKDESC_RGBA_UNORM_SRGB,
       {1, 1, 1},  4, 4,
-      32, {{8}, {8}, {8}, {8}},
+      {{8}, {8}, {8}, {8}},
       {{16}, {8}, {0}, {24}}},
 
-   {SVGA3D_R8G8B8A8_UINT, SVGA3DBLOCKDESC_RGBA,
+   {SVGA3D_R8G8B8A8_UINT, SVGA3DBLOCKDESC_RGBA_UINT,
       {1, 1, 1},  4, 4,
-      32, {{8}, {8}, {8}, {8}},
+      {{8}, {8}, {8}, {8}},
       {{16}, {8}, {0}, {24}}},
 
-   {SVGA3D_R8G8B8A8_SINT, SVGA3DBLOCKDESC_RGBA,
+   {SVGA3D_R8G8B8A8_SINT, SVGA3DBLOCKDESC_RGBA_SINT,
       {1, 1, 1},  4, 4,
-      32, {{8}, {8}, {8}, {8}},
+      {{8}, {8}, {8}, {8}},
       {{16}, {8}, {0}, {24}}},
 
-   {SVGA3D_R16G16_TYPELESS, SVGA3DBLOCKDESC_RG,
+   {SVGA3D_R16G16_TYPELESS, SVGA3DBLOCKDESC_TYPELESS,
       {1, 1, 1},  4, 4,
-      32, {{0}, {16}, {16}, {0}},
+      {{0}, {16}, {16}, {0}},
       {{0}, {16}, {0}, {0}}},
 
-   {SVGA3D_R16G16_UINT, SVGA3DBLOCKDESC_RG_FP,
+   {SVGA3D_R16G16_UINT, SVGA3DBLOCKDESC_RG_UINT,
       {1, 1, 1},  4, 4,
-      32, {{0}, {16}, {16}, {0}},
+      {{0}, {16}, {16}, {0}},
       {{0}, {16}, {0}, {0}}},
 
-   {SVGA3D_R16G16_SINT, SVGA3DBLOCKDESC_UV,
+   {SVGA3D_R16G16_SINT, SVGA3DBLOCKDESC_RG_SINT,
       {1, 1, 1},  4, 4,
-      32, {{0}, {16}, {16}, {0}},
+      {{0}, {16}, {16}, {0}},
       {{0}, {16}, {0}, {0}}},
 
-   {SVGA3D_R32_TYPELESS, SVGA3DBLOCKDESC_RED,
+   {SVGA3D_R32_TYPELESS, SVGA3DBLOCKDESC_TYPELESS,
       {1, 1, 1},  4, 4,
-      32, {{0}, {0}, {32}, {0}},
+      {{0}, {0}, {32}, {0}},
       {{0}, {0}, {0}, {0}}},
 
-   {SVGA3D_D32_FLOAT, SVGA3DBLOCKDESC_DEPTH,
+   {SVGA3D_D32_FLOAT, SVGA3DBLOCKDESC_DEPTH_FP,
       {1, 1, 1},  4, 4,
-      32, {{0}, {0}, {32}, {0}},
+      {{0}, {0}, {32}, {0}},
       {{0}, {0}, {0}, {0}}},
 
-   {SVGA3D_R32_UINT, SVGA3DBLOCKDESC_RED,
+   {SVGA3D_R32_UINT, SVGA3DBLOCKDESC_R_UINT,
       {1, 1, 1},  4, 4,
-      32, {{0}, {0}, {32}, {0}},
+      {{0}, {0}, {32}, {0}},
       {{0}, {0}, {0}, {0}}},
 
-   {SVGA3D_R32_SINT, SVGA3DBLOCKDESC_RED,
+   {SVGA3D_R32_SINT, SVGA3DBLOCKDESC_R_SINT,
       {1, 1, 1},  4, 4,
-      32, {{0}, {0}, {32}, {0}},
+      {{0}, {0}, {32}, {0}},
       {{0}, {0}, {0}, {0}}},
 
-   {SVGA3D_R24G8_TYPELESS, SVGA3DBLOCKDESC_RG,
+   {SVGA3D_R24G8_TYPELESS, SVGA3DBLOCKDESC_TYPELESS,
       {1, 1, 1},  4, 4,
-      32, {{0}, {8}, {24}, {0}},
+      {{0}, {8}, {24}, {0}},
       {{0}, {24}, {0}, {0}}},
 
-   {SVGA3D_D24_UNORM_S8_UINT, SVGA3DBLOCKDESC_DS,
+   {SVGA3D_D24_UNORM_S8_UINT, SVGA3DBLOCKDESC_DS_UNORM,
       {1, 1, 1},  4, 4,
-      32, {{0}, {8}, {24}, {0}},
+      {{0}, {8}, {24}, {0}},
       {{0}, {24}, {0}, {0}}},
 
-   {SVGA3D_R24_UNORM_X8_TYPELESS, SVGA3DBLOCKDESC_RED,
+   {SVGA3D_R24_UNORM_X8, SVGA3DBLOCKDESC_R_UNORM,
       {1, 1, 1},  4, 4,
-      32, {{0}, {0}, {24}, {0}},
+      {{0}, {0}, {24}, {0}},
       {{0}, {0}, {0}, {0}}},
 
-   {SVGA3D_X24_TYPELESS_G8_UINT, SVGA3DBLOCKDESC_GREEN,
+   {SVGA3D_X24_G8_UINT, SVGA3DBLOCKDESC_G_UINT,
       {1, 1, 1},  4, 4,
-      32, {{0}, {8}, {0}, {0}},
+      {{0}, {8}, {0}, {0}},
       {{0}, {24}, {0}, {0}}},
 
-   {SVGA3D_R8G8_TYPELESS, SVGA3DBLOCKDESC_RG,
+   {SVGA3D_R8G8_TYPELESS, SVGA3DBLOCKDESC_TYPELESS,
       {1, 1, 1},  2, 2,
-      16, {{0}, {8}, {8}, {0}},
+      {{0}, {8}, {8}, {0}},
       {{0}, {8}, {0}, {0}}},
 
-   {SVGA3D_R8G8_UNORM, SVGA3DBLOCKDESC_RG,
+   {SVGA3D_R8G8_UNORM, SVGA3DBLOCKDESC_RG_UNORM,
       {1, 1, 1},  2, 2,
-      16, {{0}, {8}, {8}, {0}},
+      {{0}, {8}, {8}, {0}},
       {{0}, {8}, {0}, {0}}},
 
-   {SVGA3D_R8G8_UINT, SVGA3DBLOCKDESC_RG,
+   {SVGA3D_R8G8_UINT, SVGA3DBLOCKDESC_RG_UINT,
       {1, 1, 1},  2, 2,
-      16, {{0}, {8}, {8}, {0}},
+      {{0}, {8}, {8}, {0}},
       {{0}, {8}, {0}, {0}}},
 
-   {SVGA3D_R8G8_SINT, SVGA3DBLOCKDESC_UV,
+   {SVGA3D_R8G8_SINT, SVGA3DBLOCKDESC_RG_SINT,
       {1, 1, 1},  2, 2,
-      16, {{0}, {8}, {8}, {0}},
+      {{0}, {8}, {8}, {0}},
       {{0}, {8}, {0}, {0}}},
 
-   {SVGA3D_R16_TYPELESS, SVGA3DBLOCKDESC_RED,
+   {SVGA3D_R16_TYPELESS, SVGA3DBLOCKDESC_TYPELESS,
       {1, 1, 1},  2, 2,
-      16, {{0}, {0}, {16}, {0}},
+      {{0}, {0}, {16}, {0}},
       {{0}, {0}, {0}, {0}}},
 
-   {SVGA3D_R16_UNORM, SVGA3DBLOCKDESC_RED,
+   {SVGA3D_R16_UNORM, SVGA3DBLOCKDESC_R_UNORM,
       {1, 1, 1},  2, 2,
-      16, {{0}, {0}, {16}, {0}},
+      {{0}, {0}, {16}, {0}},
       {{0}, {0}, {0}, {0}}},
 
-   {SVGA3D_R16_UINT, SVGA3DBLOCKDESC_RED,
+   {SVGA3D_R16_UINT, SVGA3DBLOCKDESC_R_UINT,
       {1, 1, 1},  2, 2,
-      16, {{0}, {0}, {16}, {0}},
+      {{0}, {0}, {16}, {0}},
       {{0}, {0}, {0}, {0}}},
 
-   {SVGA3D_R16_SNORM, SVGA3DBLOCKDESC_U,
+   {SVGA3D_R16_SNORM, SVGA3DBLOCKDESC_R_SNORM,
       {1, 1, 1},  2, 2,
-      16, {{0}, {0}, {16}, {0}},
+      {{0}, {0}, {16}, {0}},
       {{0}, {0}, {0}, {0}}},
 
-   {SVGA3D_R16_SINT, SVGA3DBLOCKDESC_U,
+   {SVGA3D_R16_SINT, SVGA3DBLOCKDESC_R_SINT,
       {1, 1, 1},  2, 2,
-      16, {{0}, {0}, {16}, {0}},
+      {{0}, {0}, {16}, {0}},
       {{0}, {0}, {0}, {0}}},
 
-   {SVGA3D_R8_TYPELESS, SVGA3DBLOCKDESC_RED,
+   {SVGA3D_R8_TYPELESS, SVGA3DBLOCKDESC_TYPELESS,
       {1, 1, 1},  1, 1,
-      8, {{0}, {0}, {8}, {0}},
+      {{0}, {0}, {8}, {0}},
       {{0}, {0}, {0}, {0}}},
 
-   {SVGA3D_R8_UNORM, SVGA3DBLOCKDESC_RED,
+   {SVGA3D_R8_UNORM, SVGA3DBLOCKDESC_R_UNORM,
       {1, 1, 1},  1, 1,
-      8, {{0}, {0}, {8}, {0}},
+      {{0}, {0}, {8}, {0}},
       {{0}, {0}, {0}, {0}}},
 
-   {SVGA3D_R8_UINT, SVGA3DBLOCKDESC_RED,
+   {SVGA3D_R8_UINT, SVGA3DBLOCKDESC_R_UINT,
       {1, 1, 1},  1, 1,
-      8, {{0}, {0}, {8}, {0}},
+      {{0}, {0}, {8}, {0}},
       {{0}, {0}, {0}, {0}}},
 
-   {SVGA3D_R8_SNORM, SVGA3DBLOCKDESC_U,
+   {SVGA3D_R8_SNORM, SVGA3DBLOCKDESC_R_SNORM,
       {1, 1, 1},  1, 1,
-      8, {{0}, {0}, {8}, {0}},
+      {{0}, {0}, {8}, {0}},
       {{0}, {0}, {0}, {0}}},
 
-   {SVGA3D_R8_SINT, SVGA3DBLOCKDESC_U,
+   {SVGA3D_R8_SINT, SVGA3DBLOCKDESC_R_SINT,
       {1, 1, 1},  1, 1,
-      8, {{0}, {0}, {8}, {0}},
+      {{0}, {0}, {8}, {0}},
       {{0}, {0}, {0}, {0}}},
 
-   {SVGA3D_P8, SVGA3DBLOCKDESC_RED,
+   {SVGA3D_P8, SVGA3DBLOCKDESC_NONE,
       {1, 1, 1},  1, 1,
-      8, {{0}, {0}, {8}, {0}},
+      {{0}, {0}, {8}, {0}},
       {{0}, {0}, {0}, {0}}},
 
-   {SVGA3D_R9G9B9E5_SHAREDEXP, SVGA3DBLOCKDESC_RGBE,
+   {SVGA3D_R9G9B9E5_SHAREDEXP, SVGA3DBLOCKDESC_RGB_EXP,
       {1, 1, 1},  4, 4,
-      32, {{9}, {9}, {9}, {5}},
+      {{9}, {9}, {9}, {5}},
       {{18}, {9}, {0}, {27}}},
 
-   {SVGA3D_R8G8_B8G8_UNORM, SVGA3DBLOCKDESC_RG,
-      {1, 1, 1},  2, 2,
-      16, {{0}, {8}, {8}, {0}},
-      {{0}, {8}, {0}, {0}}},
+   {SVGA3D_R8G8_B8G8_UNORM, SVGA3DBLOCKDESC_NONE,
+      {2, 1, 1},  4, 4,
+      {{0}, {8}, {8}, {0}},
+      {{0}, {0}, {8}, {0}}},
 
-   {SVGA3D_G8R8_G8B8_UNORM, SVGA3DBLOCKDESC_RG,
-      {1, 1, 1},  2, 2,
-      16, {{0}, {8}, {8}, {0}},
+   {SVGA3D_G8R8_G8B8_UNORM, SVGA3DBLOCKDESC_NONE,
+      {2, 1, 1},  4, 4,
+      {{0}, {8}, {8}, {0}},
       {{0}, {8}, {0}, {0}}},
 
-   {SVGA3D_BC1_TYPELESS, SVGA3DBLOCKDESC_COMPRESSED,
+   {SVGA3D_BC1_TYPELESS, SVGA3DBLOCKDESC_BC1_COMP_TYPELESS,
       {4, 4, 1},  8, 8,
-      64, {{0}, {0}, {64}, {0}},
+      {{0}, {0}, {64}, {0}},
       {{0}, {0}, {0}, {0}}},
 
-   {SVGA3D_BC1_UNORM_SRGB, SVGA3DBLOCKDESC_COMPRESSED_SRGB,
+   {SVGA3D_BC1_UNORM_SRGB, SVGA3DBLOCKDESC_BC1_COMP_UNORM_SRGB,
       {4, 4, 1},  8, 8,
-      64, {{0}, {0}, {64}, {0}},
+      {{0}, {0}, {64}, {0}},
       {{0}, {0}, {0}, {0}}},
 
-   {SVGA3D_BC2_TYPELESS, SVGA3DBLOCKDESC_COMPRESSED,
+   {SVGA3D_BC2_TYPELESS, SVGA3DBLOCKDESC_BC2_COMP_TYPELESS,
       {4, 4, 1},  16, 16,
-      128, {{0}, {0}, {128}, {0}},
+      {{0}, {0}, {128}, {0}},
       {{0}, {0}, {0}, {0}}},
 
-   {SVGA3D_BC2_UNORM_SRGB, SVGA3DBLOCKDESC_COMPRESSED_SRGB,
+   {SVGA3D_BC2_UNORM_SRGB, SVGA3DBLOCKDESC_BC2_COMP_UNORM_SRGB,
       {4, 4, 1},  16, 16,
-      128, {{0}, {0}, {128}, {0}},
+      {{0}, {0}, {128}, {0}},
       {{0}, {0}, {0}, {0}}},
 
-   {SVGA3D_BC3_TYPELESS, SVGA3DBLOCKDESC_COMPRESSED,
+   {SVGA3D_BC3_TYPELESS, SVGA3DBLOCKDESC_BC3_COMP_TYPELESS,
       {4, 4, 1},  16, 16,
-      128, {{0}, {0}, {128}, {0}},
+      {{0}, {0}, {128}, {0}},
       {{0}, {0}, {0}, {0}}},
 
-   {SVGA3D_BC3_UNORM_SRGB, SVGA3DBLOCKDESC_COMPRESSED_SRGB,
+   {SVGA3D_BC3_UNORM_SRGB, SVGA3DBLOCKDESC_BC3_COMP_UNORM_SRGB,
       {4, 4, 1},  16, 16,
-      128, {{0}, {0}, {128}, {0}},
+      {{0}, {0}, {128}, {0}},
       {{0}, {0}, {0}, {0}}},
 
-   {SVGA3D_BC4_TYPELESS, SVGA3DBLOCKDESC_COMPRESSED,
+   {SVGA3D_BC4_TYPELESS, SVGA3DBLOCKDESC_BC4_COMP_TYPELESS,
       {4, 4, 1},  8, 8,
-      64, {{0}, {0}, {64}, {0}},
+      {{0}, {0}, {64}, {0}},
       {{0}, {0}, {0}, {0}}},
 
-   {SVGA3D_ATI1, SVGA3DBLOCKDESC_COMPRESSED,
+   {SVGA3D_ATI1, SVGA3DBLOCKDESC_BC4_COMP_UNORM,
       {4, 4, 1},  8, 8,
-      64, {{0}, {0}, {64}, {0}},
+      {{0}, {0}, {64}, {0}},
       {{0}, {0}, {0}, {0}}},
 
-   {SVGA3D_BC4_SNORM, SVGA3DBLOCKDESC_COMPRESSED,
+   {SVGA3D_BC4_SNORM, SVGA3DBLOCKDESC_BC4_COMP_SNORM,
       {4, 4, 1},  8, 8,
-      64, {{0}, {0}, {64}, {0}},
+      {{0}, {0}, {64}, {0}},
       {{0}, {0}, {0}, {0}}},
 
-   {SVGA3D_BC5_TYPELESS, SVGA3DBLOCKDESC_COMPRESSED,
+   {SVGA3D_BC5_TYPELESS, SVGA3DBLOCKDESC_BC5_COMP_TYPELESS,
       {4, 4, 1},  16, 16,
-      128, {{0}, {0}, {128}, {0}},
+      {{0}, {0}, {128}, {0}},
       {{0}, {0}, {0}, {0}}},
 
-   {SVGA3D_ATI2, SVGA3DBLOCKDESC_COMPRESSED,
+   {SVGA3D_ATI2, SVGA3DBLOCKDESC_BC5_COMP_UNORM,
       {4, 4, 1},  16, 16,
-      128, {{0}, {0}, {128}, {0}},
+      {{0}, {0}, {128}, {0}},
       {{0}, {0}, {0}, {0}}},
 
-   {SVGA3D_BC5_SNORM, SVGA3DBLOCKDESC_COMPRESSED,
+   {SVGA3D_BC5_SNORM, SVGA3DBLOCKDESC_BC5_COMP_SNORM,
       {4, 4, 1},  16, 16,
-      128, {{0}, {0}, {128}, {0}},
+      {{0}, {0}, {128}, {0}},
       {{0}, {0}, {0}, {0}}},
 
-   {SVGA3D_R10G10B10_XR_BIAS_A2_UNORM, SVGA3DBLOCKDESC_RGBA,
+   {SVGA3D_R10G10B10_XR_BIAS_A2_UNORM, SVGA3DBLOCKDESC_RGBA_UNORM,
       {1, 1, 1},  4, 4,
-      32, {{10}, {10}, {10}, {2}},
-      {{0}, {10}, {20}, {30}}},
+      {{10}, {10}, {10}, {2}},
+     {{20}, {10}, {0}, {30}}},
 
-   {SVGA3D_B8G8R8A8_TYPELESS, SVGA3DBLOCKDESC_RGBA,
+   {SVGA3D_B8G8R8A8_TYPELESS, SVGA3DBLOCKDESC_TYPELESS,
       {1, 1, 1},  4, 4,
-      32, {{8}, {8}, {8}, {8}},
+      {{8}, {8}, {8}, {8}},
       {{0}, {8}, {16}, {24}}},
 
-   {SVGA3D_B8G8R8A8_UNORM_SRGB, SVGA3DBLOCKDESC_RGBA_SRGB,
+   {SVGA3D_B8G8R8A8_UNORM_SRGB, SVGA3DBLOCKDESC_RGBA_UNORM_SRGB,
       {1, 1, 1},  4, 4,
-      32, {{8}, {8}, {8}, {8}},
+      {{8}, {8}, {8}, {8}},
       {{0}, {8}, {16}, {24}}},
 
-   {SVGA3D_B8G8R8X8_TYPELESS, SVGA3DBLOCKDESC_RGB,
+   {SVGA3D_B8G8R8X8_TYPELESS, SVGA3DBLOCKDESC_TYPELESS,
       {1, 1, 1},  4, 4,
-      24, {{8}, {8}, {8}, {0}},
+      {{8}, {8}, {8}, {0}},
       {{0}, {8}, {16}, {24}}},
 
-   {SVGA3D_B8G8R8X8_UNORM_SRGB, SVGA3DBLOCKDESC_RGB_SRGB,
+   {SVGA3D_B8G8R8X8_UNORM_SRGB, SVGA3DBLOCKDESC_RGB_UNORM_SRGB,
       {1, 1, 1},  4, 4,
-      24, {{8}, {8}, {8}, {0}},
+      {{8}, {8}, {8}, {0}},
       {{0}, {8}, {16}, {24}}},
 
-   {SVGA3D_Z_DF16, SVGA3DBLOCKDESC_DEPTH,
+   {SVGA3D_Z_DF16, SVGA3DBLOCKDESC_DEPTH_UNORM,
       {1, 1, 1},  2, 2,
-      16, {{0}, {0}, {16}, {0}},
+      {{0}, {0}, {16}, {0}},
       {{0}, {0}, {0}, {0}}},
 
-   {SVGA3D_Z_DF24, SVGA3DBLOCKDESC_DEPTH,
+   {SVGA3D_Z_DF24, SVGA3DBLOCKDESC_DEPTH_UNORM,
       {1, 1, 1},  4, 4,
-      32, {{0}, {8}, {24}, {0}},
-      {{0}, {24}, {0}, {0}}},
+      {{0}, {0}, {24}, {0}},
+      {{0}, {0}, {8}, {0}}},
 
-   {SVGA3D_Z_D24S8_INT, SVGA3DBLOCKDESC_DS,
+   {SVGA3D_Z_D24S8_INT, SVGA3DBLOCKDESC_DS_UNORM,
       {1, 1, 1},  4, 4,
-      32, {{0}, {8}, {24}, {0}},
-      {{0}, {24}, {0}, {0}}},
+      {{0}, {8}, {24}, {0}},
+      {{0}, {0}, {8}, {0}}},
 
    {SVGA3D_YV12, SVGA3DBLOCKDESC_YV12,
       {2, 2, 1},  6, 2,
-      48, {{0}, {0}, {48}, {0}},
+      {{0}, {0}, {48}, {0}},
       {{0}, {0}, {0}, {0}}},
 
    {SVGA3D_R32G32B32A32_FLOAT, SVGA3DBLOCKDESC_RGBA_FP,
       {1, 1, 1},  16, 16,
-      128, {{32}, {32}, {32}, {32}},
+      {{32}, {32}, {32}, {32}},
       {{64}, {32}, {0}, {96}}},
 
    {SVGA3D_R16G16B16A16_FLOAT, SVGA3DBLOCKDESC_RGBA_FP,
       {1, 1, 1},  8, 8,
-      64, {{16}, {16}, {16}, {16}},
+      {{16}, {16}, {16}, {16}},
       {{32}, {16}, {0}, {48}}},
 
-   {SVGA3D_R16G16B16A16_UNORM, SVGA3DBLOCKDESC_RGBA,
+   {SVGA3D_R16G16B16A16_UNORM, SVGA3DBLOCKDESC_RGBA_UNORM,
       {1, 1, 1},  8, 8,
-      64, {{16}, {16}, {16}, {16}},
+      {{16}, {16}, {16}, {16}},
       {{32}, {16}, {0}, {48}}},
 
    {SVGA3D_R32G32_FLOAT, SVGA3DBLOCKDESC_RG_FP,
       {1, 1, 1},  8, 8,
-      64, {{0}, {32}, {32}, {0}},
+      {{0}, {32}, {32}, {0}},
       {{0}, {32}, {0}, {0}}},
 
-   {SVGA3D_R10G10B10A2_UNORM, SVGA3DBLOCKDESC_RGBA,
+   {SVGA3D_R10G10B10A2_UNORM, SVGA3DBLOCKDESC_RGBA_UNORM,
       {1, 1, 1},  4, 4,
-      32, {{10}, {10}, {10}, {2}},
-      {{0}, {10}, {20}, {30}}},
+      {{10}, {10}, {10}, {2}},
+      {{20}, {10}, {0}, {30}}},
 
-   {SVGA3D_R8G8B8A8_SNORM, SVGA3DBLOCKDESC_RGBA,
+   {SVGA3D_R8G8B8A8_SNORM, SVGA3DBLOCKDESC_RGBA_SNORM,
       {1, 1, 1},  4, 4,
-      32, {{8}, {8}, {8}, {8}},
-      {{24}, {16}, {8}, {0}}},
+      {{8}, {8}, {8}, {8}},
+      {{16}, {8}, {0}, {24}}},
 
    {SVGA3D_R16G16_FLOAT, SVGA3DBLOCKDESC_RG_FP,
       {1, 1, 1},  4, 4,
-      32, {{0}, {16}, {16}, {0}},
+      {{0}, {16}, {16}, {0}},
       {{0}, {16}, {0}, {0}}},
 
-   {SVGA3D_R16G16_UNORM, SVGA3DBLOCKDESC_RG,
+   {SVGA3D_R16G16_UNORM, SVGA3DBLOCKDESC_RG_UNORM,
       {1, 1, 1},  4, 4,
-      32, {{0}, {16}, {16}, {0}},
-      {{0}, {0}, {16}, {0}}},
+      {{0}, {16}, {16}, {0}},
+      {{0}, {16}, {0}, {0}}},
 
-   {SVGA3D_R16G16_SNORM, SVGA3DBLOCKDESC_RG,
+   {SVGA3D_R16G16_SNORM, SVGA3DBLOCKDESC_RG_SNORM,
       {1, 1, 1},  4, 4,
-      32, {{16}, {16}, {0}, {0}},
-      {{16}, {0}, {0}, {0}}},
+      {{0}, {16}, {16}, {0}},
+      {{0}, {16}, {0}, {0}}},
 
    {SVGA3D_R32_FLOAT, SVGA3DBLOCKDESC_R_FP,
       {1, 1, 1},  4, 4,
-      32, {{0}, {0}, {32}, {0}},
+      {{0}, {0}, {32}, {0}},
       {{0}, {0}, {0}, {0}}},
 
-   {SVGA3D_R8G8_SNORM, SVGA3DBLOCKDESC_RG,
+   {SVGA3D_R8G8_SNORM, SVGA3DBLOCKDESC_RG_SNORM,
       {1, 1, 1},  2, 2,
-      16, {{8}, {8}, {0}, {0}},
-      {{8}, {0}, {0}, {0}}},
+      {{0}, {8}, {8}, {0}},
+      {{0}, {8}, {0}, {0}}},
 
    {SVGA3D_R16_FLOAT, SVGA3DBLOCKDESC_R_FP,
       {1, 1, 1},  2, 2,
-      16, {{0}, {0}, {16}, {0}},
+      {{0}, {0}, {16}, {0}},
       {{0}, {0}, {0}, {0}}},
 
-   {SVGA3D_D16_UNORM, SVGA3DBLOCKDESC_DEPTH,
+   {SVGA3D_D16_UNORM, SVGA3DBLOCKDESC_DEPTH_UNORM,
       {1, 1, 1},  2, 2,
-      16, {{0}, {0}, {16}, {0}},
+      {{0}, {0}, {16}, {0}},
       {{0}, {0}, {0}, {0}}},
 
-   {SVGA3D_A8_UNORM, SVGA3DBLOCKDESC_ALPHA,
+   {SVGA3D_A8_UNORM, SVGA3DBLOCKDESC_A_UNORM,
       {1, 1, 1},  1, 1,
-      8, {{0}, {0}, {0}, {8}},
+      {{0}, {0}, {0}, {8}},
       {{0}, {0}, {0}, {0}}},
 
-   {SVGA3D_BC1_UNORM, SVGA3DBLOCKDESC_COMPRESSED,
+   {SVGA3D_BC1_UNORM, SVGA3DBLOCKDESC_BC1_COMP_UNORM,
       {4, 4, 1},  8, 8,
-      64, {{0}, {0}, {64}, {0}},
+      {{0}, {0}, {64}, {0}},
       {{0}, {0}, {0}, {0}}},
 
-   {SVGA3D_BC2_UNORM, SVGA3DBLOCKDESC_COMPRESSED,
+   {SVGA3D_BC2_UNORM, SVGA3DBLOCKDESC_BC2_COMP_UNORM,
       {4, 4, 1},  16, 16,
-      128, {{0}, {0}, {128}, {0}},
+      {{0}, {0}, {128}, {0}},
       {{0}, {0}, {0}, {0}}},
 
-   {SVGA3D_BC3_UNORM, SVGA3DBLOCKDESC_COMPRESSED,
+   {SVGA3D_BC3_UNORM, SVGA3DBLOCKDESC_BC3_COMP_UNORM,
       {4, 4, 1},  16, 16,
-      128, {{0}, {0}, {128}, {0}},
+      {{0}, {0}, {128}, {0}},
       {{0}, {0}, {0}, {0}}},
 
-   {SVGA3D_B5G6R5_UNORM, SVGA3DBLOCKDESC_RGB,
+   {SVGA3D_B5G6R5_UNORM, SVGA3DBLOCKDESC_RGB_UNORM,
       {1, 1, 1},  2, 2,
-      16, {{5}, {6}, {5}, {0}},
+      {{5}, {6}, {5}, {0}},
       {{0}, {5}, {11}, {0}}},
 
-   {SVGA3D_B5G5R5A1_UNORM, SVGA3DBLOCKDESC_RGBA,
+   {SVGA3D_B5G5R5A1_UNORM, SVGA3DBLOCKDESC_RGBA_UNORM,
       {1, 1, 1},  2, 2,
-      16, {{5}, {5}, {5}, {1}},
+      {{5}, {5}, {5}, {1}},
       {{0}, {5}, {10}, {15}}},
 
-   {SVGA3D_B8G8R8A8_UNORM, SVGA3DBLOCKDESC_RGBA,
+   {SVGA3D_B8G8R8A8_UNORM, SVGA3DBLOCKDESC_RGBA_UNORM,
       {1, 1, 1},  4, 4,
-      32, {{8}, {8}, {8}, {8}},
+      {{8}, {8}, {8}, {8}},
       {{0}, {8}, {16}, {24}}},
 
-   {SVGA3D_B8G8R8X8_UNORM, SVGA3DBLOCKDESC_RGB,
+   {SVGA3D_B8G8R8X8_UNORM, SVGA3DBLOCKDESC_RGB_UNORM,
       {1, 1, 1},  4, 4,
-      24, {{8}, {8}, {8}, {0}},
+      {{8}, {8}, {8}, {0}},
       {{0}, {8}, {16}, {24}}},
 
-   {SVGA3D_BC4_UNORM, SVGA3DBLOCKDESC_COMPRESSED,
+   {SVGA3D_BC4_UNORM, SVGA3DBLOCKDESC_BC4_COMP_UNORM,
       {4, 4, 1},  8, 8,
-      64, {{0}, {0}, {64}, {0}},
+      {{0}, {0}, {64}, {0}},
       {{0}, {0}, {0}, {0}}},
 
-   {SVGA3D_BC5_UNORM, SVGA3DBLOCKDESC_COMPRESSED,
+   {SVGA3D_BC5_UNORM, SVGA3DBLOCKDESC_BC5_COMP_UNORM,
       {4, 4, 1},  16, 16,
-      128, {{0}, {0}, {128}, {0}},
+      {{0}, {0}, {128}, {0}},
       {{0}, {0}, {0}, {0}}},
-
 };
 
 static inline u32 clamped_umul32(u32 a, u32 b)
@@ -946,6 +1111,10 @@ static inline u32 clamped_umul32(u32 a, u32 b)
 	return (tmp > (uint64_t) ((u32) -1)) ? (u32) -1 : tmp;
 }
 
+/**
+ * svga3dsurface_get_desc - Look up the appropriate SVGA3dSurfaceDesc for the
+ * given format.
+ */
 static inline const struct svga3d_surface_desc *
 svga3dsurface_get_desc(SVGA3dSurfaceFormat format)
 {
@@ -955,23 +1124,10 @@ svga3dsurface_get_desc(SVGA3dSurfaceFormat format)
 	return &svga3d_surface_descs[SVGA3D_FORMAT_INVALID];
 }
 
-/*
- *----------------------------------------------------------------------
- *
- * svga3dsurface_get_mip_size --
- *
- *      Given a base level size and the mip level, compute the size of
- *      the mip level.
- *
- * Results:
- *      See above.
- *
- * Side effects:
- *      None.
- *
- *----------------------------------------------------------------------
+/**
+ * svga3dsurface_get_mip_size -  Given a base level size and the mip level,
+ * compute the size of the mip level.
  */
-
 static inline surf_size_struct
 svga3dsurface_get_mip_size(surf_size_struct base_level, u32 mip_level)
 {
@@ -1018,28 +1174,17 @@ svga3dsurface_calculate_pitch(const struct svga3d_surface_desc *desc,
 	return pitch;
 }
 
-/*
- *-----------------------------------------------------------------------------
- *
- * svga3dsurface_get_image_buffer_size --
- *
- *      Return the number of bytes of buffer space required to store
- *      one image of a surface, optionally using the specified pitch.
- *
- *      If pitch is zero, it is assumed that rows are tightly packed.
+/**
+ * svga3dsurface_get_image_buffer_size - Calculates image buffer size.
  *
- *      This function is overflow-safe. If the result would have
- *      overflowed, instead we return MAX_UINT32.
+ * Return the number of bytes of buffer space required to store one image of a
+ * surface, optionally using the specified pitch.
  *
- * Results:
- *      Byte count.
+ * If pitch is zero, it is assumed that rows are tightly packed.
  *
- * Side effects:
- *      None.
- *
- *-----------------------------------------------------------------------------
+ * This function is overflow-safe. If the result would have overflowed, instead
+ * we return MAX_UINT32.
  */
-
 static inline u32
 svga3dsurface_get_image_buffer_size(const struct svga3d_surface_desc *desc,
 				    const surf_size_struct *size,
@@ -1067,6 +1212,9 @@ svga3dsurface_get_image_buffer_size(const struct svga3d_surface_desc *desc,
 	return total_size;
 }
 
+/**
+ * svga3dsurface_get_serialized_size - Get the serialized size for the image.
+ */
 static inline u32
 svga3dsurface_get_serialized_size(SVGA3dSurfaceFormat format,
 				  surf_size_struct base_level_size,
@@ -1087,6 +1235,26 @@ svga3dsurface_get_serialized_size(SVGA3dSurfaceFormat format,
 	return total_size * num_layers;
 }
 
+/**
+ * svga3dsurface_get_serialized_size_extended - Returns the number of bytes
+ * required for a surface with given parameters. Support for sample count.
+ */
+static inline u32
+svga3dsurface_get_serialized_size_extended(SVGA3dSurfaceFormat format,
+					   surf_size_struct base_level_size,
+					   u32 num_mip_levels,
+					   u32 num_layers,
+					   u32 num_samples)
+{
+	uint64_t total_size =
+		svga3dsurface_get_serialized_size(format,
+						  base_level_size,
+						  num_mip_levels,
+						  num_layers);
+	total_size *= max_t(u32, 1, num_samples);
+
+	return min_t(uint64_t, total_size, (uint64_t)U32_MAX);
+}
 
 /**
  * svga3dsurface_get_pixel_offset - Compute the offset (in bytes) to a pixel
@@ -1206,3 +1374,5 @@ svga3dsurface_is_screen_target_format(SVGA3dSurfaceFormat format)
 	}
 	return svga3dsurface_is_dx_screen_target_format(format);
 }
+
+#endif /* _SVGA3D_SURFACEDEFS_H_ */
diff --git a/drivers/gpu/drm/vmwgfx/device_include/svga3d_types.h b/drivers/gpu/drm/vmwgfx/device_include/svga3d_types.h
index df63f24c2e10..308370665a8e 100644
--- a/drivers/gpu/drm/vmwgfx/device_include/svga3d_types.h
+++ b/drivers/gpu/drm/vmwgfx/device_include/svga3d_types.h
@@ -45,9 +45,21 @@
 
 #define SVGA3D_INVALID_ID         ((uint32)-1)
 
+typedef uint8 SVGABool8;   /* 8-bit Bool definition */
 typedef uint32 SVGA3dBool; /* 32-bit Bool definition */
 typedef uint32 SVGA3dColor; /* a, r, g, b */
 
+typedef uint32 SVGA3dSurfaceId;
+
+typedef
+#include "vmware_pack_begin.h"
+struct {
+   uint32 numerator;
+   uint32 denominator;
+}
+#include "vmware_pack_end.h"
+SVGA3dFraction64;
+
 typedef
 #include "vmware_pack_begin.h"
 struct SVGA3dCopyRect {
@@ -146,7 +158,7 @@ typedef enum SVGA3dSurfaceFormat {
    SVGA3D_BUMPU8V8                     = 20,
    SVGA3D_BUMPL6V5U5                   = 21,
    SVGA3D_BUMPX8L8V8U8                 = 22,
-   SVGA3D_BUMPL8V8U8                   = 23,
+   SVGA3D_FORMAT_DEAD1                 = 23,
 
    SVGA3D_ARGB_S10E5                   = 24,   /* 16-bit floating-point ARGB */
    SVGA3D_ARGB_S23E8                   = 25,   /* 32-bit floating-point ARGB */
@@ -205,8 +217,8 @@ typedef enum SVGA3dSurfaceFormat {
    SVGA3D_R32G32_SINT                  = 59,
    SVGA3D_R32G8X24_TYPELESS            = 60,
    SVGA3D_D32_FLOAT_S8X24_UINT         = 61,
-   SVGA3D_R32_FLOAT_X8X24_TYPELESS     = 62,
-   SVGA3D_X32_TYPELESS_G8X24_UINT      = 63,
+   SVGA3D_R32_FLOAT_X8X24              = 62,
+   SVGA3D_X32_G8X24_UINT               = 63,
    SVGA3D_R10G10B10A2_TYPELESS         = 64,
    SVGA3D_R10G10B10A2_UINT             = 65,
    SVGA3D_R11G11B10_FLOAT              = 66,
@@ -224,8 +236,8 @@ typedef enum SVGA3dSurfaceFormat {
    SVGA3D_R32_SINT                     = 78,
    SVGA3D_R24G8_TYPELESS               = 79,
    SVGA3D_D24_UNORM_S8_UINT            = 80,
-   SVGA3D_R24_UNORM_X8_TYPELESS        = 81,
-   SVGA3D_X24_TYPELESS_G8_UINT         = 82,
+   SVGA3D_R24_UNORM_X8                 = 81,
+   SVGA3D_X24_G8_UINT                  = 82,
    SVGA3D_R8G8_TYPELESS                = 83,
    SVGA3D_R8G8_UNORM                   = 84,
    SVGA3D_R8G8_UINT                    = 85,
@@ -297,92 +309,114 @@ typedef enum SVGA3dSurfaceFormat {
    SVGA3D_FORMAT_MAX
 } SVGA3dSurfaceFormat;
 
-typedef enum SVGA3dSurfaceFlags {
-   SVGA3D_SURFACE_CUBEMAP               = (1 << 0),
+/*
+ * SVGA3d Surface Flags --
+ */
+#define SVGA3D_SURFACE_CUBEMAP                (1 << 0)
 
-   /*
-    * HINT flags are not enforced by the device but are useful for
-    * performance.
-    */
-   SVGA3D_SURFACE_HINT_STATIC           = (1 << 1),
-   SVGA3D_SURFACE_HINT_DYNAMIC          = (1 << 2),
-   SVGA3D_SURFACE_HINT_INDEXBUFFER      = (1 << 3),
-   SVGA3D_SURFACE_HINT_VERTEXBUFFER     = (1 << 4),
-   SVGA3D_SURFACE_HINT_TEXTURE          = (1 << 5),
-   SVGA3D_SURFACE_HINT_RENDERTARGET     = (1 << 6),
-   SVGA3D_SURFACE_HINT_DEPTHSTENCIL     = (1 << 7),
-   SVGA3D_SURFACE_HINT_WRITEONLY        = (1 << 8),
-   SVGA3D_SURFACE_MASKABLE_ANTIALIAS    = (1 << 9),
-   SVGA3D_SURFACE_AUTOGENMIPMAPS        = (1 << 10),
-   SVGA3D_SURFACE_DECODE_RENDERTARGET   = (1 << 11),
+/*
+ * HINT flags are not enforced by the device but are useful for
+ * performance.
+ */
+#define SVGA3D_SURFACE_HINT_STATIC            (CONST64U(1) << 1)
+#define SVGA3D_SURFACE_HINT_DYNAMIC           (CONST64U(1) << 2)
+#define SVGA3D_SURFACE_HINT_INDEXBUFFER       (CONST64U(1) << 3)
+#define SVGA3D_SURFACE_HINT_VERTEXBUFFER      (CONST64U(1) << 4)
+#define SVGA3D_SURFACE_HINT_TEXTURE           (CONST64U(1) << 5)
+#define SVGA3D_SURFACE_HINT_RENDERTARGET      (CONST64U(1) << 6)
+#define SVGA3D_SURFACE_HINT_DEPTHSTENCIL      (CONST64U(1) << 7)
+#define SVGA3D_SURFACE_HINT_WRITEONLY         (CONST64U(1) << 8)
+#define SVGA3D_SURFACE_MASKABLE_ANTIALIAS     (CONST64U(1) << 9)
+#define SVGA3D_SURFACE_AUTOGENMIPMAPS         (CONST64U(1) << 10)
+
+#define SVGA3D_SURFACE_DECODE_RENDERTARGET    (CONST64U(1) << 11)
 
-   /*
-    * Is this surface using a base-level pitch for it's mob backing?
-    *
-    * This flag is not intended to be set by guest-drivers, but is instead
-    * set by the device when the surface is bound to a mob with a specified
-    * pitch.
-    */
-   SVGA3D_SURFACE_MOB_PITCH             = (1 << 12),
+/*
+ * Is this surface using a base-level pitch for it's mob backing?
+ *
+ * This flag is not intended to be set by guest-drivers, but is instead
+ * set by the device when the surface is bound to a mob with a specified
+ * pitch.
+ */
+#define SVGA3D_SURFACE_MOB_PITCH              (CONST64U(1) << 12)
 
-   SVGA3D_SURFACE_INACTIVE              = (1 << 13),
-   SVGA3D_SURFACE_HINT_RT_LOCKABLE      = (1 << 14),
-   SVGA3D_SURFACE_VOLUME                = (1 << 15),
+#define SVGA3D_SURFACE_INACTIVE               (CONST64U(1) << 13)
+#define SVGA3D_SURFACE_HINT_RT_LOCKABLE       (CONST64U(1) << 14)
+#define SVGA3D_SURFACE_VOLUME                 (CONST64U(1) << 15)
 
-   /*
-    * Required to be set on a surface to bind it to a screen target.
-    */
-   SVGA3D_SURFACE_SCREENTARGET          = (1 << 16),
+/*
+ * Required to be set on a surface to bind it to a screen target.
+ */
+#define SVGA3D_SURFACE_SCREENTARGET           (CONST64U(1) << 16)
 
-   /*
-    * Align images in the guest-backing mob to 16-bytes.
-    */
-   SVGA3D_SURFACE_ALIGN16               = (1 << 17),
+/*
+ * Align images in the guest-backing mob to 16-bytes.
+ */
+#define SVGA3D_SURFACE_ALIGN16                (CONST64U(1) << 17)
 
-   SVGA3D_SURFACE_1D                    = (1 << 18),
-   SVGA3D_SURFACE_ARRAY                 = (1 << 19),
+#define SVGA3D_SURFACE_1D                     (CONST64U(1) << 18)
+#define SVGA3D_SURFACE_ARRAY                  (CONST64U(1) << 19)
 
-   /*
-    * Bind flags.
-    * These are enforced for any surface defined with DefineGBSurface_v2.
-    */
-   SVGA3D_SURFACE_BIND_VERTEX_BUFFER    = (1 << 20),
-   SVGA3D_SURFACE_BIND_INDEX_BUFFER     = (1 << 21),
-   SVGA3D_SURFACE_BIND_CONSTANT_BUFFER  = (1 << 22),
-   SVGA3D_SURFACE_BIND_SHADER_RESOURCE  = (1 << 23),
-   SVGA3D_SURFACE_BIND_RENDER_TARGET    = (1 << 24),
-   SVGA3D_SURFACE_BIND_DEPTH_STENCIL    = (1 << 25),
-   SVGA3D_SURFACE_BIND_STREAM_OUTPUT    = (1 << 26),
+/*
+ * Bind flags.
+ * These are enforced for any surface defined with DefineGBSurface_v2.
+ */
+#define SVGA3D_SURFACE_BIND_VERTEX_BUFFER     (CONST64U(1) << 20)
+#define SVGA3D_SURFACE_BIND_INDEX_BUFFER      (CONST64U(1) << 21)
+#define SVGA3D_SURFACE_BIND_CONSTANT_BUFFER   (CONST64U(1) << 22)
+#define SVGA3D_SURFACE_BIND_SHADER_RESOURCE   (CONST64U(1) << 23)
+#define SVGA3D_SURFACE_BIND_RENDER_TARGET     (CONST64U(1) << 24)
+#define SVGA3D_SURFACE_BIND_DEPTH_STENCIL     (CONST64U(1) << 25)
+#define SVGA3D_SURFACE_BIND_STREAM_OUTPUT     (CONST64U(1) << 26)
 
-   /*
-    * A note on staging flags:
-    *
-    * The STAGING flags notes that the surface will not be used directly by the
-    * drawing pipeline, i.e. that it will not be bound to any bind point.
-    * Staging surfaces may be used by copy operations to move data in and out
-    * of other surfaces.
-    *
-    * The HINT_INDIRECT_UPDATE flag suggests that the surface will receive
-    * updates indirectly, i.e. the surface will not be updated directly, but
-    * will receive copies from staging surfaces.
-    */
-   SVGA3D_SURFACE_STAGING_UPLOAD        = (1 << 27),
-   SVGA3D_SURFACE_STAGING_DOWNLOAD      = (1 << 28),
-   SVGA3D_SURFACE_HINT_INDIRECT_UPDATE  = (1 << 29),
+/*
+ * The STAGING flags notes that the surface will not be used directly by the
+ * drawing pipeline, i.e. that it will not be bound to any bind point.
+ * Staging surfaces may be used by copy operations to move data in and out
+ * of other surfaces.  No bind flags may be set on surfaces with this flag.
+ *
+ * The HINT_INDIRECT_UPDATE flag suggests that the surface will receive
+ * updates indirectly, i.e. the surface will not be updated directly, but
+ * will receive copies from staging surfaces.
+ */
+#define SVGA3D_SURFACE_STAGING_UPLOAD         (CONST64U(1) << 27)
+#define SVGA3D_SURFACE_STAGING_DOWNLOAD       (CONST64U(1) << 28)
+#define SVGA3D_SURFACE_HINT_INDIRECT_UPDATE   (CONST64U(1) << 29)
 
-   /*
-    * Setting this flag allow this surface to be used with the
-    * SVGA_3D_CMD_DX_TRANSFER_FROM_BUFFER command.  It is only valid for
-    * buffer surfaces, an no bind flags are allowed to be set on surfaces
-    * with this flag.
-    */
-   SVGA3D_SURFACE_TRANSFER_FROM_BUFFER  = (1 << 30),
+/*
+ * Setting this flag allow this surface to be used with the
+ * SVGA_3D_CMD_DX_TRANSFER_FROM_BUFFER command.  It is only valid for
+ * buffer surfaces, and no bind flags are allowed to be set on surfaces
+ * with this flag.
+ */
+#define SVGA3D_SURFACE_TRANSFER_FROM_BUFFER   (CONST64U(1) << 30)
 
-   /*
-    * Marker for the last defined bit.
-    */
-   SVGA3D_SURFACE_FLAG_MAX              = (1 << 31),
-} SVGA3dSurfaceFlags;
+/*
+ * Reserved for video operations.
+ */
+#define SVGA3D_SURFACE_RESERVED1              (CONST64U(1) << 31)
+
+/*
+ * Specifies that a surface is multisample, and therefore requires the full
+ * mob-backing to store all the samples.
+ */
+#define SVGA3D_SURFACE_MULTISAMPLE            (CONST64U(1) << 32)
+
+#define SVGA3D_SURFACE_FLAG_MAX               (CONST64U(1) << 33)
+
+/*
+ * Surface flags types:
+ *
+ * SVGA3dSurface1Flags:  Lower 32-bits of flags.
+ * SVGA3dSurface2Flags:  Upper 32-bits of flags.
+ * SVGA3dSurfaceAllFlags: Full 64-bits of flags.
+ */
+typedef uint32 SVGA3dSurface1Flags;
+typedef uint32 SVGA3dSurface2Flags;
+typedef uint64 SVGA3dSurfaceAllFlags;
+
+#define SVGA3D_SURFACE_FLAGS1_MASK ((uint64_t)MAX_UINT32)
+#define SVGA3D_SURFACE_FLAGS2_MASK (MAX_UINT64 & ~SVGA3D_SURFACE_FLAGS1_MASK)
 
 #define SVGA3D_SURFACE_HB_DISALLOWED_MASK        \
         (  SVGA3D_SURFACE_MOB_PITCH    |         \
@@ -393,29 +427,41 @@ typedef enum SVGA3dSurfaceFlags {
            SVGA3D_SURFACE_STAGING_UPLOAD |       \
            SVGA3D_SURFACE_STAGING_DOWNLOAD |     \
            SVGA3D_SURFACE_HINT_INDIRECT_UPDATE | \
-           SVGA3D_SURFACE_TRANSFER_FROM_BUFFER   \
+           SVGA3D_SURFACE_TRANSFER_FROM_BUFFER | \
+           SVGA3D_SURFACE_MULTISAMPLE            \
+        )
+
+#define SVGA3D_SURFACE_HB_PRESENT_DISALLOWED_MASK   \
+       (   SVGA3D_SURFACE_1D |                      \
+           SVGA3D_SURFACE_MULTISAMPLE               \
         )
 
 #define SVGA3D_SURFACE_2D_DISALLOWED_MASK           \
         (  SVGA3D_SURFACE_CUBEMAP |                 \
            SVGA3D_SURFACE_MASKABLE_ANTIALIAS |      \
            SVGA3D_SURFACE_AUTOGENMIPMAPS |          \
-           SVGA3D_SURFACE_DECODE_RENDERTARGET |     \
            SVGA3D_SURFACE_VOLUME |                  \
            SVGA3D_SURFACE_1D |                      \
-           SVGA3D_SURFACE_ARRAY |                   \
            SVGA3D_SURFACE_BIND_VERTEX_BUFFER |      \
            SVGA3D_SURFACE_BIND_INDEX_BUFFER |       \
            SVGA3D_SURFACE_BIND_CONSTANT_BUFFER |    \
            SVGA3D_SURFACE_BIND_DEPTH_STENCIL |      \
            SVGA3D_SURFACE_BIND_STREAM_OUTPUT |      \
-           SVGA3D_SURFACE_TRANSFER_FROM_BUFFER      \
+           SVGA3D_SURFACE_TRANSFER_FROM_BUFFER |    \
+           SVGA3D_SURFACE_MULTISAMPLE               \
+        )
+
+#define SVGA3D_SURFACE_BASICOPS_DISALLOWED_MASK     \
+        (  SVGA3D_SURFACE_CUBEMAP |                 \
+           SVGA3D_SURFACE_AUTOGENMIPMAPS |          \
+           SVGA3D_SURFACE_VOLUME |                  \
+           SVGA3D_SURFACE_1D |                      \
+           SVGA3D_SURFACE_MULTISAMPLE               \
         )
 
 #define SVGA3D_SURFACE_SCREENTARGET_DISALLOWED_MASK \
         (  SVGA3D_SURFACE_CUBEMAP |                 \
            SVGA3D_SURFACE_AUTOGENMIPMAPS |          \
-           SVGA3D_SURFACE_DECODE_RENDERTARGET |     \
            SVGA3D_SURFACE_VOLUME |                  \
            SVGA3D_SURFACE_1D |                      \
            SVGA3D_SURFACE_BIND_VERTEX_BUFFER |      \
@@ -427,12 +473,36 @@ typedef enum SVGA3dSurfaceFlags {
            SVGA3D_SURFACE_STAGING_UPLOAD |          \
            SVGA3D_SURFACE_STAGING_DOWNLOAD |        \
            SVGA3D_SURFACE_HINT_INDIRECT_UPDATE |    \
-           SVGA3D_SURFACE_TRANSFER_FROM_BUFFER      \
+           SVGA3D_SURFACE_TRANSFER_FROM_BUFFER |    \
+           SVGA3D_SURFACE_MULTISAMPLE               \
+        )
+
+#define SVGA3D_SURFACE_BUFFER_DISALLOWED_MASK       \
+        (  SVGA3D_SURFACE_CUBEMAP |                 \
+           SVGA3D_SURFACE_AUTOGENMIPMAPS |          \
+           SVGA3D_SURFACE_VOLUME |                  \
+           SVGA3D_SURFACE_1D |                      \
+           SVGA3D_SURFACE_MASKABLE_ANTIALIAS |      \
+           SVGA3D_SURFACE_ARRAY |                   \
+           SVGA3D_SURFACE_MULTISAMPLE |             \
+           SVGA3D_SURFACE_MOB_PITCH                 \
+        )
+
+#define SVGA3D_SURFACE_MULTISAMPLE_DISALLOWED_MASK  \
+        (  SVGA3D_SURFACE_CUBEMAP |                 \
+           SVGA3D_SURFACE_AUTOGENMIPMAPS |          \
+           SVGA3D_SURFACE_VOLUME |                  \
+           SVGA3D_SURFACE_1D |                      \
+           SVGA3D_SURFACE_SCREENTARGET |            \
+           SVGA3D_SURFACE_MOB_PITCH                 \
         )
 
 #define SVGA3D_SURFACE_DX_ONLY_MASK             \
         (  SVGA3D_SURFACE_BIND_STREAM_OUTPUT |  \
+           SVGA3D_SURFACE_STAGING_UPLOAD |      \
+           SVGA3D_SURFACE_STAGING_DOWNLOAD |    \
            SVGA3D_SURFACE_TRANSFER_FROM_BUFFER  \
+        )
 
 #define SVGA3D_SURFACE_STAGING_MASK             \
         (  SVGA3D_SURFACE_STAGING_UPLOAD |      \
@@ -488,7 +558,7 @@ typedef enum {
 
 /*
  * Indicates that this format can be converted to any RGB format for which
- * SVGA3DFORMAT_OP_MEMBEROFGROUP_ARGB is specified
+ * SVGA3DFORMAT_OP_MEMBEROFGROUP_ARGB is specified.
  */
    SVGA3DFORMAT_OP_CONVERT_TO_ARGB                       = 0x00002000,
 
@@ -499,22 +569,22 @@ typedef enum {
 
 /*
  * Indicated that this format can be read as an SRGB texture (meaning that the
- * sampler will linearize the looked up data)
+ * sampler will linearize the looked up data).
  */
    SVGA3DFORMAT_OP_SRGBREAD                              = 0x00008000,
 
 /*
- * Indicates that this format can be used in the bumpmap instructions
+ * Indicates that this format can be used in the bumpmap instructions.
  */
    SVGA3DFORMAT_OP_BUMPMAP                               = 0x00010000,
 
 /*
- * Indicates that this format can be sampled by the displacement map sampler
+ * Indicates that this format can be sampled by the displacement map sampler.
  */
    SVGA3DFORMAT_OP_DMAP                                  = 0x00020000,
 
 /*
- * Indicates that this format cannot be used with texture filtering
+ * Indicates that this format cannot be used with texture filtering.
  */
    SVGA3DFORMAT_OP_NOFILTER                              = 0x00040000,
 
@@ -531,18 +601,18 @@ typedef enum {
    SVGA3DFORMAT_OP_SRGBWRITE                             = 0x00100000,
 
 /*
- * Indicates that this format cannot be used with alpha blending
+ * Indicates that this format cannot be used with alpha blending.
  */
    SVGA3DFORMAT_OP_NOALPHABLEND                          = 0x00200000,
 
 /*
  * Indicates that the device can auto-generated sublevels for resources
- * of this format
+ * of this format.
  */
    SVGA3DFORMAT_OP_AUTOGENMIPMAP                         = 0x00400000,
 
 /*
- * Indicates that this format can be used by vertex texture sampler
+ * Indicates that this format can be used by vertex texture sampler.
  */
    SVGA3DFORMAT_OP_VERTEXTEXTURE                         = 0x00800000,
 
@@ -1502,7 +1572,6 @@ union SVGADXQueryResultUnion {
 #include "vmware_pack_end.h"
 SVGADXQueryResultUnion;
 
-
 typedef enum {
    SVGA3D_QUERYSTATE_PENDING     = 0,      /* Query is not finished yet */
    SVGA3D_QUERYSTATE_SUCCEEDED   = 1,      /* Completed successfully */
@@ -1534,9 +1603,9 @@ typedef
 struct {
    union {
       struct {
-	 uint16  function;       /* SVGA3dFogFunction */
-	 uint8   type;           /* SVGA3dFogType */
-	 uint8   base;           /* SVGA3dFogBase */
+         uint16  function;       /* SVGA3dFogFunction */
+         uint8   type;           /* SVGA3dFogType */
+         uint8   base;           /* SVGA3dFogBase */
       };
       uint32     uintValue;
    };
@@ -1548,19 +1617,27 @@ SVGA3dFogMode;
  * Uniquely identify one image (a 1D/2D/3D array) from a surface. This
  * is a surface ID as well as face/mipmap indices.
  */
-
 typedef
 #include "vmware_pack_begin.h"
 struct SVGA3dSurfaceImageId {
-   uint32               sid;
-   uint32               face;
-   uint32               mipmap;
+   uint32 sid;
+   uint32 face;
+   uint32 mipmap;
 }
 #include "vmware_pack_end.h"
 SVGA3dSurfaceImageId;
 
 typedef
 #include "vmware_pack_begin.h"
+struct SVGA3dSubSurfaceId {
+   uint32 sid;
+   uint32 subResourceId;
+}
+#include "vmware_pack_end.h"
+SVGA3dSubSurfaceId;
+
+typedef
+#include "vmware_pack_begin.h"
 struct {
    uint32               width;
    uint32               height;
@@ -1583,13 +1660,18 @@ typedef enum {
    SVGA_OTABLE_DX9_MAX         = 5,
 
    SVGA_OTABLE_DXCONTEXT       = 5,
-   SVGA_OTABLE_MAX             = 6
-} SVGAOTableType;
+   SVGA_OTABLE_DX_MAX          = 6,
 
-/*
- * Deprecated.
- */
-#define SVGA_OTABLE_COUNT 4
+   SVGA_OTABLE_RESERVED1       = 6,
+   SVGA_OTABLE_RESERVED2       = 7,
+
+   /*
+    * Additions to this table need to be tied to HW-version features and
+    * checkpointed accordingly.
+    */
+   SVGA_OTABLE_DEVEL_MAX       = 8,
+   SVGA_OTABLE_MAX             = 8
+} SVGAOTableType;
 
 typedef enum {
    SVGA_COTABLE_MIN             = 0,
@@ -1606,7 +1688,7 @@ typedef enum {
    SVGA_COTABLE_DXSHADER        = 10,
    SVGA_COTABLE_DX10_MAX        = 11,
    SVGA_COTABLE_UAVIEW          = 11,
-   SVGA_COTABLE_MAX
+   SVGA_COTABLE_MAX             = 12,
 } SVGACOTableType;
 
 /*
@@ -1627,8 +1709,37 @@ typedef enum SVGAMobFormat {
    SVGA3D_MOBFMT_PREDX_MAX   = 7,
    SVGA3D_MOBFMT_EMPTY       = 7,
    SVGA3D_MOBFMT_MAX,
+
+   /*
+    * This isn't actually used by the guest, but is a mob-format used
+    * internally by the SVGA device (and is therefore not binary compatible).
+    */
+   SVGA3D_MOBFMT_HB,
 } SVGAMobFormat;
 
 #define SVGA3D_MOB_EMPTY_BASE 1
 
+/*
+ * Multisample pattern types.
+ */
+
+typedef enum SVGA3dMSPattern {
+   SVGA3D_MS_PATTERN_NONE     = 0,
+   SVGA3D_MS_PATTERN_MIN      = 0,
+   SVGA3D_MS_PATTERN_STANDARD = 1,
+   SVGA3D_MS_PATTERN_CENTER   = 2,
+   SVGA3D_MS_PATTERN_MAX      = 3,
+} SVGA3dMSPattern;
+
+/*
+ * Precision settings for each sample.
+ */
+
+typedef enum SVGA3dMSQualityLevel {
+   SVGA3D_MS_QUALITY_NONE = 0,
+   SVGA3D_MS_QUALITY_MIN  = 0,
+   SVGA3D_MS_QUALITY_FULL = 1,
+   SVGA3D_MS_QUALITY_MAX  = 2,
+} SVGA3dMSQualityLevel;
+
 #endif /* _SVGA3D_TYPES_H_ */
diff --git a/drivers/gpu/drm/vmwgfx/device_include/svga_reg.h b/drivers/gpu/drm/vmwgfx/device_include/svga_reg.h
index 847e3301c478..056f54b35d73 100644
--- a/drivers/gpu/drm/vmwgfx/device_include/svga_reg.h
+++ b/drivers/gpu/drm/vmwgfx/device_include/svga_reg.h
@@ -64,16 +64,26 @@ typedef uint32 SVGAMobId;
 #define SVGA_MAX_BITS_PER_PIXEL         32
 #define SVGA_MAX_DEPTH                  24
 #define SVGA_MAX_DISPLAYS               10
+#define SVGA_MAX_SCREEN_SIZE            8192
+#define SVGA_SCREEN_ROOT_LIMIT (SVGA_MAX_SCREEN_SIZE * SVGA_MAX_DISPLAYS)
+
 
 /*
  * Legal values for the SVGA_REG_CURSOR_ON register in old-fashioned
  * cursor bypass mode. This is still supported, but no new guest
  * drivers should use it.
  */
-#define SVGA_CURSOR_ON_HIDE            0x0   /* Must be 0 to maintain backward compatibility */
-#define SVGA_CURSOR_ON_SHOW            0x1   /* Must be 1 to maintain backward compatibility */
-#define SVGA_CURSOR_ON_REMOVE_FROM_FB  0x2   /* Remove the cursor from the framebuffer because we need to see what's under it */
-#define SVGA_CURSOR_ON_RESTORE_TO_FB   0x3   /* Put the cursor back in the framebuffer so the user can see it */
+#define SVGA_CURSOR_ON_HIDE            0x0
+#define SVGA_CURSOR_ON_SHOW            0x1
+
+/*
+ * Remove the cursor from the framebuffer
+ * because we need to see what's under it
+ */
+#define SVGA_CURSOR_ON_REMOVE_FROM_FB  0x2
+
+/* Put the cursor back in the framebuffer so the user can see it */
+#define SVGA_CURSOR_ON_RESTORE_TO_FB   0x3
 
 /*
  * The maximum framebuffer size that can traced for guests unless the
@@ -102,7 +112,10 @@ typedef uint32 SVGAMobId;
 #define SVGA_VERSION_0     0
 #define SVGA_ID_0          SVGA_MAKE_ID(SVGA_VERSION_0)
 
-/* "Invalid" value for all SVGA IDs. (Version ID, screen object ID, surface ID...) */
+/*
+ * "Invalid" value for all SVGA IDs.
+ * (Version ID, screen object ID, surface ID...)
+ */
 #define SVGA_ID_INVALID    0xFFFFFFFF
 
 /* Port offsets, relative to BAR0 */
@@ -155,7 +168,7 @@ enum {
    SVGA_REG_CONFIG_DONE = 20,         /* Set when memory area configured */
    SVGA_REG_SYNC = 21,                /* See "FIFO Synchronization Registers" */
    SVGA_REG_BUSY = 22,                /* See "FIFO Synchronization Registers" */
-   SVGA_REG_GUEST_ID = 23,            /* Set guest OS identifier */
+   SVGA_REG_GUEST_ID = 23,            /* (Deprecated) */
    SVGA_REG_CURSOR_ID = 24,           /* (Deprecated) */
    SVGA_REG_CURSOR_X = 25,            /* (Deprecated) */
    SVGA_REG_CURSOR_Y = 26,            /* (Deprecated) */
@@ -187,7 +200,14 @@ enum {
    SVGA_REG_MEMORY_SIZE = 47,       /* Total dedicated device memory excluding FIFO */
    SVGA_REG_COMMAND_LOW = 48,       /* Lower 32 bits and submits commands */
    SVGA_REG_COMMAND_HIGH = 49,      /* Upper 32 bits of command buffer PA */
-   SVGA_REG_MAX_PRIMARY_BOUNDING_BOX_MEM = 50,   /* Max primary memory */
+
+   /*
+    * Max primary memory.
+    * See SVGA_CAP_NO_BB_RESTRICTION.
+    */
+   SVGA_REG_MAX_PRIMARY_MEM = 50,
+   SVGA_REG_MAX_PRIMARY_BOUNDING_BOX_MEM = 50,
+
    SVGA_REG_SUGGESTED_GBOBJECT_MEM_SIZE_KB = 51, /* Sugested limit on mob mem */
    SVGA_REG_DEV_CAP = 52,           /* Write dev cap index, read value */
    SVGA_REG_CMD_PREPEND_LOW = 53,
@@ -195,7 +215,10 @@ enum {
    SVGA_REG_SCREENTARGET_MAX_WIDTH = 55,
    SVGA_REG_SCREENTARGET_MAX_HEIGHT = 56,
    SVGA_REG_MOB_MAX_SIZE = 57,
-   SVGA_REG_TOP = 58,               /* Must be 1 more than the last register */
+   SVGA_REG_BLANK_SCREEN_TARGETS = 58,
+   SVGA_REG_CAP2 = 59,
+   SVGA_REG_DEVEL_CAP = 60,
+   SVGA_REG_TOP = 61,               /* Must be 1 more than the last register */
 
    SVGA_PALETTE_BASE = 1024,        /* Base of SVGA color map */
    /* Next 768 (== 256*3) registers exist for colormap */
@@ -393,6 +416,7 @@ typedef enum {
    SVGA_CB_CONTEXT_0      = 0x0,
    SVGA_CB_CONTEXT_1      = 0x1, /* Supported with SVGA_CAP_HP_CMD_QUEUE */
    SVGA_CB_CONTEXT_MAX    = 0x2,
+   SVGA_CB_CONTEXT_HP_MAX = 0x2,
 } SVGACBContext;
 
 
@@ -449,6 +473,18 @@ typedef enum {
     * due to an error.  No IRQ is raised.
     */
    SVGA_CB_STATUS_SUBMISSION_ERROR = 6,
+
+   /*
+    * Written by the host when the host finished a
+    * SVGA_DC_CMD_ASYNC_STOP_QUEUE request for this command buffer
+    * queue.  The offset of the first byte not processed is stored in
+    * the errorOffset field of the command buffer header.  All guest
+    * visible side effects of commands till that point are guaranteed
+    * to be finished before this is written.  The
+    * SVGA_IRQFLAG_COMMAND_BUFFER IRQ is raised as long as the
+    * SVGA_CB_FLAG_NO_IRQ is not set.
+    */
+   SVGA_CB_STATUS_PARTIAL_COMPLETE = 7,
 } SVGACBStatus;
 
 typedef enum {
@@ -461,8 +497,8 @@ typedef enum {
 typedef
 #include "vmware_pack_begin.h"
 struct {
-   volatile SVGACBStatus status;
-   volatile uint32 errorOffset;
+   volatile SVGACBStatus status; /* Modified by device. */
+   volatile uint32 errorOffset;  /* Modified by device. */
    uint64 id;
    SVGACBFlags flags;
    uint32 length;
@@ -473,7 +509,9 @@ struct {
          uint32 mobOffset;
       } mob;
    } ptr;
-   uint32 offset; /* Valid if CMD_BUFFERS_2 cap set, must be zero otherwise */
+   uint32 offset; /* Valid if CMD_BUFFERS_2 cap set, must be zero otherwise,
+                   * modified by device.
+                   */
    uint32 dxContext; /* Valid if DX_CONTEXT flag set, must be zero otherwise */
    uint32 mustBeZero[6];
 }
@@ -484,20 +522,26 @@ typedef enum {
    SVGA_DC_CMD_NOP                   = 0,
    SVGA_DC_CMD_START_STOP_CONTEXT    = 1,
    SVGA_DC_CMD_PREEMPT               = 2,
-   SVGA_DC_CMD_MAX                   = 3,
-   SVGA_DC_CMD_FORCE_UINT            = MAX_UINT32,
+   SVGA_DC_CMD_START_QUEUE           = 3, /* Requires SVGA_CAP_HP_CMD_QUEUE */
+   SVGA_DC_CMD_ASYNC_STOP_QUEUE      = 4, /* Requires SVGA_CAP_HP_CMD_QUEUE */
+   SVGA_DC_CMD_EMPTY_CONTEXT_QUEUE   = 5, /* Requires SVGA_CAP_HP_CMD_QUEUE */
+   SVGA_DC_CMD_MAX                   = 6,
 } SVGADeviceContextCmdId;
 
-typedef struct {
+/*
+ * Starts or stops both SVGA_CB_CONTEXT_0 and SVGA_CB_CONTEXT_1.
+ */
+
+typedef struct SVGADCCmdStartStop {
    uint32 enable;
-   SVGACBContext context;
+   SVGACBContext context; /* Must be zero */
 } SVGADCCmdStartStop;
 
 /*
  * SVGADCCmdPreempt --
  *
  * This command allows the guest to request that all command buffers
- * on the specified context be preempted that can be.  After execution
+ * on SVGA_CB_CONTEXT_0 be preempted that can be.  After execution
  * of this command all command buffers that were preempted will
  * already have SVGA_CB_STATUS_PREEMPTED written into the status
  * field.  The device might still be processing a command buffer,
@@ -507,12 +551,69 @@ typedef struct {
  * command buffer header set to zero.
  */
 
-typedef struct {
-   SVGACBContext context;
+typedef struct SVGADCCmdPreempt {
+   SVGACBContext context; /* Must be zero */
    uint32 ignoreIDZero;
 } SVGADCCmdPreempt;
 
 /*
+ * Starts the requested command buffer processing queue.  Valid only
+ * if the SVGA_CAP_HP_CMD_QUEUE cap is set.
+ *
+ * For a command queue to be considered runnable it must be enabled
+ * and any corresponding higher priority queues must also be enabled.
+ * For example in order for command buffers to be processed on
+ * SVGA_CB_CONTEXT_0 both SVGA_CB_CONTEXT_0 and SVGA_CB_CONTEXT_1 must
+ * be enabled.  But for commands to be runnable on SVGA_CB_CONTEXT_1
+ * only that queue must be enabled.
+ */
+
+typedef struct SVGADCCmdStartQueue {
+   SVGACBContext context;
+} SVGADCCmdStartQueue;
+
+/*
+ * Requests the SVGA device to stop processing the requested command
+ * buffer queue as soon as possible.  The guest knows the stop has
+ * completed when one of the following happens.
+ *
+ * 1) A command buffer status of SVGA_CB_STATUS_PARTIAL_COMPLETE is returned
+ * 2) A command buffer error is encountered with would stop the queue
+ *    regardless of the async stop request.
+ * 3) All command buffers that have been submitted complete successfully.
+ * 4) The stop completes synchronously if no command buffers are
+ *    active on the queue when it is issued.
+ *
+ * If the command queue is not in a runnable state there is no
+ * guarentee this async stop will finish.  For instance if the high
+ * priority queue is not enabled and a stop is requested on the low
+ * priority queue, the high priority queue must be reenabled to
+ * guarantee that the async stop will finish.
+ *
+ * This command along with SVGA_DC_CMD_EMPTY_CONTEXT_QUEUE can be used
+ * to implement mid command buffer preemption.
+ *
+ * Valid only if the SVGA_CAP_HP_CMD_QUEUE cap is set.
+ */
+
+typedef struct SVGADCCmdAsyncStopQueue {
+   SVGACBContext context;
+} SVGADCCmdAsyncStopQueue;
+
+/*
+ * Requests the SVGA device to throw away any full command buffers on
+ * the requested command queue that have not been started.  For a
+ * driver to know which command buffers were thrown away a driver
+ * should only issue this command when the queue is stopped, for
+ * whatever reason.
+ */
+
+typedef struct SVGADCCmdEmptyQueue {
+   SVGACBContext context;
+} SVGADCCmdEmptyQueue;
+
+
+/*
  * SVGAGMRImageFormat --
  *
  *    This is a packed representation of the source 2D image format
@@ -537,7 +638,7 @@ typedef struct SVGAGMRImageFormat {
       struct {
          uint32 bitsPerPixel : 8;
          uint32 colorDepth   : 8;
-	 uint32 reserved     : 16;  /* Must be zero */
+         uint32 reserved     : 16;  /* Must be zero */
       };
 
       uint32 value;
@@ -673,8 +774,36 @@ SVGASignedPoint;
  * SVGA_CAP_GBOBJECTS --
  *    Enable guest-backed objects and surfaces.
  *
- * SVGA_CAP_CMD_BUFFERS_3 --
- *    Enable support for command buffers in a mob.
+ * SVGA_CAP_DX --
+ *    Enable support for DX commands, and command buffers in a mob.
+ *
+ * SVGA_CAP_HP_CMD_QUEUE --
+ *    Enable support for the high priority command queue, and the
+ *    ScreenCopy command.
+ *
+ * SVGA_CAP_NO_BB_RESTRICTION --
+ *    Allow ScreenTargets to be defined without regard to the 32-bpp
+ *    bounding-box memory restrictions. ie:
+ *
+ *    The summed memory usage of all screens (assuming they were defined as
+ *    32-bpp) must always be less than the value of the
+ *    SVGA_REG_MAX_PRIMARY_MEM register.
+ *
+ *    If this cap is not present, the 32-bpp bounding box around all screens
+ *    must additionally be under the value of the SVGA_REG_MAX_PRIMARY_MEM
+ *    register.
+ *
+ *    If the cap is present, the bounding box restriction is lifted (and only
+ *    the screen-sum limit applies).
+ *
+ *    (Note that this is a slight lie... there is still a sanity limit on any
+ *     dimension of the topology to be less than SVGA_SCREEN_ROOT_LIMIT, even
+ *     when SVGA_CAP_NO_BB_RESTRICTION is present, but that should be
+ *     large enough to express any possible topology without holes between
+ *     monitors.)
+ *
+ * SVGA_CAP_CAP2_REGISTER --
+ *    If this cap is present, the SVGA_REG_CAP2 register is supported.
  */
 
 #define SVGA_CAP_NONE               0x00000000
@@ -700,8 +829,30 @@ SVGASignedPoint;
 #define SVGA_CAP_GBOBJECTS          0x08000000
 #define SVGA_CAP_DX                 0x10000000
 #define SVGA_CAP_HP_CMD_QUEUE       0x20000000
+#define SVGA_CAP_NO_BB_RESTRICTION  0x40000000
+#define SVGA_CAP_CAP2_REGISTER      0x80000000
 
-#define SVGA_CAP_CMD_RESERVED       0x80000000
+/*
+ * The SVGA_REG_CAP2 register is an additional set of SVGA capability bits.
+ *
+ * SVGA_CAP2_GROW_OTABLE --
+ *      Allow the GrowOTable/DXGrowCOTable commands.
+ *
+ * SVGA_CAP2_INTRA_SURFACE_COPY --
+ *      Allow the IntraSurfaceCopy command.
+ *
+ * SVGA_CAP2_DX2 --
+ *      Allow the DefineGBSurface_v3, WholeSurfaceCopy.
+ *
+ * SVGA_CAP2_RESERVED --
+ *      Reserve the last bit for extending the SVGA capabilities to some
+ *      future mechanisms.
+ */
+#define SVGA_CAP2_NONE               0x00000000
+#define SVGA_CAP2_GROW_OTABLE        0x00000001
+#define SVGA_CAP2_INTRA_SURFACE_COPY 0x00000002
+#define SVGA_CAP2_DX2                0x00000004
+#define SVGA_CAP2_RESERVED           0x80000000
 
 
 /*
@@ -723,7 +874,8 @@ typedef enum {
    SVGABackdoorCapDeviceCaps = 0,
    SVGABackdoorCapFifoCaps = 1,
    SVGABackdoorCap3dHWVersion = 2,
-   SVGABackdoorCapMax = 3,
+   SVGABackdoorCapDeviceCaps2 = 3,
+   SVGABackdoorCapMax = 4,
 } SVGABackdoorCapType;
 
 
@@ -1915,16 +2067,6 @@ SVGAFifoCmdRemapGMR2;
 
 #define SVGA_VRAM_SIZE_W2K          (64 * 1024 * 1024) /* 64 MB */
 
-/*
- * To simplify autoDetect display configuration, support a minimum of
- * two 1920x1200 monitors, 32bpp, side-by-side, optionally rotated:
- *   numDisplays = 2
- *   maxWidth = numDisplay * 1920 = 3840
- *   maxHeight = rotated width of single monitor = 1920
- *   vramSize = maxWidth * maxHeight * 4 = 29491200
- */
-#define SVGA_VRAM_SIZE_AUTODETECT   (32 * 1024 * 1024)
-
 #if defined(VMX86_SERVER)
 #define SVGA_VRAM_SIZE               (4 * 1024 * 1024)
 #define SVGA_VRAM_SIZE_3D           (64 * 1024 * 1024)
diff --git a/drivers/gpu/drm/vmwgfx/device_include/svga_types.h b/drivers/gpu/drm/vmwgfx/device_include/svga_types.h
index 53602a4e7957..350bbc6fab02 100644
--- a/drivers/gpu/drm/vmwgfx/device_include/svga_types.h
+++ b/drivers/gpu/drm/vmwgfx/device_include/svga_types.h
@@ -41,7 +41,10 @@ typedef uint64 PPN64;
 
 typedef bool Bool;
 
+#define MAX_UINT64 U64_MAX
 #define MAX_UINT32 U32_MAX
 #define MAX_UINT16 U16_MAX
 
+#define CONST64U(x) x##ULL
+
 #endif
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_bo.c b/drivers/gpu/drm/vmwgfx/vmwgfx_bo.c
new file mode 100644
index 000000000000..2dda03345761
--- /dev/null
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_bo.c
@@ -0,0 +1,1123 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/**************************************************************************
+ *
+ * Copyright © 2011-2018 VMware, Inc., Palo Alto, CA., USA
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sub license, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial portions
+ * of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
+ * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+ * USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ **************************************************************************/
+
+#include <drm/ttm/ttm_placement.h>
+
+#include <drm/drmP.h>
+#include "vmwgfx_drv.h"
+#include "drm/ttm/ttm_object.h"
+
+
+/**
+ * struct vmw_user_buffer_object - User-space-visible buffer object
+ *
+ * @prime: The prime object providing user visibility.
+ * @vbo: The struct vmw_buffer_object
+ */
+struct vmw_user_buffer_object {
+	struct ttm_prime_object prime;
+	struct vmw_buffer_object vbo;
+};
+
+
+/**
+ * vmw_buffer_object - Convert a struct ttm_buffer_object to a struct
+ * vmw_buffer_object.
+ *
+ * @bo: Pointer to the TTM buffer object.
+ * Return: Pointer to the struct vmw_buffer_object embedding the
+ * TTM buffer object.
+ */
+static struct vmw_buffer_object *
+vmw_buffer_object(struct ttm_buffer_object *bo)
+{
+	return container_of(bo, struct vmw_buffer_object, base);
+}
+
+
+/**
+ * vmw_user_buffer_object - Convert a struct ttm_buffer_object to a struct
+ * vmw_user_buffer_object.
+ *
+ * @bo: Pointer to the TTM buffer object.
+ * Return: Pointer to the struct vmw_buffer_object embedding the TTM buffer
+ * object.
+ */
+static struct vmw_user_buffer_object *
+vmw_user_buffer_object(struct ttm_buffer_object *bo)
+{
+	struct vmw_buffer_object *vmw_bo = vmw_buffer_object(bo);
+
+	return container_of(vmw_bo, struct vmw_user_buffer_object, vbo);
+}
+
+
+/**
+ * vmw_bo_pin_in_placement - Validate a buffer to placement.
+ *
+ * @dev_priv:  Driver private.
+ * @buf:  DMA buffer to move.
+ * @placement:  The placement to pin it.
+ * @interruptible:  Use interruptible wait.
+ * Return: Zero on success, Negative error code on failure. In particular
+ * -ERESTARTSYS if interrupted by a signal
+ */
+int vmw_bo_pin_in_placement(struct vmw_private *dev_priv,
+			    struct vmw_buffer_object *buf,
+			    struct ttm_placement *placement,
+			    bool interruptible)
+{
+	struct ttm_operation_ctx ctx = {interruptible, false };
+	struct ttm_buffer_object *bo = &buf->base;
+	int ret;
+	uint32_t new_flags;
+
+	ret = ttm_write_lock(&dev_priv->reservation_sem, interruptible);
+	if (unlikely(ret != 0))
+		return ret;
+
+	vmw_execbuf_release_pinned_bo(dev_priv);
+
+	ret = ttm_bo_reserve(bo, interruptible, false, NULL);
+	if (unlikely(ret != 0))
+		goto err;
+
+	if (buf->pin_count > 0)
+		ret = ttm_bo_mem_compat(placement, &bo->mem,
+					&new_flags) == true ? 0 : -EINVAL;
+	else
+		ret = ttm_bo_validate(bo, placement, &ctx);
+
+	if (!ret)
+		vmw_bo_pin_reserved(buf, true);
+
+	ttm_bo_unreserve(bo);
+
+err:
+	ttm_write_unlock(&dev_priv->reservation_sem);
+	return ret;
+}
+
+
+/**
+ * vmw_bo_pin_in_vram_or_gmr - Move a buffer to vram or gmr.
+ *
+ * This function takes the reservation_sem in write mode.
+ * Flushes and unpins the query bo to avoid failures.
+ *
+ * @dev_priv:  Driver private.
+ * @buf:  DMA buffer to move.
+ * @pin:  Pin buffer if true.
+ * @interruptible:  Use interruptible wait.
+ * Return: Zero on success, Negative error code on failure. In particular
+ * -ERESTARTSYS if interrupted by a signal
+ */
+int vmw_bo_pin_in_vram_or_gmr(struct vmw_private *dev_priv,
+			      struct vmw_buffer_object *buf,
+			      bool interruptible)
+{
+	struct ttm_operation_ctx ctx = {interruptible, false };
+	struct ttm_buffer_object *bo = &buf->base;
+	int ret;
+	uint32_t new_flags;
+
+	ret = ttm_write_lock(&dev_priv->reservation_sem, interruptible);
+	if (unlikely(ret != 0))
+		return ret;
+
+	vmw_execbuf_release_pinned_bo(dev_priv);
+
+	ret = ttm_bo_reserve(bo, interruptible, false, NULL);
+	if (unlikely(ret != 0))
+		goto err;
+
+	if (buf->pin_count > 0) {
+		ret = ttm_bo_mem_compat(&vmw_vram_gmr_placement, &bo->mem,
+					&new_flags) == true ? 0 : -EINVAL;
+		goto out_unreserve;
+	}
+
+	ret = ttm_bo_validate(bo, &vmw_vram_gmr_placement, &ctx);
+	if (likely(ret == 0) || ret == -ERESTARTSYS)
+		goto out_unreserve;
+
+	ret = ttm_bo_validate(bo, &vmw_vram_placement, &ctx);
+
+out_unreserve:
+	if (!ret)
+		vmw_bo_pin_reserved(buf, true);
+
+	ttm_bo_unreserve(bo);
+err:
+	ttm_write_unlock(&dev_priv->reservation_sem);
+	return ret;
+}
+
+
+/**
+ * vmw_bo_pin_in_vram - Move a buffer to vram.
+ *
+ * This function takes the reservation_sem in write mode.
+ * Flushes and unpins the query bo to avoid failures.
+ *
+ * @dev_priv:  Driver private.
+ * @buf:  DMA buffer to move.
+ * @interruptible:  Use interruptible wait.
+ * Return: Zero on success, Negative error code on failure. In particular
+ * -ERESTARTSYS if interrupted by a signal
+ */
+int vmw_bo_pin_in_vram(struct vmw_private *dev_priv,
+		       struct vmw_buffer_object *buf,
+		       bool interruptible)
+{
+	return vmw_bo_pin_in_placement(dev_priv, buf, &vmw_vram_placement,
+				       interruptible);
+}
+
+
+/**
+ * vmw_bo_pin_in_start_of_vram - Move a buffer to start of vram.
+ *
+ * This function takes the reservation_sem in write mode.
+ * Flushes and unpins the query bo to avoid failures.
+ *
+ * @dev_priv:  Driver private.
+ * @buf:  DMA buffer to pin.
+ * @interruptible:  Use interruptible wait.
+ * Return: Zero on success, Negative error code on failure. In particular
+ * -ERESTARTSYS if interrupted by a signal
+ */
+int vmw_bo_pin_in_start_of_vram(struct vmw_private *dev_priv,
+				struct vmw_buffer_object *buf,
+				bool interruptible)
+{
+	struct ttm_operation_ctx ctx = {interruptible, false };
+	struct ttm_buffer_object *bo = &buf->base;
+	struct ttm_placement placement;
+	struct ttm_place place;
+	int ret = 0;
+	uint32_t new_flags;
+
+	place = vmw_vram_placement.placement[0];
+	place.lpfn = bo->num_pages;
+	placement.num_placement = 1;
+	placement.placement = &place;
+	placement.num_busy_placement = 1;
+	placement.busy_placement = &place;
+
+	ret = ttm_write_lock(&dev_priv->reservation_sem, interruptible);
+	if (unlikely(ret != 0))
+		return ret;
+
+	vmw_execbuf_release_pinned_bo(dev_priv);
+	ret = ttm_bo_reserve(bo, interruptible, false, NULL);
+	if (unlikely(ret != 0))
+		goto err_unlock;
+
+	/*
+	 * Is this buffer already in vram but not at the start of it?
+	 * In that case, evict it first because TTM isn't good at handling
+	 * that situation.
+	 */
+	if (bo->mem.mem_type == TTM_PL_VRAM &&
+	    bo->mem.start < bo->num_pages &&
+	    bo->mem.start > 0 &&
+	    buf->pin_count == 0) {
+		ctx.interruptible = false;
+		(void) ttm_bo_validate(bo, &vmw_sys_placement, &ctx);
+	}
+
+	if (buf->pin_count > 0)
+		ret = ttm_bo_mem_compat(&placement, &bo->mem,
+					&new_flags) == true ? 0 : -EINVAL;
+	else
+		ret = ttm_bo_validate(bo, &placement, &ctx);
+
+	/* For some reason we didn't end up at the start of vram */
+	WARN_ON(ret == 0 && bo->offset != 0);
+	if (!ret)
+		vmw_bo_pin_reserved(buf, true);
+
+	ttm_bo_unreserve(bo);
+err_unlock:
+	ttm_write_unlock(&dev_priv->reservation_sem);
+
+	return ret;
+}
+
+
+/**
+ * vmw_bo_unpin - Unpin the buffer given buffer, does not move the buffer.
+ *
+ * This function takes the reservation_sem in write mode.
+ *
+ * @dev_priv:  Driver private.
+ * @buf:  DMA buffer to unpin.
+ * @interruptible:  Use interruptible wait.
+ * Return: Zero on success, Negative error code on failure. In particular
+ * -ERESTARTSYS if interrupted by a signal
+ */
+int vmw_bo_unpin(struct vmw_private *dev_priv,
+		 struct vmw_buffer_object *buf,
+		 bool interruptible)
+{
+	struct ttm_buffer_object *bo = &buf->base;
+	int ret;
+
+	ret = ttm_read_lock(&dev_priv->reservation_sem, interruptible);
+	if (unlikely(ret != 0))
+		return ret;
+
+	ret = ttm_bo_reserve(bo, interruptible, false, NULL);
+	if (unlikely(ret != 0))
+		goto err;
+
+	vmw_bo_pin_reserved(buf, false);
+
+	ttm_bo_unreserve(bo);
+
+err:
+	ttm_read_unlock(&dev_priv->reservation_sem);
+	return ret;
+}
+
+/**
+ * vmw_bo_get_guest_ptr - Get the guest ptr representing the current placement
+ * of a buffer.
+ *
+ * @bo: Pointer to a struct ttm_buffer_object. Must be pinned or reserved.
+ * @ptr: SVGAGuestPtr returning the result.
+ */
+void vmw_bo_get_guest_ptr(const struct ttm_buffer_object *bo,
+			  SVGAGuestPtr *ptr)
+{
+	if (bo->mem.mem_type == TTM_PL_VRAM) {
+		ptr->gmrId = SVGA_GMR_FRAMEBUFFER;
+		ptr->offset = bo->offset;
+	} else {
+		ptr->gmrId = bo->mem.start;
+		ptr->offset = 0;
+	}
+}
+
+
+/**
+ * vmw_bo_pin_reserved - Pin or unpin a buffer object without moving it.
+ *
+ * @vbo: The buffer object. Must be reserved.
+ * @pin: Whether to pin or unpin.
+ *
+ */
+void vmw_bo_pin_reserved(struct vmw_buffer_object *vbo, bool pin)
+{
+	struct ttm_operation_ctx ctx = { false, true };
+	struct ttm_place pl;
+	struct ttm_placement placement;
+	struct ttm_buffer_object *bo = &vbo->base;
+	uint32_t old_mem_type = bo->mem.mem_type;
+	int ret;
+
+	lockdep_assert_held(&bo->resv->lock.base);
+
+	if (pin) {
+		if (vbo->pin_count++ > 0)
+			return;
+	} else {
+		WARN_ON(vbo->pin_count <= 0);
+		if (--vbo->pin_count > 0)
+			return;
+	}
+
+	pl.fpfn = 0;
+	pl.lpfn = 0;
+	pl.flags = TTM_PL_FLAG_VRAM | VMW_PL_FLAG_GMR | VMW_PL_FLAG_MOB
+		| TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED;
+	if (pin)
+		pl.flags |= TTM_PL_FLAG_NO_EVICT;
+
+	memset(&placement, 0, sizeof(placement));
+	placement.num_placement = 1;
+	placement.placement = &pl;
+
+	ret = ttm_bo_validate(bo, &placement, &ctx);
+
+	BUG_ON(ret != 0 || bo->mem.mem_type != old_mem_type);
+}
+
+
+/**
+ * vmw_bo_map_and_cache - Map a buffer object and cache the map
+ *
+ * @vbo: The buffer object to map
+ * Return: A kernel virtual address or NULL if mapping failed.
+ *
+ * This function maps a buffer object into the kernel address space, or
+ * returns the virtual kernel address of an already existing map. The virtual
+ * address remains valid as long as the buffer object is pinned or reserved.
+ * The cached map is torn down on either
+ * 1) Buffer object move
+ * 2) Buffer object swapout
+ * 3) Buffer object destruction
+ *
+ */
+void *vmw_bo_map_and_cache(struct vmw_buffer_object *vbo)
+{
+	struct ttm_buffer_object *bo = &vbo->base;
+	bool not_used;
+	void *virtual;
+	int ret;
+
+	virtual = ttm_kmap_obj_virtual(&vbo->map, &not_used);
+	if (virtual)
+		return virtual;
+
+	ret = ttm_bo_kmap(bo, 0, bo->num_pages, &vbo->map);
+	if (ret)
+		DRM_ERROR("Buffer object map failed: %d.\n", ret);
+
+	return ttm_kmap_obj_virtual(&vbo->map, &not_used);
+}
+
+
+/**
+ * vmw_bo_unmap - Tear down a cached buffer object map.
+ *
+ * @vbo: The buffer object whose map we are tearing down.
+ *
+ * This function tears down a cached map set up using
+ * vmw_buffer_object_map_and_cache().
+ */
+void vmw_bo_unmap(struct vmw_buffer_object *vbo)
+{
+	if (vbo->map.bo == NULL)
+		return;
+
+	ttm_bo_kunmap(&vbo->map);
+}
+
+
+/**
+ * vmw_bo_acc_size - Calculate the pinned memory usage of buffers
+ *
+ * @dev_priv: Pointer to a struct vmw_private identifying the device.
+ * @size: The requested buffer size.
+ * @user: Whether this is an ordinary dma buffer or a user dma buffer.
+ */
+static size_t vmw_bo_acc_size(struct vmw_private *dev_priv, size_t size,
+			      bool user)
+{
+	static size_t struct_size, user_struct_size;
+	size_t num_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
+	size_t page_array_size = ttm_round_pot(num_pages * sizeof(void *));
+
+	if (unlikely(struct_size == 0)) {
+		size_t backend_size = ttm_round_pot(vmw_tt_size);
+
+		struct_size = backend_size +
+			ttm_round_pot(sizeof(struct vmw_buffer_object));
+		user_struct_size = backend_size +
+			ttm_round_pot(sizeof(struct vmw_user_buffer_object));
+	}
+
+	if (dev_priv->map_mode == vmw_dma_alloc_coherent)
+		page_array_size +=
+			ttm_round_pot(num_pages * sizeof(dma_addr_t));
+
+	return ((user) ? user_struct_size : struct_size) +
+		page_array_size;
+}
+
+
+/**
+ * vmw_bo_bo_free - vmw buffer object destructor
+ *
+ * @bo: Pointer to the embedded struct ttm_buffer_object
+ */
+void vmw_bo_bo_free(struct ttm_buffer_object *bo)
+{
+	struct vmw_buffer_object *vmw_bo = vmw_buffer_object(bo);
+
+	vmw_bo_unmap(vmw_bo);
+	kfree(vmw_bo);
+}
+
+
+/**
+ * vmw_user_bo_destroy - vmw buffer object destructor
+ *
+ * @bo: Pointer to the embedded struct ttm_buffer_object
+ */
+static void vmw_user_bo_destroy(struct ttm_buffer_object *bo)
+{
+	struct vmw_user_buffer_object *vmw_user_bo = vmw_user_buffer_object(bo);
+
+	vmw_bo_unmap(&vmw_user_bo->vbo);
+	ttm_prime_object_kfree(vmw_user_bo, prime);
+}
+
+
+/**
+ * vmw_bo_init - Initialize a vmw buffer object
+ *
+ * @dev_priv: Pointer to the device private struct
+ * @vmw_bo: Pointer to the struct vmw_buffer_object to initialize.
+ * @size: Buffer object size in bytes.
+ * @placement: Initial placement.
+ * @interruptible: Whether waits should be performed interruptible.
+ * @bo_free: The buffer object destructor.
+ * Returns: Zero on success, negative error code on error.
+ *
+ * Note that on error, the code will free the buffer object.
+ */
+int vmw_bo_init(struct vmw_private *dev_priv,
+		struct vmw_buffer_object *vmw_bo,
+		size_t size, struct ttm_placement *placement,
+		bool interruptible,
+		void (*bo_free)(struct ttm_buffer_object *bo))
+{
+	struct ttm_bo_device *bdev = &dev_priv->bdev;
+	size_t acc_size;
+	int ret;
+	bool user = (bo_free == &vmw_user_bo_destroy);
+
+	WARN_ON_ONCE(!bo_free && (!user && (bo_free != vmw_bo_bo_free)));
+
+	acc_size = vmw_bo_acc_size(dev_priv, size, user);
+	memset(vmw_bo, 0, sizeof(*vmw_bo));
+
+	INIT_LIST_HEAD(&vmw_bo->res_list);
+
+	ret = ttm_bo_init(bdev, &vmw_bo->base, size,
+			  ttm_bo_type_device, placement,
+			  0, interruptible, acc_size,
+			  NULL, NULL, bo_free);
+	return ret;
+}
+
+
+/**
+ * vmw_user_bo_release - TTM reference base object release callback for
+ * vmw user buffer objects
+ *
+ * @p_base: The TTM base object pointer about to be unreferenced.
+ *
+ * Clears the TTM base object pointer and drops the reference the
+ * base object has on the underlying struct vmw_buffer_object.
+ */
+static void vmw_user_bo_release(struct ttm_base_object **p_base)
+{
+	struct vmw_user_buffer_object *vmw_user_bo;
+	struct ttm_base_object *base = *p_base;
+	struct ttm_buffer_object *bo;
+
+	*p_base = NULL;
+
+	if (unlikely(base == NULL))
+		return;
+
+	vmw_user_bo = container_of(base, struct vmw_user_buffer_object,
+				   prime.base);
+	bo = &vmw_user_bo->vbo.base;
+	ttm_bo_unref(&bo);
+}
+
+
+/**
+ * vmw_user_bo_ref_obj-release - TTM synccpu reference object release callback
+ * for vmw user buffer objects
+ *
+ * @base: Pointer to the TTM base object
+ * @ref_type: Reference type of the reference reaching zero.
+ *
+ * Called when user-space drops its last synccpu reference on the buffer
+ * object, Either explicitly or as part of a cleanup file close.
+ */
+static void vmw_user_bo_ref_obj_release(struct ttm_base_object *base,
+					enum ttm_ref_type ref_type)
+{
+	struct vmw_user_buffer_object *user_bo;
+
+	user_bo = container_of(base, struct vmw_user_buffer_object, prime.base);
+
+	switch (ref_type) {
+	case TTM_REF_SYNCCPU_WRITE:
+		ttm_bo_synccpu_write_release(&user_bo->vbo.base);
+		break;
+	default:
+		WARN_ONCE(true, "Undefined buffer object reference release.\n");
+	}
+}
+
+
+/**
+ * vmw_user_bo_alloc - Allocate a user buffer object
+ *
+ * @dev_priv: Pointer to a struct device private.
+ * @tfile: Pointer to a struct ttm_object_file on which to register the user
+ * object.
+ * @size: Size of the buffer object.
+ * @shareable: Boolean whether the buffer is shareable with other open files.
+ * @handle: Pointer to where the handle value should be assigned.
+ * @p_vbo: Pointer to where the refcounted struct vmw_buffer_object pointer
+ * should be assigned.
+ * Return: Zero on success, negative error code on error.
+ */
+int vmw_user_bo_alloc(struct vmw_private *dev_priv,
+		      struct ttm_object_file *tfile,
+		      uint32_t size,
+		      bool shareable,
+		      uint32_t *handle,
+		      struct vmw_buffer_object **p_vbo,
+		      struct ttm_base_object **p_base)
+{
+	struct vmw_user_buffer_object *user_bo;
+	struct ttm_buffer_object *tmp;
+	int ret;
+
+	user_bo = kzalloc(sizeof(*user_bo), GFP_KERNEL);
+	if (unlikely(!user_bo)) {
+		DRM_ERROR("Failed to allocate a buffer.\n");
+		return -ENOMEM;
+	}
+
+	ret = vmw_bo_init(dev_priv, &user_bo->vbo, size,
+			  (dev_priv->has_mob) ?
+			  &vmw_sys_placement :
+			  &vmw_vram_sys_placement, true,
+			  &vmw_user_bo_destroy);
+	if (unlikely(ret != 0))
+		return ret;
+
+	tmp = ttm_bo_reference(&user_bo->vbo.base);
+	ret = ttm_prime_object_init(tfile,
+				    size,
+				    &user_bo->prime,
+				    shareable,
+				    ttm_buffer_type,
+				    &vmw_user_bo_release,
+				    &vmw_user_bo_ref_obj_release);
+	if (unlikely(ret != 0)) {
+		ttm_bo_unref(&tmp);
+		goto out_no_base_object;
+	}
+
+	*p_vbo = &user_bo->vbo;
+	if (p_base) {
+		*p_base = &user_bo->prime.base;
+		kref_get(&(*p_base)->refcount);
+	}
+	*handle = user_bo->prime.base.hash.key;
+
+out_no_base_object:
+	return ret;
+}
+
+
+/**
+ * vmw_user_bo_verify_access - verify access permissions on this
+ * buffer object.
+ *
+ * @bo: Pointer to the buffer object being accessed
+ * @tfile: Identifying the caller.
+ */
+int vmw_user_bo_verify_access(struct ttm_buffer_object *bo,
+			      struct ttm_object_file *tfile)
+{
+	struct vmw_user_buffer_object *vmw_user_bo;
+
+	if (unlikely(bo->destroy != vmw_user_bo_destroy))
+		return -EPERM;
+
+	vmw_user_bo = vmw_user_buffer_object(bo);
+
+	/* Check that the caller has opened the object. */
+	if (likely(ttm_ref_object_exists(tfile, &vmw_user_bo->prime.base)))
+		return 0;
+
+	DRM_ERROR("Could not grant buffer access.\n");
+	return -EPERM;
+}
+
+
+/**
+ * vmw_user_bo_synccpu_grab - Grab a struct vmw_user_buffer_object for cpu
+ * access, idling previous GPU operations on the buffer and optionally
+ * blocking it for further command submissions.
+ *
+ * @user_bo: Pointer to the buffer object being grabbed for CPU access
+ * @tfile: Identifying the caller.
+ * @flags: Flags indicating how the grab should be performed.
+ * Return: Zero on success, Negative error code on error. In particular,
+ * -EBUSY will be returned if a dontblock operation is requested and the
+ * buffer object is busy, and -ERESTARTSYS will be returned if a wait is
+ * interrupted by a signal.
+ *
+ * A blocking grab will be automatically released when @tfile is closed.
+ */
+static int vmw_user_bo_synccpu_grab(struct vmw_user_buffer_object *user_bo,
+				    struct ttm_object_file *tfile,
+				    uint32_t flags)
+{
+	struct ttm_buffer_object *bo = &user_bo->vbo.base;
+	bool existed;
+	int ret;
+
+	if (flags & drm_vmw_synccpu_allow_cs) {
+		bool nonblock = !!(flags & drm_vmw_synccpu_dontblock);
+		long lret;
+
+		lret = reservation_object_wait_timeout_rcu
+			(bo->resv, true, true,
+			 nonblock ? 0 : MAX_SCHEDULE_TIMEOUT);
+		if (!lret)
+			return -EBUSY;
+		else if (lret < 0)
+			return lret;
+		return 0;
+	}
+
+	ret = ttm_bo_synccpu_write_grab
+		(bo, !!(flags & drm_vmw_synccpu_dontblock));
+	if (unlikely(ret != 0))
+		return ret;
+
+	ret = ttm_ref_object_add(tfile, &user_bo->prime.base,
+				 TTM_REF_SYNCCPU_WRITE, &existed, false);
+	if (ret != 0 || existed)
+		ttm_bo_synccpu_write_release(&user_bo->vbo.base);
+
+	return ret;
+}
+
+/**
+ * vmw_user_bo_synccpu_release - Release a previous grab for CPU access,
+ * and unblock command submission on the buffer if blocked.
+ *
+ * @handle: Handle identifying the buffer object.
+ * @tfile: Identifying the caller.
+ * @flags: Flags indicating the type of release.
+ */
+static int vmw_user_bo_synccpu_release(uint32_t handle,
+					   struct ttm_object_file *tfile,
+					   uint32_t flags)
+{
+	if (!(flags & drm_vmw_synccpu_allow_cs))
+		return ttm_ref_object_base_unref(tfile, handle,
+						 TTM_REF_SYNCCPU_WRITE);
+
+	return 0;
+}
+
+
+/**
+ * vmw_user_bo_synccpu_ioctl - ioctl function implementing the synccpu
+ * functionality.
+ *
+ * @dev: Identifies the drm device.
+ * @data: Pointer to the ioctl argument.
+ * @file_priv: Identifies the caller.
+ * Return: Zero on success, negative error code on error.
+ *
+ * This function checks the ioctl arguments for validity and calls the
+ * relevant synccpu functions.
+ */
+int vmw_user_bo_synccpu_ioctl(struct drm_device *dev, void *data,
+			      struct drm_file *file_priv)
+{
+	struct drm_vmw_synccpu_arg *arg =
+		(struct drm_vmw_synccpu_arg *) data;
+	struct vmw_buffer_object *vbo;
+	struct vmw_user_buffer_object *user_bo;
+	struct ttm_object_file *tfile = vmw_fpriv(file_priv)->tfile;
+	struct ttm_base_object *buffer_base;
+	int ret;
+
+	if ((arg->flags & (drm_vmw_synccpu_read | drm_vmw_synccpu_write)) == 0
+	    || (arg->flags & ~(drm_vmw_synccpu_read | drm_vmw_synccpu_write |
+			       drm_vmw_synccpu_dontblock |
+			       drm_vmw_synccpu_allow_cs)) != 0) {
+		DRM_ERROR("Illegal synccpu flags.\n");
+		return -EINVAL;
+	}
+
+	switch (arg->op) {
+	case drm_vmw_synccpu_grab:
+		ret = vmw_user_bo_lookup(tfile, arg->handle, &vbo,
+					     &buffer_base);
+		if (unlikely(ret != 0))
+			return ret;
+
+		user_bo = container_of(vbo, struct vmw_user_buffer_object,
+				       vbo);
+		ret = vmw_user_bo_synccpu_grab(user_bo, tfile, arg->flags);
+		vmw_bo_unreference(&vbo);
+		ttm_base_object_unref(&buffer_base);
+		if (unlikely(ret != 0 && ret != -ERESTARTSYS &&
+			     ret != -EBUSY)) {
+			DRM_ERROR("Failed synccpu grab on handle 0x%08x.\n",
+				  (unsigned int) arg->handle);
+			return ret;
+		}
+		break;
+	case drm_vmw_synccpu_release:
+		ret = vmw_user_bo_synccpu_release(arg->handle, tfile,
+						  arg->flags);
+		if (unlikely(ret != 0)) {
+			DRM_ERROR("Failed synccpu release on handle 0x%08x.\n",
+				  (unsigned int) arg->handle);
+			return ret;
+		}
+		break;
+	default:
+		DRM_ERROR("Invalid synccpu operation.\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+
+/**
+ * vmw_bo_alloc_ioctl - ioctl function implementing the buffer object
+ * allocation functionality.
+ *
+ * @dev: Identifies the drm device.
+ * @data: Pointer to the ioctl argument.
+ * @file_priv: Identifies the caller.
+ * Return: Zero on success, negative error code on error.
+ *
+ * This function checks the ioctl arguments for validity and allocates a
+ * struct vmw_user_buffer_object bo.
+ */
+int vmw_bo_alloc_ioctl(struct drm_device *dev, void *data,
+		       struct drm_file *file_priv)
+{
+	struct vmw_private *dev_priv = vmw_priv(dev);
+	union drm_vmw_alloc_dmabuf_arg *arg =
+	    (union drm_vmw_alloc_dmabuf_arg *)data;
+	struct drm_vmw_alloc_dmabuf_req *req = &arg->req;
+	struct drm_vmw_dmabuf_rep *rep = &arg->rep;
+	struct vmw_buffer_object *vbo;
+	uint32_t handle;
+	int ret;
+
+	ret = ttm_read_lock(&dev_priv->reservation_sem, true);
+	if (unlikely(ret != 0))
+		return ret;
+
+	ret = vmw_user_bo_alloc(dev_priv, vmw_fpriv(file_priv)->tfile,
+				req->size, false, &handle, &vbo,
+				NULL);
+	if (unlikely(ret != 0))
+		goto out_no_bo;
+
+	rep->handle = handle;
+	rep->map_handle = drm_vma_node_offset_addr(&vbo->base.vma_node);
+	rep->cur_gmr_id = handle;
+	rep->cur_gmr_offset = 0;
+
+	vmw_bo_unreference(&vbo);
+
+out_no_bo:
+	ttm_read_unlock(&dev_priv->reservation_sem);
+
+	return ret;
+}
+
+
+/**
+ * vmw_bo_unref_ioctl - Generic handle close ioctl.
+ *
+ * @dev: Identifies the drm device.
+ * @data: Pointer to the ioctl argument.
+ * @file_priv: Identifies the caller.
+ * Return: Zero on success, negative error code on error.
+ *
+ * This function checks the ioctl arguments for validity and closes a
+ * handle to a TTM base object, optionally freeing the object.
+ */
+int vmw_bo_unref_ioctl(struct drm_device *dev, void *data,
+		       struct drm_file *file_priv)
+{
+	struct drm_vmw_unref_dmabuf_arg *arg =
+	    (struct drm_vmw_unref_dmabuf_arg *)data;
+
+	return ttm_ref_object_base_unref(vmw_fpriv(file_priv)->tfile,
+					 arg->handle,
+					 TTM_REF_USAGE);
+}
+
+
+/**
+ * vmw_user_bo_lookup - Look up a vmw user buffer object from a handle.
+ *
+ * @tfile: The TTM object file the handle is registered with.
+ * @handle: The user buffer object handle
+ * @out: Pointer to a where a pointer to the embedded
+ * struct vmw_buffer_object should be placed.
+ * @p_base: Pointer to where a pointer to the TTM base object should be
+ * placed, or NULL if no such pointer is required.
+ * Return: Zero on success, Negative error code on error.
+ *
+ * Both the output base object pointer and the vmw buffer object pointer
+ * will be refcounted.
+ */
+int vmw_user_bo_lookup(struct ttm_object_file *tfile,
+		       uint32_t handle, struct vmw_buffer_object **out,
+		       struct ttm_base_object **p_base)
+{
+	struct vmw_user_buffer_object *vmw_user_bo;
+	struct ttm_base_object *base;
+
+	base = ttm_base_object_lookup(tfile, handle);
+	if (unlikely(base == NULL)) {
+		DRM_ERROR("Invalid buffer object handle 0x%08lx.\n",
+			  (unsigned long)handle);
+		return -ESRCH;
+	}
+
+	if (unlikely(ttm_base_object_type(base) != ttm_buffer_type)) {
+		ttm_base_object_unref(&base);
+		DRM_ERROR("Invalid buffer object handle 0x%08lx.\n",
+			  (unsigned long)handle);
+		return -EINVAL;
+	}
+
+	vmw_user_bo = container_of(base, struct vmw_user_buffer_object,
+				   prime.base);
+	(void)ttm_bo_reference(&vmw_user_bo->vbo.base);
+	if (p_base)
+		*p_base = base;
+	else
+		ttm_base_object_unref(&base);
+	*out = &vmw_user_bo->vbo;
+
+	return 0;
+}
+
+
+/**
+ * vmw_user_bo_reference - Open a handle to a vmw user buffer object.
+ *
+ * @tfile: The TTM object file to register the handle with.
+ * @vbo: The embedded vmw buffer object.
+ * @handle: Pointer to where the new handle should be placed.
+ * Return: Zero on success, Negative error code on error.
+ */
+int vmw_user_bo_reference(struct ttm_object_file *tfile,
+			  struct vmw_buffer_object *vbo,
+			  uint32_t *handle)
+{
+	struct vmw_user_buffer_object *user_bo;
+
+	if (vbo->base.destroy != vmw_user_bo_destroy)
+		return -EINVAL;
+
+	user_bo = container_of(vbo, struct vmw_user_buffer_object, vbo);
+
+	*handle = user_bo->prime.base.hash.key;
+	return ttm_ref_object_add(tfile, &user_bo->prime.base,
+				  TTM_REF_USAGE, NULL, false);
+}
+
+
+/**
+ * vmw_bo_fence_single - Utility function to fence a single TTM buffer
+ *                       object without unreserving it.
+ *
+ * @bo:             Pointer to the struct ttm_buffer_object to fence.
+ * @fence:          Pointer to the fence. If NULL, this function will
+ *                  insert a fence into the command stream..
+ *
+ * Contrary to the ttm_eu version of this function, it takes only
+ * a single buffer object instead of a list, and it also doesn't
+ * unreserve the buffer object, which needs to be done separately.
+ */
+void vmw_bo_fence_single(struct ttm_buffer_object *bo,
+			 struct vmw_fence_obj *fence)
+{
+	struct ttm_bo_device *bdev = bo->bdev;
+
+	struct vmw_private *dev_priv =
+		container_of(bdev, struct vmw_private, bdev);
+
+	if (fence == NULL) {
+		vmw_execbuf_fence_commands(NULL, dev_priv, &fence, NULL);
+		reservation_object_add_excl_fence(bo->resv, &fence->base);
+		dma_fence_put(&fence->base);
+	} else
+		reservation_object_add_excl_fence(bo->resv, &fence->base);
+}
+
+
+/**
+ * vmw_dumb_create - Create a dumb kms buffer
+ *
+ * @file_priv: Pointer to a struct drm_file identifying the caller.
+ * @dev: Pointer to the drm device.
+ * @args: Pointer to a struct drm_mode_create_dumb structure
+ * Return: Zero on success, negative error code on failure.
+ *
+ * This is a driver callback for the core drm create_dumb functionality.
+ * Note that this is very similar to the vmw_bo_alloc ioctl, except
+ * that the arguments have a different format.
+ */
+int vmw_dumb_create(struct drm_file *file_priv,
+		    struct drm_device *dev,
+		    struct drm_mode_create_dumb *args)
+{
+	struct vmw_private *dev_priv = vmw_priv(dev);
+	struct vmw_buffer_object *vbo;
+	int ret;
+
+	args->pitch = args->width * ((args->bpp + 7) / 8);
+	args->size = args->pitch * args->height;
+
+	ret = ttm_read_lock(&dev_priv->reservation_sem, true);
+	if (unlikely(ret != 0))
+		return ret;
+
+	ret = vmw_user_bo_alloc(dev_priv, vmw_fpriv(file_priv)->tfile,
+				    args->size, false, &args->handle,
+				    &vbo, NULL);
+	if (unlikely(ret != 0))
+		goto out_no_bo;
+
+	vmw_bo_unreference(&vbo);
+out_no_bo:
+	ttm_read_unlock(&dev_priv->reservation_sem);
+	return ret;
+}
+
+
+/**
+ * vmw_dumb_map_offset - Return the address space offset of a dumb buffer
+ *
+ * @file_priv: Pointer to a struct drm_file identifying the caller.
+ * @dev: Pointer to the drm device.
+ * @handle: Handle identifying the dumb buffer.
+ * @offset: The address space offset returned.
+ * Return: Zero on success, negative error code on failure.
+ *
+ * This is a driver callback for the core drm dumb_map_offset functionality.
+ */
+int vmw_dumb_map_offset(struct drm_file *file_priv,
+			struct drm_device *dev, uint32_t handle,
+			uint64_t *offset)
+{
+	struct ttm_object_file *tfile = vmw_fpriv(file_priv)->tfile;
+	struct vmw_buffer_object *out_buf;
+	int ret;
+
+	ret = vmw_user_bo_lookup(tfile, handle, &out_buf, NULL);
+	if (ret != 0)
+		return -EINVAL;
+
+	*offset = drm_vma_node_offset_addr(&out_buf->base.vma_node);
+	vmw_bo_unreference(&out_buf);
+	return 0;
+}
+
+
+/**
+ * vmw_dumb_destroy - Destroy a dumb boffer
+ *
+ * @file_priv: Pointer to a struct drm_file identifying the caller.
+ * @dev: Pointer to the drm device.
+ * @handle: Handle identifying the dumb buffer.
+ * Return: Zero on success, negative error code on failure.
+ *
+ * This is a driver callback for the core drm dumb_destroy functionality.
+ */
+int vmw_dumb_destroy(struct drm_file *file_priv,
+		     struct drm_device *dev,
+		     uint32_t handle)
+{
+	return ttm_ref_object_base_unref(vmw_fpriv(file_priv)->tfile,
+					 handle, TTM_REF_USAGE);
+}
+
+
+/**
+ * vmw_bo_swap_notify - swapout notify callback.
+ *
+ * @bo: The buffer object to be swapped out.
+ */
+void vmw_bo_swap_notify(struct ttm_buffer_object *bo)
+{
+	/* Is @bo embedded in a struct vmw_buffer_object? */
+	if (bo->destroy != vmw_bo_bo_free &&
+	    bo->destroy != vmw_user_bo_destroy)
+		return;
+
+	/* Kill any cached kernel maps before swapout */
+	vmw_bo_unmap(vmw_buffer_object(bo));
+}
+
+
+/**
+ * vmw_bo_move_notify - TTM move_notify_callback
+ *
+ * @bo: The TTM buffer object about to move.
+ * @mem: The struct ttm_mem_reg indicating to what memory
+ *       region the move is taking place.
+ *
+ * Detaches cached maps and device bindings that require that the
+ * buffer doesn't move.
+ */
+void vmw_bo_move_notify(struct ttm_buffer_object *bo,
+			struct ttm_mem_reg *mem)
+{
+	struct vmw_buffer_object *vbo;
+
+	if (mem == NULL)
+		return;
+
+	/* Make sure @bo is embedded in a struct vmw_buffer_object? */
+	if (bo->destroy != vmw_bo_bo_free &&
+	    bo->destroy != vmw_user_bo_destroy)
+		return;
+
+	vbo = container_of(bo, struct vmw_buffer_object, base);
+
+	/*
+	 * Kill any cached kernel maps before move to or from VRAM.
+	 * With other types of moves, the underlying pages stay the same,
+	 * and the map can be kept.
+	 */
+	if (mem->mem_type == TTM_PL_VRAM || bo->mem.mem_type == TTM_PL_VRAM)
+		vmw_bo_unmap(vbo);
+
+	/*
+	 * If we're moving a backup MOB out of MOB placement, then make sure we
+	 * read back all resource content first, and unbind the MOB from
+	 * the resource.
+	 */
+	if (mem->mem_type != VMW_PL_MOB && bo->mem.mem_type == VMW_PL_MOB)
+		vmw_resource_unbind_list(vbo);
+}
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_context.c b/drivers/gpu/drm/vmwgfx/vmwgfx_context.c
index 750319d95458..7c3cb8efd11a 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_context.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_context.c
@@ -38,7 +38,7 @@ struct vmw_user_context {
 	struct vmw_cmdbuf_res_manager *man;
 	struct vmw_resource *cotables[SVGA_COTABLE_DX10_MAX];
 	spinlock_t cotable_lock;
-	struct vmw_dma_buffer *dx_query_mob;
+	struct vmw_buffer_object *dx_query_mob;
 };
 
 static void vmw_user_context_free(struct vmw_resource *res);
@@ -424,7 +424,7 @@ static int vmw_gb_context_unbind(struct vmw_resource *res,
 	(void) vmw_execbuf_fence_commands(NULL, dev_priv,
 					  &fence, NULL);
 
-	vmw_fence_single_bo(bo, fence);
+	vmw_bo_fence_single(bo, fence);
 
 	if (likely(fence != NULL))
 		vmw_fence_obj_unreference(&fence);
@@ -648,7 +648,7 @@ static int vmw_dx_context_unbind(struct vmw_resource *res,
 	(void) vmw_execbuf_fence_commands(NULL, dev_priv,
 					  &fence, NULL);
 
-	vmw_fence_single_bo(bo, fence);
+	vmw_bo_fence_single(bo, fence);
 
 	if (likely(fence != NULL))
 		vmw_fence_obj_unreference(&fence);
@@ -900,7 +900,7 @@ vmw_context_binding_state(struct vmw_resource *ctx)
  * specified in the parameter.  0 otherwise.
  */
 int vmw_context_bind_dx_query(struct vmw_resource *ctx_res,
-			      struct vmw_dma_buffer *mob)
+			      struct vmw_buffer_object *mob)
 {
 	struct vmw_user_context *uctx =
 		container_of(ctx_res, struct vmw_user_context, res);
@@ -908,7 +908,7 @@ int vmw_context_bind_dx_query(struct vmw_resource *ctx_res,
 	if (mob == NULL) {
 		if (uctx->dx_query_mob) {
 			uctx->dx_query_mob->dx_query_ctx = NULL;
-			vmw_dmabuf_unreference(&uctx->dx_query_mob);
+			vmw_bo_unreference(&uctx->dx_query_mob);
 			uctx->dx_query_mob = NULL;
 		}
 
@@ -922,7 +922,7 @@ int vmw_context_bind_dx_query(struct vmw_resource *ctx_res,
 	mob->dx_query_ctx  = ctx_res;
 
 	if (!uctx->dx_query_mob)
-		uctx->dx_query_mob = vmw_dmabuf_reference(mob);
+		uctx->dx_query_mob = vmw_bo_reference(mob);
 
 	return 0;
 }
@@ -932,7 +932,7 @@ int vmw_context_bind_dx_query(struct vmw_resource *ctx_res,
  *
  * @ctx_res: The context resource
  */
-struct vmw_dma_buffer *
+struct vmw_buffer_object *
 vmw_context_get_dx_query_mob(struct vmw_resource *ctx_res)
 {
 	struct vmw_user_context *uctx =
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_cotable.c b/drivers/gpu/drm/vmwgfx/vmwgfx_cotable.c
index df55e7d8b4cf..1d45714e1d5a 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_cotable.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_cotable.c
@@ -324,7 +324,7 @@ static int vmw_cotable_unbind(struct vmw_resource *res,
 		vmw_dx_context_scrub_cotables(vcotbl->ctx, readback);
 	mutex_unlock(&dev_priv->binding_mutex);
 	(void) vmw_execbuf_fence_commands(NULL, dev_priv, &fence, NULL);
-	vmw_fence_single_bo(bo, fence);
+	vmw_bo_fence_single(bo, fence);
 	if (likely(fence != NULL))
 		vmw_fence_obj_unreference(&fence);
 
@@ -367,7 +367,7 @@ static int vmw_cotable_readback(struct vmw_resource *res)
 	}
 
 	(void) vmw_execbuf_fence_commands(NULL, dev_priv, &fence, NULL);
-	vmw_fence_single_bo(&res->backup->base, fence);
+	vmw_bo_fence_single(&res->backup->base, fence);
 	vmw_fence_obj_unreference(&fence);
 
 	return 0;
@@ -390,7 +390,7 @@ static int vmw_cotable_resize(struct vmw_resource *res, size_t new_size)
 	struct ttm_operation_ctx ctx = { false, false };
 	struct vmw_private *dev_priv = res->dev_priv;
 	struct vmw_cotable *vcotbl = vmw_cotable(res);
-	struct vmw_dma_buffer *buf, *old_buf = res->backup;
+	struct vmw_buffer_object *buf, *old_buf = res->backup;
 	struct ttm_buffer_object *bo, *old_bo = &res->backup->base;
 	size_t old_size = res->backup_size;
 	size_t old_size_read_back = vcotbl->size_read_back;
@@ -415,8 +415,8 @@ static int vmw_cotable_resize(struct vmw_resource *res, size_t new_size)
 	if (!buf)
 		return -ENOMEM;
 
-	ret = vmw_dmabuf_init(dev_priv, buf, new_size, &vmw_mob_ne_placement,
-			      true, vmw_dmabuf_bo_free);
+	ret = vmw_bo_init(dev_priv, buf, new_size, &vmw_mob_ne_placement,
+			  true, vmw_bo_bo_free);
 	if (ret) {
 		DRM_ERROR("Failed initializing new cotable MOB.\n");
 		return ret;
@@ -482,7 +482,7 @@ static int vmw_cotable_resize(struct vmw_resource *res, size_t new_size)
 	/* Let go of the old mob. */
 	list_del(&res->mob_head);
 	list_add_tail(&res->mob_head, &buf->res_list);
-	vmw_dmabuf_unreference(&old_buf);
+	vmw_bo_unreference(&old_buf);
 	res->id = vcotbl->type;
 
 	return 0;
@@ -491,7 +491,7 @@ out_map_new:
 	ttm_bo_kunmap(&old_map);
 out_wait:
 	ttm_bo_unreserve(bo);
-	vmw_dmabuf_unreference(&buf);
+	vmw_bo_unreference(&buf);
 
 	return ret;
 }
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_dmabuf.c b/drivers/gpu/drm/vmwgfx/vmwgfx_dmabuf.c
deleted file mode 100644
index df7d1c97eb82..000000000000
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_dmabuf.c
+++ /dev/null
@@ -1,376 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0 OR MIT
-/**************************************************************************
- *
- * Copyright 2011-2015 VMware, Inc., Palo Alto, CA., USA
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the
- * "Software"), to deal in the Software without restriction, including
- * without limitation the rights to use, copy, modify, merge, publish,
- * distribute, sub license, and/or sell copies of the Software, and to
- * permit persons to whom the Software is furnished to do so, subject to
- * the following conditions:
- *
- * The above copyright notice and this permission notice (including the
- * next paragraph) shall be included in all copies or substantial portions
- * of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
- * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
- * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
- * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
- * USE OR OTHER DEALINGS IN THE SOFTWARE.
- *
- **************************************************************************/
-
-#include <drm/ttm/ttm_placement.h>
-
-#include <drm/drmP.h>
-#include "vmwgfx_drv.h"
-
-
-/**
- * vmw_dmabuf_pin_in_placement - Validate a buffer to placement.
- *
- * @dev_priv:  Driver private.
- * @buf:  DMA buffer to move.
- * @placement:  The placement to pin it.
- * @interruptible:  Use interruptible wait.
- *
- * Returns
- *  -ERESTARTSYS if interrupted by a signal.
- */
-int vmw_dmabuf_pin_in_placement(struct vmw_private *dev_priv,
-				struct vmw_dma_buffer *buf,
-				struct ttm_placement *placement,
-				bool interruptible)
-{
-	struct ttm_operation_ctx ctx = {interruptible, false };
-	struct ttm_buffer_object *bo = &buf->base;
-	int ret;
-	uint32_t new_flags;
-
-	ret = ttm_write_lock(&dev_priv->reservation_sem, interruptible);
-	if (unlikely(ret != 0))
-		return ret;
-
-	vmw_execbuf_release_pinned_bo(dev_priv);
-
-	ret = ttm_bo_reserve(bo, interruptible, false, NULL);
-	if (unlikely(ret != 0))
-		goto err;
-
-	if (buf->pin_count > 0)
-		ret = ttm_bo_mem_compat(placement, &bo->mem,
-					&new_flags) == true ? 0 : -EINVAL;
-	else
-		ret = ttm_bo_validate(bo, placement, &ctx);
-
-	if (!ret)
-		vmw_bo_pin_reserved(buf, true);
-
-	ttm_bo_unreserve(bo);
-
-err:
-	ttm_write_unlock(&dev_priv->reservation_sem);
-	return ret;
-}
-
-/**
- * vmw_dmabuf_pin_in_vram_or_gmr - Move a buffer to vram or gmr.
- *
- * This function takes the reservation_sem in write mode.
- * Flushes and unpins the query bo to avoid failures.
- *
- * @dev_priv:  Driver private.
- * @buf:  DMA buffer to move.
- * @pin:  Pin buffer if true.
- * @interruptible:  Use interruptible wait.
- *
- * Returns
- * -ERESTARTSYS if interrupted by a signal.
- */
-int vmw_dmabuf_pin_in_vram_or_gmr(struct vmw_private *dev_priv,
-				  struct vmw_dma_buffer *buf,
-				  bool interruptible)
-{
-	struct ttm_operation_ctx ctx = {interruptible, false };
-	struct ttm_buffer_object *bo = &buf->base;
-	int ret;
-	uint32_t new_flags;
-
-	ret = ttm_write_lock(&dev_priv->reservation_sem, interruptible);
-	if (unlikely(ret != 0))
-		return ret;
-
-	vmw_execbuf_release_pinned_bo(dev_priv);
-
-	ret = ttm_bo_reserve(bo, interruptible, false, NULL);
-	if (unlikely(ret != 0))
-		goto err;
-
-	if (buf->pin_count > 0) {
-		ret = ttm_bo_mem_compat(&vmw_vram_gmr_placement, &bo->mem,
-					&new_flags) == true ? 0 : -EINVAL;
-		goto out_unreserve;
-	}
-
-	ret = ttm_bo_validate(bo, &vmw_vram_gmr_placement, &ctx);
-	if (likely(ret == 0) || ret == -ERESTARTSYS)
-		goto out_unreserve;
-
-	ret = ttm_bo_validate(bo, &vmw_vram_placement, &ctx);
-
-out_unreserve:
-	if (!ret)
-		vmw_bo_pin_reserved(buf, true);
-
-	ttm_bo_unreserve(bo);
-err:
-	ttm_write_unlock(&dev_priv->reservation_sem);
-	return ret;
-}
-
-/**
- * vmw_dmabuf_pin_in_vram - Move a buffer to vram.
- *
- * This function takes the reservation_sem in write mode.
- * Flushes and unpins the query bo to avoid failures.
- *
- * @dev_priv:  Driver private.
- * @buf:  DMA buffer to move.
- * @interruptible:  Use interruptible wait.
- *
- * Returns
- * -ERESTARTSYS if interrupted by a signal.
- */
-int vmw_dmabuf_pin_in_vram(struct vmw_private *dev_priv,
-			   struct vmw_dma_buffer *buf,
-			   bool interruptible)
-{
-	return vmw_dmabuf_pin_in_placement(dev_priv, buf, &vmw_vram_placement,
-					   interruptible);
-}
-
-/**
- * vmw_dmabuf_pin_in_start_of_vram - Move a buffer to start of vram.
- *
- * This function takes the reservation_sem in write mode.
- * Flushes and unpins the query bo to avoid failures.
- *
- * @dev_priv:  Driver private.
- * @buf:  DMA buffer to pin.
- * @interruptible:  Use interruptible wait.
- *
- * Returns
- * -ERESTARTSYS if interrupted by a signal.
- */
-int vmw_dmabuf_pin_in_start_of_vram(struct vmw_private *dev_priv,
-				    struct vmw_dma_buffer *buf,
-				    bool interruptible)
-{
-	struct ttm_operation_ctx ctx = {interruptible, false };
-	struct ttm_buffer_object *bo = &buf->base;
-	struct ttm_placement placement;
-	struct ttm_place place;
-	int ret = 0;
-	uint32_t new_flags;
-
-	place = vmw_vram_placement.placement[0];
-	place.lpfn = bo->num_pages;
-	placement.num_placement = 1;
-	placement.placement = &place;
-	placement.num_busy_placement = 1;
-	placement.busy_placement = &place;
-
-	ret = ttm_write_lock(&dev_priv->reservation_sem, interruptible);
-	if (unlikely(ret != 0))
-		return ret;
-
-	vmw_execbuf_release_pinned_bo(dev_priv);
-	ret = ttm_bo_reserve(bo, interruptible, false, NULL);
-	if (unlikely(ret != 0))
-		goto err_unlock;
-
-	/*
-	 * Is this buffer already in vram but not at the start of it?
-	 * In that case, evict it first because TTM isn't good at handling
-	 * that situation.
-	 */
-	if (bo->mem.mem_type == TTM_PL_VRAM &&
-	    bo->mem.start < bo->num_pages &&
-	    bo->mem.start > 0 &&
-	    buf->pin_count == 0) {
-		ctx.interruptible = false;
-		(void) ttm_bo_validate(bo, &vmw_sys_placement, &ctx);
-	}
-
-	if (buf->pin_count > 0)
-		ret = ttm_bo_mem_compat(&placement, &bo->mem,
-					&new_flags) == true ? 0 : -EINVAL;
-	else
-		ret = ttm_bo_validate(bo, &placement, &ctx);
-
-	/* For some reason we didn't end up at the start of vram */
-	WARN_ON(ret == 0 && bo->offset != 0);
-	if (!ret)
-		vmw_bo_pin_reserved(buf, true);
-
-	ttm_bo_unreserve(bo);
-err_unlock:
-	ttm_write_unlock(&dev_priv->reservation_sem);
-
-	return ret;
-}
-
-/**
- * vmw_dmabuf_unpin - Unpin the buffer given buffer, does not move the buffer.
- *
- * This function takes the reservation_sem in write mode.
- *
- * @dev_priv:  Driver private.
- * @buf:  DMA buffer to unpin.
- * @interruptible:  Use interruptible wait.
- *
- * Returns
- * -ERESTARTSYS if interrupted by a signal.
- */
-int vmw_dmabuf_unpin(struct vmw_private *dev_priv,
-		     struct vmw_dma_buffer *buf,
-		     bool interruptible)
-{
-	struct ttm_buffer_object *bo = &buf->base;
-	int ret;
-
-	ret = ttm_read_lock(&dev_priv->reservation_sem, interruptible);
-	if (unlikely(ret != 0))
-		return ret;
-
-	ret = ttm_bo_reserve(bo, interruptible, false, NULL);
-	if (unlikely(ret != 0))
-		goto err;
-
-	vmw_bo_pin_reserved(buf, false);
-
-	ttm_bo_unreserve(bo);
-
-err:
-	ttm_read_unlock(&dev_priv->reservation_sem);
-	return ret;
-}
-
-/**
- * vmw_bo_get_guest_ptr - Get the guest ptr representing the current placement
- * of a buffer.
- *
- * @bo: Pointer to a struct ttm_buffer_object. Must be pinned or reserved.
- * @ptr: SVGAGuestPtr returning the result.
- */
-void vmw_bo_get_guest_ptr(const struct ttm_buffer_object *bo,
-			  SVGAGuestPtr *ptr)
-{
-	if (bo->mem.mem_type == TTM_PL_VRAM) {
-		ptr->gmrId = SVGA_GMR_FRAMEBUFFER;
-		ptr->offset = bo->offset;
-	} else {
-		ptr->gmrId = bo->mem.start;
-		ptr->offset = 0;
-	}
-}
-
-
-/**
- * vmw_bo_pin_reserved - Pin or unpin a buffer object without moving it.
- *
- * @vbo: The buffer object. Must be reserved.
- * @pin: Whether to pin or unpin.
- *
- */
-void vmw_bo_pin_reserved(struct vmw_dma_buffer *vbo, bool pin)
-{
-	struct ttm_operation_ctx ctx = { false, true };
-	struct ttm_place pl;
-	struct ttm_placement placement;
-	struct ttm_buffer_object *bo = &vbo->base;
-	uint32_t old_mem_type = bo->mem.mem_type;
-	int ret;
-
-	lockdep_assert_held(&bo->resv->lock.base);
-
-	if (pin) {
-		if (vbo->pin_count++ > 0)
-			return;
-	} else {
-		WARN_ON(vbo->pin_count <= 0);
-		if (--vbo->pin_count > 0)
-			return;
-	}
-
-	pl.fpfn = 0;
-	pl.lpfn = 0;
-	pl.flags = TTM_PL_FLAG_VRAM | VMW_PL_FLAG_GMR | VMW_PL_FLAG_MOB
-		| TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED;
-	if (pin)
-		pl.flags |= TTM_PL_FLAG_NO_EVICT;
-
-	memset(&placement, 0, sizeof(placement));
-	placement.num_placement = 1;
-	placement.placement = &pl;
-
-	ret = ttm_bo_validate(bo, &placement, &ctx);
-
-	BUG_ON(ret != 0 || bo->mem.mem_type != old_mem_type);
-}
-
-
-/*
- * vmw_dma_buffer_unmap - Tear down a cached buffer object map.
- *
- * @vbo: The buffer object whose map we are tearing down.
- *
- * This function tears down a cached map set up using
- * vmw_dma_buffer_map_and_cache().
- */
-void vmw_dma_buffer_unmap(struct vmw_dma_buffer *vbo)
-{
-	if (vbo->map.bo == NULL)
-		return;
-
-	ttm_bo_kunmap(&vbo->map);
-}
-
-
-/*
- * vmw_dma_buffer_map_and_cache - Map a buffer object and cache the map
- *
- * @vbo: The buffer object to map
- * Return: A kernel virtual address or NULL if mapping failed.
- *
- * This function maps a buffer object into the kernel address space, or
- * returns the virtual kernel address of an already existing map. The virtual
- * address remains valid as long as the buffer object is pinned or reserved.
- * The cached map is torn down on either
- * 1) Buffer object move
- * 2) Buffer object swapout
- * 3) Buffer object destruction
- *
- */
-void *vmw_dma_buffer_map_and_cache(struct vmw_dma_buffer *vbo)
-{
-	struct ttm_buffer_object *bo = &vbo->base;
-	bool not_used;
-	void *virtual;
-	int ret;
-
-	virtual = ttm_kmap_obj_virtual(&vbo->map, &not_used);
-	if (virtual)
-		return virtual;
-
-	ret = ttm_bo_kmap(bo, 0, bo->num_pages, &vbo->map);
-	if (ret)
-		DRM_ERROR("Buffer object map failed: %d.\n", ret);
-
-	return ttm_kmap_obj_virtual(&vbo->map, &not_used);
-}
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
index af88cb737be3..bb6dbbe18835 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
@@ -137,6 +137,12 @@
 #define DRM_IOCTL_VMW_CREATE_EXTENDED_CONTEXT			\
 	DRM_IOWR(DRM_COMMAND_BASE + DRM_VMW_CREATE_EXTENDED_CONTEXT,	\
 		struct drm_vmw_context_arg)
+#define DRM_IOCTL_VMW_GB_SURFACE_CREATE_EXT				\
+	DRM_IOWR(DRM_COMMAND_BASE + DRM_VMW_GB_SURFACE_CREATE_EXT,	\
+		union drm_vmw_gb_surface_create_ext_arg)
+#define DRM_IOCTL_VMW_GB_SURFACE_REF_EXT				\
+	DRM_IOWR(DRM_COMMAND_BASE + DRM_VMW_GB_SURFACE_REF_EXT,		\
+		union drm_vmw_gb_surface_reference_ext_arg)
 
 /**
  * The core DRM version of this macro doesn't account for
@@ -153,9 +159,9 @@
 static const struct drm_ioctl_desc vmw_ioctls[] = {
 	VMW_IOCTL_DEF(VMW_GET_PARAM, vmw_getparam_ioctl,
 		      DRM_AUTH | DRM_RENDER_ALLOW),
-	VMW_IOCTL_DEF(VMW_ALLOC_DMABUF, vmw_dmabuf_alloc_ioctl,
+	VMW_IOCTL_DEF(VMW_ALLOC_DMABUF, vmw_bo_alloc_ioctl,
 		      DRM_AUTH | DRM_RENDER_ALLOW),
-	VMW_IOCTL_DEF(VMW_UNREF_DMABUF, vmw_dmabuf_unref_ioctl,
+	VMW_IOCTL_DEF(VMW_UNREF_DMABUF, vmw_bo_unref_ioctl,
 		      DRM_RENDER_ALLOW),
 	VMW_IOCTL_DEF(VMW_CURSOR_BYPASS,
 		      vmw_kms_cursor_bypass_ioctl,
@@ -219,11 +225,17 @@ static const struct drm_ioctl_desc vmw_ioctls[] = {
 		      vmw_gb_surface_reference_ioctl,
 		      DRM_AUTH | DRM_RENDER_ALLOW),
 	VMW_IOCTL_DEF(VMW_SYNCCPU,
-		      vmw_user_dmabuf_synccpu_ioctl,
+		      vmw_user_bo_synccpu_ioctl,
 		      DRM_RENDER_ALLOW),
 	VMW_IOCTL_DEF(VMW_CREATE_EXTENDED_CONTEXT,
 		      vmw_extended_context_define_ioctl,
 		      DRM_AUTH | DRM_RENDER_ALLOW),
+	VMW_IOCTL_DEF(VMW_GB_SURFACE_CREATE_EXT,
+		      vmw_gb_surface_define_ext_ioctl,
+		      DRM_AUTH | DRM_RENDER_ALLOW),
+	VMW_IOCTL_DEF(VMW_GB_SURFACE_REF_EXT,
+		      vmw_gb_surface_reference_ext_ioctl,
+		      DRM_AUTH | DRM_RENDER_ALLOW),
 };
 
 static const struct pci_device_id vmw_pci_id_list[] = {
@@ -258,6 +270,15 @@ MODULE_PARM_DESC(assume_16bpp, "Assume 16-bpp when filtering modes");
 module_param_named(assume_16bpp, vmw_assume_16bpp, int, 0600);
 
 
+static void vmw_print_capabilities2(uint32_t capabilities2)
+{
+	DRM_INFO("Capabilities2:\n");
+	if (capabilities2 & SVGA_CAP2_GROW_OTABLE)
+		DRM_INFO("  Grow oTable.\n");
+	if (capabilities2 & SVGA_CAP2_INTRA_SURFACE_COPY)
+		DRM_INFO("  IntraSurface copy.\n");
+}
+
 static void vmw_print_capabilities(uint32_t capabilities)
 {
 	DRM_INFO("Capabilities:\n");
@@ -321,7 +342,7 @@ static void vmw_print_capabilities(uint32_t capabilities)
 static int vmw_dummy_query_bo_create(struct vmw_private *dev_priv)
 {
 	int ret;
-	struct vmw_dma_buffer *vbo;
+	struct vmw_buffer_object *vbo;
 	struct ttm_bo_kmap_obj map;
 	volatile SVGA3dQueryResult *result;
 	bool dummy;
@@ -335,9 +356,9 @@ static int vmw_dummy_query_bo_create(struct vmw_private *dev_priv)
 	if (!vbo)
 		return -ENOMEM;
 
-	ret = vmw_dmabuf_init(dev_priv, vbo, PAGE_SIZE,
-			      &vmw_sys_ne_placement, false,
-			      &vmw_dmabuf_bo_free);
+	ret = vmw_bo_init(dev_priv, vbo, PAGE_SIZE,
+			  &vmw_sys_ne_placement, false,
+			  &vmw_bo_bo_free);
 	if (unlikely(ret != 0))
 		return ret;
 
@@ -358,7 +379,7 @@ static int vmw_dummy_query_bo_create(struct vmw_private *dev_priv)
 
 	if (unlikely(ret != 0)) {
 		DRM_ERROR("Dummy query buffer map failed.\n");
-		vmw_dmabuf_unreference(&vbo);
+		vmw_bo_unreference(&vbo);
 	} else
 		dev_priv->dummy_query_bo = vbo;
 
@@ -460,7 +481,7 @@ static void vmw_release_device_early(struct vmw_private *dev_priv)
 
 	BUG_ON(dev_priv->pinned_bo != NULL);
 
-	vmw_dmabuf_unreference(&dev_priv->dummy_query_bo);
+	vmw_bo_unreference(&dev_priv->dummy_query_bo);
 	if (dev_priv->cman)
 		vmw_cmdbuf_remove_pool(dev_priv->cman);
 
@@ -644,6 +665,7 @@ static int vmw_driver_load(struct drm_device *dev, unsigned long chipset)
 	mutex_init(&dev_priv->cmdbuf_mutex);
 	mutex_init(&dev_priv->release_mutex);
 	mutex_init(&dev_priv->binding_mutex);
+	mutex_init(&dev_priv->requested_layout_mutex);
 	mutex_init(&dev_priv->global_kms_state_mutex);
 	rwlock_init(&dev_priv->resource_lock);
 	ttm_lock_init(&dev_priv->reservation_sem);
@@ -683,6 +705,12 @@ static int vmw_driver_load(struct drm_device *dev, unsigned long chipset)
 	}
 
 	dev_priv->capabilities = vmw_read(dev_priv, SVGA_REG_CAPABILITIES);
+
+	if (dev_priv->capabilities & SVGA_CAP_CAP2_REGISTER) {
+		dev_priv->capabilities2 = vmw_read(dev_priv, SVGA_REG_CAP2);
+	}
+
+
 	ret = vmw_dma_select_mode(dev_priv);
 	if (unlikely(ret != 0)) {
 		DRM_INFO("Restricting capabilities due to IOMMU setup.\n");
@@ -751,6 +779,8 @@ static int vmw_driver_load(struct drm_device *dev, unsigned long chipset)
 	}
 
 	vmw_print_capabilities(dev_priv->capabilities);
+	if (dev_priv->capabilities & SVGA_CAP_CAP2_REGISTER)
+		vmw_print_capabilities2(dev_priv->capabilities2);
 
 	ret = vmw_dma_masks(dev_priv);
 	if (unlikely(ret != 0))
@@ -883,7 +913,7 @@ static int vmw_driver_load(struct drm_device *dev, unsigned long chipset)
 
 	if (dev_priv->has_mob) {
 		spin_lock(&dev_priv->cap_lock);
-		vmw_write(dev_priv, SVGA_REG_DEV_CAP, SVGA3D_DEVCAP_DX);
+		vmw_write(dev_priv, SVGA_REG_DEV_CAP, SVGA3D_DEVCAP_DXCONTEXT);
 		dev_priv->has_dx = !!vmw_read(dev_priv, SVGA_REG_DEV_CAP);
 		spin_unlock(&dev_priv->cap_lock);
 	}
@@ -898,9 +928,23 @@ static int vmw_driver_load(struct drm_device *dev, unsigned long chipset)
 	if (ret)
 		goto out_no_fifo;
 
+	if (dev_priv->has_dx) {
+		/*
+		 * SVGA_CAP2_DX2 (DefineGBSurface_v3) is needed for SM4_1
+		 * support
+		 */
+		if ((dev_priv->capabilities2 & SVGA_CAP2_DX2) != 0) {
+			vmw_write(dev_priv, SVGA_REG_DEV_CAP,
+					SVGA3D_DEVCAP_SM41);
+			dev_priv->has_sm4_1 = vmw_read(dev_priv,
+							SVGA_REG_DEV_CAP);
+		}
+	}
+
 	DRM_INFO("DX: %s\n", dev_priv->has_dx ? "yes." : "no.");
-	DRM_INFO("Atomic: %s\n",
-		 (dev->driver->driver_features & DRIVER_ATOMIC) ? "yes" : "no");
+	DRM_INFO("Atomic: %s\n", (dev->driver->driver_features & DRIVER_ATOMIC)
+		 ? "yes." : "no.");
+	DRM_INFO("SM4_1: %s\n", dev_priv->has_sm4_1 ? "yes." : "no.");
 
 	snprintf(host_log, sizeof(host_log), "vmwgfx: %s-%s",
 		VMWGFX_REPO, VMWGFX_GIT_VERSION);
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
index 422eef114b98..1abe21758b0d 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
@@ -43,10 +43,10 @@
 #include <linux/sync_file.h>
 
 #define VMWGFX_DRIVER_NAME "vmwgfx"
-#define VMWGFX_DRIVER_DATE "20180322"
+#define VMWGFX_DRIVER_DATE "20180704"
 #define VMWGFX_DRIVER_MAJOR 2
-#define VMWGFX_DRIVER_MINOR 14
-#define VMWGFX_DRIVER_PATCHLEVEL 1
+#define VMWGFX_DRIVER_MINOR 15
+#define VMWGFX_DRIVER_PATCHLEVEL 0
 #define VMWGFX_FILE_PAGE_OFFSET 0x00100000
 #define VMWGFX_FIFO_STATIC_SIZE (1024*1024)
 #define VMWGFX_MAX_RELOCATIONS 2048
@@ -83,10 +83,10 @@
 struct vmw_fpriv {
 	struct drm_master *locked_master;
 	struct ttm_object_file *tfile;
-	bool gb_aware;
+	bool gb_aware; /* user-space is guest-backed aware */
 };
 
-struct vmw_dma_buffer {
+struct vmw_buffer_object {
 	struct ttm_buffer_object base;
 	struct list_head res_list;
 	s32 pin_count;
@@ -120,7 +120,7 @@ struct vmw_resource {
 	unsigned long backup_size;
 	bool res_dirty; /* Protected by backup buffer reserved */
 	bool backup_dirty; /* Protected by backup buffer reserved */
-	struct vmw_dma_buffer *backup;
+	struct vmw_buffer_object *backup;
 	unsigned long backup_offset;
 	unsigned long pin_count; /* Protected by resource reserved */
 	const struct vmw_res_func *func;
@@ -166,7 +166,7 @@ struct vmw_surface_offset;
 
 struct vmw_surface {
 	struct vmw_resource res;
-	uint32_t flags;
+	SVGA3dSurfaceAllFlags flags;
 	uint32_t format;
 	uint32_t mip_levels[DRM_VMW_MAX_SURFACE_FACES];
 	struct drm_vmw_size base_size;
@@ -180,6 +180,8 @@ struct vmw_surface {
 	SVGA3dTextureFilter autogen_filter;
 	uint32_t multisample_count;
 	struct list_head view_list;
+	SVGA3dMSPattern multisample_pattern;
+	SVGA3dMSQualityLevel quality_level;
 };
 
 struct vmw_marker_queue {
@@ -304,7 +306,7 @@ struct vmw_sw_context{
 	uint32_t cmd_bounce_size;
 	struct list_head resource_list;
 	struct list_head ctx_resource_list; /* For contexts and cotables */
-	struct vmw_dma_buffer *cur_query_bo;
+	struct vmw_buffer_object *cur_query_bo;
 	struct list_head res_relocations;
 	uint32_t *buf_start;
 	struct vmw_res_cache_entry res_cache[vmw_res_max];
@@ -315,7 +317,7 @@ struct vmw_sw_context{
 	bool staged_bindings_inuse;
 	struct list_head staged_cmd_res;
 	struct vmw_resource_val_node *dx_ctx_node;
-	struct vmw_dma_buffer *dx_query_mob;
+	struct vmw_buffer_object *dx_query_mob;
 	struct vmw_resource *dx_query_ctx;
 	struct vmw_cmdbuf_res_manager *man;
 };
@@ -386,6 +388,7 @@ struct vmw_private {
 	uint32_t initial_height;
 	u32 *mmio_virt;
 	uint32_t capabilities;
+	uint32_t capabilities2;
 	uint32_t max_gmr_ids;
 	uint32_t max_gmr_pages;
 	uint32_t max_mob_pages;
@@ -397,6 +400,7 @@ struct vmw_private {
 	spinlock_t cap_lock;
 	bool has_dx;
 	bool assume_16bpp;
+	bool has_sm4_1;
 
 	/*
 	 * VGA registers.
@@ -412,6 +416,15 @@ struct vmw_private {
 	uint32_t num_displays;
 
 	/*
+	 * Currently requested_layout_mutex is used to protect the gui
+	 * positionig state in display unit. With that use case currently this
+	 * mutex is only taken during layout ioctl and atomic check_modeset.
+	 * Other display unit state can be protected with this mutex but that
+	 * needs careful consideration.
+	 */
+	struct mutex requested_layout_mutex;
+
+	/*
 	 * Framebuffer info.
 	 */
 
@@ -513,8 +526,8 @@ struct vmw_private {
 	 * are protected by the cmdbuf mutex.
 	 */
 
-	struct vmw_dma_buffer *dummy_query_bo;
-	struct vmw_dma_buffer *pinned_bo;
+	struct vmw_buffer_object *dummy_query_bo;
+	struct vmw_buffer_object *pinned_bo;
 	uint32_t query_cid;
 	uint32_t query_cid_valid;
 	bool dummy_query_bo_pinned;
@@ -623,43 +636,13 @@ extern int vmw_user_lookup_handle(struct vmw_private *dev_priv,
 				  struct ttm_object_file *tfile,
 				  uint32_t handle,
 				  struct vmw_surface **out_surf,
-				  struct vmw_dma_buffer **out_buf);
+				  struct vmw_buffer_object **out_buf);
 extern int vmw_user_resource_lookup_handle(
 	struct vmw_private *dev_priv,
 	struct ttm_object_file *tfile,
 	uint32_t handle,
 	const struct vmw_user_resource_conv *converter,
 	struct vmw_resource **p_res);
-extern void vmw_dmabuf_bo_free(struct ttm_buffer_object *bo);
-extern int vmw_dmabuf_init(struct vmw_private *dev_priv,
-			   struct vmw_dma_buffer *vmw_bo,
-			   size_t size, struct ttm_placement *placement,
-			   bool interuptable,
-			   void (*bo_free) (struct ttm_buffer_object *bo));
-extern int vmw_user_dmabuf_verify_access(struct ttm_buffer_object *bo,
-				  struct ttm_object_file *tfile);
-extern int vmw_user_dmabuf_alloc(struct vmw_private *dev_priv,
-				 struct ttm_object_file *tfile,
-				 uint32_t size,
-				 bool shareable,
-				 uint32_t *handle,
-				 struct vmw_dma_buffer **p_dma_buf,
-				 struct ttm_base_object **p_base);
-extern int vmw_user_dmabuf_reference(struct ttm_object_file *tfile,
-				     struct vmw_dma_buffer *dma_buf,
-				     uint32_t *handle);
-extern int vmw_dmabuf_alloc_ioctl(struct drm_device *dev, void *data,
-				  struct drm_file *file_priv);
-extern int vmw_dmabuf_unref_ioctl(struct drm_device *dev, void *data,
-				  struct drm_file *file_priv);
-extern int vmw_user_dmabuf_synccpu_ioctl(struct drm_device *dev, void *data,
-					 struct drm_file *file_priv);
-extern uint32_t vmw_dmabuf_validate_node(struct ttm_buffer_object *bo,
-					 uint32_t cur_validate_node);
-extern void vmw_dmabuf_validate_clear(struct ttm_buffer_object *bo);
-extern int vmw_user_dmabuf_lookup(struct ttm_object_file *tfile,
-				  uint32_t id, struct vmw_dma_buffer **out,
-				  struct ttm_base_object **base);
 extern int vmw_stream_claim_ioctl(struct drm_device *dev, void *data,
 				  struct drm_file *file_priv);
 extern int vmw_stream_unref_ioctl(struct drm_device *dev, void *data,
@@ -670,43 +653,70 @@ extern int vmw_user_stream_lookup(struct vmw_private *dev_priv,
 				  struct vmw_resource **out);
 extern void vmw_resource_unreserve(struct vmw_resource *res,
 				   bool switch_backup,
-				   struct vmw_dma_buffer *new_backup,
+				   struct vmw_buffer_object *new_backup,
 				   unsigned long new_backup_offset);
-extern void vmw_resource_move_notify(struct ttm_buffer_object *bo,
-				     struct ttm_mem_reg *mem);
 extern void vmw_query_move_notify(struct ttm_buffer_object *bo,
 				  struct ttm_mem_reg *mem);
-extern void vmw_resource_swap_notify(struct ttm_buffer_object *bo);
-extern int vmw_query_readback_all(struct vmw_dma_buffer *dx_query_mob);
-extern void vmw_fence_single_bo(struct ttm_buffer_object *bo,
-				struct vmw_fence_obj *fence);
+extern int vmw_query_readback_all(struct vmw_buffer_object *dx_query_mob);
 extern void vmw_resource_evict_all(struct vmw_private *dev_priv);
-
+extern void vmw_resource_unbind_list(struct vmw_buffer_object *vbo);
 
 /**
- * DMA buffer helper routines - vmwgfx_dmabuf.c
+ * Buffer object helper functions - vmwgfx_bo.c
  */
-extern int vmw_dmabuf_pin_in_placement(struct vmw_private *vmw_priv,
-				       struct vmw_dma_buffer *bo,
-				       struct ttm_placement *placement,
+extern int vmw_bo_pin_in_placement(struct vmw_private *vmw_priv,
+				   struct vmw_buffer_object *bo,
+				   struct ttm_placement *placement,
+				   bool interruptible);
+extern int vmw_bo_pin_in_vram(struct vmw_private *dev_priv,
+			      struct vmw_buffer_object *buf,
+			      bool interruptible);
+extern int vmw_bo_pin_in_vram_or_gmr(struct vmw_private *dev_priv,
+				     struct vmw_buffer_object *buf,
+				     bool interruptible);
+extern int vmw_bo_pin_in_start_of_vram(struct vmw_private *vmw_priv,
+				       struct vmw_buffer_object *bo,
 				       bool interruptible);
-extern int vmw_dmabuf_pin_in_vram(struct vmw_private *dev_priv,
-				  struct vmw_dma_buffer *buf,
-				  bool interruptible);
-extern int vmw_dmabuf_pin_in_vram_or_gmr(struct vmw_private *dev_priv,
-					 struct vmw_dma_buffer *buf,
-					 bool interruptible);
-extern int vmw_dmabuf_pin_in_start_of_vram(struct vmw_private *vmw_priv,
-					   struct vmw_dma_buffer *bo,
-					   bool interruptible);
-extern int vmw_dmabuf_unpin(struct vmw_private *vmw_priv,
-			    struct vmw_dma_buffer *bo,
-			    bool interruptible);
+extern int vmw_bo_unpin(struct vmw_private *vmw_priv,
+			struct vmw_buffer_object *bo,
+			bool interruptible);
 extern void vmw_bo_get_guest_ptr(const struct ttm_buffer_object *buf,
 				 SVGAGuestPtr *ptr);
-extern void vmw_bo_pin_reserved(struct vmw_dma_buffer *bo, bool pin);
-extern void *vmw_dma_buffer_map_and_cache(struct vmw_dma_buffer *vbo);
-extern void vmw_dma_buffer_unmap(struct vmw_dma_buffer *vbo);
+extern void vmw_bo_pin_reserved(struct vmw_buffer_object *bo, bool pin);
+extern void vmw_bo_bo_free(struct ttm_buffer_object *bo);
+extern int vmw_bo_init(struct vmw_private *dev_priv,
+		       struct vmw_buffer_object *vmw_bo,
+		       size_t size, struct ttm_placement *placement,
+		       bool interuptable,
+		       void (*bo_free)(struct ttm_buffer_object *bo));
+extern int vmw_user_bo_verify_access(struct ttm_buffer_object *bo,
+				     struct ttm_object_file *tfile);
+extern int vmw_user_bo_alloc(struct vmw_private *dev_priv,
+			     struct ttm_object_file *tfile,
+			     uint32_t size,
+			     bool shareable,
+			     uint32_t *handle,
+			     struct vmw_buffer_object **p_dma_buf,
+			     struct ttm_base_object **p_base);
+extern int vmw_user_bo_reference(struct ttm_object_file *tfile,
+				 struct vmw_buffer_object *dma_buf,
+				 uint32_t *handle);
+extern int vmw_bo_alloc_ioctl(struct drm_device *dev, void *data,
+			      struct drm_file *file_priv);
+extern int vmw_bo_unref_ioctl(struct drm_device *dev, void *data,
+			      struct drm_file *file_priv);
+extern int vmw_user_bo_synccpu_ioctl(struct drm_device *dev, void *data,
+				     struct drm_file *file_priv);
+extern int vmw_user_bo_lookup(struct ttm_object_file *tfile,
+			      uint32_t id, struct vmw_buffer_object **out,
+			      struct ttm_base_object **base);
+extern void vmw_bo_fence_single(struct ttm_buffer_object *bo,
+				struct vmw_fence_obj *fence);
+extern void *vmw_bo_map_and_cache(struct vmw_buffer_object *vbo);
+extern void vmw_bo_unmap(struct vmw_buffer_object *vbo);
+extern void vmw_bo_move_notify(struct ttm_buffer_object *bo,
+			       struct ttm_mem_reg *mem);
+extern void vmw_bo_swap_notify(struct ttm_buffer_object *bo);
 
 /**
  * Misc Ioctl functionality - vmwgfx_ioctl.c
@@ -758,7 +768,7 @@ extern void vmw_ttm_global_release(struct vmw_private *dev_priv);
 extern int vmw_mmap(struct file *filp, struct vm_area_struct *vma);
 
 /**
- * TTM buffer object driver - vmwgfx_buffer.c
+ * TTM buffer object driver - vmwgfx_ttm_buffer.c
  */
 
 extern const size_t vmw_tt_size;
@@ -1041,8 +1051,8 @@ vmw_context_binding_state(struct vmw_resource *ctx);
 extern void vmw_dx_context_scrub_cotables(struct vmw_resource *ctx,
 					  bool readback);
 extern int vmw_context_bind_dx_query(struct vmw_resource *ctx_res,
-				     struct vmw_dma_buffer *mob);
-extern struct vmw_dma_buffer *
+				     struct vmw_buffer_object *mob);
+extern struct vmw_buffer_object *
 vmw_context_get_dx_query_mob(struct vmw_resource *ctx_res);
 
 
@@ -1070,14 +1080,22 @@ extern int vmw_surface_validate(struct vmw_private *dev_priv,
 				struct vmw_surface *srf);
 int vmw_surface_gb_priv_define(struct drm_device *dev,
 			       uint32_t user_accounting_size,
-			       uint32_t svga3d_flags,
+			       SVGA3dSurfaceAllFlags svga3d_flags,
 			       SVGA3dSurfaceFormat format,
 			       bool for_scanout,
 			       uint32_t num_mip_levels,
 			       uint32_t multisample_count,
 			       uint32_t array_size,
 			       struct drm_vmw_size size,
+			       SVGA3dMSPattern multisample_pattern,
+			       SVGA3dMSQualityLevel quality_level,
 			       struct vmw_surface **srf_out);
+extern int vmw_gb_surface_define_ext_ioctl(struct drm_device *dev,
+					   void *data,
+					   struct drm_file *file_priv);
+extern int vmw_gb_surface_reference_ext_ioctl(struct drm_device *dev,
+					      void *data,
+					      struct drm_file *file_priv);
 
 /*
  * Shader management - vmwgfx_shader.c
@@ -1224,6 +1242,11 @@ int vmw_bo_cpu_blit(struct ttm_buffer_object *dst,
 		    u32 w, u32 h,
 		    struct vmw_diff_cpy *diff);
 
+/* Host messaging -vmwgfx_msg.c: */
+int vmw_host_get_guestinfo(const char *guest_info_param,
+			   char *buffer, size_t *length);
+int vmw_host_log(const char *log);
+
 /**
  * Inline helper functions
  */
@@ -1243,9 +1266,9 @@ static inline struct vmw_surface *vmw_surface_reference(struct vmw_surface *srf)
 	return srf;
 }
 
-static inline void vmw_dmabuf_unreference(struct vmw_dma_buffer **buf)
+static inline void vmw_bo_unreference(struct vmw_buffer_object **buf)
 {
-	struct vmw_dma_buffer *tmp_buf = *buf;
+	struct vmw_buffer_object *tmp_buf = *buf;
 
 	*buf = NULL;
 	if (tmp_buf != NULL) {
@@ -1255,7 +1278,8 @@ static inline void vmw_dmabuf_unreference(struct vmw_dma_buffer **buf)
 	}
 }
 
-static inline struct vmw_dma_buffer *vmw_dmabuf_reference(struct vmw_dma_buffer *buf)
+static inline struct vmw_buffer_object *
+vmw_bo_reference(struct vmw_buffer_object *buf)
 {
 	if (ttm_bo_reference(&buf->base))
 		return buf;
@@ -1302,10 +1326,4 @@ static inline void vmw_mmio_write(u32 value, u32 *addr)
 {
 	WRITE_ONCE(*addr, value);
 }
-
-/**
- * Add vmw_msg module function
- */
-extern int vmw_host_log(const char *log);
-
 #endif
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c b/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c
index 47bb7e58a368..1f134570b759 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c
@@ -92,7 +92,7 @@ struct vmw_resource_val_node {
 	struct list_head head;
 	struct drm_hash_item hash;
 	struct vmw_resource *res;
-	struct vmw_dma_buffer *new_backup;
+	struct vmw_buffer_object *new_backup;
 	struct vmw_ctx_binding_state *staged_bindings;
 	unsigned long new_backup_offset;
 	u32 first_usage : 1;
@@ -126,9 +126,9 @@ static int vmw_resource_context_res_add(struct vmw_private *dev_priv,
 static int vmw_translate_mob_ptr(struct vmw_private *dev_priv,
 				 struct vmw_sw_context *sw_context,
 				 SVGAMobId *id,
-				 struct vmw_dma_buffer **vmw_bo_p);
+				 struct vmw_buffer_object **vmw_bo_p);
 static int vmw_bo_to_validate_list(struct vmw_sw_context *sw_context,
-				   struct vmw_dma_buffer *vbo,
+				   struct vmw_buffer_object *vbo,
 				   bool validate_as_mob,
 				   uint32_t *p_val_node);
 /**
@@ -185,7 +185,7 @@ static void vmw_resources_unreserve(struct vmw_sw_context *sw_context,
 		}
 		vmw_resource_unreserve(res, switch_backup, val->new_backup,
 				       val->new_backup_offset);
-		vmw_dmabuf_unreference(&val->new_backup);
+		vmw_bo_unreference(&val->new_backup);
 	}
 }
 
@@ -423,7 +423,7 @@ static int vmw_resource_context_res_add(struct vmw_private *dev_priv,
 	}
 
 	if (dev_priv->has_dx && vmw_res_type(ctx) == vmw_res_dx_context) {
-		struct vmw_dma_buffer *dx_query_mob;
+		struct vmw_buffer_object *dx_query_mob;
 
 		dx_query_mob = vmw_context_get_dx_query_mob(ctx);
 		if (dx_query_mob)
@@ -544,7 +544,7 @@ static int vmw_cmd_ok(struct vmw_private *dev_priv,
  * submission is reached.
  */
 static int vmw_bo_to_validate_list(struct vmw_sw_context *sw_context,
-				   struct vmw_dma_buffer *vbo,
+				   struct vmw_buffer_object *vbo,
 				   bool validate_as_mob,
 				   uint32_t *p_val_node)
 {
@@ -616,7 +616,7 @@ static int vmw_resources_reserve(struct vmw_sw_context *sw_context)
 			return ret;
 
 		if (res->backup) {
-			struct vmw_dma_buffer *vbo = res->backup;
+			struct vmw_buffer_object *vbo = res->backup;
 
 			ret = vmw_bo_to_validate_list
 				(sw_context, vbo,
@@ -628,7 +628,7 @@ static int vmw_resources_reserve(struct vmw_sw_context *sw_context)
 	}
 
 	if (sw_context->dx_query_mob) {
-		struct vmw_dma_buffer *expected_dx_query_mob;
+		struct vmw_buffer_object *expected_dx_query_mob;
 
 		expected_dx_query_mob =
 			vmw_context_get_dx_query_mob(sw_context->dx_query_ctx);
@@ -657,7 +657,7 @@ static int vmw_resources_validate(struct vmw_sw_context *sw_context)
 
 	list_for_each_entry(val, &sw_context->resource_list, head) {
 		struct vmw_resource *res = val->res;
-		struct vmw_dma_buffer *backup = res->backup;
+		struct vmw_buffer_object *backup = res->backup;
 
 		ret = vmw_resource_validate(res);
 		if (unlikely(ret != 0)) {
@@ -668,7 +668,7 @@ static int vmw_resources_validate(struct vmw_sw_context *sw_context)
 
 		/* Check if the resource switched backup buffer */
 		if (backup && res->backup && (backup != res->backup)) {
-			struct vmw_dma_buffer *vbo = res->backup;
+			struct vmw_buffer_object *vbo = res->backup;
 
 			ret = vmw_bo_to_validate_list
 				(sw_context, vbo,
@@ -821,7 +821,7 @@ out_no_reloc:
 static int vmw_rebind_all_dx_query(struct vmw_resource *ctx_res)
 {
 	struct vmw_private *dev_priv = ctx_res->dev_priv;
-	struct vmw_dma_buffer *dx_query_mob;
+	struct vmw_buffer_object *dx_query_mob;
 	struct {
 		SVGA3dCmdHeader header;
 		SVGA3dCmdDXBindAllQuery body;
@@ -1152,7 +1152,7 @@ static int vmw_cmd_present_check(struct vmw_private *dev_priv,
  * command batch.
  */
 static int vmw_query_bo_switch_prepare(struct vmw_private *dev_priv,
-				       struct vmw_dma_buffer *new_query_bo,
+				       struct vmw_buffer_object *new_query_bo,
 				       struct vmw_sw_context *sw_context)
 {
 	struct vmw_res_cache_entry *ctx_entry =
@@ -1234,7 +1234,7 @@ static void vmw_query_bo_switch_commit(struct vmw_private *dev_priv,
 	if (dev_priv->pinned_bo != sw_context->cur_query_bo) {
 		if (dev_priv->pinned_bo) {
 			vmw_bo_pin_reserved(dev_priv->pinned_bo, false);
-			vmw_dmabuf_unreference(&dev_priv->pinned_bo);
+			vmw_bo_unreference(&dev_priv->pinned_bo);
 		}
 
 		if (!sw_context->needs_post_query_barrier) {
@@ -1256,7 +1256,7 @@ static void vmw_query_bo_switch_commit(struct vmw_private *dev_priv,
 			dev_priv->query_cid = sw_context->last_query_ctx->id;
 			dev_priv->query_cid_valid = true;
 			dev_priv->pinned_bo =
-				vmw_dmabuf_reference(sw_context->cur_query_bo);
+				vmw_bo_reference(sw_context->cur_query_bo);
 		}
 	}
 }
@@ -1282,15 +1282,14 @@ static void vmw_query_bo_switch_commit(struct vmw_private *dev_priv,
 static int vmw_translate_mob_ptr(struct vmw_private *dev_priv,
 				 struct vmw_sw_context *sw_context,
 				 SVGAMobId *id,
-				 struct vmw_dma_buffer **vmw_bo_p)
+				 struct vmw_buffer_object **vmw_bo_p)
 {
-	struct vmw_dma_buffer *vmw_bo = NULL;
+	struct vmw_buffer_object *vmw_bo = NULL;
 	uint32_t handle = *id;
 	struct vmw_relocation *reloc;
 	int ret;
 
-	ret = vmw_user_dmabuf_lookup(sw_context->fp->tfile, handle, &vmw_bo,
-				     NULL);
+	ret = vmw_user_bo_lookup(sw_context->fp->tfile, handle, &vmw_bo, NULL);
 	if (unlikely(ret != 0)) {
 		DRM_ERROR("Could not find or use MOB buffer.\n");
 		ret = -EINVAL;
@@ -1316,7 +1315,7 @@ static int vmw_translate_mob_ptr(struct vmw_private *dev_priv,
 	return 0;
 
 out_no_reloc:
-	vmw_dmabuf_unreference(&vmw_bo);
+	vmw_bo_unreference(&vmw_bo);
 	*vmw_bo_p = NULL;
 	return ret;
 }
@@ -1343,15 +1342,14 @@ out_no_reloc:
 static int vmw_translate_guest_ptr(struct vmw_private *dev_priv,
 				   struct vmw_sw_context *sw_context,
 				   SVGAGuestPtr *ptr,
-				   struct vmw_dma_buffer **vmw_bo_p)
+				   struct vmw_buffer_object **vmw_bo_p)
 {
-	struct vmw_dma_buffer *vmw_bo = NULL;
+	struct vmw_buffer_object *vmw_bo = NULL;
 	uint32_t handle = ptr->gmrId;
 	struct vmw_relocation *reloc;
 	int ret;
 
-	ret = vmw_user_dmabuf_lookup(sw_context->fp->tfile, handle, &vmw_bo,
-				     NULL);
+	ret = vmw_user_bo_lookup(sw_context->fp->tfile, handle, &vmw_bo, NULL);
 	if (unlikely(ret != 0)) {
 		DRM_ERROR("Could not find or use GMR region.\n");
 		ret = -EINVAL;
@@ -1376,7 +1374,7 @@ static int vmw_translate_guest_ptr(struct vmw_private *dev_priv,
 	return 0;
 
 out_no_reloc:
-	vmw_dmabuf_unreference(&vmw_bo);
+	vmw_bo_unreference(&vmw_bo);
 	*vmw_bo_p = NULL;
 	return ret;
 }
@@ -1447,7 +1445,7 @@ static int vmw_cmd_dx_bind_query(struct vmw_private *dev_priv,
 		SVGA3dCmdDXBindQuery q;
 	} *cmd;
 
-	struct vmw_dma_buffer *vmw_bo;
+	struct vmw_buffer_object *vmw_bo;
 	int    ret;
 
 
@@ -1466,7 +1464,7 @@ static int vmw_cmd_dx_bind_query(struct vmw_private *dev_priv,
 	sw_context->dx_query_mob = vmw_bo;
 	sw_context->dx_query_ctx = sw_context->dx_ctx_node->res;
 
-	vmw_dmabuf_unreference(&vmw_bo);
+	vmw_bo_unreference(&vmw_bo);
 
 	return ret;
 }
@@ -1549,7 +1547,7 @@ static int vmw_cmd_end_gb_query(struct vmw_private *dev_priv,
 				struct vmw_sw_context *sw_context,
 				SVGA3dCmdHeader *header)
 {
-	struct vmw_dma_buffer *vmw_bo;
+	struct vmw_buffer_object *vmw_bo;
 	struct vmw_query_cmd {
 		SVGA3dCmdHeader header;
 		SVGA3dCmdEndGBQuery q;
@@ -1569,7 +1567,7 @@ static int vmw_cmd_end_gb_query(struct vmw_private *dev_priv,
 
 	ret = vmw_query_bo_switch_prepare(dev_priv, vmw_bo, sw_context);
 
-	vmw_dmabuf_unreference(&vmw_bo);
+	vmw_bo_unreference(&vmw_bo);
 	return ret;
 }
 
@@ -1584,7 +1582,7 @@ static int vmw_cmd_end_query(struct vmw_private *dev_priv,
 			     struct vmw_sw_context *sw_context,
 			     SVGA3dCmdHeader *header)
 {
-	struct vmw_dma_buffer *vmw_bo;
+	struct vmw_buffer_object *vmw_bo;
 	struct vmw_query_cmd {
 		SVGA3dCmdHeader header;
 		SVGA3dCmdEndQuery q;
@@ -1623,7 +1621,7 @@ static int vmw_cmd_end_query(struct vmw_private *dev_priv,
 
 	ret = vmw_query_bo_switch_prepare(dev_priv, vmw_bo, sw_context);
 
-	vmw_dmabuf_unreference(&vmw_bo);
+	vmw_bo_unreference(&vmw_bo);
 	return ret;
 }
 
@@ -1638,7 +1636,7 @@ static int vmw_cmd_wait_gb_query(struct vmw_private *dev_priv,
 				 struct vmw_sw_context *sw_context,
 				 SVGA3dCmdHeader *header)
 {
-	struct vmw_dma_buffer *vmw_bo;
+	struct vmw_buffer_object *vmw_bo;
 	struct vmw_query_cmd {
 		SVGA3dCmdHeader header;
 		SVGA3dCmdWaitForGBQuery q;
@@ -1656,7 +1654,7 @@ static int vmw_cmd_wait_gb_query(struct vmw_private *dev_priv,
 	if (unlikely(ret != 0))
 		return ret;
 
-	vmw_dmabuf_unreference(&vmw_bo);
+	vmw_bo_unreference(&vmw_bo);
 	return 0;
 }
 
@@ -1671,7 +1669,7 @@ static int vmw_cmd_wait_query(struct vmw_private *dev_priv,
 			      struct vmw_sw_context *sw_context,
 			      SVGA3dCmdHeader *header)
 {
-	struct vmw_dma_buffer *vmw_bo;
+	struct vmw_buffer_object *vmw_bo;
 	struct vmw_query_cmd {
 		SVGA3dCmdHeader header;
 		SVGA3dCmdWaitForQuery q;
@@ -1708,7 +1706,7 @@ static int vmw_cmd_wait_query(struct vmw_private *dev_priv,
 	if (unlikely(ret != 0))
 		return ret;
 
-	vmw_dmabuf_unreference(&vmw_bo);
+	vmw_bo_unreference(&vmw_bo);
 	return 0;
 }
 
@@ -1716,7 +1714,7 @@ static int vmw_cmd_dma(struct vmw_private *dev_priv,
 		       struct vmw_sw_context *sw_context,
 		       SVGA3dCmdHeader *header)
 {
-	struct vmw_dma_buffer *vmw_bo = NULL;
+	struct vmw_buffer_object *vmw_bo = NULL;
 	struct vmw_surface *srf = NULL;
 	struct vmw_dma_cmd {
 		SVGA3dCmdHeader header;
@@ -1768,7 +1766,7 @@ static int vmw_cmd_dma(struct vmw_private *dev_priv,
 			     header);
 
 out_no_surface:
-	vmw_dmabuf_unreference(&vmw_bo);
+	vmw_bo_unreference(&vmw_bo);
 	return ret;
 }
 
@@ -1887,7 +1885,7 @@ static int vmw_cmd_check_define_gmrfb(struct vmw_private *dev_priv,
 				      struct vmw_sw_context *sw_context,
 				      void *buf)
 {
-	struct vmw_dma_buffer *vmw_bo;
+	struct vmw_buffer_object *vmw_bo;
 	int ret;
 
 	struct {
@@ -1901,7 +1899,7 @@ static int vmw_cmd_check_define_gmrfb(struct vmw_private *dev_priv,
 	if (unlikely(ret != 0))
 		return ret;
 
-	vmw_dmabuf_unreference(&vmw_bo);
+	vmw_bo_unreference(&vmw_bo);
 
 	return ret;
 }
@@ -1928,7 +1926,7 @@ static int vmw_cmd_res_switch_backup(struct vmw_private *dev_priv,
 				     uint32_t *buf_id,
 				     unsigned long backup_offset)
 {
-	struct vmw_dma_buffer *dma_buf;
+	struct vmw_buffer_object *dma_buf;
 	int ret;
 
 	ret = vmw_translate_mob_ptr(dev_priv, sw_context, buf_id, &dma_buf);
@@ -1939,7 +1937,7 @@ static int vmw_cmd_res_switch_backup(struct vmw_private *dev_priv,
 	if (val_node->first_usage)
 		val_node->no_buffer_needed = true;
 
-	vmw_dmabuf_unreference(&val_node->new_backup);
+	vmw_bo_unreference(&val_node->new_backup);
 	val_node->new_backup = dma_buf;
 	val_node->new_backup_offset = backup_offset;
 
@@ -3118,6 +3116,32 @@ static int vmw_cmd_dx_transfer_from_buffer(struct vmw_private *dev_priv,
 				 &cmd->body.destSid, NULL);
 }
 
+/**
+ * vmw_cmd_intra_surface_copy -
+ * Validate an SVGA_3D_CMD_INTRA_SURFACE_COPY command
+ *
+ * @dev_priv: Pointer to a device private struct.
+ * @sw_context: The software context being used for this batch.
+ * @header: Pointer to the command header in the command stream.
+ */
+static int vmw_cmd_intra_surface_copy(struct vmw_private *dev_priv,
+					   struct vmw_sw_context *sw_context,
+					   SVGA3dCmdHeader *header)
+{
+	struct {
+		SVGA3dCmdHeader header;
+		SVGA3dCmdIntraSurfaceCopy body;
+	} *cmd = container_of(header, typeof(*cmd), header);
+
+	if (!(dev_priv->capabilities2 & SVGA_CAP2_INTRA_SURFACE_COPY))
+		return -EINVAL;
+
+	return vmw_cmd_res_check(dev_priv, sw_context, vmw_res_surface,
+				user_surface_converter,
+				&cmd->body.surface.sid, NULL);
+}
+
+
 static int vmw_cmd_check_not_3d(struct vmw_private *dev_priv,
 				struct vmw_sw_context *sw_context,
 				void *buf, uint32_t *size)
@@ -3232,9 +3256,9 @@ static const struct vmw_cmd_entry vmw_cmd_entries[SVGA_3D_CMD_MAX] = {
 		    false, false, false),
 	VMW_CMD_DEF(SVGA_3D_CMD_SCREEN_DMA, &vmw_cmd_invalid,
 		    false, false, false),
-	VMW_CMD_DEF(SVGA_3D_CMD_SET_UNITY_SURFACE_COOKIE, &vmw_cmd_invalid,
+	VMW_CMD_DEF(SVGA_3D_CMD_DEAD1, &vmw_cmd_invalid,
 		    false, false, false),
-	VMW_CMD_DEF(SVGA_3D_CMD_OPEN_CONTEXT_SURFACE, &vmw_cmd_invalid,
+	VMW_CMD_DEF(SVGA_3D_CMD_DEAD2, &vmw_cmd_invalid,
 		    false, false, false),
 	VMW_CMD_DEF(SVGA_3D_CMD_LOGICOPS_BITBLT, &vmw_cmd_invalid,
 		    false, false, false),
@@ -3473,6 +3497,8 @@ static const struct vmw_cmd_entry vmw_cmd_entries[SVGA_3D_CMD_MAX] = {
 	VMW_CMD_DEF(SVGA_3D_CMD_DX_TRANSFER_FROM_BUFFER,
 		    &vmw_cmd_dx_transfer_from_buffer,
 		    true, false, true),
+	VMW_CMD_DEF(SVGA_3D_CMD_INTRA_SURFACE_COPY, &vmw_cmd_intra_surface_copy,
+		    true, false, true),
 };
 
 bool vmw_cmd_describe(const void *buf, u32 *size, char const **cmd)
@@ -3701,8 +3727,8 @@ int vmw_validate_single_buffer(struct vmw_private *dev_priv,
 			       bool interruptible,
 			       bool validate_as_mob)
 {
-	struct vmw_dma_buffer *vbo = container_of(bo, struct vmw_dma_buffer,
-						  base);
+	struct vmw_buffer_object *vbo =
+		container_of(bo, struct vmw_buffer_object, base);
 	struct ttm_operation_ctx ctx = { interruptible, true };
 	int ret;
 
@@ -4423,7 +4449,7 @@ void __vmw_execbuf_release_pinned_bo(struct vmw_private *dev_priv,
 
 	ttm_bo_unref(&query_val.bo);
 	ttm_bo_unref(&pinned_val.bo);
-	vmw_dmabuf_unreference(&dev_priv->pinned_bo);
+	vmw_bo_unreference(&dev_priv->pinned_bo);
 out_unlock:
 	return;
 
@@ -4432,7 +4458,7 @@ out_no_emit:
 out_no_reserve:
 	ttm_bo_unref(&query_val.bo);
 	ttm_bo_unref(&pinned_val.bo);
-	vmw_dmabuf_unreference(&dev_priv->pinned_bo);
+	vmw_bo_unreference(&dev_priv->pinned_bo);
 }
 
 /**
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_fb.c b/drivers/gpu/drm/vmwgfx/vmwgfx_fb.c
index 9b7e0aca5f84..b913a56f3426 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_fb.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_fb.c
@@ -42,7 +42,7 @@ struct vmw_fb_par {
 	void *vmalloc;
 
 	struct mutex bo_mutex;
-	struct vmw_dma_buffer *vmw_bo;
+	struct vmw_buffer_object *vmw_bo;
 	unsigned bo_size;
 	struct drm_framebuffer *set_fb;
 	struct drm_display_mode *set_mode;
@@ -184,7 +184,7 @@ static void vmw_fb_dirty_flush(struct work_struct *work)
 	struct drm_clip_rect clip;
 	struct drm_framebuffer *cur_fb;
 	u8 *src_ptr, *dst_ptr;
-	struct vmw_dma_buffer *vbo = par->vmw_bo;
+	struct vmw_buffer_object *vbo = par->vmw_bo;
 	void *virtual;
 
 	if (!READ_ONCE(par->dirty.active))
@@ -197,7 +197,7 @@ static void vmw_fb_dirty_flush(struct work_struct *work)
 
 	(void) ttm_read_lock(&vmw_priv->reservation_sem, false);
 	(void) ttm_bo_reserve(&vbo->base, false, false, NULL);
-	virtual = vmw_dma_buffer_map_and_cache(vbo);
+	virtual = vmw_bo_map_and_cache(vbo);
 	if (!virtual)
 		goto out_unreserve;
 
@@ -391,9 +391,9 @@ static void vmw_fb_imageblit(struct fb_info *info, const struct fb_image *image)
  */
 
 static int vmw_fb_create_bo(struct vmw_private *vmw_priv,
-			    size_t size, struct vmw_dma_buffer **out)
+			    size_t size, struct vmw_buffer_object **out)
 {
-	struct vmw_dma_buffer *vmw_bo;
+	struct vmw_buffer_object *vmw_bo;
 	int ret;
 
 	(void) ttm_write_lock(&vmw_priv->reservation_sem, false);
@@ -404,10 +404,10 @@ static int vmw_fb_create_bo(struct vmw_private *vmw_priv,
 		goto err_unlock;
 	}
 
-	ret = vmw_dmabuf_init(vmw_priv, vmw_bo, size,
+	ret = vmw_bo_init(vmw_priv, vmw_bo, size,
 			      &vmw_sys_placement,
 			      false,
-			      &vmw_dmabuf_bo_free);
+			      &vmw_bo_bo_free);
 	if (unlikely(ret != 0))
 		goto err_unlock; /* init frees the buffer on failure */
 
@@ -491,7 +491,7 @@ static int vmw_fb_kms_detach(struct vmw_fb_par *par,
 	}
 
 	if (par->vmw_bo && detach_bo && unref_bo)
-		vmw_dmabuf_unreference(&par->vmw_bo);
+		vmw_bo_unreference(&par->vmw_bo);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_fence.c b/drivers/gpu/drm/vmwgfx/vmwgfx_fence.c
index c2eeb677cdfc..3d546d409334 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_fence.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_fence.c
@@ -175,7 +175,6 @@ static long vmw_fence_wait(struct dma_fence *f, bool intr, signed long timeout)
 	struct vmw_private *dev_priv = fman->dev_priv;
 	struct vmwgfx_wait_cb cb;
 	long ret = timeout;
-	unsigned long irq_flags;
 
 	if (likely(vmw_fence_obj_signaled(fence)))
 		return timeout;
@@ -183,7 +182,7 @@ static long vmw_fence_wait(struct dma_fence *f, bool intr, signed long timeout)
 	vmw_fifo_ping_host(dev_priv, SVGA_SYNC_GENERIC);
 	vmw_seqno_waiter_add(dev_priv);
 
-	spin_lock_irqsave(f->lock, irq_flags);
+	spin_lock(f->lock);
 
 	if (intr && signal_pending(current)) {
 		ret = -ERESTARTSYS;
@@ -194,30 +193,45 @@ static long vmw_fence_wait(struct dma_fence *f, bool intr, signed long timeout)
 	cb.task = current;
 	list_add(&cb.base.node, &f->cb_list);
 
-	while (ret > 0) {
+	for (;;) {
 		__vmw_fences_update(fman);
-		if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &f->flags))
-			break;
 
+		/*
+		 * We can use the barrier free __set_current_state() since
+		 * DMA_FENCE_FLAG_SIGNALED_BIT + wakeup is protected by the
+		 * fence spinlock.
+		 */
 		if (intr)
 			__set_current_state(TASK_INTERRUPTIBLE);
 		else
 			__set_current_state(TASK_UNINTERRUPTIBLE);
-		spin_unlock_irqrestore(f->lock, irq_flags);
 
-		ret = schedule_timeout(ret);
+		if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &f->flags)) {
+			if (ret == 0 && timeout > 0)
+				ret = 1;
+			break;
+		}
 
-		spin_lock_irqsave(f->lock, irq_flags);
-		if (ret > 0 && intr && signal_pending(current))
+		if (intr && signal_pending(current)) {
 			ret = -ERESTARTSYS;
-	}
+			break;
+		}
 
+		if (ret == 0)
+			break;
+
+		spin_unlock(f->lock);
+
+		ret = schedule_timeout(ret);
+
+		spin_lock(f->lock);
+	}
+	__set_current_state(TASK_RUNNING);
 	if (!list_empty(&cb.base.node))
 		list_del(&cb.base.node);
-	__set_current_state(TASK_RUNNING);
 
 out:
-	spin_unlock_irqrestore(f->lock, irq_flags);
+	spin_unlock(f->lock);
 
 	vmw_seqno_waiter_remove(dev_priv);
 
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_gmrid_manager.c b/drivers/gpu/drm/vmwgfx/vmwgfx_gmrid_manager.c
index ddb1e9365a3e..b93c558dd86e 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_gmrid_manager.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_gmrid_manager.c
@@ -51,51 +51,34 @@ static int vmw_gmrid_man_get_node(struct ttm_mem_type_manager *man,
 {
 	struct vmwgfx_gmrid_man *gman =
 		(struct vmwgfx_gmrid_man *)man->priv;
-	int ret = 0;
 	int id;
 
 	mem->mm_node = NULL;
 
+	id = ida_alloc_max(&gman->gmr_ida, gman->max_gmr_ids - 1, GFP_KERNEL);
+	if (id < 0)
+		return id;
+
 	spin_lock(&gman->lock);
 
 	if (gman->max_gmr_pages > 0) {
 		gman->used_gmr_pages += bo->num_pages;
 		if (unlikely(gman->used_gmr_pages > gman->max_gmr_pages))
-			goto out_err_locked;
+			goto nospace;
 	}
 
-	do {
-		spin_unlock(&gman->lock);
-		if (unlikely(ida_pre_get(&gman->gmr_ida, GFP_KERNEL) == 0)) {
-			ret = -ENOMEM;
-			goto out_err;
-		}
-		spin_lock(&gman->lock);
-
-		ret = ida_get_new(&gman->gmr_ida, &id);
-		if (unlikely(ret == 0 && id >= gman->max_gmr_ids)) {
-			ida_remove(&gman->gmr_ida, id);
-			ret = 0;
-			goto out_err_locked;
-		}
-	} while (ret == -EAGAIN);
-
-	if (likely(ret == 0)) {
-		mem->mm_node = gman;
-		mem->start = id;
-		mem->num_pages = bo->num_pages;
-	} else
-		goto out_err_locked;
+	mem->mm_node = gman;
+	mem->start = id;
+	mem->num_pages = bo->num_pages;
 
 	spin_unlock(&gman->lock);
 	return 0;
 
-out_err:
-	spin_lock(&gman->lock);
-out_err_locked:
+nospace:
 	gman->used_gmr_pages -= bo->num_pages;
 	spin_unlock(&gman->lock);
-	return ret;
+	ida_free(&gman->gmr_ida, id);
+	return 0;
 }
 
 static void vmw_gmrid_man_put_node(struct ttm_mem_type_manager *man,
@@ -105,8 +88,8 @@ static void vmw_gmrid_man_put_node(struct ttm_mem_type_manager *man,
 		(struct vmwgfx_gmrid_man *)man->priv;
 
 	if (mem->mm_node) {
+		ida_free(&gman->gmr_ida, mem->start);
 		spin_lock(&gman->lock);
-		ida_remove(&gman->gmr_ida, mem->start);
 		gman->used_gmr_pages -= mem->num_pages;
 		spin_unlock(&gman->lock);
 		mem->mm_node = NULL;
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_ioctl.c b/drivers/gpu/drm/vmwgfx/vmwgfx_ioctl.c
index 78fce406e1d4..172a6ba6539c 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_ioctl.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_ioctl.c
@@ -56,6 +56,9 @@ int vmw_getparam_ioctl(struct drm_device *dev, void *data,
 	case DRM_VMW_PARAM_HW_CAPS:
 		param->value = dev_priv->capabilities;
 		break;
+	case DRM_VMW_PARAM_HW_CAPS2:
+		param->value = dev_priv->capabilities2;
+		break;
 	case DRM_VMW_PARAM_FIFO_CAPS:
 		param->value = dev_priv->fifo.capabilities;
 		break;
@@ -113,6 +116,9 @@ int vmw_getparam_ioctl(struct drm_device *dev, void *data,
 	case DRM_VMW_PARAM_DX:
 		param->value = dev_priv->has_dx;
 		break;
+	case DRM_VMW_PARAM_SM4_1:
+		param->value = dev_priv->has_sm4_1;
+		break;
 	default:
 		return -EINVAL;
 	}
@@ -122,15 +128,12 @@ int vmw_getparam_ioctl(struct drm_device *dev, void *data,
 
 static u32 vmw_mask_multisample(unsigned int cap, u32 fmt_value)
 {
-	/* If the header is updated, update the format test as well! */
-	BUILD_BUG_ON(SVGA3D_DEVCAP_DXFMT_BC5_UNORM + 1 != SVGA3D_DEVCAP_MAX);
-
-	if (cap >= SVGA3D_DEVCAP_DXFMT_X8R8G8B8 &&
-	    cap <= SVGA3D_DEVCAP_DXFMT_BC5_UNORM)
-		fmt_value &= ~(SVGADX_DXFMT_MULTISAMPLE_2 |
-			       SVGADX_DXFMT_MULTISAMPLE_4 |
-			       SVGADX_DXFMT_MULTISAMPLE_8);
-	else if (cap == SVGA3D_DEVCAP_MULTISAMPLE_MASKABLESAMPLES)
+	/*
+	 * A version of user-space exists which use MULTISAMPLE_MASKABLESAMPLES
+	 * to check the sample count supported by virtual device. Since there
+	 * never was support for multisample count for backing MOB return 0.
+	 */
+	if (cap == SVGA3D_DEVCAP_MULTISAMPLE_MASKABLESAMPLES)
 		return 0;
 
 	return fmt_value;
@@ -377,8 +380,8 @@ int vmw_present_readback_ioctl(struct drm_device *dev, void *data,
 	}
 
 	vfb = vmw_framebuffer_to_vfb(fb);
-	if (!vfb->dmabuf) {
-		DRM_ERROR("Framebuffer not dmabuf backed.\n");
+	if (!vfb->bo) {
+		DRM_ERROR("Framebuffer not buffer backed.\n");
 		ret = -EINVAL;
 		goto out_no_ttm_lock;
 	}
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
index 61824e360619..0c25bb8faf80 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
@@ -85,10 +85,10 @@ static int vmw_cursor_update_image(struct vmw_private *dev_priv,
 	return 0;
 }
 
-static int vmw_cursor_update_dmabuf(struct vmw_private *dev_priv,
-				    struct vmw_dma_buffer *dmabuf,
-				    u32 width, u32 height,
-				    u32 hotspotX, u32 hotspotY)
+static int vmw_cursor_update_bo(struct vmw_private *dev_priv,
+				struct vmw_buffer_object *bo,
+				u32 width, u32 height,
+				u32 hotspotX, u32 hotspotY)
 {
 	struct ttm_bo_kmap_obj map;
 	unsigned long kmap_offset;
@@ -100,13 +100,13 @@ static int vmw_cursor_update_dmabuf(struct vmw_private *dev_priv,
 	kmap_offset = 0;
 	kmap_num = (width*height*4 + PAGE_SIZE - 1) >> PAGE_SHIFT;
 
-	ret = ttm_bo_reserve(&dmabuf->base, true, false, NULL);
+	ret = ttm_bo_reserve(&bo->base, true, false, NULL);
 	if (unlikely(ret != 0)) {
 		DRM_ERROR("reserve failed\n");
 		return -EINVAL;
 	}
 
-	ret = ttm_bo_kmap(&dmabuf->base, kmap_offset, kmap_num, &map);
+	ret = ttm_bo_kmap(&bo->base, kmap_offset, kmap_num, &map);
 	if (unlikely(ret != 0))
 		goto err_unreserve;
 
@@ -116,7 +116,7 @@ static int vmw_cursor_update_dmabuf(struct vmw_private *dev_priv,
 
 	ttm_bo_kunmap(&map);
 err_unreserve:
-	ttm_bo_unreserve(&dmabuf->base);
+	ttm_bo_unreserve(&bo->base);
 
 	return ret;
 }
@@ -352,13 +352,13 @@ vmw_du_cursor_plane_prepare_fb(struct drm_plane *plane,
 	if (vps->surf)
 		vmw_surface_unreference(&vps->surf);
 
-	if (vps->dmabuf)
-		vmw_dmabuf_unreference(&vps->dmabuf);
+	if (vps->bo)
+		vmw_bo_unreference(&vps->bo);
 
 	if (fb) {
-		if (vmw_framebuffer_to_vfb(fb)->dmabuf) {
-			vps->dmabuf = vmw_framebuffer_to_vfbd(fb)->buffer;
-			vmw_dmabuf_reference(vps->dmabuf);
+		if (vmw_framebuffer_to_vfb(fb)->bo) {
+			vps->bo = vmw_framebuffer_to_vfbd(fb)->buffer;
+			vmw_bo_reference(vps->bo);
 		} else {
 			vps->surf = vmw_framebuffer_to_vfbs(fb)->surface;
 			vmw_surface_reference(vps->surf);
@@ -390,7 +390,7 @@ vmw_du_cursor_plane_atomic_update(struct drm_plane *plane,
 	}
 
 	du->cursor_surface = vps->surf;
-	du->cursor_dmabuf = vps->dmabuf;
+	du->cursor_bo = vps->bo;
 
 	if (vps->surf) {
 		du->cursor_age = du->cursor_surface->snooper.age;
@@ -399,11 +399,11 @@ vmw_du_cursor_plane_atomic_update(struct drm_plane *plane,
 					      vps->surf->snooper.image,
 					      64, 64, hotspot_x,
 					      hotspot_y);
-	} else if (vps->dmabuf) {
-		ret = vmw_cursor_update_dmabuf(dev_priv, vps->dmabuf,
-					       plane->state->crtc_w,
-					       plane->state->crtc_h,
-					       hotspot_x, hotspot_y);
+	} else if (vps->bo) {
+		ret = vmw_cursor_update_bo(dev_priv, vps->bo,
+					   plane->state->crtc_w,
+					   plane->state->crtc_h,
+					   hotspot_x, hotspot_y);
 	} else {
 		vmw_cursor_update_position(dev_priv, false, 0, 0);
 		return;
@@ -519,7 +519,7 @@ int vmw_du_cursor_plane_atomic_check(struct drm_plane *plane,
 		ret = -EINVAL;
 	}
 
-	if (!vmw_framebuffer_to_vfb(fb)->dmabuf)
+	if (!vmw_framebuffer_to_vfb(fb)->bo)
 		surface = vmw_framebuffer_to_vfbs(fb)->surface;
 
 	if (surface && !surface->snooper.image) {
@@ -687,8 +687,8 @@ vmw_du_plane_duplicate_state(struct drm_plane *plane)
 	if (vps->surf)
 		(void) vmw_surface_reference(vps->surf);
 
-	if (vps->dmabuf)
-		(void) vmw_dmabuf_reference(vps->dmabuf);
+	if (vps->bo)
+		(void) vmw_bo_reference(vps->bo);
 
 	state = &vps->base;
 
@@ -743,8 +743,8 @@ vmw_du_plane_destroy_state(struct drm_plane *plane,
 	if (vps->surf)
 		vmw_surface_unreference(&vps->surf);
 
-	if (vps->dmabuf)
-		vmw_dmabuf_unreference(&vps->dmabuf);
+	if (vps->bo)
+		vmw_bo_unreference(&vps->bo);
 
 	drm_atomic_helper_plane_destroy_state(plane, state);
 }
@@ -900,12 +900,12 @@ static int vmw_framebuffer_surface_dirty(struct drm_framebuffer *framebuffer,
 
 /**
  * vmw_kms_readback - Perform a readback from the screen system to
- * a dma-buffer backed framebuffer.
+ * a buffer-object backed framebuffer.
  *
  * @dev_priv: Pointer to the device private structure.
  * @file_priv: Pointer to a struct drm_file identifying the caller.
  * Must be set to NULL if @user_fence_rep is NULL.
- * @vfb: Pointer to the dma-buffer backed framebuffer.
+ * @vfb: Pointer to the buffer-object backed framebuffer.
  * @user_fence_rep: User-space provided structure for fence information.
  * Must be set to non-NULL if @file_priv is non-NULL.
  * @vclips: Array of clip rects.
@@ -949,7 +949,7 @@ static int vmw_kms_new_framebuffer_surface(struct vmw_private *dev_priv,
 					   struct vmw_framebuffer **out,
 					   const struct drm_mode_fb_cmd2
 					   *mode_cmd,
-					   bool is_dmabuf_proxy)
+					   bool is_bo_proxy)
 
 {
 	struct drm_device *dev = dev_priv->dev;
@@ -1017,7 +1017,7 @@ static int vmw_kms_new_framebuffer_surface(struct vmw_private *dev_priv,
 	drm_helper_mode_fill_fb_struct(dev, &vfbs->base.base, mode_cmd);
 	vfbs->surface = vmw_surface_reference(surface);
 	vfbs->base.user_handle = mode_cmd->handles[0];
-	vfbs->is_dmabuf_proxy = is_dmabuf_proxy;
+	vfbs->is_bo_proxy = is_bo_proxy;
 
 	*out = &vfbs->base;
 
@@ -1036,30 +1036,30 @@ out_err1:
 }
 
 /*
- * Dmabuf framebuffer code
+ * Buffer-object framebuffer code
  */
 
-static void vmw_framebuffer_dmabuf_destroy(struct drm_framebuffer *framebuffer)
+static void vmw_framebuffer_bo_destroy(struct drm_framebuffer *framebuffer)
 {
-	struct vmw_framebuffer_dmabuf *vfbd =
+	struct vmw_framebuffer_bo *vfbd =
 		vmw_framebuffer_to_vfbd(framebuffer);
 
 	drm_framebuffer_cleanup(framebuffer);
-	vmw_dmabuf_unreference(&vfbd->buffer);
+	vmw_bo_unreference(&vfbd->buffer);
 	if (vfbd->base.user_obj)
 		ttm_base_object_unref(&vfbd->base.user_obj);
 
 	kfree(vfbd);
 }
 
-static int vmw_framebuffer_dmabuf_dirty(struct drm_framebuffer *framebuffer,
-				 struct drm_file *file_priv,
-				 unsigned flags, unsigned color,
-				 struct drm_clip_rect *clips,
-				 unsigned num_clips)
+static int vmw_framebuffer_bo_dirty(struct drm_framebuffer *framebuffer,
+				    struct drm_file *file_priv,
+				    unsigned int flags, unsigned int color,
+				    struct drm_clip_rect *clips,
+				    unsigned int num_clips)
 {
 	struct vmw_private *dev_priv = vmw_priv(framebuffer->dev);
-	struct vmw_framebuffer_dmabuf *vfbd =
+	struct vmw_framebuffer_bo *vfbd =
 		vmw_framebuffer_to_vfbd(framebuffer);
 	struct drm_clip_rect norect;
 	int ret, increment = 1;
@@ -1090,13 +1090,13 @@ static int vmw_framebuffer_dmabuf_dirty(struct drm_framebuffer *framebuffer,
 				       true, true, NULL);
 		break;
 	case vmw_du_screen_object:
-		ret = vmw_kms_sou_do_dmabuf_dirty(dev_priv, &vfbd->base,
-						  clips, NULL, num_clips,
-						  increment, true, NULL, NULL);
+		ret = vmw_kms_sou_do_bo_dirty(dev_priv, &vfbd->base,
+					      clips, NULL, num_clips,
+					      increment, true, NULL, NULL);
 		break;
 	case vmw_du_legacy:
-		ret = vmw_kms_ldu_do_dmabuf_dirty(dev_priv, &vfbd->base, 0, 0,
-						  clips, num_clips, increment);
+		ret = vmw_kms_ldu_do_bo_dirty(dev_priv, &vfbd->base, 0, 0,
+					      clips, num_clips, increment);
 		break;
 	default:
 		ret = -EINVAL;
@@ -1112,23 +1112,23 @@ static int vmw_framebuffer_dmabuf_dirty(struct drm_framebuffer *framebuffer,
 	return ret;
 }
 
-static const struct drm_framebuffer_funcs vmw_framebuffer_dmabuf_funcs = {
-	.destroy = vmw_framebuffer_dmabuf_destroy,
-	.dirty = vmw_framebuffer_dmabuf_dirty,
+static const struct drm_framebuffer_funcs vmw_framebuffer_bo_funcs = {
+	.destroy = vmw_framebuffer_bo_destroy,
+	.dirty = vmw_framebuffer_bo_dirty,
 };
 
 /**
- * Pin the dmabuffer in a location suitable for access by the
+ * Pin the bofer in a location suitable for access by the
  * display system.
  */
 static int vmw_framebuffer_pin(struct vmw_framebuffer *vfb)
 {
 	struct vmw_private *dev_priv = vmw_priv(vfb->base.dev);
-	struct vmw_dma_buffer *buf;
+	struct vmw_buffer_object *buf;
 	struct ttm_placement *placement;
 	int ret;
 
-	buf = vfb->dmabuf ?  vmw_framebuffer_to_vfbd(&vfb->base)->buffer :
+	buf = vfb->bo ?  vmw_framebuffer_to_vfbd(&vfb->base)->buffer :
 		vmw_framebuffer_to_vfbs(&vfb->base)->surface->res.backup;
 
 	if (!buf)
@@ -1137,12 +1137,12 @@ static int vmw_framebuffer_pin(struct vmw_framebuffer *vfb)
 	switch (dev_priv->active_display_unit) {
 	case vmw_du_legacy:
 		vmw_overlay_pause_all(dev_priv);
-		ret = vmw_dmabuf_pin_in_start_of_vram(dev_priv, buf, false);
+		ret = vmw_bo_pin_in_start_of_vram(dev_priv, buf, false);
 		vmw_overlay_resume_all(dev_priv);
 		break;
 	case vmw_du_screen_object:
 	case vmw_du_screen_target:
-		if (vfb->dmabuf) {
+		if (vfb->bo) {
 			if (dev_priv->capabilities & SVGA_CAP_3D) {
 				/*
 				 * Use surface DMA to get content to
@@ -1158,8 +1158,7 @@ static int vmw_framebuffer_pin(struct vmw_framebuffer *vfb)
 			placement = &vmw_mob_placement;
 		}
 
-		return vmw_dmabuf_pin_in_placement(dev_priv, buf, placement,
-						   false);
+		return vmw_bo_pin_in_placement(dev_priv, buf, placement, false);
 	default:
 		return -EINVAL;
 	}
@@ -1170,36 +1169,36 @@ static int vmw_framebuffer_pin(struct vmw_framebuffer *vfb)
 static int vmw_framebuffer_unpin(struct vmw_framebuffer *vfb)
 {
 	struct vmw_private *dev_priv = vmw_priv(vfb->base.dev);
-	struct vmw_dma_buffer *buf;
+	struct vmw_buffer_object *buf;
 
-	buf = vfb->dmabuf ?  vmw_framebuffer_to_vfbd(&vfb->base)->buffer :
+	buf = vfb->bo ?  vmw_framebuffer_to_vfbd(&vfb->base)->buffer :
 		vmw_framebuffer_to_vfbs(&vfb->base)->surface->res.backup;
 
 	if (WARN_ON(!buf))
 		return 0;
 
-	return vmw_dmabuf_unpin(dev_priv, buf, false);
+	return vmw_bo_unpin(dev_priv, buf, false);
 }
 
 /**
- * vmw_create_dmabuf_proxy - create a proxy surface for the DMA buf
+ * vmw_create_bo_proxy - create a proxy surface for the buffer object
  *
  * @dev: DRM device
  * @mode_cmd: parameters for the new surface
- * @dmabuf_mob: MOB backing the DMA buf
+ * @bo_mob: MOB backing the buffer object
  * @srf_out: newly created surface
  *
- * When the content FB is a DMA buf, we create a surface as a proxy to the
+ * When the content FB is a buffer object, we create a surface as a proxy to the
  * same buffer.  This way we can do a surface copy rather than a surface DMA.
  * This is a more efficient approach
  *
  * RETURNS:
  * 0 on success, error code otherwise
  */
-static int vmw_create_dmabuf_proxy(struct drm_device *dev,
-				   const struct drm_mode_fb_cmd2 *mode_cmd,
-				   struct vmw_dma_buffer *dmabuf_mob,
-				   struct vmw_surface **srf_out)
+static int vmw_create_bo_proxy(struct drm_device *dev,
+			       const struct drm_mode_fb_cmd2 *mode_cmd,
+			       struct vmw_buffer_object *bo_mob,
+			       struct vmw_surface **srf_out)
 {
 	uint32_t format;
 	struct drm_vmw_size content_base_size = {0};
@@ -1237,15 +1236,17 @@ static int vmw_create_dmabuf_proxy(struct drm_device *dev,
 	content_base_size.depth  = 1;
 
 	ret = vmw_surface_gb_priv_define(dev,
-			0, /* kernel visible only */
-			0, /* flags */
-			format,
-			true, /* can be a scanout buffer */
-			1, /* num of mip levels */
-			0,
-			0,
-			content_base_size,
-			srf_out);
+					 0, /* kernel visible only */
+					 0, /* flags */
+					 format,
+					 true, /* can be a scanout buffer */
+					 1, /* num of mip levels */
+					 0,
+					 0,
+					 content_base_size,
+					 SVGA3D_MS_PATTERN_NONE,
+					 SVGA3D_MS_QUALITY_NONE,
+					 srf_out);
 	if (ret) {
 		DRM_ERROR("Failed to allocate proxy content buffer\n");
 		return ret;
@@ -1256,8 +1257,8 @@ static int vmw_create_dmabuf_proxy(struct drm_device *dev,
 	/* Reserve and switch the backing mob. */
 	mutex_lock(&res->dev_priv->cmdbuf_mutex);
 	(void) vmw_resource_reserve(res, false, true);
-	vmw_dmabuf_unreference(&res->backup);
-	res->backup = vmw_dmabuf_reference(dmabuf_mob);
+	vmw_bo_unreference(&res->backup);
+	res->backup = vmw_bo_reference(bo_mob);
 	res->backup_offset = 0;
 	vmw_resource_unreserve(res, false, NULL, 0);
 	mutex_unlock(&res->dev_priv->cmdbuf_mutex);
@@ -1267,21 +1268,21 @@ static int vmw_create_dmabuf_proxy(struct drm_device *dev,
 
 
 
-static int vmw_kms_new_framebuffer_dmabuf(struct vmw_private *dev_priv,
-					  struct vmw_dma_buffer *dmabuf,
-					  struct vmw_framebuffer **out,
-					  const struct drm_mode_fb_cmd2
-					  *mode_cmd)
+static int vmw_kms_new_framebuffer_bo(struct vmw_private *dev_priv,
+				      struct vmw_buffer_object *bo,
+				      struct vmw_framebuffer **out,
+				      const struct drm_mode_fb_cmd2
+				      *mode_cmd)
 
 {
 	struct drm_device *dev = dev_priv->dev;
-	struct vmw_framebuffer_dmabuf *vfbd;
+	struct vmw_framebuffer_bo *vfbd;
 	unsigned int requested_size;
 	struct drm_format_name_buf format_name;
 	int ret;
 
 	requested_size = mode_cmd->height * mode_cmd->pitches[0];
-	if (unlikely(requested_size > dmabuf->base.num_pages * PAGE_SIZE)) {
+	if (unlikely(requested_size > bo->base.num_pages * PAGE_SIZE)) {
 		DRM_ERROR("Screen buffer object size is too small "
 			  "for requested mode.\n");
 		return -EINVAL;
@@ -1310,20 +1311,20 @@ static int vmw_kms_new_framebuffer_dmabuf(struct vmw_private *dev_priv,
 	}
 
 	drm_helper_mode_fill_fb_struct(dev, &vfbd->base.base, mode_cmd);
-	vfbd->base.dmabuf = true;
-	vfbd->buffer = vmw_dmabuf_reference(dmabuf);
+	vfbd->base.bo = true;
+	vfbd->buffer = vmw_bo_reference(bo);
 	vfbd->base.user_handle = mode_cmd->handles[0];
 	*out = &vfbd->base;
 
 	ret = drm_framebuffer_init(dev, &vfbd->base.base,
-				   &vmw_framebuffer_dmabuf_funcs);
+				   &vmw_framebuffer_bo_funcs);
 	if (ret)
 		goto out_err2;
 
 	return 0;
 
 out_err2:
-	vmw_dmabuf_unreference(&dmabuf);
+	vmw_bo_unreference(&bo);
 	kfree(vfbd);
 out_err1:
 	return ret;
@@ -1352,57 +1353,57 @@ vmw_kms_srf_ok(struct vmw_private *dev_priv, uint32_t width, uint32_t height)
  * vmw_kms_new_framebuffer - Create a new framebuffer.
  *
  * @dev_priv: Pointer to device private struct.
- * @dmabuf: Pointer to dma buffer to wrap the kms framebuffer around.
- * Either @dmabuf or @surface must be NULL.
+ * @bo: Pointer to buffer object to wrap the kms framebuffer around.
+ * Either @bo or @surface must be NULL.
  * @surface: Pointer to a surface to wrap the kms framebuffer around.
- * Either @dmabuf or @surface must be NULL.
- * @only_2d: No presents will occur to this dma buffer based framebuffer. This
- * Helps the code to do some important optimizations.
+ * Either @bo or @surface must be NULL.
+ * @only_2d: No presents will occur to this buffer object based framebuffer.
+ * This helps the code to do some important optimizations.
  * @mode_cmd: Frame-buffer metadata.
  */
 struct vmw_framebuffer *
 vmw_kms_new_framebuffer(struct vmw_private *dev_priv,
-			struct vmw_dma_buffer *dmabuf,
+			struct vmw_buffer_object *bo,
 			struct vmw_surface *surface,
 			bool only_2d,
 			const struct drm_mode_fb_cmd2 *mode_cmd)
 {
 	struct vmw_framebuffer *vfb = NULL;
-	bool is_dmabuf_proxy = false;
+	bool is_bo_proxy = false;
 	int ret;
 
 	/*
 	 * We cannot use the SurfaceDMA command in an non-accelerated VM,
-	 * therefore, wrap the DMA buf in a surface so we can use the
+	 * therefore, wrap the buffer object in a surface so we can use the
 	 * SurfaceCopy command.
 	 */
 	if (vmw_kms_srf_ok(dev_priv, mode_cmd->width, mode_cmd->height)  &&
-	    dmabuf && only_2d &&
+	    bo && only_2d &&
 	    mode_cmd->width > 64 &&  /* Don't create a proxy for cursor */
 	    dev_priv->active_display_unit == vmw_du_screen_target) {
-		ret = vmw_create_dmabuf_proxy(dev_priv->dev, mode_cmd,
-					      dmabuf, &surface);
+		ret = vmw_create_bo_proxy(dev_priv->dev, mode_cmd,
+					  bo, &surface);
 		if (ret)
 			return ERR_PTR(ret);
 
-		is_dmabuf_proxy = true;
+		is_bo_proxy = true;
 	}
 
 	/* Create the new framebuffer depending one what we have */
 	if (surface) {
 		ret = vmw_kms_new_framebuffer_surface(dev_priv, surface, &vfb,
 						      mode_cmd,
-						      is_dmabuf_proxy);
+						      is_bo_proxy);
 
 		/*
-		 * vmw_create_dmabuf_proxy() adds a reference that is no longer
+		 * vmw_create_bo_proxy() adds a reference that is no longer
 		 * needed
 		 */
-		if (is_dmabuf_proxy)
+		if (is_bo_proxy)
 			vmw_surface_unreference(&surface);
-	} else if (dmabuf) {
-		ret = vmw_kms_new_framebuffer_dmabuf(dev_priv, dmabuf, &vfb,
-						     mode_cmd);
+	} else if (bo) {
+		ret = vmw_kms_new_framebuffer_bo(dev_priv, bo, &vfb,
+						 mode_cmd);
 	} else {
 		BUG();
 	}
@@ -1428,23 +1429,10 @@ static struct drm_framebuffer *vmw_kms_fb_create(struct drm_device *dev,
 	struct ttm_object_file *tfile = vmw_fpriv(file_priv)->tfile;
 	struct vmw_framebuffer *vfb = NULL;
 	struct vmw_surface *surface = NULL;
-	struct vmw_dma_buffer *bo = NULL;
+	struct vmw_buffer_object *bo = NULL;
 	struct ttm_base_object *user_obj;
 	int ret;
 
-	/**
-	 * This code should be conditioned on Screen Objects not being used.
-	 * If screen objects are used, we can allocate a GMR to hold the
-	 * requested framebuffer.
-	 */
-
-	if (!vmw_kms_validate_mode_vram(dev_priv,
-					mode_cmd->pitches[0],
-					mode_cmd->height)) {
-		DRM_ERROR("Requested mode exceed bounding box limit.\n");
-		return ERR_PTR(-ENOMEM);
-	}
-
 	/*
 	 * Take a reference on the user object of the resource
 	 * backing the kms fb. This ensures that user-space handle
@@ -1464,7 +1452,7 @@ static struct drm_framebuffer *vmw_kms_fb_create(struct drm_device *dev,
 	 * End conditioned code.
 	 */
 
-	/* returns either a dmabuf or surface */
+	/* returns either a bo or surface */
 	ret = vmw_user_lookup_handle(dev_priv, tfile,
 				     mode_cmd->handles[0],
 				     &surface, &bo);
@@ -1492,7 +1480,7 @@ static struct drm_framebuffer *vmw_kms_fb_create(struct drm_device *dev,
 err_out:
 	/* vmw_user_lookup_handle takes one ref so does new_fb */
 	if (bo)
-		vmw_dmabuf_unreference(&bo);
+		vmw_bo_unreference(&bo);
 	if (surface)
 		vmw_surface_unreference(&surface);
 
@@ -1506,7 +1494,168 @@ err_out:
 	return &vfb->base;
 }
 
+/**
+ * vmw_kms_check_display_memory - Validates display memory required for a
+ * topology
+ * @dev: DRM device
+ * @num_rects: number of drm_rect in rects
+ * @rects: array of drm_rect representing the topology to validate indexed by
+ * crtc index.
+ *
+ * Returns:
+ * 0 on success otherwise negative error code
+ */
+static int vmw_kms_check_display_memory(struct drm_device *dev,
+					uint32_t num_rects,
+					struct drm_rect *rects)
+{
+	struct vmw_private *dev_priv = vmw_priv(dev);
+	struct drm_mode_config *mode_config = &dev->mode_config;
+	struct drm_rect bounding_box = {0};
+	u64 total_pixels = 0, pixel_mem, bb_mem;
+	int i;
+
+	for (i = 0; i < num_rects; i++) {
+		/*
+		 * Currently this check is limiting the topology within max
+		 * texture/screentarget size. This should change in future when
+		 * user-space support multiple fb with topology.
+		 */
+		if (rects[i].x1 < 0 ||  rects[i].y1 < 0 ||
+		    rects[i].x2 > mode_config->max_width ||
+		    rects[i].y2 > mode_config->max_height) {
+			DRM_ERROR("Invalid GUI layout.\n");
+			return -EINVAL;
+		}
+
+		/* Bounding box upper left is at (0,0). */
+		if (rects[i].x2 > bounding_box.x2)
+			bounding_box.x2 = rects[i].x2;
+
+		if (rects[i].y2 > bounding_box.y2)
+			bounding_box.y2 = rects[i].y2;
+
+		total_pixels += (u64) drm_rect_width(&rects[i]) *
+			(u64) drm_rect_height(&rects[i]);
+	}
+
+	/* Virtual svga device primary limits are always in 32-bpp. */
+	pixel_mem = total_pixels * 4;
+
+	/*
+	 * For HV10 and below prim_bb_mem is vram size. When
+	 * SVGA_REG_MAX_PRIMARY_BOUNDING_BOX_MEM is not present vram size is
+	 * limit on primary bounding box
+	 */
+	if (pixel_mem > dev_priv->prim_bb_mem) {
+		DRM_ERROR("Combined output size too large.\n");
+		return -EINVAL;
+	}
+
+	/* SVGA_CAP_NO_BB_RESTRICTION is available for STDU only. */
+	if (dev_priv->active_display_unit != vmw_du_screen_target ||
+	    !(dev_priv->capabilities & SVGA_CAP_NO_BB_RESTRICTION)) {
+		bb_mem = (u64) bounding_box.x2 * bounding_box.y2 * 4;
+
+		if (bb_mem > dev_priv->prim_bb_mem) {
+			DRM_ERROR("Topology is beyond supported limits.\n");
+			return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
+/**
+ * vmw_kms_check_topology - Validates topology in drm_atomic_state
+ * @dev: DRM device
+ * @state: the driver state object
+ *
+ * Returns:
+ * 0 on success otherwise negative error code
+ */
+static int vmw_kms_check_topology(struct drm_device *dev,
+				  struct drm_atomic_state *state)
+{
+	struct vmw_private *dev_priv = vmw_priv(dev);
+	struct drm_crtc_state *old_crtc_state, *new_crtc_state;
+	struct drm_rect *rects;
+	struct drm_crtc *crtc;
+	uint32_t i;
+	int ret = 0;
+
+	rects = kcalloc(dev->mode_config.num_crtc, sizeof(struct drm_rect),
+			GFP_KERNEL);
+	if (!rects)
+		return -ENOMEM;
+
+	mutex_lock(&dev_priv->requested_layout_mutex);
+
+	drm_for_each_crtc(crtc, dev) {
+		struct vmw_display_unit *du = vmw_crtc_to_du(crtc);
+		struct drm_crtc_state *crtc_state = crtc->state;
+
+		i = drm_crtc_index(crtc);
+
+		if (crtc_state && crtc_state->enable) {
+			rects[i].x1 = du->gui_x;
+			rects[i].y1 = du->gui_y;
+			rects[i].x2 = du->gui_x + crtc_state->mode.hdisplay;
+			rects[i].y2 = du->gui_y + crtc_state->mode.vdisplay;
+		}
+	}
+
+	/* Determine change to topology due to new atomic state */
+	for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state,
+				      new_crtc_state, i) {
+		struct vmw_display_unit *du = vmw_crtc_to_du(crtc);
+		struct drm_connector *connector;
+		struct drm_connector_state *conn_state;
+		struct vmw_connector_state *vmw_conn_state;
+
+		if (!new_crtc_state->enable && old_crtc_state->enable) {
+			rects[i].x1 = 0;
+			rects[i].y1 = 0;
+			rects[i].x2 = 0;
+			rects[i].y2 = 0;
+			continue;
+		}
+
+		if (!du->pref_active) {
+			ret = -EINVAL;
+			goto clean;
+		}
+
+		/*
+		 * For vmwgfx each crtc has only one connector attached and it
+		 * is not changed so don't really need to check the
+		 * crtc->connector_mask and iterate over it.
+		 */
+		connector = &du->connector;
+		conn_state = drm_atomic_get_connector_state(state, connector);
+		if (IS_ERR(conn_state)) {
+			ret = PTR_ERR(conn_state);
+			goto clean;
+		}
+
+		vmw_conn_state = vmw_connector_state_to_vcs(conn_state);
+		vmw_conn_state->gui_x = du->gui_x;
+		vmw_conn_state->gui_y = du->gui_y;
+
+		rects[i].x1 = du->gui_x;
+		rects[i].y1 = du->gui_y;
+		rects[i].x2 = du->gui_x + new_crtc_state->mode.hdisplay;
+		rects[i].y2 = du->gui_y + new_crtc_state->mode.vdisplay;
+	}
+
+	ret = vmw_kms_check_display_memory(dev, dev->mode_config.num_crtc,
+					   rects);
 
+clean:
+	mutex_unlock(&dev_priv->requested_layout_mutex);
+	kfree(rects);
+	return ret;
+}
 
 /**
  * vmw_kms_atomic_check_modeset- validate state object for modeset changes
@@ -1518,40 +1667,39 @@ err_out:
  * us to assign a value to mode->crtc_clock so that
  * drm_calc_timestamping_constants() won't throw an error message
  *
- * RETURNS
+ * Returns:
  * Zero for success or -errno
  */
 static int
 vmw_kms_atomic_check_modeset(struct drm_device *dev,
 			     struct drm_atomic_state *state)
 {
-	struct drm_crtc_state *crtc_state;
 	struct drm_crtc *crtc;
-	struct vmw_private *dev_priv = vmw_priv(dev);
-	int i;
-
-	for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
-		unsigned long requested_bb_mem = 0;
-
-		if (dev_priv->active_display_unit == vmw_du_screen_target) {
-			struct drm_plane *plane = crtc->primary;
-			struct drm_plane_state *plane_state;
-
-			plane_state = drm_atomic_get_new_plane_state(state, plane);
+	struct drm_crtc_state *crtc_state;
+	bool need_modeset = false;
+	int i, ret;
 
-			if (plane_state && plane_state->fb) {
-				int cpp = plane_state->fb->format->cpp[0];
+	ret = drm_atomic_helper_check(dev, state);
+	if (ret)
+		return ret;
 
-				requested_bb_mem += crtc->mode.hdisplay * cpp *
-						    crtc->mode.vdisplay;
-			}
+	if (!state->allow_modeset)
+		return ret;
 
-			if (requested_bb_mem > dev_priv->prim_bb_mem)
-				return -EINVAL;
-		}
+	/*
+	 * Legacy path do not set allow_modeset properly like
+	 * @drm_atomic_helper_update_plane, This will result in unnecessary call
+	 * to vmw_kms_check_topology. So extra set of check.
+	 */
+	for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
+		if (drm_atomic_crtc_needs_modeset(crtc_state))
+			need_modeset = true;
 	}
 
-	return drm_atomic_helper_check(dev, state);
+	if (need_modeset)
+		return vmw_kms_check_topology(dev, state);
+
+	return ret;
 }
 
 static const struct drm_mode_config_funcs vmw_kms_funcs = {
@@ -1843,40 +1991,49 @@ void vmw_disable_vblank(struct drm_device *dev, unsigned int pipe)
 {
 }
 
-
-/*
- * Small shared kms functions.
+/**
+ * vmw_du_update_layout - Update the display unit with topology from resolution
+ * plugin and generate DRM uevent
+ * @dev_priv: device private
+ * @num_rects: number of drm_rect in rects
+ * @rects: toplogy to update
  */
-
-static int vmw_du_update_layout(struct vmw_private *dev_priv, unsigned num,
-			 struct drm_vmw_rect *rects)
+static int vmw_du_update_layout(struct vmw_private *dev_priv,
+				unsigned int num_rects, struct drm_rect *rects)
 {
 	struct drm_device *dev = dev_priv->dev;
 	struct vmw_display_unit *du;
 	struct drm_connector *con;
+	struct drm_connector_list_iter conn_iter;
 
-	mutex_lock(&dev->mode_config.mutex);
-
-#if 0
-	{
-		unsigned int i;
-
-		DRM_INFO("%s: new layout ", __func__);
-		for (i = 0; i < num; i++)
-			DRM_INFO("(%i, %i %ux%u) ", rects[i].x, rects[i].y,
-				 rects[i].w, rects[i].h);
-		DRM_INFO("\n");
+	/*
+	 * Currently only gui_x/y is protected with requested_layout_mutex.
+	 */
+	mutex_lock(&dev_priv->requested_layout_mutex);
+	drm_connector_list_iter_begin(dev, &conn_iter);
+	drm_for_each_connector_iter(con, &conn_iter) {
+		du = vmw_connector_to_du(con);
+		if (num_rects > du->unit) {
+			du->pref_width = drm_rect_width(&rects[du->unit]);
+			du->pref_height = drm_rect_height(&rects[du->unit]);
+			du->pref_active = true;
+			du->gui_x = rects[du->unit].x1;
+			du->gui_y = rects[du->unit].y1;
+		} else {
+			du->pref_width = 800;
+			du->pref_height = 600;
+			du->pref_active = false;
+			du->gui_x = 0;
+			du->gui_y = 0;
+		}
 	}
-#endif
+	drm_connector_list_iter_end(&conn_iter);
+	mutex_unlock(&dev_priv->requested_layout_mutex);
 
+	mutex_lock(&dev->mode_config.mutex);
 	list_for_each_entry(con, &dev->mode_config.connector_list, head) {
 		du = vmw_connector_to_du(con);
-		if (num > du->unit) {
-			du->pref_width = rects[du->unit].w;
-			du->pref_height = rects[du->unit].h;
-			du->pref_active = true;
-			du->gui_x = rects[du->unit].x;
-			du->gui_y = rects[du->unit].y;
+		if (num_rects > du->unit) {
 			drm_object_property_set_value
 			  (&con->base, dev->mode_config.suggested_x_property,
 			   du->gui_x);
@@ -1884,9 +2041,6 @@ static int vmw_du_update_layout(struct vmw_private *dev_priv, unsigned num,
 			  (&con->base, dev->mode_config.suggested_y_property,
 			   du->gui_y);
 		} else {
-			du->pref_width = 800;
-			du->pref_height = 600;
-			du->pref_active = false;
 			drm_object_property_set_value
 			  (&con->base, dev->mode_config.suggested_x_property,
 			   0);
@@ -1896,8 +2050,8 @@ static int vmw_du_update_layout(struct vmw_private *dev_priv, unsigned num,
 		}
 		con->status = vmw_du_connector_detect(con, true);
 	}
-
 	mutex_unlock(&dev->mode_config.mutex);
+
 	drm_sysfs_hotplug_event(dev);
 
 	return 0;
@@ -2197,7 +2351,25 @@ vmw_du_connector_atomic_get_property(struct drm_connector *connector,
 	return 0;
 }
 
-
+/**
+ * vmw_kms_update_layout_ioctl - Handler for DRM_VMW_UPDATE_LAYOUT ioctl
+ * @dev: drm device for the ioctl
+ * @data: data pointer for the ioctl
+ * @file_priv: drm file for the ioctl call
+ *
+ * Update preferred topology of display unit as per ioctl request. The topology
+ * is expressed as array of drm_vmw_rect.
+ * e.g.
+ * [0 0 640 480] [640 0 800 600] [0 480 640 480]
+ *
+ * NOTE:
+ * The x and y offset (upper left) in drm_vmw_rect cannot be less than 0. Beside
+ * device limit on topology, x + w and y + h (lower right) cannot be greater
+ * than INT_MAX. So topology beyond these limits will return with error.
+ *
+ * Returns:
+ * Zero on success, negative errno on failure.
+ */
 int vmw_kms_update_layout_ioctl(struct drm_device *dev, void *data,
 				struct drm_file *file_priv)
 {
@@ -2206,15 +2378,12 @@ int vmw_kms_update_layout_ioctl(struct drm_device *dev, void *data,
 		(struct drm_vmw_update_layout_arg *)data;
 	void __user *user_rects;
 	struct drm_vmw_rect *rects;
+	struct drm_rect *drm_rects;
 	unsigned rects_size;
-	int ret;
-	int i;
-	u64 total_pixels = 0;
-	struct drm_mode_config *mode_config = &dev->mode_config;
-	struct drm_vmw_rect bounding_box = {0};
+	int ret, i;
 
 	if (!arg->num_outputs) {
-		struct drm_vmw_rect def_rect = {0, 0, 800, 600};
+		struct drm_rect def_rect = {0, 0, 800, 600};
 		vmw_du_update_layout(dev_priv, 1, &def_rect);
 		return 0;
 	}
@@ -2233,52 +2402,29 @@ int vmw_kms_update_layout_ioctl(struct drm_device *dev, void *data,
 		goto out_free;
 	}
 
-	for (i = 0; i < arg->num_outputs; ++i) {
-		if (rects[i].x < 0 ||
-		    rects[i].y < 0 ||
-		    rects[i].x + rects[i].w > mode_config->max_width ||
-		    rects[i].y + rects[i].h > mode_config->max_height) {
-			DRM_ERROR("Invalid GUI layout.\n");
-			ret = -EINVAL;
-			goto out_free;
-		}
-
-		/*
-		 * bounding_box.w and bunding_box.h are used as
-		 * lower-right coordinates
-		 */
-		if (rects[i].x + rects[i].w > bounding_box.w)
-			bounding_box.w = rects[i].x + rects[i].w;
-
-		if (rects[i].y + rects[i].h > bounding_box.h)
-			bounding_box.h = rects[i].y + rects[i].h;
+	drm_rects = (struct drm_rect *)rects;
 
-		total_pixels += (u64) rects[i].w * (u64) rects[i].h;
-	}
+	for (i = 0; i < arg->num_outputs; i++) {
+		struct drm_vmw_rect curr_rect;
 
-	if (dev_priv->active_display_unit == vmw_du_screen_target) {
-		/*
-		 * For Screen Targets, the limits for a toplogy are:
-		 *	1. Bounding box (assuming 32bpp) must be < prim_bb_mem
-		 *      2. Total pixels (assuming 32bpp) must be < prim_bb_mem
-		 */
-		u64 bb_mem    = (u64) bounding_box.w * bounding_box.h * 4;
-		u64 pixel_mem = total_pixels * 4;
-
-		if (bb_mem > dev_priv->prim_bb_mem) {
-			DRM_ERROR("Topology is beyond supported limits.\n");
-			ret = -EINVAL;
+		/* Verify user-space for overflow as kernel use drm_rect */
+		if ((rects[i].x + rects[i].w > INT_MAX) ||
+		    (rects[i].y + rects[i].h > INT_MAX)) {
+			ret = -ERANGE;
 			goto out_free;
 		}
 
-		if (pixel_mem > dev_priv->prim_bb_mem) {
-			DRM_ERROR("Combined output size too large\n");
-			ret = -EINVAL;
-			goto out_free;
-		}
+		curr_rect = rects[i];
+		drm_rects[i].x1 = curr_rect.x;
+		drm_rects[i].y1 = curr_rect.y;
+		drm_rects[i].x2 = curr_rect.x + curr_rect.w;
+		drm_rects[i].y2 = curr_rect.y + curr_rect.h;
 	}
 
-	vmw_du_update_layout(dev_priv, arg->num_outputs, rects);
+	ret = vmw_kms_check_display_memory(dev, arg->num_outputs, drm_rects);
+
+	if (ret == 0)
+		vmw_du_update_layout(dev_priv, arg->num_outputs, drm_rects);
 
 out_free:
 	kfree(rects);
@@ -2425,7 +2571,7 @@ int vmw_kms_helper_dirty(struct vmw_private *dev_priv,
  * interrupted by a signal.
  */
 int vmw_kms_helper_buffer_prepare(struct vmw_private *dev_priv,
-				  struct vmw_dma_buffer *buf,
+				  struct vmw_buffer_object *buf,
 				  bool interruptible,
 				  bool validate_as_mob,
 				  bool for_cpu_blit)
@@ -2457,7 +2603,7 @@ int vmw_kms_helper_buffer_prepare(struct vmw_private *dev_priv,
  * Helper to be used if an error forces the caller to undo the actions of
  * vmw_kms_helper_buffer_prepare.
  */
-void vmw_kms_helper_buffer_revert(struct vmw_dma_buffer *buf)
+void vmw_kms_helper_buffer_revert(struct vmw_buffer_object *buf)
 {
 	if (buf)
 		ttm_bo_unreserve(&buf->base);
@@ -2480,7 +2626,7 @@ void vmw_kms_helper_buffer_revert(struct vmw_dma_buffer *buf)
  */
 void vmw_kms_helper_buffer_finish(struct vmw_private *dev_priv,
 				  struct drm_file *file_priv,
-				  struct vmw_dma_buffer *buf,
+				  struct vmw_buffer_object *buf,
 				  struct vmw_fence_obj **out_fence,
 				  struct drm_vmw_fence_rep __user *
 				  user_fence_rep)
@@ -2492,7 +2638,7 @@ void vmw_kms_helper_buffer_finish(struct vmw_private *dev_priv,
 	ret = vmw_execbuf_fence_commands(file_priv, dev_priv, &fence,
 					 file_priv ? &handle : NULL);
 	if (buf)
-		vmw_fence_single_bo(&buf->base, fence);
+		vmw_bo_fence_single(&buf->base, fence);
 	if (file_priv)
 		vmw_execbuf_copy_fence_user(dev_priv, vmw_fpriv(file_priv),
 					    ret, user_fence_rep, fence,
@@ -2520,7 +2666,7 @@ void vmw_kms_helper_resource_revert(struct vmw_validation_ctx *ctx)
 	struct vmw_resource *res = ctx->res;
 
 	vmw_kms_helper_buffer_revert(ctx->buf);
-	vmw_dmabuf_unreference(&ctx->buf);
+	vmw_bo_unreference(&ctx->buf);
 	vmw_resource_unreserve(res, false, NULL, 0);
 	mutex_unlock(&res->dev_priv->cmdbuf_mutex);
 }
@@ -2565,7 +2711,7 @@ int vmw_kms_helper_resource_prepare(struct vmw_resource *res,
 		if (ret)
 			goto out_unreserve;
 
-		ctx->buf = vmw_dmabuf_reference(res->backup);
+		ctx->buf = vmw_bo_reference(res->backup);
 	}
 	ret = vmw_resource_validate(res);
 	if (ret)
@@ -2598,7 +2744,7 @@ void vmw_kms_helper_resource_finish(struct vmw_validation_ctx *ctx,
 		vmw_kms_helper_buffer_finish(res->dev_priv, NULL, ctx->buf,
 					     out_fence, NULL);
 
-	vmw_dmabuf_unreference(&ctx->buf);
+	vmw_bo_unreference(&ctx->buf);
 	vmw_resource_unreserve(res, false, NULL, 0);
 	mutex_unlock(&res->dev_priv->cmdbuf_mutex);
 }
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.h b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.h
index 27c41c227120..31311298ec0b 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.h
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.h
@@ -90,7 +90,7 @@ struct vmw_kms_dirty {
 #define vmw_framebuffer_to_vfbs(x) \
 	container_of(x, struct vmw_framebuffer_surface, base.base)
 #define vmw_framebuffer_to_vfbd(x) \
-	container_of(x, struct vmw_framebuffer_dmabuf, base.base)
+	container_of(x, struct vmw_framebuffer_bo, base.base)
 
 /**
  * Base class for framebuffers
@@ -102,7 +102,7 @@ struct vmw_framebuffer {
 	struct drm_framebuffer base;
 	int (*pin)(struct vmw_framebuffer *fb);
 	int (*unpin)(struct vmw_framebuffer *fb);
-	bool dmabuf;
+	bool bo;
 	struct ttm_base_object *user_obj;
 	uint32_t user_handle;
 };
@@ -117,15 +117,15 @@ struct vmw_clip_rect {
 struct vmw_framebuffer_surface {
 	struct vmw_framebuffer base;
 	struct vmw_surface *surface;
-	struct vmw_dma_buffer *buffer;
+	struct vmw_buffer_object *buffer;
 	struct list_head head;
-	bool is_dmabuf_proxy;  /* true if this is proxy surface for DMA buf */
+	bool is_bo_proxy;  /* true if this is proxy surface for DMA buf */
 };
 
 
-struct vmw_framebuffer_dmabuf {
+struct vmw_framebuffer_bo {
 	struct vmw_framebuffer base;
-	struct vmw_dma_buffer *buffer;
+	struct vmw_buffer_object *buffer;
 };
 
 
@@ -161,18 +161,18 @@ struct vmw_crtc_state {
  *
  * @base DRM plane object
  * @surf Display surface for STDU
- * @dmabuf display dmabuf for SOU
+ * @bo display bo for SOU
  * @content_fb_type Used by STDU.
- * @dmabuf_size Size of the dmabuf, used by Screen Object Display Unit
+ * @bo_size Size of the bo, used by Screen Object Display Unit
  * @pinned pin count for STDU display surface
  */
 struct vmw_plane_state {
 	struct drm_plane_state base;
 	struct vmw_surface *surf;
-	struct vmw_dma_buffer *dmabuf;
+	struct vmw_buffer_object *bo;
 
 	int content_fb_type;
-	unsigned long dmabuf_size;
+	unsigned long bo_size;
 
 	int pinned;
 
@@ -192,6 +192,24 @@ struct vmw_connector_state {
 	struct drm_connector_state base;
 
 	bool is_implicit;
+
+	/**
+	 * @gui_x:
+	 *
+	 * vmwgfx connector property representing the x position of this display
+	 * unit (connector is synonymous to display unit) in overall topology.
+	 * This is what the device expect as xRoot while creating screen.
+	 */
+	int gui_x;
+
+	/**
+	 * @gui_y:
+	 *
+	 * vmwgfx connector property representing the y position of this display
+	 * unit (connector is synonymous to display unit) in overall topology.
+	 * This is what the device expect as yRoot while creating screen.
+	 */
+	int gui_y;
 };
 
 /**
@@ -209,7 +227,7 @@ struct vmw_display_unit {
 	struct drm_plane cursor;
 
 	struct vmw_surface *cursor_surface;
-	struct vmw_dma_buffer *cursor_dmabuf;
+	struct vmw_buffer_object *cursor_bo;
 	size_t cursor_age;
 
 	int cursor_x;
@@ -243,7 +261,7 @@ struct vmw_display_unit {
 
 struct vmw_validation_ctx {
 	struct vmw_resource *res;
-	struct vmw_dma_buffer *buf;
+	struct vmw_buffer_object *buf;
 };
 
 #define vmw_crtc_to_du(x) \
@@ -291,14 +309,14 @@ int vmw_kms_helper_dirty(struct vmw_private *dev_priv,
 			 struct vmw_kms_dirty *dirty);
 
 int vmw_kms_helper_buffer_prepare(struct vmw_private *dev_priv,
-				  struct vmw_dma_buffer *buf,
+				  struct vmw_buffer_object *buf,
 				  bool interruptible,
 				  bool validate_as_mob,
 				  bool for_cpu_blit);
-void vmw_kms_helper_buffer_revert(struct vmw_dma_buffer *buf);
+void vmw_kms_helper_buffer_revert(struct vmw_buffer_object *buf);
 void vmw_kms_helper_buffer_finish(struct vmw_private *dev_priv,
 				  struct drm_file *file_priv,
-				  struct vmw_dma_buffer *buf,
+				  struct vmw_buffer_object *buf,
 				  struct vmw_fence_obj **out_fence,
 				  struct drm_vmw_fence_rep __user *
 				  user_fence_rep);
@@ -316,7 +334,7 @@ int vmw_kms_readback(struct vmw_private *dev_priv,
 		     uint32_t num_clips);
 struct vmw_framebuffer *
 vmw_kms_new_framebuffer(struct vmw_private *dev_priv,
-			struct vmw_dma_buffer *dmabuf,
+			struct vmw_buffer_object *bo,
 			struct vmw_surface *surface,
 			bool only_2d,
 			const struct drm_mode_fb_cmd2 *mode_cmd);
@@ -384,11 +402,11 @@ void vmw_du_connector_destroy_state(struct drm_connector *connector,
  */
 int vmw_kms_ldu_init_display(struct vmw_private *dev_priv);
 int vmw_kms_ldu_close_display(struct vmw_private *dev_priv);
-int vmw_kms_ldu_do_dmabuf_dirty(struct vmw_private *dev_priv,
-				struct vmw_framebuffer *framebuffer,
-				unsigned flags, unsigned color,
-				struct drm_clip_rect *clips,
-				unsigned num_clips, int increment);
+int vmw_kms_ldu_do_bo_dirty(struct vmw_private *dev_priv,
+			    struct vmw_framebuffer *framebuffer,
+			    unsigned int flags, unsigned int color,
+			    struct drm_clip_rect *clips,
+			    unsigned int num_clips, int increment);
 int vmw_kms_update_proxy(struct vmw_resource *res,
 			 const struct drm_clip_rect *clips,
 			 unsigned num_clips,
@@ -408,14 +426,14 @@ int vmw_kms_sou_do_surface_dirty(struct vmw_private *dev_priv,
 				 unsigned num_clips, int inc,
 				 struct vmw_fence_obj **out_fence,
 				 struct drm_crtc *crtc);
-int vmw_kms_sou_do_dmabuf_dirty(struct vmw_private *dev_priv,
-				struct vmw_framebuffer *framebuffer,
-				struct drm_clip_rect *clips,
-				struct drm_vmw_rect *vclips,
-				unsigned num_clips, int increment,
-				bool interruptible,
-				struct vmw_fence_obj **out_fence,
-				struct drm_crtc *crtc);
+int vmw_kms_sou_do_bo_dirty(struct vmw_private *dev_priv,
+			    struct vmw_framebuffer *framebuffer,
+			    struct drm_clip_rect *clips,
+			    struct drm_vmw_rect *vclips,
+			    unsigned int num_clips, int increment,
+			    bool interruptible,
+			    struct vmw_fence_obj **out_fence,
+			    struct drm_crtc *crtc);
 int vmw_kms_sou_readback(struct vmw_private *dev_priv,
 			 struct drm_file *file_priv,
 			 struct vmw_framebuffer *vfb,
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c b/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c
index a234fe0e1061..723578117191 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c
@@ -547,11 +547,11 @@ int vmw_kms_ldu_close_display(struct vmw_private *dev_priv)
 }
 
 
-int vmw_kms_ldu_do_dmabuf_dirty(struct vmw_private *dev_priv,
-				struct vmw_framebuffer *framebuffer,
-				unsigned flags, unsigned color,
-				struct drm_clip_rect *clips,
-				unsigned num_clips, int increment)
+int vmw_kms_ldu_do_bo_dirty(struct vmw_private *dev_priv,
+			    struct vmw_framebuffer *framebuffer,
+			    unsigned int flags, unsigned int color,
+			    struct drm_clip_rect *clips,
+			    unsigned int num_clips, int increment)
 {
 	size_t fifo_size;
 	int i;
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_mob.c b/drivers/gpu/drm/vmwgfx/vmwgfx_mob.c
index 0963d0438190..7ed179d30ec5 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_mob.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_mob.c
@@ -225,7 +225,7 @@ static void vmw_takedown_otable_base(struct vmw_private *dev_priv,
 		ret = ttm_bo_reserve(bo, false, true, NULL);
 		BUG_ON(ret != 0);
 
-		vmw_fence_single_bo(bo, NULL);
+		vmw_bo_fence_single(bo, NULL);
 		ttm_bo_unreserve(bo);
 	}
 
@@ -362,7 +362,7 @@ static void vmw_otable_batch_takedown(struct vmw_private *dev_priv,
 	ret = ttm_bo_reserve(bo, false, true, NULL);
 	BUG_ON(ret != 0);
 
-	vmw_fence_single_bo(bo, NULL);
+	vmw_bo_fence_single(bo, NULL);
 	ttm_bo_unreserve(bo);
 
 	ttm_bo_unref(&batch->otable_bo);
@@ -620,7 +620,7 @@ void vmw_mob_unbind(struct vmw_private *dev_priv,
 		vmw_fifo_commit(dev_priv, sizeof(*cmd));
 	}
 	if (bo) {
-		vmw_fence_single_bo(bo, NULL);
+		vmw_bo_fence_single(bo, NULL);
 		ttm_bo_unreserve(bo);
 	}
 	vmw_fifo_resource_dec(dev_priv);
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_msg.c b/drivers/gpu/drm/vmwgfx/vmwgfx_msg.c
index 4437c41c585a..8b9270f31409 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_msg.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_msg.c
@@ -31,6 +31,7 @@
 #include <linux/frame.h>
 #include <asm/hypervisor.h>
 #include <drm/drmP.h>
+#include "vmwgfx_drv.h"
 #include "vmwgfx_msg.h"
 
 
@@ -234,7 +235,7 @@ static int vmw_recv_msg(struct rpc_channel *channel, void **msg,
 
 		if ((HIGH_WORD(ecx) & MESSAGE_STATUS_SUCCESS) == 0 ||
 		    (HIGH_WORD(ecx) & MESSAGE_STATUS_HB) == 0) {
-			DRM_ERROR("Failed to get reply size\n");
+			DRM_ERROR("Failed to get reply size for host message.\n");
 			return -EINVAL;
 		}
 
@@ -245,7 +246,7 @@ static int vmw_recv_msg(struct rpc_channel *channel, void **msg,
 		reply_len = ebx;
 		reply     = kzalloc(reply_len + 1, GFP_KERNEL);
 		if (!reply) {
-			DRM_ERROR("Cannot allocate memory for reply\n");
+			DRM_ERROR("Cannot allocate memory for host message reply.\n");
 			return -ENOMEM;
 		}
 
@@ -338,7 +339,8 @@ int vmw_host_get_guestinfo(const char *guest_info_param,
 
 	msg = kasprintf(GFP_KERNEL, "info-get %s", guest_info_param);
 	if (!msg) {
-		DRM_ERROR("Cannot allocate memory to get %s", guest_info_param);
+		DRM_ERROR("Cannot allocate memory to get guest info \"%s\".",
+			  guest_info_param);
 		return -ENOMEM;
 	}
 
@@ -374,7 +376,7 @@ out_msg:
 out_open:
 	*length = 0;
 	kfree(msg);
-	DRM_ERROR("Failed to get %s", guest_info_param);
+	DRM_ERROR("Failed to get guest info \"%s\".", guest_info_param);
 
 	return -EINVAL;
 }
@@ -403,7 +405,7 @@ int vmw_host_log(const char *log)
 
 	msg = kasprintf(GFP_KERNEL, "log %s", log);
 	if (!msg) {
-		DRM_ERROR("Cannot allocate memory for log message\n");
+		DRM_ERROR("Cannot allocate memory for host log message.\n");
 		return -ENOMEM;
 	}
 
@@ -422,7 +424,7 @@ out_msg:
 	vmw_close_channel(&channel);
 out_open:
 	kfree(msg);
-	DRM_ERROR("Failed to send log\n");
+	DRM_ERROR("Failed to send host log message.\n");
 
 	return -EINVAL;
 }
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_overlay.c b/drivers/gpu/drm/vmwgfx/vmwgfx_overlay.c
index 3ced89e70a64..9f1b9d289bec 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_overlay.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_overlay.c
@@ -38,7 +38,7 @@
 #define VMW_OVERLAY_CAP_MASK (SVGA_FIFO_CAP_VIDEO | SVGA_FIFO_CAP_ESCAPE)
 
 struct vmw_stream {
-	struct vmw_dma_buffer *buf;
+	struct vmw_buffer_object *buf;
 	bool claimed;
 	bool paused;
 	struct drm_vmw_control_stream_arg saved;
@@ -94,7 +94,7 @@ static inline void fill_flush(struct vmw_escape_video_flush *cmd,
  * -ERESTARTSYS if interrupted by a signal.
  */
 static int vmw_overlay_send_put(struct vmw_private *dev_priv,
-				struct vmw_dma_buffer *buf,
+				struct vmw_buffer_object *buf,
 				struct drm_vmw_control_stream_arg *arg,
 				bool interruptible)
 {
@@ -225,16 +225,16 @@ static int vmw_overlay_send_stop(struct vmw_private *dev_priv,
  * used with GMRs instead of being locked to vram.
  */
 static int vmw_overlay_move_buffer(struct vmw_private *dev_priv,
-				   struct vmw_dma_buffer *buf,
+				   struct vmw_buffer_object *buf,
 				   bool pin, bool inter)
 {
 	if (!pin)
-		return vmw_dmabuf_unpin(dev_priv, buf, inter);
+		return vmw_bo_unpin(dev_priv, buf, inter);
 
 	if (dev_priv->active_display_unit == vmw_du_legacy)
-		return vmw_dmabuf_pin_in_vram(dev_priv, buf, inter);
+		return vmw_bo_pin_in_vram(dev_priv, buf, inter);
 
-	return vmw_dmabuf_pin_in_vram_or_gmr(dev_priv, buf, inter);
+	return vmw_bo_pin_in_vram_or_gmr(dev_priv, buf, inter);
 }
 
 /**
@@ -278,7 +278,7 @@ static int vmw_overlay_stop(struct vmw_private *dev_priv,
 	}
 
 	if (!pause) {
-		vmw_dmabuf_unreference(&stream->buf);
+		vmw_bo_unreference(&stream->buf);
 		stream->paused = false;
 	} else {
 		stream->paused = true;
@@ -297,7 +297,7 @@ static int vmw_overlay_stop(struct vmw_private *dev_priv,
  * -ERESTARTSYS if interrupted.
  */
 static int vmw_overlay_update_stream(struct vmw_private *dev_priv,
-				     struct vmw_dma_buffer *buf,
+				     struct vmw_buffer_object *buf,
 				     struct drm_vmw_control_stream_arg *arg,
 				     bool interruptible)
 {
@@ -347,7 +347,7 @@ static int vmw_overlay_update_stream(struct vmw_private *dev_priv,
 	}
 
 	if (stream->buf != buf)
-		stream->buf = vmw_dmabuf_reference(buf);
+		stream->buf = vmw_bo_reference(buf);
 	stream->saved = *arg;
 	/* stream is no longer stopped/paused */
 	stream->paused = false;
@@ -466,7 +466,7 @@ int vmw_overlay_ioctl(struct drm_device *dev, void *data,
 	struct vmw_overlay *overlay = dev_priv->overlay_priv;
 	struct drm_vmw_control_stream_arg *arg =
 	    (struct drm_vmw_control_stream_arg *)data;
-	struct vmw_dma_buffer *buf;
+	struct vmw_buffer_object *buf;
 	struct vmw_resource *res;
 	int ret;
 
@@ -484,13 +484,13 @@ int vmw_overlay_ioctl(struct drm_device *dev, void *data,
 		goto out_unlock;
 	}
 
-	ret = vmw_user_dmabuf_lookup(tfile, arg->handle, &buf, NULL);
+	ret = vmw_user_bo_lookup(tfile, arg->handle, &buf, NULL);
 	if (ret)
 		goto out_unlock;
 
 	ret = vmw_overlay_update_stream(dev_priv, buf, arg, true);
 
-	vmw_dmabuf_unreference(&buf);
+	vmw_bo_unreference(&buf);
 
 out_unlock:
 	mutex_unlock(&overlay->mutex);
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c b/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c
index 3063a3e4bd40..92003ea5a219 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c
@@ -27,7 +27,6 @@
 
 #include "vmwgfx_drv.h"
 #include <drm/vmwgfx_drm.h>
-#include <drm/ttm/ttm_object.h>
 #include <drm/ttm/ttm_placement.h>
 #include <drm/drmP.h>
 #include "vmwgfx_resource_priv.h"
@@ -35,29 +34,6 @@
 
 #define VMW_RES_EVICT_ERR_COUNT 10
 
-struct vmw_user_dma_buffer {
-	struct ttm_prime_object prime;
-	struct vmw_dma_buffer dma;
-};
-
-struct vmw_bo_user_rep {
-	uint32_t handle;
-	uint64_t map_handle;
-};
-
-static inline struct vmw_dma_buffer *
-vmw_dma_buffer(struct ttm_buffer_object *bo)
-{
-	return container_of(bo, struct vmw_dma_buffer, base);
-}
-
-static inline struct vmw_user_dma_buffer *
-vmw_user_dma_buffer(struct ttm_buffer_object *bo)
-{
-	struct vmw_dma_buffer *vmw_bo = vmw_dma_buffer(bo);
-	return container_of(vmw_bo, struct vmw_user_dma_buffer, dma);
-}
-
 struct vmw_resource *vmw_resource_reference(struct vmw_resource *res)
 {
 	kref_get(&res->kref);
@@ -116,7 +92,7 @@ static void vmw_resource_release(struct kref *kref)
 		res->backup_dirty = false;
 		list_del_init(&res->mob_head);
 		ttm_bo_unreserve(bo);
-		vmw_dmabuf_unreference(&res->backup);
+		vmw_bo_unreference(&res->backup);
 	}
 
 	if (likely(res->hw_destroy != NULL)) {
@@ -287,7 +263,7 @@ out_bad_resource:
 }
 
 /**
- * Helper function that looks either a surface or dmabuf.
+ * Helper function that looks either a surface or bo.
  *
  * The pointer this pointed at by out_surf and out_buf needs to be null.
  */
@@ -295,7 +271,7 @@ int vmw_user_lookup_handle(struct vmw_private *dev_priv,
 			   struct ttm_object_file *tfile,
 			   uint32_t handle,
 			   struct vmw_surface **out_surf,
-			   struct vmw_dma_buffer **out_buf)
+			   struct vmw_buffer_object **out_buf)
 {
 	struct vmw_resource *res;
 	int ret;
@@ -311,513 +287,11 @@ int vmw_user_lookup_handle(struct vmw_private *dev_priv,
 	}
 
 	*out_surf = NULL;
-	ret = vmw_user_dmabuf_lookup(tfile, handle, out_buf, NULL);
+	ret = vmw_user_bo_lookup(tfile, handle, out_buf, NULL);
 	return ret;
 }
 
 /**
- * Buffer management.
- */
-
-/**
- * vmw_dmabuf_acc_size - Calculate the pinned memory usage of buffers
- *
- * @dev_priv: Pointer to a struct vmw_private identifying the device.
- * @size: The requested buffer size.
- * @user: Whether this is an ordinary dma buffer or a user dma buffer.
- */
-static size_t vmw_dmabuf_acc_size(struct vmw_private *dev_priv, size_t size,
-				  bool user)
-{
-	static size_t struct_size, user_struct_size;
-	size_t num_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
-	size_t page_array_size = ttm_round_pot(num_pages * sizeof(void *));
-
-	if (unlikely(struct_size == 0)) {
-		size_t backend_size = ttm_round_pot(vmw_tt_size);
-
-		struct_size = backend_size +
-			ttm_round_pot(sizeof(struct vmw_dma_buffer));
-		user_struct_size = backend_size +
-			ttm_round_pot(sizeof(struct vmw_user_dma_buffer));
-	}
-
-	if (dev_priv->map_mode == vmw_dma_alloc_coherent)
-		page_array_size +=
-			ttm_round_pot(num_pages * sizeof(dma_addr_t));
-
-	return ((user) ? user_struct_size : struct_size) +
-		page_array_size;
-}
-
-void vmw_dmabuf_bo_free(struct ttm_buffer_object *bo)
-{
-	struct vmw_dma_buffer *vmw_bo = vmw_dma_buffer(bo);
-
-	vmw_dma_buffer_unmap(vmw_bo);
-	kfree(vmw_bo);
-}
-
-static void vmw_user_dmabuf_destroy(struct ttm_buffer_object *bo)
-{
-	struct vmw_user_dma_buffer *vmw_user_bo = vmw_user_dma_buffer(bo);
-
-	vmw_dma_buffer_unmap(&vmw_user_bo->dma);
-	ttm_prime_object_kfree(vmw_user_bo, prime);
-}
-
-int vmw_dmabuf_init(struct vmw_private *dev_priv,
-		    struct vmw_dma_buffer *vmw_bo,
-		    size_t size, struct ttm_placement *placement,
-		    bool interruptible,
-		    void (*bo_free) (struct ttm_buffer_object *bo))
-{
-	struct ttm_bo_device *bdev = &dev_priv->bdev;
-	size_t acc_size;
-	int ret;
-	bool user = (bo_free == &vmw_user_dmabuf_destroy);
-
-	BUG_ON(!bo_free && (!user && (bo_free != vmw_dmabuf_bo_free)));
-
-	acc_size = vmw_dmabuf_acc_size(dev_priv, size, user);
-	memset(vmw_bo, 0, sizeof(*vmw_bo));
-
-	INIT_LIST_HEAD(&vmw_bo->res_list);
-
-	ret = ttm_bo_init(bdev, &vmw_bo->base, size,
-			  ttm_bo_type_device, placement,
-			  0, interruptible, acc_size,
-			  NULL, NULL, bo_free);
-	return ret;
-}
-
-static void vmw_user_dmabuf_release(struct ttm_base_object **p_base)
-{
-	struct vmw_user_dma_buffer *vmw_user_bo;
-	struct ttm_base_object *base = *p_base;
-	struct ttm_buffer_object *bo;
-
-	*p_base = NULL;
-
-	if (unlikely(base == NULL))
-		return;
-
-	vmw_user_bo = container_of(base, struct vmw_user_dma_buffer,
-				   prime.base);
-	bo = &vmw_user_bo->dma.base;
-	ttm_bo_unref(&bo);
-}
-
-static void vmw_user_dmabuf_ref_obj_release(struct ttm_base_object *base,
-					    enum ttm_ref_type ref_type)
-{
-	struct vmw_user_dma_buffer *user_bo;
-	user_bo = container_of(base, struct vmw_user_dma_buffer, prime.base);
-
-	switch (ref_type) {
-	case TTM_REF_SYNCCPU_WRITE:
-		ttm_bo_synccpu_write_release(&user_bo->dma.base);
-		break;
-	default:
-		BUG();
-	}
-}
-
-/**
- * vmw_user_dmabuf_alloc - Allocate a user dma buffer
- *
- * @dev_priv: Pointer to a struct device private.
- * @tfile: Pointer to a struct ttm_object_file on which to register the user
- * object.
- * @size: Size of the dma buffer.
- * @shareable: Boolean whether the buffer is shareable with other open files.
- * @handle: Pointer to where the handle value should be assigned.
- * @p_dma_buf: Pointer to where the refcounted struct vmw_dma_buffer pointer
- * should be assigned.
- */
-int vmw_user_dmabuf_alloc(struct vmw_private *dev_priv,
-			  struct ttm_object_file *tfile,
-			  uint32_t size,
-			  bool shareable,
-			  uint32_t *handle,
-			  struct vmw_dma_buffer **p_dma_buf,
-			  struct ttm_base_object **p_base)
-{
-	struct vmw_user_dma_buffer *user_bo;
-	struct ttm_buffer_object *tmp;
-	int ret;
-
-	user_bo = kzalloc(sizeof(*user_bo), GFP_KERNEL);
-	if (unlikely(!user_bo)) {
-		DRM_ERROR("Failed to allocate a buffer.\n");
-		return -ENOMEM;
-	}
-
-	ret = vmw_dmabuf_init(dev_priv, &user_bo->dma, size,
-			      (dev_priv->has_mob) ?
-			      &vmw_sys_placement :
-			      &vmw_vram_sys_placement, true,
-			      &vmw_user_dmabuf_destroy);
-	if (unlikely(ret != 0))
-		return ret;
-
-	tmp = ttm_bo_reference(&user_bo->dma.base);
-	ret = ttm_prime_object_init(tfile,
-				    size,
-				    &user_bo->prime,
-				    shareable,
-				    ttm_buffer_type,
-				    &vmw_user_dmabuf_release,
-				    &vmw_user_dmabuf_ref_obj_release);
-	if (unlikely(ret != 0)) {
-		ttm_bo_unref(&tmp);
-		goto out_no_base_object;
-	}
-
-	*p_dma_buf = &user_bo->dma;
-	if (p_base) {
-		*p_base = &user_bo->prime.base;
-		kref_get(&(*p_base)->refcount);
-	}
-	*handle = user_bo->prime.base.hash.key;
-
-out_no_base_object:
-	return ret;
-}
-
-/**
- * vmw_user_dmabuf_verify_access - verify access permissions on this
- * buffer object.
- *
- * @bo: Pointer to the buffer object being accessed
- * @tfile: Identifying the caller.
- */
-int vmw_user_dmabuf_verify_access(struct ttm_buffer_object *bo,
-				  struct ttm_object_file *tfile)
-{
-	struct vmw_user_dma_buffer *vmw_user_bo;
-
-	if (unlikely(bo->destroy != vmw_user_dmabuf_destroy))
-		return -EPERM;
-
-	vmw_user_bo = vmw_user_dma_buffer(bo);
-
-	/* Check that the caller has opened the object. */
-	if (likely(ttm_ref_object_exists(tfile, &vmw_user_bo->prime.base)))
-		return 0;
-
-	DRM_ERROR("Could not grant buffer access.\n");
-	return -EPERM;
-}
-
-/**
- * vmw_user_dmabuf_synccpu_grab - Grab a struct vmw_user_dma_buffer for cpu
- * access, idling previous GPU operations on the buffer and optionally
- * blocking it for further command submissions.
- *
- * @user_bo: Pointer to the buffer object being grabbed for CPU access
- * @tfile: Identifying the caller.
- * @flags: Flags indicating how the grab should be performed.
- *
- * A blocking grab will be automatically released when @tfile is closed.
- */
-static int vmw_user_dmabuf_synccpu_grab(struct vmw_user_dma_buffer *user_bo,
-					struct ttm_object_file *tfile,
-					uint32_t flags)
-{
-	struct ttm_buffer_object *bo = &user_bo->dma.base;
-	bool existed;
-	int ret;
-
-	if (flags & drm_vmw_synccpu_allow_cs) {
-		bool nonblock = !!(flags & drm_vmw_synccpu_dontblock);
-		long lret;
-
-		lret = reservation_object_wait_timeout_rcu(bo->resv, true, true,
-							   nonblock ? 0 : MAX_SCHEDULE_TIMEOUT);
-		if (!lret)
-			return -EBUSY;
-		else if (lret < 0)
-			return lret;
-		return 0;
-	}
-
-	ret = ttm_bo_synccpu_write_grab
-		(bo, !!(flags & drm_vmw_synccpu_dontblock));
-	if (unlikely(ret != 0))
-		return ret;
-
-	ret = ttm_ref_object_add(tfile, &user_bo->prime.base,
-				 TTM_REF_SYNCCPU_WRITE, &existed, false);
-	if (ret != 0 || existed)
-		ttm_bo_synccpu_write_release(&user_bo->dma.base);
-
-	return ret;
-}
-
-/**
- * vmw_user_dmabuf_synccpu_release - Release a previous grab for CPU access,
- * and unblock command submission on the buffer if blocked.
- *
- * @handle: Handle identifying the buffer object.
- * @tfile: Identifying the caller.
- * @flags: Flags indicating the type of release.
- */
-static int vmw_user_dmabuf_synccpu_release(uint32_t handle,
-					   struct ttm_object_file *tfile,
-					   uint32_t flags)
-{
-	if (!(flags & drm_vmw_synccpu_allow_cs))
-		return ttm_ref_object_base_unref(tfile, handle,
-						 TTM_REF_SYNCCPU_WRITE);
-
-	return 0;
-}
-
-/**
- * vmw_user_dmabuf_synccpu_release - ioctl function implementing the synccpu
- * functionality.
- *
- * @dev: Identifies the drm device.
- * @data: Pointer to the ioctl argument.
- * @file_priv: Identifies the caller.
- *
- * This function checks the ioctl arguments for validity and calls the
- * relevant synccpu functions.
- */
-int vmw_user_dmabuf_synccpu_ioctl(struct drm_device *dev, void *data,
-				  struct drm_file *file_priv)
-{
-	struct drm_vmw_synccpu_arg *arg =
-		(struct drm_vmw_synccpu_arg *) data;
-	struct vmw_dma_buffer *dma_buf;
-	struct vmw_user_dma_buffer *user_bo;
-	struct ttm_object_file *tfile = vmw_fpriv(file_priv)->tfile;
-	struct ttm_base_object *buffer_base;
-	int ret;
-
-	if ((arg->flags & (drm_vmw_synccpu_read | drm_vmw_synccpu_write)) == 0
-	    || (arg->flags & ~(drm_vmw_synccpu_read | drm_vmw_synccpu_write |
-			       drm_vmw_synccpu_dontblock |
-			       drm_vmw_synccpu_allow_cs)) != 0) {
-		DRM_ERROR("Illegal synccpu flags.\n");
-		return -EINVAL;
-	}
-
-	switch (arg->op) {
-	case drm_vmw_synccpu_grab:
-		ret = vmw_user_dmabuf_lookup(tfile, arg->handle, &dma_buf,
-					     &buffer_base);
-		if (unlikely(ret != 0))
-			return ret;
-
-		user_bo = container_of(dma_buf, struct vmw_user_dma_buffer,
-				       dma);
-		ret = vmw_user_dmabuf_synccpu_grab(user_bo, tfile, arg->flags);
-		vmw_dmabuf_unreference(&dma_buf);
-		ttm_base_object_unref(&buffer_base);
-		if (unlikely(ret != 0 && ret != -ERESTARTSYS &&
-			     ret != -EBUSY)) {
-			DRM_ERROR("Failed synccpu grab on handle 0x%08x.\n",
-				  (unsigned int) arg->handle);
-			return ret;
-		}
-		break;
-	case drm_vmw_synccpu_release:
-		ret = vmw_user_dmabuf_synccpu_release(arg->handle, tfile,
-						      arg->flags);
-		if (unlikely(ret != 0)) {
-			DRM_ERROR("Failed synccpu release on handle 0x%08x.\n",
-				  (unsigned int) arg->handle);
-			return ret;
-		}
-		break;
-	default:
-		DRM_ERROR("Invalid synccpu operation.\n");
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-int vmw_dmabuf_alloc_ioctl(struct drm_device *dev, void *data,
-			   struct drm_file *file_priv)
-{
-	struct vmw_private *dev_priv = vmw_priv(dev);
-	union drm_vmw_alloc_dmabuf_arg *arg =
-	    (union drm_vmw_alloc_dmabuf_arg *)data;
-	struct drm_vmw_alloc_dmabuf_req *req = &arg->req;
-	struct drm_vmw_dmabuf_rep *rep = &arg->rep;
-	struct vmw_dma_buffer *dma_buf;
-	uint32_t handle;
-	int ret;
-
-	ret = ttm_read_lock(&dev_priv->reservation_sem, true);
-	if (unlikely(ret != 0))
-		return ret;
-
-	ret = vmw_user_dmabuf_alloc(dev_priv, vmw_fpriv(file_priv)->tfile,
-				    req->size, false, &handle, &dma_buf,
-				    NULL);
-	if (unlikely(ret != 0))
-		goto out_no_dmabuf;
-
-	rep->handle = handle;
-	rep->map_handle = drm_vma_node_offset_addr(&dma_buf->base.vma_node);
-	rep->cur_gmr_id = handle;
-	rep->cur_gmr_offset = 0;
-
-	vmw_dmabuf_unreference(&dma_buf);
-
-out_no_dmabuf:
-	ttm_read_unlock(&dev_priv->reservation_sem);
-
-	return ret;
-}
-
-int vmw_dmabuf_unref_ioctl(struct drm_device *dev, void *data,
-			   struct drm_file *file_priv)
-{
-	struct drm_vmw_unref_dmabuf_arg *arg =
-	    (struct drm_vmw_unref_dmabuf_arg *)data;
-
-	return ttm_ref_object_base_unref(vmw_fpriv(file_priv)->tfile,
-					 arg->handle,
-					 TTM_REF_USAGE);
-}
-
-int vmw_user_dmabuf_lookup(struct ttm_object_file *tfile,
-			   uint32_t handle, struct vmw_dma_buffer **out,
-			   struct ttm_base_object **p_base)
-{
-	struct vmw_user_dma_buffer *vmw_user_bo;
-	struct ttm_base_object *base;
-
-	base = ttm_base_object_lookup(tfile, handle);
-	if (unlikely(base == NULL)) {
-		pr_err("Invalid buffer object handle 0x%08lx\n",
-		       (unsigned long)handle);
-		return -ESRCH;
-	}
-
-	if (unlikely(ttm_base_object_type(base) != ttm_buffer_type)) {
-		ttm_base_object_unref(&base);
-		pr_err("Invalid buffer object handle 0x%08lx\n",
-		       (unsigned long)handle);
-		return -EINVAL;
-	}
-
-	vmw_user_bo = container_of(base, struct vmw_user_dma_buffer,
-				   prime.base);
-	(void)ttm_bo_reference(&vmw_user_bo->dma.base);
-	if (p_base)
-		*p_base = base;
-	else
-		ttm_base_object_unref(&base);
-	*out = &vmw_user_bo->dma;
-
-	return 0;
-}
-
-int vmw_user_dmabuf_reference(struct ttm_object_file *tfile,
-			      struct vmw_dma_buffer *dma_buf,
-			      uint32_t *handle)
-{
-	struct vmw_user_dma_buffer *user_bo;
-
-	if (dma_buf->base.destroy != vmw_user_dmabuf_destroy)
-		return -EINVAL;
-
-	user_bo = container_of(dma_buf, struct vmw_user_dma_buffer, dma);
-
-	*handle = user_bo->prime.base.hash.key;
-	return ttm_ref_object_add(tfile, &user_bo->prime.base,
-				  TTM_REF_USAGE, NULL, false);
-}
-
-/**
- * vmw_dumb_create - Create a dumb kms buffer
- *
- * @file_priv: Pointer to a struct drm_file identifying the caller.
- * @dev: Pointer to the drm device.
- * @args: Pointer to a struct drm_mode_create_dumb structure
- *
- * This is a driver callback for the core drm create_dumb functionality.
- * Note that this is very similar to the vmw_dmabuf_alloc ioctl, except
- * that the arguments have a different format.
- */
-int vmw_dumb_create(struct drm_file *file_priv,
-		    struct drm_device *dev,
-		    struct drm_mode_create_dumb *args)
-{
-	struct vmw_private *dev_priv = vmw_priv(dev);
-	struct vmw_dma_buffer *dma_buf;
-	int ret;
-
-	args->pitch = args->width * ((args->bpp + 7) / 8);
-	args->size = args->pitch * args->height;
-
-	ret = ttm_read_lock(&dev_priv->reservation_sem, true);
-	if (unlikely(ret != 0))
-		return ret;
-
-	ret = vmw_user_dmabuf_alloc(dev_priv, vmw_fpriv(file_priv)->tfile,
-				    args->size, false, &args->handle,
-				    &dma_buf, NULL);
-	if (unlikely(ret != 0))
-		goto out_no_dmabuf;
-
-	vmw_dmabuf_unreference(&dma_buf);
-out_no_dmabuf:
-	ttm_read_unlock(&dev_priv->reservation_sem);
-	return ret;
-}
-
-/**
- * vmw_dumb_map_offset - Return the address space offset of a dumb buffer
- *
- * @file_priv: Pointer to a struct drm_file identifying the caller.
- * @dev: Pointer to the drm device.
- * @handle: Handle identifying the dumb buffer.
- * @offset: The address space offset returned.
- *
- * This is a driver callback for the core drm dumb_map_offset functionality.
- */
-int vmw_dumb_map_offset(struct drm_file *file_priv,
-			struct drm_device *dev, uint32_t handle,
-			uint64_t *offset)
-{
-	struct ttm_object_file *tfile = vmw_fpriv(file_priv)->tfile;
-	struct vmw_dma_buffer *out_buf;
-	int ret;
-
-	ret = vmw_user_dmabuf_lookup(tfile, handle, &out_buf, NULL);
-	if (ret != 0)
-		return -EINVAL;
-
-	*offset = drm_vma_node_offset_addr(&out_buf->base.vma_node);
-	vmw_dmabuf_unreference(&out_buf);
-	return 0;
-}
-
-/**
- * vmw_dumb_destroy - Destroy a dumb boffer
- *
- * @file_priv: Pointer to a struct drm_file identifying the caller.
- * @dev: Pointer to the drm device.
- * @handle: Handle identifying the dumb buffer.
- *
- * This is a driver callback for the core drm dumb_destroy functionality.
- */
-int vmw_dumb_destroy(struct drm_file *file_priv,
-		     struct drm_device *dev,
-		     uint32_t handle)
-{
-	return ttm_ref_object_base_unref(vmw_fpriv(file_priv)->tfile,
-					 handle, TTM_REF_USAGE);
-}
-
-/**
  * vmw_resource_buf_alloc - Allocate a backup buffer for a resource.
  *
  * @res:            The resource for which to allocate a backup buffer.
@@ -829,7 +303,7 @@ static int vmw_resource_buf_alloc(struct vmw_resource *res,
 {
 	unsigned long size =
 		(res->backup_size + PAGE_SIZE - 1) & PAGE_MASK;
-	struct vmw_dma_buffer *backup;
+	struct vmw_buffer_object *backup;
 	int ret;
 
 	if (likely(res->backup)) {
@@ -841,16 +315,16 @@ static int vmw_resource_buf_alloc(struct vmw_resource *res,
 	if (unlikely(!backup))
 		return -ENOMEM;
 
-	ret = vmw_dmabuf_init(res->dev_priv, backup, res->backup_size,
+	ret = vmw_bo_init(res->dev_priv, backup, res->backup_size,
 			      res->func->backup_placement,
 			      interruptible,
-			      &vmw_dmabuf_bo_free);
+			      &vmw_bo_bo_free);
 	if (unlikely(ret != 0))
-		goto out_no_dmabuf;
+		goto out_no_bo;
 
 	res->backup = backup;
 
-out_no_dmabuf:
+out_no_bo:
 	return ret;
 }
 
@@ -919,7 +393,7 @@ out_bind_failed:
  */
 void vmw_resource_unreserve(struct vmw_resource *res,
 			    bool switch_backup,
-			    struct vmw_dma_buffer *new_backup,
+			    struct vmw_buffer_object *new_backup,
 			    unsigned long new_backup_offset)
 {
 	struct vmw_private *dev_priv = res->dev_priv;
@@ -931,11 +405,11 @@ void vmw_resource_unreserve(struct vmw_resource *res,
 		if (res->backup) {
 			lockdep_assert_held(&res->backup->base.resv->lock.base);
 			list_del_init(&res->mob_head);
-			vmw_dmabuf_unreference(&res->backup);
+			vmw_bo_unreference(&res->backup);
 		}
 
 		if (new_backup) {
-			res->backup = vmw_dmabuf_reference(new_backup);
+			res->backup = vmw_bo_reference(new_backup);
 			lockdep_assert_held(&new_backup->base.resv->lock.base);
 			list_add_tail(&res->mob_head, &new_backup->res_list);
 		} else {
@@ -959,6 +433,7 @@ void vmw_resource_unreserve(struct vmw_resource *res,
  *                             for a resource and in that case, allocate
  *                             one, reserve and validate it.
  *
+ * @ticket:         The ww aqcquire context to use, or NULL if trylocking.
  * @res:            The resource for which to allocate a backup buffer.
  * @interruptible:  Whether any sleeps during allocation should be
  *                  performed while interruptible.
@@ -966,7 +441,8 @@ void vmw_resource_unreserve(struct vmw_resource *res,
  *                  reserved and validated backup buffer.
  */
 static int
-vmw_resource_check_buffer(struct vmw_resource *res,
+vmw_resource_check_buffer(struct ww_acquire_ctx *ticket,
+			  struct vmw_resource *res,
 			  bool interruptible,
 			  struct ttm_validate_buffer *val_buf)
 {
@@ -985,7 +461,7 @@ vmw_resource_check_buffer(struct vmw_resource *res,
 	val_buf->bo = ttm_bo_reference(&res->backup->base);
 	val_buf->shared = false;
 	list_add_tail(&val_buf->head, &val_list);
-	ret = ttm_eu_reserve_buffers(NULL, &val_list, interruptible, NULL);
+	ret = ttm_eu_reserve_buffers(ticket, &val_list, interruptible, NULL);
 	if (unlikely(ret != 0))
 		goto out_no_reserve;
 
@@ -1003,11 +479,11 @@ vmw_resource_check_buffer(struct vmw_resource *res,
 	return 0;
 
 out_no_validate:
-	ttm_eu_backoff_reservation(NULL, &val_list);
+	ttm_eu_backoff_reservation(ticket, &val_list);
 out_no_reserve:
 	ttm_bo_unref(&val_buf->bo);
 	if (backup_dirty)
-		vmw_dmabuf_unreference(&res->backup);
+		vmw_bo_unreference(&res->backup);
 
 	return ret;
 }
@@ -1050,10 +526,12 @@ int vmw_resource_reserve(struct vmw_resource *res, bool interruptible,
  * vmw_resource_backoff_reservation - Unreserve and unreference a
  *                                    backup buffer
  *.
+ * @ticket:         The ww acquire ctx used for reservation.
  * @val_buf:        Backup buffer information.
  */
 static void
-vmw_resource_backoff_reservation(struct ttm_validate_buffer *val_buf)
+vmw_resource_backoff_reservation(struct ww_acquire_ctx *ticket,
+				 struct ttm_validate_buffer *val_buf)
 {
 	struct list_head val_list;
 
@@ -1062,7 +540,7 @@ vmw_resource_backoff_reservation(struct ttm_validate_buffer *val_buf)
 
 	INIT_LIST_HEAD(&val_list);
 	list_add_tail(&val_buf->head, &val_list);
-	ttm_eu_backoff_reservation(NULL, &val_list);
+	ttm_eu_backoff_reservation(ticket, &val_list);
 	ttm_bo_unref(&val_buf->bo);
 }
 
@@ -1070,10 +548,12 @@ vmw_resource_backoff_reservation(struct ttm_validate_buffer *val_buf)
  * vmw_resource_do_evict - Evict a resource, and transfer its data
  *                         to a backup buffer.
  *
+ * @ticket:         The ww acquire ticket to use, or NULL if trylocking.
  * @res:            The resource to evict.
  * @interruptible:  Whether to wait interruptible.
  */
-static int vmw_resource_do_evict(struct vmw_resource *res, bool interruptible)
+static int vmw_resource_do_evict(struct ww_acquire_ctx *ticket,
+				 struct vmw_resource *res, bool interruptible)
 {
 	struct ttm_validate_buffer val_buf;
 	const struct vmw_res_func *func = res->func;
@@ -1083,7 +563,7 @@ static int vmw_resource_do_evict(struct vmw_resource *res, bool interruptible)
 
 	val_buf.bo = NULL;
 	val_buf.shared = false;
-	ret = vmw_resource_check_buffer(res, interruptible, &val_buf);
+	ret = vmw_resource_check_buffer(ticket, res, interruptible, &val_buf);
 	if (unlikely(ret != 0))
 		return ret;
 
@@ -1098,7 +578,7 @@ static int vmw_resource_do_evict(struct vmw_resource *res, bool interruptible)
 	res->backup_dirty = true;
 	res->res_dirty = false;
 out_no_unbind:
-	vmw_resource_backoff_reservation(&val_buf);
+	vmw_resource_backoff_reservation(ticket, &val_buf);
 
 	return ret;
 }
@@ -1152,7 +632,8 @@ int vmw_resource_validate(struct vmw_resource *res)
 
 		write_unlock(&dev_priv->resource_lock);
 
-		ret = vmw_resource_do_evict(evict_res, true);
+		/* Trylock backup buffers with a NULL ticket. */
+		ret = vmw_resource_do_evict(NULL, evict_res, true);
 		if (unlikely(ret != 0)) {
 			write_lock(&dev_priv->resource_lock);
 			list_add_tail(&evict_res->lru_head, lru_list);
@@ -1171,7 +652,7 @@ int vmw_resource_validate(struct vmw_resource *res)
 		goto out_no_validate;
 	else if (!res->func->needs_backup && res->backup) {
 		list_del_init(&res->mob_head);
-		vmw_dmabuf_unreference(&res->backup);
+		vmw_bo_unreference(&res->backup);
 	}
 
 	return 0;
@@ -1180,109 +661,39 @@ out_no_validate:
 	return ret;
 }
 
-/**
- * vmw_fence_single_bo - Utility function to fence a single TTM buffer
- *                       object without unreserving it.
- *
- * @bo:             Pointer to the struct ttm_buffer_object to fence.
- * @fence:          Pointer to the fence. If NULL, this function will
- *                  insert a fence into the command stream..
- *
- * Contrary to the ttm_eu version of this function, it takes only
- * a single buffer object instead of a list, and it also doesn't
- * unreserve the buffer object, which needs to be done separately.
- */
-void vmw_fence_single_bo(struct ttm_buffer_object *bo,
-			 struct vmw_fence_obj *fence)
-{
-	struct ttm_bo_device *bdev = bo->bdev;
-
-	struct vmw_private *dev_priv =
-		container_of(bdev, struct vmw_private, bdev);
-
-	if (fence == NULL) {
-		vmw_execbuf_fence_commands(NULL, dev_priv, &fence, NULL);
-		reservation_object_add_excl_fence(bo->resv, &fence->base);
-		dma_fence_put(&fence->base);
-	} else
-		reservation_object_add_excl_fence(bo->resv, &fence->base);
-}
 
 /**
- * vmw_resource_move_notify - TTM move_notify_callback
+ * vmw_resource_unbind_list
  *
- * @bo: The TTM buffer object about to move.
- * @mem: The struct ttm_mem_reg indicating to what memory
- *       region the move is taking place.
+ * @vbo: Pointer to the current backing MOB.
  *
  * Evicts the Guest Backed hardware resource if the backup
  * buffer is being moved out of MOB memory.
- * Note that this function should not race with the resource
- * validation code as long as it accesses only members of struct
- * resource that remain static while bo::res is !NULL and
- * while we have @bo reserved. struct resource::backup is *not* a
- * static member. The resource validation code will take care
- * to set @bo::res to NULL, while having @bo reserved when the
- * buffer is no longer bound to the resource, so @bo:res can be
- * used to determine whether there is a need to unbind and whether
- * it is safe to unbind.
+ * Note that this function will not race with the resource
+ * validation code, since resource validation and eviction
+ * both require the backup buffer to be reserved.
  */
-void vmw_resource_move_notify(struct ttm_buffer_object *bo,
-			      struct ttm_mem_reg *mem)
+void vmw_resource_unbind_list(struct vmw_buffer_object *vbo)
 {
-	struct vmw_dma_buffer *dma_buf;
-
-	if (mem == NULL)
-		return;
-
-	if (bo->destroy != vmw_dmabuf_bo_free &&
-	    bo->destroy != vmw_user_dmabuf_destroy)
-		return;
-
-	dma_buf = container_of(bo, struct vmw_dma_buffer, base);
-
-	/*
-	 * Kill any cached kernel maps before move. An optimization could
-	 * be to do this iff source or destination memory type is VRAM.
-	 */
-	vmw_dma_buffer_unmap(dma_buf);
 
-	if (mem->mem_type != VMW_PL_MOB) {
-		struct vmw_resource *res, *n;
-		struct ttm_validate_buffer val_buf;
+	struct vmw_resource *res, *next;
+	struct ttm_validate_buffer val_buf = {
+		.bo = &vbo->base,
+		.shared = false
+	};
 
-		val_buf.bo = bo;
-		val_buf.shared = false;
+	lockdep_assert_held(&vbo->base.resv->lock.base);
+	list_for_each_entry_safe(res, next, &vbo->res_list, mob_head) {
+		if (!res->func->unbind)
+			continue;
 
-		list_for_each_entry_safe(res, n, &dma_buf->res_list, mob_head) {
-
-			if (unlikely(res->func->unbind == NULL))
-				continue;
-
-			(void) res->func->unbind(res, true, &val_buf);
-			res->backup_dirty = true;
-			res->res_dirty = false;
-			list_del_init(&res->mob_head);
-		}
-
-		(void) ttm_bo_wait(bo, false, false);
+		(void) res->func->unbind(res, true, &val_buf);
+		res->backup_dirty = true;
+		res->res_dirty = false;
+		list_del_init(&res->mob_head);
 	}
-}
-
-
-/**
- * vmw_resource_swap_notify - swapout notify callback.
- *
- * @bo: The buffer object to be swapped out.
- */
-void vmw_resource_swap_notify(struct ttm_buffer_object *bo)
-{
-	if (bo->destroy != vmw_dmabuf_bo_free &&
-	    bo->destroy != vmw_user_dmabuf_destroy)
-		return;
 
-	/* Kill any cached kernel maps before swapout */
-	vmw_dma_buffer_unmap(vmw_dma_buffer(bo));
+	(void) ttm_bo_wait(&vbo->base, false, false);
 }
 
 
@@ -1294,7 +705,7 @@ void vmw_resource_swap_notify(struct ttm_buffer_object *bo)
  * Read back cached states from the device if they exist.  This function
  * assumings binding_mutex is held.
  */
-int vmw_query_readback_all(struct vmw_dma_buffer *dx_query_mob)
+int vmw_query_readback_all(struct vmw_buffer_object *dx_query_mob)
 {
 	struct vmw_resource *dx_query_ctx;
 	struct vmw_private *dev_priv;
@@ -1344,7 +755,7 @@ int vmw_query_readback_all(struct vmw_dma_buffer *dx_query_mob)
 void vmw_query_move_notify(struct ttm_buffer_object *bo,
 			   struct ttm_mem_reg *mem)
 {
-	struct vmw_dma_buffer *dx_query_mob;
+	struct vmw_buffer_object *dx_query_mob;
 	struct ttm_bo_device *bdev = bo->bdev;
 	struct vmw_private *dev_priv;
 
@@ -1353,7 +764,7 @@ void vmw_query_move_notify(struct ttm_buffer_object *bo,
 
 	mutex_lock(&dev_priv->binding_mutex);
 
-	dx_query_mob = container_of(bo, struct vmw_dma_buffer, base);
+	dx_query_mob = container_of(bo, struct vmw_buffer_object, base);
 	if (mem == NULL || !dx_query_mob || !dx_query_mob->dx_query_ctx) {
 		mutex_unlock(&dev_priv->binding_mutex);
 		return;
@@ -1368,7 +779,7 @@ void vmw_query_move_notify(struct ttm_buffer_object *bo,
 
 		/* Create a fence and attach the BO to it */
 		(void) vmw_execbuf_fence_commands(NULL, dev_priv, &fence, NULL);
-		vmw_fence_single_bo(bo, fence);
+		vmw_bo_fence_single(bo, fence);
 
 		if (fence != NULL)
 			vmw_fence_obj_unreference(&fence);
@@ -1405,6 +816,7 @@ static void vmw_resource_evict_type(struct vmw_private *dev_priv,
 	struct vmw_resource *evict_res;
 	unsigned err_count = 0;
 	int ret;
+	struct ww_acquire_ctx ticket;
 
 	do {
 		write_lock(&dev_priv->resource_lock);
@@ -1418,7 +830,8 @@ static void vmw_resource_evict_type(struct vmw_private *dev_priv,
 		list_del_init(&evict_res->lru_head);
 		write_unlock(&dev_priv->resource_lock);
 
-		ret = vmw_resource_do_evict(evict_res, false);
+		/* Wait lock backup buffers with a ticket. */
+		ret = vmw_resource_do_evict(&ticket, evict_res, false);
 		if (unlikely(ret != 0)) {
 			write_lock(&dev_priv->resource_lock);
 			list_add_tail(&evict_res->lru_head, lru_list);
@@ -1481,7 +894,7 @@ int vmw_resource_pin(struct vmw_resource *res, bool interruptible)
 		goto out_no_reserve;
 
 	if (res->pin_count == 0) {
-		struct vmw_dma_buffer *vbo = NULL;
+		struct vmw_buffer_object *vbo = NULL;
 
 		if (res->backup) {
 			vbo = res->backup;
@@ -1539,7 +952,7 @@ void vmw_resource_unpin(struct vmw_resource *res)
 
 	WARN_ON(res->pin_count == 0);
 	if (--res->pin_count == 0 && res->backup) {
-		struct vmw_dma_buffer *vbo = res->backup;
+		struct vmw_buffer_object *vbo = res->backup;
 
 		(void) ttm_bo_reserve(&vbo->base, false, false, NULL);
 		vmw_bo_pin_reserved(vbo, false);
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_scrn.c b/drivers/gpu/drm/vmwgfx/vmwgfx_scrn.c
index dd417eb74524..ad0de7f0cd60 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_scrn.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_scrn.c
@@ -66,7 +66,7 @@ struct vmw_kms_sou_readback_blit {
 	SVGAFifoCmdBlitScreenToGMRFB body;
 };
 
-struct vmw_kms_sou_dmabuf_blit {
+struct vmw_kms_sou_bo_blit {
 	uint32 header;
 	SVGAFifoCmdBlitGMRFBToScreen body;
 };
@@ -83,7 +83,7 @@ struct vmw_screen_object_unit {
 	struct vmw_display_unit base;
 
 	unsigned long buffer_size; /**< Size of allocated buffer */
-	struct vmw_dma_buffer *buffer; /**< Backing store buffer */
+	struct vmw_buffer_object *buffer; /**< Backing store buffer */
 
 	bool defined;
 };
@@ -109,7 +109,7 @@ static void vmw_sou_crtc_destroy(struct drm_crtc *crtc)
  */
 static int vmw_sou_fifo_create(struct vmw_private *dev_priv,
 			       struct vmw_screen_object_unit *sou,
-			       uint32_t x, uint32_t y,
+			       int x, int y,
 			       struct drm_display_mode *mode)
 {
 	size_t fifo_size;
@@ -139,13 +139,8 @@ static int vmw_sou_fifo_create(struct vmw_private *dev_priv,
 		(sou->base.unit == 0 ? SVGA_SCREEN_IS_PRIMARY : 0);
 	cmd->obj.size.width = mode->hdisplay;
 	cmd->obj.size.height = mode->vdisplay;
-	if (sou->base.is_implicit) {
-		cmd->obj.root.x = x;
-		cmd->obj.root.y = y;
-	} else {
-		cmd->obj.root.x = sou->base.gui_x;
-		cmd->obj.root.y = sou->base.gui_y;
-	}
+	cmd->obj.root.x = x;
+	cmd->obj.root.y = y;
 	sou->base.set_gui_x = cmd->obj.root.x;
 	sou->base.set_gui_y = cmd->obj.root.y;
 
@@ -222,12 +217,11 @@ static void vmw_sou_crtc_mode_set_nofb(struct drm_crtc *crtc)
 	struct vmw_plane_state *vps;
 	int ret;
 
-
-	sou      = vmw_crtc_to_sou(crtc);
+	sou = vmw_crtc_to_sou(crtc);
 	dev_priv = vmw_priv(crtc->dev);
-	ps       = crtc->primary->state;
-	fb       = ps->fb;
-	vps      = vmw_plane_state_to_vps(ps);
+	ps = crtc->primary->state;
+	fb = ps->fb;
+	vps = vmw_plane_state_to_vps(ps);
 
 	vfb = (fb) ? vmw_framebuffer_to_vfb(fb) : NULL;
 
@@ -240,11 +234,25 @@ static void vmw_sou_crtc_mode_set_nofb(struct drm_crtc *crtc)
 	}
 
 	if (vfb) {
-		sou->buffer = vps->dmabuf;
-		sou->buffer_size = vps->dmabuf_size;
+		struct drm_connector_state *conn_state;
+		struct vmw_connector_state *vmw_conn_state;
+		int x, y;
+
+		sou->buffer = vps->bo;
+		sou->buffer_size = vps->bo_size;
+
+		if (sou->base.is_implicit) {
+			x = crtc->x;
+			y = crtc->y;
+		} else {
+			conn_state = sou->base.connector.state;
+			vmw_conn_state = vmw_connector_state_to_vcs(conn_state);
+
+			x = vmw_conn_state->gui_x;
+			y = vmw_conn_state->gui_y;
+		}
 
-		ret = vmw_sou_fifo_create(dev_priv, sou, crtc->x, crtc->y,
-					  &crtc->mode);
+		ret = vmw_sou_fifo_create(dev_priv, sou, x, y, &crtc->mode);
 		if (ret)
 			DRM_ERROR("Failed to define Screen Object %dx%d\n",
 				  crtc->x, crtc->y);
@@ -408,10 +416,10 @@ vmw_sou_primary_plane_cleanup_fb(struct drm_plane *plane,
 	struct drm_crtc *crtc = plane->state->crtc ?
 		plane->state->crtc : old_state->crtc;
 
-	if (vps->dmabuf)
-		vmw_dmabuf_unpin(vmw_priv(crtc->dev), vps->dmabuf, false);
-	vmw_dmabuf_unreference(&vps->dmabuf);
-	vps->dmabuf_size = 0;
+	if (vps->bo)
+		vmw_bo_unpin(vmw_priv(crtc->dev), vps->bo, false);
+	vmw_bo_unreference(&vps->bo);
+	vps->bo_size = 0;
 
 	vmw_du_plane_cleanup_fb(plane, old_state);
 }
@@ -440,8 +448,8 @@ vmw_sou_primary_plane_prepare_fb(struct drm_plane *plane,
 
 
 	if (!new_fb) {
-		vmw_dmabuf_unreference(&vps->dmabuf);
-		vps->dmabuf_size = 0;
+		vmw_bo_unreference(&vps->bo);
+		vps->bo_size = 0;
 
 		return 0;
 	}
@@ -449,22 +457,22 @@ vmw_sou_primary_plane_prepare_fb(struct drm_plane *plane,
 	size = new_state->crtc_w * new_state->crtc_h * 4;
 	dev_priv = vmw_priv(crtc->dev);
 
-	if (vps->dmabuf) {
-		if (vps->dmabuf_size == size) {
+	if (vps->bo) {
+		if (vps->bo_size == size) {
 			/*
 			 * Note that this might temporarily up the pin-count
 			 * to 2, until cleanup_fb() is called.
 			 */
-			return vmw_dmabuf_pin_in_vram(dev_priv, vps->dmabuf,
+			return vmw_bo_pin_in_vram(dev_priv, vps->bo,
 						      true);
 		}
 
-		vmw_dmabuf_unreference(&vps->dmabuf);
-		vps->dmabuf_size = 0;
+		vmw_bo_unreference(&vps->bo);
+		vps->bo_size = 0;
 	}
 
-	vps->dmabuf = kzalloc(sizeof(*vps->dmabuf), GFP_KERNEL);
-	if (!vps->dmabuf)
+	vps->bo = kzalloc(sizeof(*vps->bo), GFP_KERNEL);
+	if (!vps->bo)
 		return -ENOMEM;
 
 	vmw_svga_enable(dev_priv);
@@ -473,22 +481,22 @@ vmw_sou_primary_plane_prepare_fb(struct drm_plane *plane,
 	 * resume the overlays, this is preferred to failing to alloc.
 	 */
 	vmw_overlay_pause_all(dev_priv);
-	ret = vmw_dmabuf_init(dev_priv, vps->dmabuf, size,
+	ret = vmw_bo_init(dev_priv, vps->bo, size,
 			      &vmw_vram_ne_placement,
-			      false, &vmw_dmabuf_bo_free);
+			      false, &vmw_bo_bo_free);
 	vmw_overlay_resume_all(dev_priv);
 	if (ret) {
-		vps->dmabuf = NULL; /* vmw_dmabuf_init frees on error */
+		vps->bo = NULL; /* vmw_bo_init frees on error */
 		return ret;
 	}
 
-	vps->dmabuf_size = size;
+	vps->bo_size = size;
 
 	/*
 	 * TTM already thinks the buffer is pinned, but make sure the
 	 * pin_count is upped.
 	 */
-	return vmw_dmabuf_pin_in_vram(dev_priv, vps->dmabuf, true);
+	return vmw_bo_pin_in_vram(dev_priv, vps->bo, true);
 }
 
 
@@ -512,10 +520,10 @@ vmw_sou_primary_plane_atomic_update(struct drm_plane *plane,
 		vclips.w = crtc->mode.hdisplay;
 		vclips.h = crtc->mode.vdisplay;
 
-		if (vfb->dmabuf)
-			ret = vmw_kms_sou_do_dmabuf_dirty(dev_priv, vfb, NULL,
-							  &vclips, 1, 1, true,
-							  &fence, crtc);
+		if (vfb->bo)
+			ret = vmw_kms_sou_do_bo_dirty(dev_priv, vfb, NULL,
+						      &vclips, 1, 1, true,
+						      &fence, crtc);
 		else
 			ret = vmw_kms_sou_do_surface_dirty(dev_priv, vfb, NULL,
 							   &vclips, NULL, 0, 0,
@@ -775,11 +783,11 @@ int vmw_kms_sou_init_display(struct vmw_private *dev_priv)
 	return 0;
 }
 
-static int do_dmabuf_define_gmrfb(struct vmw_private *dev_priv,
+static int do_bo_define_gmrfb(struct vmw_private *dev_priv,
 				  struct vmw_framebuffer *framebuffer)
 {
-	struct vmw_dma_buffer *buf =
-		container_of(framebuffer, struct vmw_framebuffer_dmabuf,
+	struct vmw_buffer_object *buf =
+		container_of(framebuffer, struct vmw_framebuffer_bo,
 			     base)->buffer;
 	int depth = framebuffer->base.format->depth;
 	struct {
@@ -970,13 +978,13 @@ int vmw_kms_sou_do_surface_dirty(struct vmw_private *dev_priv,
 }
 
 /**
- * vmw_sou_dmabuf_fifo_commit - Callback to submit a set of readback clips.
+ * vmw_sou_bo_fifo_commit - Callback to submit a set of readback clips.
  *
  * @dirty: The closure structure.
  *
  * Commits a previously built command buffer of readback clips.
  */
-static void vmw_sou_dmabuf_fifo_commit(struct vmw_kms_dirty *dirty)
+static void vmw_sou_bo_fifo_commit(struct vmw_kms_dirty *dirty)
 {
 	if (!dirty->num_hits) {
 		vmw_fifo_commit(dirty->dev_priv, 0);
@@ -984,20 +992,20 @@ static void vmw_sou_dmabuf_fifo_commit(struct vmw_kms_dirty *dirty)
 	}
 
 	vmw_fifo_commit(dirty->dev_priv,
-			sizeof(struct vmw_kms_sou_dmabuf_blit) *
+			sizeof(struct vmw_kms_sou_bo_blit) *
 			dirty->num_hits);
 }
 
 /**
- * vmw_sou_dmabuf_clip - Callback to encode a readback cliprect.
+ * vmw_sou_bo_clip - Callback to encode a readback cliprect.
  *
  * @dirty: The closure structure
  *
  * Encodes a BLIT_GMRFB_TO_SCREEN cliprect.
  */
-static void vmw_sou_dmabuf_clip(struct vmw_kms_dirty *dirty)
+static void vmw_sou_bo_clip(struct vmw_kms_dirty *dirty)
 {
-	struct vmw_kms_sou_dmabuf_blit *blit = dirty->cmd;
+	struct vmw_kms_sou_bo_blit *blit = dirty->cmd;
 
 	blit += dirty->num_hits;
 	blit->header = SVGA_CMD_BLIT_GMRFB_TO_SCREEN;
@@ -1012,10 +1020,10 @@ static void vmw_sou_dmabuf_clip(struct vmw_kms_dirty *dirty)
 }
 
 /**
- * vmw_kms_do_dmabuf_dirty - Dirty part of a dma-buffer backed framebuffer
+ * vmw_kms_do_bo_dirty - Dirty part of a buffer-object backed framebuffer
  *
  * @dev_priv: Pointer to the device private structure.
- * @framebuffer: Pointer to the dma-buffer backed framebuffer.
+ * @framebuffer: Pointer to the buffer-object backed framebuffer.
  * @clips: Array of clip rects.
  * @vclips: Alternate array of clip rects. Either @clips or @vclips must
  * be NULL.
@@ -1025,12 +1033,12 @@ static void vmw_sou_dmabuf_clip(struct vmw_kms_dirty *dirty)
  * @out_fence: If non-NULL, will return a ref-counted pointer to a
  * struct vmw_fence_obj. The returned fence pointer may be NULL in which
  * case the device has already synchronized.
- * @crtc: If crtc is passed, perform dmabuf dirty on that crtc only.
+ * @crtc: If crtc is passed, perform bo dirty on that crtc only.
  *
  * Returns 0 on success, negative error code on failure. -ERESTARTSYS if
  * interrupted.
  */
-int vmw_kms_sou_do_dmabuf_dirty(struct vmw_private *dev_priv,
+int vmw_kms_sou_do_bo_dirty(struct vmw_private *dev_priv,
 				struct vmw_framebuffer *framebuffer,
 				struct drm_clip_rect *clips,
 				struct drm_vmw_rect *vclips,
@@ -1039,8 +1047,8 @@ int vmw_kms_sou_do_dmabuf_dirty(struct vmw_private *dev_priv,
 				struct vmw_fence_obj **out_fence,
 				struct drm_crtc *crtc)
 {
-	struct vmw_dma_buffer *buf =
-		container_of(framebuffer, struct vmw_framebuffer_dmabuf,
+	struct vmw_buffer_object *buf =
+		container_of(framebuffer, struct vmw_framebuffer_bo,
 			     base)->buffer;
 	struct vmw_kms_dirty dirty;
 	int ret;
@@ -1050,14 +1058,14 @@ int vmw_kms_sou_do_dmabuf_dirty(struct vmw_private *dev_priv,
 	if (ret)
 		return ret;
 
-	ret = do_dmabuf_define_gmrfb(dev_priv, framebuffer);
+	ret = do_bo_define_gmrfb(dev_priv, framebuffer);
 	if (unlikely(ret != 0))
 		goto out_revert;
 
 	dirty.crtc = crtc;
-	dirty.fifo_commit = vmw_sou_dmabuf_fifo_commit;
-	dirty.clip = vmw_sou_dmabuf_clip;
-	dirty.fifo_reserve_size = sizeof(struct vmw_kms_sou_dmabuf_blit) *
+	dirty.fifo_commit = vmw_sou_bo_fifo_commit;
+	dirty.clip = vmw_sou_bo_clip;
+	dirty.fifo_reserve_size = sizeof(struct vmw_kms_sou_bo_blit) *
 		num_clips;
 	ret = vmw_kms_helper_dirty(dev_priv, framebuffer, clips, vclips,
 				   0, 0, num_clips, increment, &dirty);
@@ -1116,12 +1124,12 @@ static void vmw_sou_readback_clip(struct vmw_kms_dirty *dirty)
 
 /**
  * vmw_kms_sou_readback - Perform a readback from the screen object system to
- * a dma-buffer backed framebuffer.
+ * a buffer-object backed framebuffer.
  *
  * @dev_priv: Pointer to the device private structure.
  * @file_priv: Pointer to a struct drm_file identifying the caller.
  * Must be set to NULL if @user_fence_rep is NULL.
- * @vfb: Pointer to the dma-buffer backed framebuffer.
+ * @vfb: Pointer to the buffer-object backed framebuffer.
  * @user_fence_rep: User-space provided structure for fence information.
  * Must be set to non-NULL if @file_priv is non-NULL.
  * @vclips: Array of clip rects.
@@ -1139,8 +1147,8 @@ int vmw_kms_sou_readback(struct vmw_private *dev_priv,
 			 uint32_t num_clips,
 			 struct drm_crtc *crtc)
 {
-	struct vmw_dma_buffer *buf =
-		container_of(vfb, struct vmw_framebuffer_dmabuf, base)->buffer;
+	struct vmw_buffer_object *buf =
+		container_of(vfb, struct vmw_framebuffer_bo, base)->buffer;
 	struct vmw_kms_dirty dirty;
 	int ret;
 
@@ -1149,7 +1157,7 @@ int vmw_kms_sou_readback(struct vmw_private *dev_priv,
 	if (ret)
 		return ret;
 
-	ret = do_dmabuf_define_gmrfb(dev_priv, vfb);
+	ret = do_bo_define_gmrfb(dev_priv, vfb);
 	if (unlikely(ret != 0))
 		goto out_revert;
 
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_shader.c b/drivers/gpu/drm/vmwgfx/vmwgfx_shader.c
index e3ba63da396d..fe4842ca3b6e 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_shader.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_shader.c
@@ -159,7 +159,7 @@ static int vmw_gb_shader_init(struct vmw_private *dev_priv,
 			      SVGA3dShaderType type,
 			      uint8_t num_input_sig,
 			      uint8_t num_output_sig,
-			      struct vmw_dma_buffer *byte_code,
+			      struct vmw_buffer_object *byte_code,
 			      void (*res_free) (struct vmw_resource *res))
 {
 	struct vmw_shader *shader = vmw_res_to_shader(res);
@@ -178,7 +178,7 @@ static int vmw_gb_shader_init(struct vmw_private *dev_priv,
 
 	res->backup_size = size;
 	if (byte_code) {
-		res->backup = vmw_dmabuf_reference(byte_code);
+		res->backup = vmw_bo_reference(byte_code);
 		res->backup_offset = offset;
 	}
 	shader->size = size;
@@ -306,7 +306,7 @@ static int vmw_gb_shader_unbind(struct vmw_resource *res,
 	(void) vmw_execbuf_fence_commands(NULL, dev_priv,
 					  &fence, NULL);
 
-	vmw_fence_single_bo(val_buf->bo, fence);
+	vmw_bo_fence_single(val_buf->bo, fence);
 
 	if (likely(fence != NULL))
 		vmw_fence_obj_unreference(&fence);
@@ -537,7 +537,7 @@ static int vmw_dx_shader_unbind(struct vmw_resource *res,
 
 	(void) vmw_execbuf_fence_commands(NULL, dev_priv,
 					  &fence, NULL);
-	vmw_fence_single_bo(val_buf->bo, fence);
+	vmw_bo_fence_single(val_buf->bo, fence);
 
 	if (likely(fence != NULL))
 		vmw_fence_obj_unreference(&fence);
@@ -723,7 +723,7 @@ int vmw_shader_destroy_ioctl(struct drm_device *dev, void *data,
 }
 
 static int vmw_user_shader_alloc(struct vmw_private *dev_priv,
-				 struct vmw_dma_buffer *buffer,
+				 struct vmw_buffer_object *buffer,
 				 size_t shader_size,
 				 size_t offset,
 				 SVGA3dShaderType shader_type,
@@ -801,7 +801,7 @@ out:
 
 
 static struct vmw_resource *vmw_shader_alloc(struct vmw_private *dev_priv,
-					     struct vmw_dma_buffer *buffer,
+					     struct vmw_buffer_object *buffer,
 					     size_t shader_size,
 					     size_t offset,
 					     SVGA3dShaderType shader_type)
@@ -862,12 +862,12 @@ static int vmw_shader_define(struct drm_device *dev, struct drm_file *file_priv,
 {
 	struct vmw_private *dev_priv = vmw_priv(dev);
 	struct ttm_object_file *tfile = vmw_fpriv(file_priv)->tfile;
-	struct vmw_dma_buffer *buffer = NULL;
+	struct vmw_buffer_object *buffer = NULL;
 	SVGA3dShaderType shader_type;
 	int ret;
 
 	if (buffer_handle != SVGA3D_INVALID_ID) {
-		ret = vmw_user_dmabuf_lookup(tfile, buffer_handle,
+		ret = vmw_user_bo_lookup(tfile, buffer_handle,
 					     &buffer, NULL);
 		if (unlikely(ret != 0)) {
 			DRM_ERROR("Could not find buffer for shader "
@@ -906,7 +906,7 @@ static int vmw_shader_define(struct drm_device *dev, struct drm_file *file_priv,
 
 	ttm_read_unlock(&dev_priv->reservation_sem);
 out_bad_arg:
-	vmw_dmabuf_unreference(&buffer);
+	vmw_bo_unreference(&buffer);
 	return ret;
 }
 
@@ -983,7 +983,7 @@ int vmw_compat_shader_add(struct vmw_private *dev_priv,
 			  struct list_head *list)
 {
 	struct ttm_operation_ctx ctx = { false, true };
-	struct vmw_dma_buffer *buf;
+	struct vmw_buffer_object *buf;
 	struct ttm_bo_kmap_obj map;
 	bool is_iomem;
 	int ret;
@@ -997,8 +997,8 @@ int vmw_compat_shader_add(struct vmw_private *dev_priv,
 	if (unlikely(!buf))
 		return -ENOMEM;
 
-	ret = vmw_dmabuf_init(dev_priv, buf, size, &vmw_sys_ne_placement,
-			      true, vmw_dmabuf_bo_free);
+	ret = vmw_bo_init(dev_priv, buf, size, &vmw_sys_ne_placement,
+			      true, vmw_bo_bo_free);
 	if (unlikely(ret != 0))
 		goto out;
 
@@ -1031,7 +1031,7 @@ int vmw_compat_shader_add(struct vmw_private *dev_priv,
 				 res, list);
 	vmw_resource_unreference(&res);
 no_reserve:
-	vmw_dmabuf_unreference(&buf);
+	vmw_bo_unreference(&buf);
 out:
 	return ret;
 }
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c b/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c
index 4cca78cab03f..93f6b96ca7bb 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c
@@ -44,7 +44,7 @@
 enum stdu_content_type {
 	SAME_AS_DISPLAY = 0,
 	SEPARATE_SURFACE,
-	SEPARATE_DMA
+	SEPARATE_BO
 };
 
 /**
@@ -58,7 +58,7 @@ enum stdu_content_type {
  * @bottom: Bottom side of bounding box.
  * @fb_left: Left side of the framebuffer/content bounding box
  * @fb_top: Top of the framebuffer/content bounding box
- * @buf: DMA buffer when DMA-ing between buffer and screen targets.
+ * @buf: buffer object when DMA-ing between buffer and screen targets.
  * @sid: Surface ID when copying between surface and screen targets.
  */
 struct vmw_stdu_dirty {
@@ -68,7 +68,7 @@ struct vmw_stdu_dirty {
 	s32 fb_left, fb_top;
 	u32 pitch;
 	union {
-		struct vmw_dma_buffer *buf;
+		struct vmw_buffer_object *buf;
 		u32 sid;
 	};
 };
@@ -178,13 +178,9 @@ static int vmw_stdu_define_st(struct vmw_private *dev_priv,
 	cmd->body.height = mode->vdisplay;
 	cmd->body.flags  = (0 == cmd->body.stid) ? SVGA_STFLAG_PRIMARY : 0;
 	cmd->body.dpi    = 0;
-	if (stdu->base.is_implicit) {
-		cmd->body.xRoot  = crtc_x;
-		cmd->body.yRoot  = crtc_y;
-	} else {
-		cmd->body.xRoot  = stdu->base.gui_x;
-		cmd->body.yRoot  = stdu->base.gui_y;
-	}
+	cmd->body.xRoot  = crtc_x;
+	cmd->body.yRoot  = crtc_y;
+
 	stdu->base.set_gui_x = cmd->body.xRoot;
 	stdu->base.set_gui_y = cmd->body.yRoot;
 
@@ -374,11 +370,14 @@ static void vmw_stdu_crtc_mode_set_nofb(struct drm_crtc *crtc)
 {
 	struct vmw_private *dev_priv;
 	struct vmw_screen_target_display_unit *stdu;
-	int ret;
-
+	struct drm_connector_state *conn_state;
+	struct vmw_connector_state *vmw_conn_state;
+	int x, y, ret;
 
-	stdu     = vmw_crtc_to_stdu(crtc);
+	stdu = vmw_crtc_to_stdu(crtc);
 	dev_priv = vmw_priv(crtc->dev);
+	conn_state = stdu->base.connector.state;
+	vmw_conn_state = vmw_connector_state_to_vcs(conn_state);
 
 	if (stdu->defined) {
 		ret = vmw_stdu_bind_st(dev_priv, stdu, NULL);
@@ -397,8 +396,16 @@ static void vmw_stdu_crtc_mode_set_nofb(struct drm_crtc *crtc)
 	if (!crtc->state->enable)
 		return;
 
+	if (stdu->base.is_implicit) {
+		x = crtc->x;
+		y = crtc->y;
+	} else {
+		x = vmw_conn_state->gui_x;
+		y = vmw_conn_state->gui_y;
+	}
+
 	vmw_svga_enable(dev_priv);
-	ret = vmw_stdu_define_st(dev_priv, stdu, &crtc->mode, crtc->x, crtc->y);
+	ret = vmw_stdu_define_st(dev_priv, stdu, &crtc->mode, x, y);
 
 	if (ret)
 		DRM_ERROR("Failed to define Screen Target of size %dx%d\n",
@@ -508,14 +515,14 @@ static int vmw_stdu_crtc_page_flip(struct drm_crtc *crtc,
 
 
 /**
- * vmw_stdu_dmabuf_clip - Callback to encode a suface DMA command cliprect
+ * vmw_stdu_bo_clip - Callback to encode a suface DMA command cliprect
  *
  * @dirty: The closure structure.
  *
  * Encodes a surface DMA command cliprect and updates the bounding box
  * for the DMA.
  */
-static void vmw_stdu_dmabuf_clip(struct vmw_kms_dirty *dirty)
+static void vmw_stdu_bo_clip(struct vmw_kms_dirty *dirty)
 {
 	struct vmw_stdu_dirty *ddirty =
 		container_of(dirty, struct vmw_stdu_dirty, base);
@@ -543,14 +550,14 @@ static void vmw_stdu_dmabuf_clip(struct vmw_kms_dirty *dirty)
 }
 
 /**
- * vmw_stdu_dmabuf_fifo_commit - Callback to fill in and submit a DMA command.
+ * vmw_stdu_bo_fifo_commit - Callback to fill in and submit a DMA command.
  *
  * @dirty: The closure structure.
  *
  * Fills in the missing fields in a DMA command, and optionally encodes
  * a screen target update command, depending on transfer direction.
  */
-static void vmw_stdu_dmabuf_fifo_commit(struct vmw_kms_dirty *dirty)
+static void vmw_stdu_bo_fifo_commit(struct vmw_kms_dirty *dirty)
 {
 	struct vmw_stdu_dirty *ddirty =
 		container_of(dirty, struct vmw_stdu_dirty, base);
@@ -594,13 +601,13 @@ static void vmw_stdu_dmabuf_fifo_commit(struct vmw_kms_dirty *dirty)
 
 
 /**
- * vmw_stdu_dmabuf_cpu_clip - Callback to encode a CPU blit
+ * vmw_stdu_bo_cpu_clip - Callback to encode a CPU blit
  *
  * @dirty: The closure structure.
  *
  * This function calculates the bounding box for all the incoming clips.
  */
-static void vmw_stdu_dmabuf_cpu_clip(struct vmw_kms_dirty *dirty)
+static void vmw_stdu_bo_cpu_clip(struct vmw_kms_dirty *dirty)
 {
 	struct vmw_stdu_dirty *ddirty =
 		container_of(dirty, struct vmw_stdu_dirty, base);
@@ -624,14 +631,14 @@ static void vmw_stdu_dmabuf_cpu_clip(struct vmw_kms_dirty *dirty)
 
 
 /**
- * vmw_stdu_dmabuf_cpu_commit - Callback to do a CPU blit from DMAbuf
+ * vmw_stdu_bo_cpu_commit - Callback to do a CPU blit from buffer object
  *
  * @dirty: The closure structure.
  *
  * For the special case when we cannot create a proxy surface in a
  * 2D VM, we have to do a CPU blit ourselves.
  */
-static void vmw_stdu_dmabuf_cpu_commit(struct vmw_kms_dirty *dirty)
+static void vmw_stdu_bo_cpu_commit(struct vmw_kms_dirty *dirty)
 {
 	struct vmw_stdu_dirty *ddirty =
 		container_of(dirty, struct vmw_stdu_dirty, base);
@@ -652,7 +659,7 @@ static void vmw_stdu_dmabuf_cpu_commit(struct vmw_kms_dirty *dirty)
 	if (width == 0 || height == 0)
 		return;
 
-	/* Assume we are blitting from Guest (dmabuf) to Host (display_srf) */
+	/* Assume we are blitting from Guest (bo) to Host (display_srf) */
 	dst_pitch = stdu->display_srf->base_size.width * stdu->cpp;
 	dst_bo = &stdu->display_srf->res.backup->base;
 	dst_offset = ddirty->top * dst_pitch + ddirty->left * stdu->cpp;
@@ -712,13 +719,13 @@ out_cleanup:
 }
 
 /**
- * vmw_kms_stdu_dma - Perform a DMA transfer between a dma-buffer backed
+ * vmw_kms_stdu_dma - Perform a DMA transfer between a buffer-object backed
  * framebuffer and the screen target system.
  *
  * @dev_priv: Pointer to the device private structure.
  * @file_priv: Pointer to a struct drm-file identifying the caller. May be
  * set to NULL, but then @user_fence_rep must also be set to NULL.
- * @vfb: Pointer to the dma-buffer backed framebuffer.
+ * @vfb: Pointer to the buffer-object backed framebuffer.
  * @clips: Array of clip rects. Either @clips or @vclips must be NULL.
  * @vclips: Alternate array of clip rects. Either @clips or @vclips must
  * be NULL.
@@ -747,8 +754,8 @@ int vmw_kms_stdu_dma(struct vmw_private *dev_priv,
 		     bool interruptible,
 		     struct drm_crtc *crtc)
 {
-	struct vmw_dma_buffer *buf =
-		container_of(vfb, struct vmw_framebuffer_dmabuf, base)->buffer;
+	struct vmw_buffer_object *buf =
+		container_of(vfb, struct vmw_framebuffer_bo, base)->buffer;
 	struct vmw_stdu_dirty ddirty;
 	int ret;
 	bool cpu_blit = !(dev_priv->capabilities & SVGA_CAP_3D);
@@ -770,8 +777,8 @@ int vmw_kms_stdu_dma(struct vmw_private *dev_priv,
 	ddirty.fb_left = ddirty.fb_top = S32_MAX;
 	ddirty.pitch = vfb->base.pitches[0];
 	ddirty.buf = buf;
-	ddirty.base.fifo_commit = vmw_stdu_dmabuf_fifo_commit;
-	ddirty.base.clip = vmw_stdu_dmabuf_clip;
+	ddirty.base.fifo_commit = vmw_stdu_bo_fifo_commit;
+	ddirty.base.clip = vmw_stdu_bo_clip;
 	ddirty.base.fifo_reserve_size = sizeof(struct vmw_stdu_dma) +
 		num_clips * sizeof(SVGA3dCopyBox) +
 		sizeof(SVGA3dCmdSurfaceDMASuffix);
@@ -780,8 +787,8 @@ int vmw_kms_stdu_dma(struct vmw_private *dev_priv,
 
 
 	if (cpu_blit) {
-		ddirty.base.fifo_commit = vmw_stdu_dmabuf_cpu_commit;
-		ddirty.base.clip = vmw_stdu_dmabuf_cpu_clip;
+		ddirty.base.fifo_commit = vmw_stdu_bo_cpu_commit;
+		ddirty.base.clip = vmw_stdu_bo_cpu_clip;
 		ddirty.base.fifo_reserve_size = 0;
 	}
 
@@ -927,7 +934,7 @@ int vmw_kms_stdu_surface_dirty(struct vmw_private *dev_priv,
 	if (ret)
 		return ret;
 
-	if (vfbs->is_dmabuf_proxy) {
+	if (vfbs->is_bo_proxy) {
 		ret = vmw_kms_update_proxy(srf, clips, num_clips, inc);
 		if (ret)
 			goto out_finish;
@@ -1075,7 +1082,7 @@ vmw_stdu_primary_plane_cleanup_fb(struct drm_plane *plane,
  * @new_state: info on the new plane state, including the FB
  *
  * This function allocates a new display surface if the content is
- * backed by a DMA.  The display surface is pinned here, and it'll
+ * backed by a buffer object.  The display surface is pinned here, and it'll
  * be unpinned in .cleanup_fb()
  *
  * Returns 0 on success
@@ -1105,13 +1112,13 @@ vmw_stdu_primary_plane_prepare_fb(struct drm_plane *plane,
 	}
 
 	vfb = vmw_framebuffer_to_vfb(new_fb);
-	new_vfbs = (vfb->dmabuf) ? NULL : vmw_framebuffer_to_vfbs(new_fb);
+	new_vfbs = (vfb->bo) ? NULL : vmw_framebuffer_to_vfbs(new_fb);
 
 	if (new_vfbs && new_vfbs->surface->base_size.width == hdisplay &&
 	    new_vfbs->surface->base_size.height == vdisplay)
 		new_content_type = SAME_AS_DISPLAY;
-	else if (vfb->dmabuf)
-		new_content_type = SEPARATE_DMA;
+	else if (vfb->bo)
+		new_content_type = SEPARATE_BO;
 	else
 		new_content_type = SEPARATE_SURFACE;
 
@@ -1124,10 +1131,10 @@ vmw_stdu_primary_plane_prepare_fb(struct drm_plane *plane,
 		display_base_size.depth  = 1;
 
 		/*
-		 * If content buffer is a DMA buf, then we have to construct
-		 * surface info
+		 * If content buffer is a buffer object, then we have to
+		 * construct surface info
 		 */
-		if (new_content_type == SEPARATE_DMA) {
+		if (new_content_type == SEPARATE_BO) {
 
 			switch (new_fb->format->cpp[0]*8) {
 			case 32:
@@ -1150,6 +1157,9 @@ vmw_stdu_primary_plane_prepare_fb(struct drm_plane *plane,
 			content_srf.flags             = 0;
 			content_srf.mip_levels[0]     = 1;
 			content_srf.multisample_count = 0;
+			content_srf.multisample_pattern =
+				SVGA3D_MS_PATTERN_NONE;
+			content_srf.quality_level = SVGA3D_MS_QUALITY_NONE;
 		} else {
 			content_srf = *new_vfbs->surface;
 		}
@@ -1178,6 +1188,8 @@ vmw_stdu_primary_plane_prepare_fb(struct drm_plane *plane,
 				 content_srf.multisample_count,
 				 0,
 				 display_base_size,
+				 content_srf.multisample_pattern,
+				 content_srf.quality_level,
 				 &vps->surf);
 			if (ret != 0) {
 				DRM_ERROR("Couldn't allocate STDU surface.\n");
@@ -1212,12 +1224,12 @@ vmw_stdu_primary_plane_prepare_fb(struct drm_plane *plane,
 	vps->content_fb_type = new_content_type;
 
 	/*
-	 * This should only happen if the DMA buf is too large to create a
+	 * This should only happen if the buffer object is too large to create a
 	 * proxy surface for.
-	 * If we are a 2D VM with a DMA buffer then we have to use CPU blit
+	 * If we are a 2D VM with a buffer object then we have to use CPU blit
 	 * so cache these mappings
 	 */
-	if (vps->content_fb_type == SEPARATE_DMA &&
+	if (vps->content_fb_type == SEPARATE_BO &&
 	    !(dev_priv->capabilities & SVGA_CAP_3D))
 		vps->cpp = new_fb->pitches[0] / new_fb->width;
 
@@ -1276,7 +1288,7 @@ vmw_stdu_primary_plane_atomic_update(struct drm_plane *plane,
 		if (ret)
 			DRM_ERROR("Failed to bind surface to STDU.\n");
 
-		if (vfb->dmabuf)
+		if (vfb->bo)
 			ret = vmw_kms_stdu_dma(dev_priv, NULL, vfb, NULL, NULL,
 					       &vclips, 1, 1, true, false,
 					       crtc);
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_surface.c b/drivers/gpu/drm/vmwgfx/vmwgfx_surface.c
index 40a439797ae2..e125233e074b 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_surface.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_surface.c
@@ -33,6 +33,10 @@
 #include "vmwgfx_binding.h"
 #include "device_include/svga3d_surfacedefs.h"
 
+#define SVGA3D_FLAGS_64(upper32, lower32) (((uint64_t)upper32 << 32) | lower32)
+#define SVGA3D_FLAGS_UPPER_32(svga3d_flags) (svga3d_flags >> 32)
+#define SVGA3D_FLAGS_LOWER_32(svga3d_flags) \
+	(svga3d_flags & ((uint64_t)U32_MAX))
 
 /**
  * struct vmw_user_surface - User-space visible surface resource
@@ -81,7 +85,16 @@ static int vmw_gb_surface_unbind(struct vmw_resource *res,
 				 bool readback,
 				 struct ttm_validate_buffer *val_buf);
 static int vmw_gb_surface_destroy(struct vmw_resource *res);
-
+static int
+vmw_gb_surface_define_internal(struct drm_device *dev,
+			       struct drm_vmw_gb_surface_create_ext_req *req,
+			       struct drm_vmw_gb_surface_create_rep *rep,
+			       struct drm_file *file_priv);
+static int
+vmw_gb_surface_reference_internal(struct drm_device *dev,
+				  struct drm_vmw_surface_arg *req,
+				  struct drm_vmw_gb_surface_ref_ext_rep *rep,
+				  struct drm_file *file_priv);
 
 static const struct vmw_user_resource_conv user_surface_conv = {
 	.object_type = VMW_RES_SURFACE,
@@ -224,7 +237,12 @@ static void vmw_surface_define_encode(const struct vmw_surface *srf,
 	cmd->header.id = SVGA_3D_CMD_SURFACE_DEFINE;
 	cmd->header.size = cmd_len;
 	cmd->body.sid = srf->res.id;
-	cmd->body.surfaceFlags = srf->flags;
+	/*
+	 * Downcast of surfaceFlags, was upcasted when received from user-space,
+	 * since driver internally stores as 64 bit.
+	 * For legacy surface define only 32 bit flag is supported.
+	 */
+	cmd->body.surfaceFlags = (SVGA3dSurface1Flags)srf->flags;
 	cmd->body.format = srf->format;
 	for (i = 0; i < DRM_VMW_MAX_SURFACE_FACES; ++i)
 		cmd->body.face[i].numMipLevels = srf->mip_levels[i];
@@ -468,7 +486,7 @@ static int vmw_legacy_srf_dma(struct vmw_resource *res,
 	(void) vmw_execbuf_fence_commands(NULL, dev_priv,
 					  &fence, NULL);
 
-	vmw_fence_single_bo(val_buf->bo, fence);
+	vmw_bo_fence_single(val_buf->bo, fence);
 
 	if (likely(fence != NULL))
 		vmw_fence_obj_unreference(&fence);
@@ -760,7 +778,8 @@ int vmw_surface_define_ioctl(struct drm_device *dev, void *data,
 	srf = &user_srf->srf;
 	res = &srf->res;
 
-	srf->flags = req->flags;
+	/* Driver internally stores as 64-bit flags */
+	srf->flags = (SVGA3dSurfaceAllFlags)req->flags;
 	srf->format = req->format;
 	srf->scanout = req->scanout;
 
@@ -785,6 +804,8 @@ int vmw_surface_define_ioctl(struct drm_device *dev, void *data,
 	srf->base_size = *srf->sizes;
 	srf->autogen_filter = SVGA3D_TEX_FILTER_NONE;
 	srf->multisample_count = 0;
+	srf->multisample_pattern = SVGA3D_MS_PATTERN_NONE;
+	srf->quality_level = SVGA3D_MS_QUALITY_NONE;
 
 	cur_bo_offset = 0;
 	cur_offset = srf->offsets;
@@ -842,12 +863,12 @@ int vmw_surface_define_ioctl(struct drm_device *dev, void *data,
 	if (dev_priv->has_mob && req->shareable) {
 		uint32_t backup_handle;
 
-		ret = vmw_user_dmabuf_alloc(dev_priv, tfile,
-					    res->backup_size,
-					    true,
-					    &backup_handle,
-					    &res->backup,
-					    &user_srf->backup_base);
+		ret = vmw_user_bo_alloc(dev_priv, tfile,
+					res->backup_size,
+					true,
+					&backup_handle,
+					&res->backup,
+					&user_srf->backup_base);
 		if (unlikely(ret != 0)) {
 			vmw_resource_unreference(&res);
 			goto out_unlock;
@@ -990,7 +1011,8 @@ int vmw_surface_reference_ioctl(struct drm_device *dev, void *data,
 	user_srf = container_of(base, struct vmw_user_surface, prime.base);
 	srf = &user_srf->srf;
 
-	rep->flags = srf->flags;
+	/* Downcast of flags when sending back to user space */
+	rep->flags = (uint32_t)srf->flags;
 	rep->format = srf->format;
 	memcpy(rep->mip_levels, srf->mip_levels, sizeof(srf->mip_levels));
 	user_sizes = (struct drm_vmw_size __user *)(unsigned long)
@@ -1031,6 +1053,10 @@ static int vmw_gb_surface_create(struct vmw_resource *res)
 		SVGA3dCmdHeader header;
 		SVGA3dCmdDefineGBSurface_v2 body;
 	} *cmd2;
+	struct {
+		SVGA3dCmdHeader header;
+		SVGA3dCmdDefineGBSurface_v3 body;
+	} *cmd3;
 
 	if (likely(res->id != -1))
 		return 0;
@@ -1047,7 +1073,11 @@ static int vmw_gb_surface_create(struct vmw_resource *res)
 		goto out_no_fifo;
 	}
 
-	if (srf->array_size > 0) {
+	if (dev_priv->has_sm4_1 && srf->array_size > 0) {
+		cmd_id = SVGA_3D_CMD_DEFINE_GB_SURFACE_V3;
+		cmd_len = sizeof(cmd3->body);
+		submit_len = sizeof(*cmd3);
+	} else if (srf->array_size > 0) {
 		/* has_dx checked on creation time. */
 		cmd_id = SVGA_3D_CMD_DEFINE_GB_SURFACE_V2;
 		cmd_len = sizeof(cmd2->body);
@@ -1060,6 +1090,7 @@ static int vmw_gb_surface_create(struct vmw_resource *res)
 
 	cmd = vmw_fifo_reserve(dev_priv, submit_len);
 	cmd2 = (typeof(cmd2))cmd;
+	cmd3 = (typeof(cmd3))cmd;
 	if (unlikely(!cmd)) {
 		DRM_ERROR("Failed reserving FIFO space for surface "
 			  "creation.\n");
@@ -1067,12 +1098,27 @@ static int vmw_gb_surface_create(struct vmw_resource *res)
 		goto out_no_fifo;
 	}
 
-	if (srf->array_size > 0) {
+	if (dev_priv->has_sm4_1 && srf->array_size > 0) {
+		cmd3->header.id = cmd_id;
+		cmd3->header.size = cmd_len;
+		cmd3->body.sid = srf->res.id;
+		cmd3->body.surfaceFlags = srf->flags;
+		cmd3->body.format = srf->format;
+		cmd3->body.numMipLevels = srf->mip_levels[0];
+		cmd3->body.multisampleCount = srf->multisample_count;
+		cmd3->body.multisamplePattern = srf->multisample_pattern;
+		cmd3->body.qualityLevel = srf->quality_level;
+		cmd3->body.autogenFilter = srf->autogen_filter;
+		cmd3->body.size.width = srf->base_size.width;
+		cmd3->body.size.height = srf->base_size.height;
+		cmd3->body.size.depth = srf->base_size.depth;
+		cmd3->body.arraySize = srf->array_size;
+	} else if (srf->array_size > 0) {
 		cmd2->header.id = cmd_id;
 		cmd2->header.size = cmd_len;
 		cmd2->body.sid = srf->res.id;
 		cmd2->body.surfaceFlags = srf->flags;
-		cmd2->body.format = cpu_to_le32(srf->format);
+		cmd2->body.format = srf->format;
 		cmd2->body.numMipLevels = srf->mip_levels[0];
 		cmd2->body.multisampleCount = srf->multisample_count;
 		cmd2->body.autogenFilter = srf->autogen_filter;
@@ -1085,7 +1131,7 @@ static int vmw_gb_surface_create(struct vmw_resource *res)
 		cmd->header.size = cmd_len;
 		cmd->body.sid = srf->res.id;
 		cmd->body.surfaceFlags = srf->flags;
-		cmd->body.format = cpu_to_le32(srf->format);
+		cmd->body.format = srf->format;
 		cmd->body.numMipLevels = srf->mip_levels[0];
 		cmd->body.multisampleCount = srf->multisample_count;
 		cmd->body.autogenFilter = srf->autogen_filter;
@@ -1210,7 +1256,7 @@ static int vmw_gb_surface_unbind(struct vmw_resource *res,
 	(void) vmw_execbuf_fence_commands(NULL, dev_priv,
 					  &fence, NULL);
 
-	vmw_fence_single_bo(val_buf->bo, fence);
+	vmw_bo_fence_single(val_buf->bo, fence);
 
 	if (likely(fence != NULL))
 		vmw_fence_obj_unreference(&fence);
@@ -1256,194 +1302,55 @@ static int vmw_gb_surface_destroy(struct vmw_resource *res)
 
 /**
  * vmw_gb_surface_define_ioctl - Ioctl function implementing
- *                               the user surface define functionality.
+ * the user surface define functionality.
  *
- * @dev:            Pointer to a struct drm_device.
- * @data:           Pointer to data copied from / to user-space.
- * @file_priv:      Pointer to a drm file private structure.
+ * @dev: Pointer to a struct drm_device.
+ * @data: Pointer to data copied from / to user-space.
+ * @file_priv: Pointer to a drm file private structure.
  */
 int vmw_gb_surface_define_ioctl(struct drm_device *dev, void *data,
 				struct drm_file *file_priv)
 {
-	struct vmw_private *dev_priv = vmw_priv(dev);
-	struct vmw_user_surface *user_srf;
-	struct vmw_surface *srf;
-	struct vmw_resource *res;
-	struct vmw_resource *tmp;
 	union drm_vmw_gb_surface_create_arg *arg =
 	    (union drm_vmw_gb_surface_create_arg *)data;
-	struct drm_vmw_gb_surface_create_req *req = &arg->req;
 	struct drm_vmw_gb_surface_create_rep *rep = &arg->rep;
-	struct ttm_object_file *tfile = vmw_fpriv(file_priv)->tfile;
-	int ret;
-	uint32_t size;
-	uint32_t backup_handle = 0;
-
-	if (req->multisample_count != 0)
-		return -EINVAL;
-
-	if (req->mip_levels > DRM_VMW_MAX_MIP_LEVELS)
-		return -EINVAL;
+	struct drm_vmw_gb_surface_create_ext_req req_ext;
 
-	if (unlikely(vmw_user_surface_size == 0))
-		vmw_user_surface_size = ttm_round_pot(sizeof(*user_srf)) +
-			128;
-
-	size = vmw_user_surface_size + 128;
-
-	/* Define a surface based on the parameters. */
-	ret = vmw_surface_gb_priv_define(dev,
-			size,
-			req->svga3d_flags,
-			req->format,
-			req->drm_surface_flags & drm_vmw_surface_flag_scanout,
-			req->mip_levels,
-			req->multisample_count,
-			req->array_size,
-			req->base_size,
-			&srf);
-	if (unlikely(ret != 0))
-		return ret;
-
-	user_srf = container_of(srf, struct vmw_user_surface, srf);
-	if (drm_is_primary_client(file_priv))
-		user_srf->master = drm_master_get(file_priv->master);
+	req_ext.base = arg->req;
+	req_ext.version = drm_vmw_gb_surface_v1;
+	req_ext.svga3d_flags_upper_32_bits = 0;
+	req_ext.multisample_pattern = SVGA3D_MS_PATTERN_NONE;
+	req_ext.quality_level = SVGA3D_MS_QUALITY_NONE;
+	req_ext.must_be_zero = 0;
 
-	ret = ttm_read_lock(&dev_priv->reservation_sem, true);
-	if (unlikely(ret != 0))
-		return ret;
-
-	res = &user_srf->srf.res;
-
-
-	if (req->buffer_handle != SVGA3D_INVALID_ID) {
-		ret = vmw_user_dmabuf_lookup(tfile, req->buffer_handle,
-					     &res->backup,
-					     &user_srf->backup_base);
-		if (ret == 0) {
-			if (res->backup->base.num_pages * PAGE_SIZE <
-			    res->backup_size) {
-				DRM_ERROR("Surface backup buffer is too small.\n");
-				vmw_dmabuf_unreference(&res->backup);
-				ret = -EINVAL;
-				goto out_unlock;
-			} else {
-				backup_handle = req->buffer_handle;
-			}
-		}
-	} else if (req->drm_surface_flags & drm_vmw_surface_flag_create_buffer)
-		ret = vmw_user_dmabuf_alloc(dev_priv, tfile,
-					    res->backup_size,
-					    req->drm_surface_flags &
-					    drm_vmw_surface_flag_shareable,
-					    &backup_handle,
-					    &res->backup,
-					    &user_srf->backup_base);
-
-	if (unlikely(ret != 0)) {
-		vmw_resource_unreference(&res);
-		goto out_unlock;
-	}
-
-	tmp = vmw_resource_reference(res);
-	ret = ttm_prime_object_init(tfile, res->backup_size, &user_srf->prime,
-				    req->drm_surface_flags &
-				    drm_vmw_surface_flag_shareable,
-				    VMW_RES_SURFACE,
-				    &vmw_user_surface_base_release, NULL);
-
-	if (unlikely(ret != 0)) {
-		vmw_resource_unreference(&tmp);
-		vmw_resource_unreference(&res);
-		goto out_unlock;
-	}
-
-	rep->handle      = user_srf->prime.base.hash.key;
-	rep->backup_size = res->backup_size;
-	if (res->backup) {
-		rep->buffer_map_handle =
-			drm_vma_node_offset_addr(&res->backup->base.vma_node);
-		rep->buffer_size = res->backup->base.num_pages * PAGE_SIZE;
-		rep->buffer_handle = backup_handle;
-	} else {
-		rep->buffer_map_handle = 0;
-		rep->buffer_size = 0;
-		rep->buffer_handle = SVGA3D_INVALID_ID;
-	}
-
-	vmw_resource_unreference(&res);
-
-out_unlock:
-	ttm_read_unlock(&dev_priv->reservation_sem);
-	return ret;
+	return vmw_gb_surface_define_internal(dev, &req_ext, rep, file_priv);
 }
 
 /**
  * vmw_gb_surface_reference_ioctl - Ioctl function implementing
- *                                  the user surface reference functionality.
+ * the user surface reference functionality.
  *
- * @dev:            Pointer to a struct drm_device.
- * @data:           Pointer to data copied from / to user-space.
- * @file_priv:      Pointer to a drm file private structure.
+ * @dev: Pointer to a struct drm_device.
+ * @data: Pointer to data copied from / to user-space.
+ * @file_priv: Pointer to a drm file private structure.
  */
 int vmw_gb_surface_reference_ioctl(struct drm_device *dev, void *data,
 				   struct drm_file *file_priv)
 {
-	struct vmw_private *dev_priv = vmw_priv(dev);
 	union drm_vmw_gb_surface_reference_arg *arg =
 	    (union drm_vmw_gb_surface_reference_arg *)data;
 	struct drm_vmw_surface_arg *req = &arg->req;
 	struct drm_vmw_gb_surface_ref_rep *rep = &arg->rep;
-	struct ttm_object_file *tfile = vmw_fpriv(file_priv)->tfile;
-	struct vmw_surface *srf;
-	struct vmw_user_surface *user_srf;
-	struct ttm_base_object *base;
-	uint32_t backup_handle;
-	int ret = -EINVAL;
+	struct drm_vmw_gb_surface_ref_ext_rep rep_ext;
+	int ret;
+
+	ret = vmw_gb_surface_reference_internal(dev, req, &rep_ext, file_priv);
 
-	ret = vmw_surface_handle_reference(dev_priv, file_priv, req->sid,
-					   req->handle_type, &base);
 	if (unlikely(ret != 0))
 		return ret;
 
-	user_srf = container_of(base, struct vmw_user_surface, prime.base);
-	srf = &user_srf->srf;
-	if (!srf->res.backup) {
-		DRM_ERROR("Shared GB surface is missing a backup buffer.\n");
-		goto out_bad_resource;
-	}
-
-	mutex_lock(&dev_priv->cmdbuf_mutex); /* Protect res->backup */
-	ret = vmw_user_dmabuf_reference(tfile, srf->res.backup,
-					&backup_handle);
-	mutex_unlock(&dev_priv->cmdbuf_mutex);
-
-	if (unlikely(ret != 0)) {
-		DRM_ERROR("Could not add a reference to a GB surface "
-			  "backup buffer.\n");
-		(void) ttm_ref_object_base_unref(tfile, base->hash.key,
-						 TTM_REF_USAGE);
-		goto out_bad_resource;
-	}
-
-	rep->creq.svga3d_flags = srf->flags;
-	rep->creq.format = srf->format;
-	rep->creq.mip_levels = srf->mip_levels[0];
-	rep->creq.drm_surface_flags = 0;
-	rep->creq.multisample_count = srf->multisample_count;
-	rep->creq.autogen_filter = srf->autogen_filter;
-	rep->creq.array_size = srf->array_size;
-	rep->creq.buffer_handle = backup_handle;
-	rep->creq.base_size = srf->base_size;
-	rep->crep.handle = user_srf->prime.base.hash.key;
-	rep->crep.backup_size = srf->res.backup_size;
-	rep->crep.buffer_handle = backup_handle;
-	rep->crep.buffer_map_handle =
-		drm_vma_node_offset_addr(&srf->res.backup->base.vma_node);
-	rep->crep.buffer_size = srf->res.backup->base.num_pages * PAGE_SIZE;
-
-out_bad_resource:
-	ttm_base_object_unref(&base);
+	rep->creq = rep_ext.creq.base;
+	rep->crep = rep_ext.crep;
 
 	return ret;
 }
@@ -1461,6 +1368,8 @@ out_bad_resource:
  * @multisample_count:
  * @array_size: Surface array size.
  * @size: width, heigh, depth of the surface requested
+ * @multisample_pattern: Multisampling pattern when msaa is supported
+ * @quality_level: Precision settings
  * @user_srf_out: allocated user_srf.  Set to NULL on failure.
  *
  * GB surfaces allocated by this function will not have a user mode handle, and
@@ -1470,13 +1379,15 @@ out_bad_resource:
  */
 int vmw_surface_gb_priv_define(struct drm_device *dev,
 			       uint32_t user_accounting_size,
-			       uint32_t svga3d_flags,
+			       SVGA3dSurfaceAllFlags svga3d_flags,
 			       SVGA3dSurfaceFormat format,
 			       bool for_scanout,
 			       uint32_t num_mip_levels,
 			       uint32_t multisample_count,
 			       uint32_t array_size,
 			       struct drm_vmw_size size,
+			       SVGA3dMSPattern multisample_pattern,
+			       SVGA3dMSQualityLevel quality_level,
 			       struct vmw_surface **srf_out)
 {
 	struct vmw_private *dev_priv = vmw_priv(dev);
@@ -1487,7 +1398,8 @@ int vmw_surface_gb_priv_define(struct drm_device *dev,
 	};
 	struct vmw_surface *srf;
 	int ret;
-	u32 num_layers;
+	u32 num_layers = 1;
+	u32 sample_count = 1;
 
 	*srf_out = NULL;
 
@@ -1562,19 +1474,23 @@ int vmw_surface_gb_priv_define(struct drm_device *dev,
 	srf->autogen_filter    = SVGA3D_TEX_FILTER_NONE;
 	srf->array_size        = array_size;
 	srf->multisample_count = multisample_count;
+	srf->multisample_pattern = multisample_pattern;
+	srf->quality_level = quality_level;
 
 	if (array_size)
 		num_layers = array_size;
 	else if (svga3d_flags & SVGA3D_SURFACE_CUBEMAP)
 		num_layers = SVGA3D_MAX_SURFACE_FACES;
-	else
-		num_layers = 1;
+
+	if (srf->flags & SVGA3D_SURFACE_MULTISAMPLE)
+		sample_count = srf->multisample_count;
 
 	srf->res.backup_size   =
-		svga3dsurface_get_serialized_size(srf->format,
-						  srf->base_size,
-						  srf->mip_levels[0],
-						  num_layers);
+		svga3dsurface_get_serialized_size_extended(srf->format,
+							   srf->base_size,
+							   srf->mip_levels[0],
+							   num_layers,
+							   sample_count);
 
 	if (srf->flags & SVGA3D_SURFACE_BIND_STREAM_OUTPUT)
 		srf->res.backup_size += sizeof(SVGA3dDXSOState);
@@ -1599,3 +1515,266 @@ out_unlock:
 	ttm_read_unlock(&dev_priv->reservation_sem);
 	return ret;
 }
+
+/**
+ * vmw_gb_surface_define_ext_ioctl - Ioctl function implementing
+ * the user surface define functionality.
+ *
+ * @dev: Pointer to a struct drm_device.
+ * @data: Pointer to data copied from / to user-space.
+ * @file_priv: Pointer to a drm file private structure.
+ */
+int vmw_gb_surface_define_ext_ioctl(struct drm_device *dev, void *data,
+				struct drm_file *file_priv)
+{
+	union drm_vmw_gb_surface_create_ext_arg *arg =
+	    (union drm_vmw_gb_surface_create_ext_arg *)data;
+	struct drm_vmw_gb_surface_create_ext_req *req = &arg->req;
+	struct drm_vmw_gb_surface_create_rep *rep = &arg->rep;
+
+	return vmw_gb_surface_define_internal(dev, req, rep, file_priv);
+}
+
+/**
+ * vmw_gb_surface_reference_ext_ioctl - Ioctl function implementing
+ * the user surface reference functionality.
+ *
+ * @dev: Pointer to a struct drm_device.
+ * @data: Pointer to data copied from / to user-space.
+ * @file_priv: Pointer to a drm file private structure.
+ */
+int vmw_gb_surface_reference_ext_ioctl(struct drm_device *dev, void *data,
+				   struct drm_file *file_priv)
+{
+	union drm_vmw_gb_surface_reference_ext_arg *arg =
+	    (union drm_vmw_gb_surface_reference_ext_arg *)data;
+	struct drm_vmw_surface_arg *req = &arg->req;
+	struct drm_vmw_gb_surface_ref_ext_rep *rep = &arg->rep;
+
+	return vmw_gb_surface_reference_internal(dev, req, rep, file_priv);
+}
+
+/**
+ * vmw_gb_surface_define_internal - Ioctl function implementing
+ * the user surface define functionality.
+ *
+ * @dev: Pointer to a struct drm_device.
+ * @req: Request argument from user-space.
+ * @rep: Response argument to user-space.
+ * @file_priv: Pointer to a drm file private structure.
+ */
+static int
+vmw_gb_surface_define_internal(struct drm_device *dev,
+			       struct drm_vmw_gb_surface_create_ext_req *req,
+			       struct drm_vmw_gb_surface_create_rep *rep,
+			       struct drm_file *file_priv)
+{
+	struct vmw_private *dev_priv = vmw_priv(dev);
+	struct vmw_user_surface *user_srf;
+	struct vmw_surface *srf;
+	struct vmw_resource *res;
+	struct vmw_resource *tmp;
+	struct ttm_object_file *tfile = vmw_fpriv(file_priv)->tfile;
+	int ret;
+	uint32_t size;
+	uint32_t backup_handle = 0;
+	SVGA3dSurfaceAllFlags svga3d_flags_64 =
+		SVGA3D_FLAGS_64(req->svga3d_flags_upper_32_bits,
+				req->base.svga3d_flags);
+
+	if (!dev_priv->has_sm4_1) {
+		/*
+		 * If SM4_1 is not support then cannot send 64-bit flag to
+		 * device.
+		 */
+		if (req->svga3d_flags_upper_32_bits != 0)
+			return -EINVAL;
+
+		if (req->base.multisample_count != 0)
+			return -EINVAL;
+
+		if (req->multisample_pattern != SVGA3D_MS_PATTERN_NONE)
+			return -EINVAL;
+
+		if (req->quality_level != SVGA3D_MS_QUALITY_NONE)
+			return -EINVAL;
+	}
+
+	if ((svga3d_flags_64 & SVGA3D_SURFACE_MULTISAMPLE) &&
+	    req->base.multisample_count == 0)
+		return -EINVAL;
+
+	if (req->base.mip_levels > DRM_VMW_MAX_MIP_LEVELS)
+		return -EINVAL;
+
+	if (unlikely(vmw_user_surface_size == 0))
+		vmw_user_surface_size = ttm_round_pot(sizeof(*user_srf)) +
+			128;
+
+	size = vmw_user_surface_size + 128;
+
+	/* Define a surface based on the parameters. */
+	ret = vmw_surface_gb_priv_define(dev,
+					 size,
+					 svga3d_flags_64,
+					 req->base.format,
+					 req->base.drm_surface_flags &
+					 drm_vmw_surface_flag_scanout,
+					 req->base.mip_levels,
+					 req->base.multisample_count,
+					 req->base.array_size,
+					 req->base.base_size,
+					 req->multisample_pattern,
+					 req->quality_level,
+					 &srf);
+	if (unlikely(ret != 0))
+		return ret;
+
+	user_srf = container_of(srf, struct vmw_user_surface, srf);
+	if (drm_is_primary_client(file_priv))
+		user_srf->master = drm_master_get(file_priv->master);
+
+	ret = ttm_read_lock(&dev_priv->reservation_sem, true);
+	if (unlikely(ret != 0))
+		return ret;
+
+	res = &user_srf->srf.res;
+
+	if (req->base.buffer_handle != SVGA3D_INVALID_ID) {
+		ret = vmw_user_bo_lookup(tfile, req->base.buffer_handle,
+					 &res->backup,
+					 &user_srf->backup_base);
+		if (ret == 0) {
+			if (res->backup->base.num_pages * PAGE_SIZE <
+			    res->backup_size) {
+				DRM_ERROR("Surface backup buffer too small.\n");
+				vmw_bo_unreference(&res->backup);
+				ret = -EINVAL;
+				goto out_unlock;
+			} else {
+				backup_handle = req->base.buffer_handle;
+			}
+		}
+	} else if (req->base.drm_surface_flags &
+		   drm_vmw_surface_flag_create_buffer)
+		ret = vmw_user_bo_alloc(dev_priv, tfile,
+					res->backup_size,
+					req->base.drm_surface_flags &
+					drm_vmw_surface_flag_shareable,
+					&backup_handle,
+					&res->backup,
+					&user_srf->backup_base);
+
+	if (unlikely(ret != 0)) {
+		vmw_resource_unreference(&res);
+		goto out_unlock;
+	}
+
+	tmp = vmw_resource_reference(res);
+	ret = ttm_prime_object_init(tfile, res->backup_size, &user_srf->prime,
+				    req->base.drm_surface_flags &
+				    drm_vmw_surface_flag_shareable,
+				    VMW_RES_SURFACE,
+				    &vmw_user_surface_base_release, NULL);
+
+	if (unlikely(ret != 0)) {
+		vmw_resource_unreference(&tmp);
+		vmw_resource_unreference(&res);
+		goto out_unlock;
+	}
+
+	rep->handle      = user_srf->prime.base.hash.key;
+	rep->backup_size = res->backup_size;
+	if (res->backup) {
+		rep->buffer_map_handle =
+			drm_vma_node_offset_addr(&res->backup->base.vma_node);
+		rep->buffer_size = res->backup->base.num_pages * PAGE_SIZE;
+		rep->buffer_handle = backup_handle;
+	} else {
+		rep->buffer_map_handle = 0;
+		rep->buffer_size = 0;
+		rep->buffer_handle = SVGA3D_INVALID_ID;
+	}
+
+	vmw_resource_unreference(&res);
+
+out_unlock:
+	ttm_read_unlock(&dev_priv->reservation_sem);
+	return ret;
+}
+
+/**
+ * vmw_gb_surface_reference_internal - Ioctl function implementing
+ * the user surface reference functionality.
+ *
+ * @dev: Pointer to a struct drm_device.
+ * @req: Pointer to user-space request surface arg.
+ * @rep: Pointer to response to user-space.
+ * @file_priv: Pointer to a drm file private structure.
+ */
+static int
+vmw_gb_surface_reference_internal(struct drm_device *dev,
+				  struct drm_vmw_surface_arg *req,
+				  struct drm_vmw_gb_surface_ref_ext_rep *rep,
+				  struct drm_file *file_priv)
+{
+	struct vmw_private *dev_priv = vmw_priv(dev);
+	struct ttm_object_file *tfile = vmw_fpriv(file_priv)->tfile;
+	struct vmw_surface *srf;
+	struct vmw_user_surface *user_srf;
+	struct ttm_base_object *base;
+	uint32_t backup_handle;
+	int ret = -EINVAL;
+
+	ret = vmw_surface_handle_reference(dev_priv, file_priv, req->sid,
+					   req->handle_type, &base);
+	if (unlikely(ret != 0))
+		return ret;
+
+	user_srf = container_of(base, struct vmw_user_surface, prime.base);
+	srf = &user_srf->srf;
+	if (!srf->res.backup) {
+		DRM_ERROR("Shared GB surface is missing a backup buffer.\n");
+		goto out_bad_resource;
+	}
+
+	mutex_lock(&dev_priv->cmdbuf_mutex); /* Protect res->backup */
+	ret = vmw_user_bo_reference(tfile, srf->res.backup, &backup_handle);
+	mutex_unlock(&dev_priv->cmdbuf_mutex);
+
+	if (unlikely(ret != 0)) {
+		DRM_ERROR("Could not add a reference to a GB surface "
+			  "backup buffer.\n");
+		(void) ttm_ref_object_base_unref(tfile, base->hash.key,
+						 TTM_REF_USAGE);
+		goto out_bad_resource;
+	}
+
+	rep->creq.base.svga3d_flags = SVGA3D_FLAGS_LOWER_32(srf->flags);
+	rep->creq.base.format = srf->format;
+	rep->creq.base.mip_levels = srf->mip_levels[0];
+	rep->creq.base.drm_surface_flags = 0;
+	rep->creq.base.multisample_count = srf->multisample_count;
+	rep->creq.base.autogen_filter = srf->autogen_filter;
+	rep->creq.base.array_size = srf->array_size;
+	rep->creq.base.buffer_handle = backup_handle;
+	rep->creq.base.base_size = srf->base_size;
+	rep->crep.handle = user_srf->prime.base.hash.key;
+	rep->crep.backup_size = srf->res.backup_size;
+	rep->crep.buffer_handle = backup_handle;
+	rep->crep.buffer_map_handle =
+		drm_vma_node_offset_addr(&srf->res.backup->base.vma_node);
+	rep->crep.buffer_size = srf->res.backup->base.num_pages * PAGE_SIZE;
+
+	rep->creq.version = drm_vmw_gb_surface_v1;
+	rep->creq.svga3d_flags_upper_32_bits =
+		SVGA3D_FLAGS_UPPER_32(srf->flags);
+	rep->creq.multisample_pattern = srf->multisample_pattern;
+	rep->creq.quality_level = srf->quality_level;
+	rep->creq.must_be_zero = 0;
+
+out_bad_resource:
+	ttm_base_object_unref(&base);
+
+	return ret;
+}
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c b/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_buffer.c
index 02de216d55a1..31786b200afc 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_buffer.c
@@ -798,7 +798,7 @@ static int vmw_verify_access(struct ttm_buffer_object *bo, struct file *filp)
 	struct ttm_object_file *tfile =
 		vmw_fpriv((struct drm_file *)filp->private_data)->tfile;
 
-	return vmw_user_dmabuf_verify_access(bo, tfile);
+	return vmw_user_bo_verify_access(bo, tfile);
 }
 
 static int vmw_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
@@ -852,7 +852,7 @@ static void vmw_move_notify(struct ttm_buffer_object *bo,
 			    bool evict,
 			    struct ttm_mem_reg *mem)
 {
-	vmw_resource_move_notify(bo, mem);
+	vmw_bo_move_notify(bo, mem);
 	vmw_query_move_notify(bo, mem);
 }
 
@@ -864,7 +864,7 @@ static void vmw_move_notify(struct ttm_buffer_object *bo,
  */
 static void vmw_swap_notify(struct ttm_buffer_object *bo)
 {
-	vmw_resource_swap_notify(bo);
+	vmw_bo_swap_notify(bo);
 	(void) ttm_bo_wait(bo, false, false);
 }